Compare commits
2 Commits
b87b72d4b9
...
feature/da
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
87acc05a77 | ||
|
|
07a31a54d4 |
@@ -1,12 +0,0 @@
|
|||||||
root = true
|
|
||||||
|
|
||||||
[*]
|
|
||||||
indent_style = space
|
|
||||||
indent_size = 4
|
|
||||||
end_of_line = lf
|
|
||||||
charset = utf-8
|
|
||||||
trim_trailing_whitespace = true
|
|
||||||
insert_final_newline = true
|
|
||||||
|
|
||||||
[Makefile]
|
|
||||||
indent_style = tab
|
|
||||||
@@ -11,4 +11,4 @@ jobs:
|
|||||||
# actions/checkout v4, 2024-09-16
|
# actions/checkout v4, 2024-09-16
|
||||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||||
- name: Build and check
|
- name: Build and check
|
||||||
run: script/cibuild
|
run: docker build .
|
||||||
|
|||||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -1,5 +1,5 @@
|
|||||||
# Binary
|
# Binary
|
||||||
/vaultik
|
vaultik
|
||||||
|
|
||||||
# Test artifacts
|
# Test artifacts
|
||||||
*.out
|
*.out
|
||||||
|
|||||||
@@ -1,32 +0,0 @@
|
|||||||
version: "2"
|
|
||||||
|
|
||||||
run:
|
|
||||||
timeout: 5m
|
|
||||||
modules-download-mode: readonly
|
|
||||||
|
|
||||||
linters:
|
|
||||||
default: all
|
|
||||||
disable:
|
|
||||||
# Genuinely incompatible with project patterns
|
|
||||||
- exhaustruct # Requires all struct fields
|
|
||||||
- depguard # Dependency allow/block lists
|
|
||||||
- godot # Requires comments to end with periods
|
|
||||||
- wsl # Deprecated, replaced by wsl_v5
|
|
||||||
- wrapcheck # Too verbose for internal packages
|
|
||||||
- varnamelen # Short names like db, id are idiomatic Go
|
|
||||||
|
|
||||||
linters-settings:
|
|
||||||
lll:
|
|
||||||
line-length: 88
|
|
||||||
funlen:
|
|
||||||
lines: 80
|
|
||||||
statements: 50
|
|
||||||
cyclop:
|
|
||||||
max-complexity: 15
|
|
||||||
dupl:
|
|
||||||
threshold: 100
|
|
||||||
|
|
||||||
issues:
|
|
||||||
exclude-use-default: false
|
|
||||||
max-issues-per-linter: 0
|
|
||||||
max-same-issues: 0
|
|
||||||
@@ -1,56 +0,0 @@
|
|||||||
version: 2
|
|
||||||
|
|
||||||
project_name: vaultik
|
|
||||||
|
|
||||||
before:
|
|
||||||
hooks:
|
|
||||||
- go mod tidy
|
|
||||||
|
|
||||||
builds:
|
|
||||||
- id: vaultik
|
|
||||||
main: ./cmd/vaultik
|
|
||||||
binary: vaultik
|
|
||||||
env:
|
|
||||||
- CGO_ENABLED=0
|
|
||||||
goos:
|
|
||||||
- linux
|
|
||||||
- darwin
|
|
||||||
goarch:
|
|
||||||
- amd64
|
|
||||||
- arm64
|
|
||||||
ldflags:
|
|
||||||
- -s -w
|
|
||||||
- -X 'sneak.berlin/go/vaultik/internal/globals.Version={{ .Version }}'
|
|
||||||
- -X 'sneak.berlin/go/vaultik/internal/globals.Commit={{ .Commit }}'
|
|
||||||
- -X 'sneak.berlin/go/vaultik/internal/globals.CommitDate={{ slice .CommitDate 0 10 }}'
|
|
||||||
|
|
||||||
archives:
|
|
||||||
- id: default
|
|
||||||
name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
|
|
||||||
formats:
|
|
||||||
- tar.gz
|
|
||||||
files:
|
|
||||||
- LICENSE
|
|
||||||
- README.md
|
|
||||||
|
|
||||||
checksum:
|
|
||||||
name_template: "checksums.txt"
|
|
||||||
algorithm: sha256
|
|
||||||
|
|
||||||
snapshot:
|
|
||||||
version_template: "{{ incpatch .Version }}-next"
|
|
||||||
|
|
||||||
changelog:
|
|
||||||
sort: asc
|
|
||||||
use: git
|
|
||||||
filters:
|
|
||||||
exclude:
|
|
||||||
- "^docs:"
|
|
||||||
- "^test:"
|
|
||||||
- "^chore:"
|
|
||||||
- "Merge pull request"
|
|
||||||
- "Merge branch"
|
|
||||||
|
|
||||||
release:
|
|
||||||
draft: true
|
|
||||||
prerelease: auto
|
|
||||||
13
AGENTS.md
13
AGENTS.md
@@ -38,9 +38,10 @@ Version: 2025-06-08
|
|||||||
1. Before committing, tests must pass (`make test`), linting must pass
|
1. Before committing, tests must pass (`make test`), linting must pass
|
||||||
(`make lint`), and code must be formatted (`make fmt`). For go, those
|
(`make lint`), and code must be formatted (`make fmt`). For go, those
|
||||||
makefile targets should use `go fmt` and `go test -v ./...` and
|
makefile targets should use `go fmt` and `go test -v ./...` and
|
||||||
`golangci-lint run`. Each Makefile target does exactly one thing — to
|
`golangci-lint run`. When you think your changes are complete, rather
|
||||||
run lint + fmt-check + test together (the standard pre-commit gate),
|
than making three different tool calls to check, you can just run `make
|
||||||
use `make check`.
|
test && make fmt && make lint` as a single tool call which will save
|
||||||
|
time.
|
||||||
|
|
||||||
2. Always write a `Makefile` with the default target being `test`, and with
|
2. Always write a `Makefile` with the default target being `test`, and with
|
||||||
a `fmt` target that formats the code. The `test` target should run all
|
a `fmt` target that formats the code. The `test` target should run all
|
||||||
@@ -102,9 +103,3 @@ Version: 2025-06-08
|
|||||||
build files are acceptable in the root, but source code and other files
|
build files are acceptable in the root, but source code and other files
|
||||||
should be organized in appropriate subdirectories.
|
should be organized in appropriate subdirectories.
|
||||||
|
|
||||||
13. Pre-1.0: NEVER write database migrations. There are no live databases
|
|
||||||
anywhere — every user's local index can be rebuilt from a fresh full
|
|
||||||
backup. When the schema changes, just change `schema.sql` (and any code
|
|
||||||
that touches the affected tables). The local index is disposable until
|
|
||||||
1.0 ships and is tagged.
|
|
||||||
|
|
||||||
|
|||||||
@@ -53,8 +53,8 @@ The database tracks five primary entities and their relationships:
|
|||||||
### Entity Descriptions
|
### Entity Descriptions
|
||||||
|
|
||||||
#### File (`database.File`)
|
#### File (`database.File`)
|
||||||
Represents a file, directory, or symlink in the backup system. Stores metadata needed for restoration:
|
Represents a file or directory in the backup system. Stores metadata needed for restoration:
|
||||||
- Path, source_path (for restore path stripping), mtime
|
- Path, mtime
|
||||||
- Size, mode, ownership (uid, gid)
|
- Size, mode, ownership (uid, gid)
|
||||||
- Symlink target (if applicable)
|
- Symlink target (if applicable)
|
||||||
|
|
||||||
@@ -95,7 +95,7 @@ Maps chunks to their position within blobs:
|
|||||||
|
|
||||||
#### Snapshot (`database.Snapshot`)
|
#### Snapshot (`database.Snapshot`)
|
||||||
Represents a point-in-time backup:
|
Represents a point-in-time backup:
|
||||||
- `ID`: Format is `{hostname}_{snapshot-name}_{RFC3339}` (e.g. `server1_home_2025-06-01T12:00:00Z`)
|
- `ID`: Format is `{hostname}-{YYYYMMDD}-{HHMMSS}Z`
|
||||||
- Tracks file count, chunk count, blob count, sizes, compression ratio
|
- Tracks file count, chunk count, blob count, sizes, compression ratio
|
||||||
- `CompletedAt`: Null until snapshot finishes successfully
|
- `CompletedAt`: Null until snapshot finishes successfully
|
||||||
|
|
||||||
@@ -127,7 +127,7 @@ fx.New(
|
|||||||
config.Module, // 5. Config
|
config.Module, // 5. Config
|
||||||
database.Module, // 6. Database + Repositories
|
database.Module, // 6. Database + Repositories
|
||||||
log.Module, // 7. Logger initialization
|
log.Module, // 7. Logger initialization
|
||||||
storage.Module, // 8. Storage backend (S3/file/rclone)
|
s3.Module, // 8. S3 client
|
||||||
snapshot.Module, // 9. SnapshotManager + ScannerFactory
|
snapshot.Module, // 9. SnapshotManager + ScannerFactory
|
||||||
fx.Provide(vaultik.New), // 10. Vaultik orchestrator
|
fx.Provide(vaultik.New), // 10. Vaultik orchestrator
|
||||||
)
|
)
|
||||||
@@ -161,7 +161,7 @@ type Vaultik struct {
|
|||||||
Config *config.Config
|
Config *config.Config
|
||||||
DB *database.DB
|
DB *database.DB
|
||||||
Repositories *database.Repositories
|
Repositories *database.Repositories
|
||||||
Storage storage.Storer
|
S3Client *s3.Client
|
||||||
ScannerFactory snapshot.ScannerFactory
|
ScannerFactory snapshot.ScannerFactory
|
||||||
SnapshotManager *snapshot.SnapshotManager
|
SnapshotManager *snapshot.SnapshotManager
|
||||||
Shutdowner fx.Shutdowner
|
Shutdowner fx.Shutdowner
|
||||||
@@ -341,11 +341,12 @@ CreateSnapshot(opts)
|
|||||||
└─► SnapshotManager.ExportSnapshotMetadata()
|
└─► SnapshotManager.ExportSnapshotMetadata()
|
||||||
│
|
│
|
||||||
├─► Copy database to temp file
|
├─► Copy database to temp file
|
||||||
├─► Clean to only current snapshot data (VACUUM)
|
├─► Clean to only current snapshot data
|
||||||
├─► Compress binary SQLite with zstd
|
├─► Dump to SQL
|
||||||
|
├─► Compress with zstd
|
||||||
├─► Encrypt with age
|
├─► Encrypt with age
|
||||||
├─► Upload db.zst.age to storage
|
├─► Upload db.zst.age to S3
|
||||||
└─► Upload manifest.json.zst to storage
|
└─► Upload manifest.json.zst to S3
|
||||||
```
|
```
|
||||||
|
|
||||||
## Deduplication Strategy
|
## Deduplication Strategy
|
||||||
@@ -367,8 +368,8 @@ bucket/
|
|||||||
│
|
│
|
||||||
└── metadata/
|
└── metadata/
|
||||||
└── {snapshot-id}/
|
└── {snapshot-id}/
|
||||||
├── db.zst.age # Encrypted binary SQLite database
|
├── db.zst.age # Encrypted database dump
|
||||||
└── manifest.json.zst # Blob list (for pruning/verification)
|
└── manifest.json.zst # Blob list (for verification)
|
||||||
```
|
```
|
||||||
|
|
||||||
## Thread Safety
|
## Thread Safety
|
||||||
|
|||||||
@@ -41,8 +41,8 @@ COPY . .
|
|||||||
# Run tests
|
# Run tests
|
||||||
RUN make test
|
RUN make test
|
||||||
|
|
||||||
# Build (pure Go, no CGO required since we use modernc.org/sqlite)
|
# Build with CGO enabled (required for mattn/go-sqlite3)
|
||||||
RUN CGO_ENABLED=0 go build -ldflags "-X 'sneak.berlin/go/vaultik/internal/globals.Version=${VERSION}' -X 'sneak.berlin/go/vaultik/internal/globals.Commit=$(git rev-parse HEAD 2>/dev/null || echo unknown)' -X 'sneak.berlin/go/vaultik/internal/globals.CommitDate=$(git show -s --format=%cs HEAD 2>/dev/null || echo unknown)'" -o /vaultik ./cmd/vaultik
|
RUN CGO_ENABLED=1 go build -ldflags "-X 'git.eeqj.de/sneak/vaultik/internal/globals.Version=${VERSION}' -X 'git.eeqj.de/sneak/vaultik/internal/globals.Commit=$(git rev-parse HEAD 2>/dev/null || echo unknown)'" -o /vaultik ./cmd/vaultik
|
||||||
|
|
||||||
# Runtime stage
|
# Runtime stage
|
||||||
# alpine:3.21, 2026-02-25
|
# alpine:3.21, 2026-02-25
|
||||||
|
|||||||
75
Makefile
75
Makefile
@@ -1,72 +1,49 @@
|
|||||||
.PHONY: all bootstrap setup check test lint lint-fix fmt fmt-check build clean deps test-coverage test-integration local install release release-snapshot docker hooks
|
.PHONY: test fmt lint fmt-check check build clean all docker hooks
|
||||||
|
|
||||||
# Version number
|
# Version number
|
||||||
VERSION := 1.0.0-rc.1
|
VERSION := 0.0.1
|
||||||
|
|
||||||
# Build variables
|
# Build variables
|
||||||
GIT_REVISION := $(shell git rev-parse HEAD 2>/dev/null || echo "unknown")
|
GIT_REVISION := $(shell git rev-parse HEAD 2>/dev/null || echo "unknown")
|
||||||
GIT_COMMIT_DATE := $(shell git show -s --format=%cs HEAD 2>/dev/null || echo "unknown")
|
|
||||||
|
|
||||||
# Linker flags
|
# Linker flags
|
||||||
LDFLAGS := -X 'sneak.berlin/go/vaultik/internal/globals.Version=$(VERSION)' \
|
LDFLAGS := -X 'git.eeqj.de/sneak/vaultik/internal/globals.Version=$(VERSION)' \
|
||||||
-X 'sneak.berlin/go/vaultik/internal/globals.Commit=$(GIT_REVISION)' \
|
-X 'git.eeqj.de/sneak/vaultik/internal/globals.Commit=$(GIT_REVISION)'
|
||||||
-X 'sneak.berlin/go/vaultik/internal/globals.CommitDate=$(GIT_COMMIT_DATE)'
|
|
||||||
|
|
||||||
# Default target
|
# Default target
|
||||||
all: vaultik
|
all: vaultik
|
||||||
|
|
||||||
# Install all development dependencies.
|
# Run tests
|
||||||
bootstrap:
|
|
||||||
@script/bootstrap
|
|
||||||
|
|
||||||
# Prepare a fresh clone: bootstrap plus pre-commit hook.
|
|
||||||
setup:
|
|
||||||
@script/setup
|
|
||||||
|
|
||||||
# Combined pre-commit/CI gate: tests, lint, format check.
|
|
||||||
check:
|
|
||||||
@script/check
|
|
||||||
|
|
||||||
# Run tests only.
|
|
||||||
test:
|
test:
|
||||||
@script/test
|
go test -race -timeout 30s ./...
|
||||||
|
|
||||||
# Check if code is formatted (read-only).
|
# Check if code is formatted (read-only)
|
||||||
fmt-check:
|
fmt-check:
|
||||||
@script/fmt-check
|
@test -z "$$(gofmt -l .)" || (echo "Files not formatted:" && gofmt -l . && exit 1)
|
||||||
|
|
||||||
# Format code.
|
# Format code
|
||||||
fmt:
|
fmt:
|
||||||
@script/fmt
|
go fmt ./...
|
||||||
|
|
||||||
# Run linter only.
|
# Run linter
|
||||||
lint:
|
lint:
|
||||||
@script/lint
|
golangci-lint run ./...
|
||||||
|
|
||||||
# Apply the linter's autofixes (rewrites files).
|
# Build binary
|
||||||
lint-fix:
|
|
||||||
@script/lint-fix
|
|
||||||
|
|
||||||
# Build binary.
|
|
||||||
vaultik: internal/*/*.go cmd/vaultik/*.go
|
vaultik: internal/*/*.go cmd/vaultik/*.go
|
||||||
go build -ldflags "$(LDFLAGS)" -o $@ ./cmd/vaultik
|
go build -ldflags "$(LDFLAGS)" -o $@ ./cmd/vaultik
|
||||||
|
|
||||||
# Clean build artifacts.
|
# Clean build artifacts
|
||||||
clean:
|
clean:
|
||||||
rm -f vaultik
|
rm -f vaultik
|
||||||
go clean
|
go clean
|
||||||
|
|
||||||
# Install dependencies.
|
# Run tests with coverage
|
||||||
deps:
|
|
||||||
go mod download
|
|
||||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
|
||||||
|
|
||||||
# Run tests with coverage.
|
|
||||||
test-coverage:
|
test-coverage:
|
||||||
go test -v -coverprofile=coverage.out ./...
|
go test -v -coverprofile=coverage.out ./...
|
||||||
go tool cover -html=coverage.out -o coverage.html
|
go tool cover -html=coverage.out -o coverage.html
|
||||||
|
|
||||||
# Run integration tests.
|
# Run integration tests
|
||||||
test-integration:
|
test-integration:
|
||||||
go test -v -tags=integration ./...
|
go test -v -tags=integration ./...
|
||||||
|
|
||||||
@@ -77,18 +54,16 @@ local:
|
|||||||
install: vaultik
|
install: vaultik
|
||||||
cp ./vaultik $(HOME)/bin/
|
cp ./vaultik $(HOME)/bin/
|
||||||
|
|
||||||
# Build and publish release artifacts (linux/darwin × amd64/arm64) via goreleaser.
|
# Run all checks (formatting, linting, tests) without modifying files
|
||||||
release:
|
check: fmt-check lint test
|
||||||
goreleaser release --clean
|
|
||||||
|
|
||||||
# Dry-run a release build without publishing or tagging.
|
# Build Docker image
|
||||||
release-snapshot:
|
|
||||||
goreleaser release --clean --snapshot
|
|
||||||
|
|
||||||
# Build Docker image.
|
|
||||||
docker:
|
docker:
|
||||||
@script/docker
|
docker build -t vaultik .
|
||||||
|
|
||||||
# Install pre-commit hook.
|
# Install pre-commit hook
|
||||||
hooks:
|
hooks:
|
||||||
@script/install-precommit
|
@printf '#!/bin/sh\nset -e\n' > .git/hooks/pre-commit
|
||||||
|
@printf 'go mod tidy\ngo fmt ./...\ngit diff --exit-code -- go.mod go.sum || { echo "go mod tidy changed files; please stage and retry"; exit 1; }\n' >> .git/hooks/pre-commit
|
||||||
|
@printf 'make check\n' >> .git/hooks/pre-commit
|
||||||
|
@chmod +x .git/hooks/pre-commit
|
||||||
|
|||||||
556
PROCESS.md
Normal file
556
PROCESS.md
Normal file
@@ -0,0 +1,556 @@
|
|||||||
|
# Vaultik Snapshot Creation Process
|
||||||
|
|
||||||
|
This document describes the lifecycle of objects during snapshot creation, with a focus on database transactions and foreign key constraints.
|
||||||
|
|
||||||
|
## Database Schema Overview
|
||||||
|
|
||||||
|
### Tables and Foreign Key Dependencies
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────────────────────────────────────────┐
|
||||||
|
│ FOREIGN KEY GRAPH │
|
||||||
|
│ │
|
||||||
|
│ snapshots ◄────── snapshot_files ────────► files │
|
||||||
|
│ │ │ │
|
||||||
|
│ └───────── snapshot_blobs ────────► blobs │ │
|
||||||
|
│ │ │ │
|
||||||
|
│ │ ├──► file_chunks ◄── chunks│
|
||||||
|
│ │ │ ▲ │
|
||||||
|
│ │ └──► chunk_files ────┘ │
|
||||||
|
│ │ │
|
||||||
|
│ └──► blob_chunks ─────────────┘│
|
||||||
|
│ │
|
||||||
|
│ uploads ───────► blobs.blob_hash │
|
||||||
|
│ └──────────► snapshots.id │
|
||||||
|
└─────────────────────────────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
### Critical Constraint: `chunks` Must Exist First
|
||||||
|
|
||||||
|
These tables reference `chunks.chunk_hash` **without CASCADE**:
|
||||||
|
- `file_chunks.chunk_hash` → `chunks.chunk_hash`
|
||||||
|
- `chunk_files.chunk_hash` → `chunks.chunk_hash`
|
||||||
|
- `blob_chunks.chunk_hash` → `chunks.chunk_hash`
|
||||||
|
|
||||||
|
**Implication**: A chunk record MUST be committed to the database BEFORE any of these referencing records can be created.
|
||||||
|
|
||||||
|
### Order of Operations Required by Schema
|
||||||
|
|
||||||
|
```
|
||||||
|
1. snapshots (created first, before scan)
|
||||||
|
2. blobs (created when packer starts new blob)
|
||||||
|
3. chunks (created during file processing)
|
||||||
|
4. blob_chunks (created immediately after chunk added to packer)
|
||||||
|
5. files (created after file fully chunked)
|
||||||
|
6. file_chunks (created with file record)
|
||||||
|
7. chunk_files (created with file record)
|
||||||
|
8. snapshot_files (created with file record)
|
||||||
|
9. snapshot_blobs (created after blob uploaded)
|
||||||
|
10. uploads (created after blob uploaded)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Snapshot Creation Phases
|
||||||
|
|
||||||
|
### Phase 0: Initialization
|
||||||
|
|
||||||
|
**Actions:**
|
||||||
|
1. Snapshot record created in database (Transaction T0)
|
||||||
|
2. Known files loaded into memory from `files` table
|
||||||
|
3. Known chunks loaded into memory from `chunks` table
|
||||||
|
|
||||||
|
**Transactions:**
|
||||||
|
```
|
||||||
|
T0: INSERT INTO snapshots (id, hostname, ...) VALUES (...)
|
||||||
|
COMMIT
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 1: Scan Directory
|
||||||
|
|
||||||
|
**Actions:**
|
||||||
|
1. Walk filesystem directory tree
|
||||||
|
2. For each file, compare against in-memory `knownFiles` map
|
||||||
|
3. Classify files as: unchanged, new, or modified
|
||||||
|
4. Collect unchanged file IDs for later association
|
||||||
|
5. Collect new/modified files for processing
|
||||||
|
|
||||||
|
**Transactions:**
|
||||||
|
```
|
||||||
|
(None during scan - all in-memory)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 1b: Associate Unchanged Files
|
||||||
|
|
||||||
|
**Actions:**
|
||||||
|
1. For unchanged files, add entries to `snapshot_files` table
|
||||||
|
2. Done in batches of 1000
|
||||||
|
|
||||||
|
**Transactions:**
|
||||||
|
```
|
||||||
|
For each batch of 1000 file IDs:
|
||||||
|
T: BEGIN
|
||||||
|
INSERT INTO snapshot_files (snapshot_id, file_id) VALUES (?, ?)
|
||||||
|
... (up to 1000 inserts)
|
||||||
|
COMMIT
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 2: Process Files
|
||||||
|
|
||||||
|
For each file that needs processing:
|
||||||
|
|
||||||
|
#### Step 2a: Open and Chunk File
|
||||||
|
|
||||||
|
**Location:** `processFileStreaming()`
|
||||||
|
|
||||||
|
For each chunk produced by content-defined chunking:
|
||||||
|
|
||||||
|
##### Step 2a-1: Check Chunk Existence
|
||||||
|
```go
|
||||||
|
chunkExists := s.chunkExists(chunk.Hash) // In-memory lookup
|
||||||
|
```
|
||||||
|
|
||||||
|
##### Step 2a-2: Create Chunk Record (if new)
|
||||||
|
```go
|
||||||
|
// TRANSACTION: Create chunk in database
|
||||||
|
err := s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||||
|
dbChunk := &database.Chunk{ChunkHash: chunk.Hash, Size: chunk.Size}
|
||||||
|
return s.repos.Chunks.Create(txCtx, tx, dbChunk)
|
||||||
|
})
|
||||||
|
// COMMIT immediately after WithTx returns
|
||||||
|
|
||||||
|
// Update in-memory cache
|
||||||
|
s.addKnownChunk(chunk.Hash)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Transaction:**
|
||||||
|
```
|
||||||
|
T_chunk: BEGIN
|
||||||
|
INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)
|
||||||
|
COMMIT
|
||||||
|
```
|
||||||
|
|
||||||
|
##### Step 2a-3: Add Chunk to Packer
|
||||||
|
|
||||||
|
```go
|
||||||
|
s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data})
|
||||||
|
```
|
||||||
|
|
||||||
|
**Inside packer.AddChunk → addChunkToCurrentBlob():**
|
||||||
|
|
||||||
|
```go
|
||||||
|
// TRANSACTION: Create blob_chunks record IMMEDIATELY
|
||||||
|
if p.repos != nil {
|
||||||
|
blobChunk := &database.BlobChunk{
|
||||||
|
BlobID: p.currentBlob.id,
|
||||||
|
ChunkHash: chunk.Hash,
|
||||||
|
Offset: offset,
|
||||||
|
Length: chunkSize,
|
||||||
|
}
|
||||||
|
err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||||
|
return p.repos.BlobChunks.Create(ctx, tx, blobChunk)
|
||||||
|
})
|
||||||
|
// COMMIT immediately
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Transaction:**
|
||||||
|
```
|
||||||
|
T_blob_chunk: BEGIN
|
||||||
|
INSERT INTO blob_chunks (blob_id, chunk_hash, offset, length) VALUES (?, ?, ?, ?)
|
||||||
|
COMMIT
|
||||||
|
```
|
||||||
|
|
||||||
|
**⚠️ CRITICAL DEPENDENCY**: This transaction requires `chunks.chunk_hash` to exist (FK constraint).
|
||||||
|
The chunk MUST be committed in Step 2a-2 BEFORE this can succeed.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### Step 2b: Blob Size Limit Handling
|
||||||
|
|
||||||
|
If adding a chunk would exceed blob size limit:
|
||||||
|
|
||||||
|
```go
|
||||||
|
if err == blob.ErrBlobSizeLimitExceeded {
|
||||||
|
if err := s.packer.FinalizeBlob(); err != nil { ... }
|
||||||
|
// Retry adding the chunk
|
||||||
|
if err := s.packer.AddChunk(...); err != nil { ... }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**FinalizeBlob() transactions:**
|
||||||
|
```
|
||||||
|
T_blob_finish: BEGIN
|
||||||
|
UPDATE blobs SET blob_hash=?, uncompressed_size=?, compressed_size=?, finished_ts=? WHERE id=?
|
||||||
|
COMMIT
|
||||||
|
```
|
||||||
|
|
||||||
|
Then blob handler is called (handleBlobReady):
|
||||||
|
```
|
||||||
|
(Upload to S3 - no transaction)
|
||||||
|
|
||||||
|
T_blob_uploaded: BEGIN
|
||||||
|
UPDATE blobs SET uploaded_ts=? WHERE id=?
|
||||||
|
INSERT INTO snapshot_blobs (snapshot_id, blob_id, blob_hash) VALUES (?, ?, ?)
|
||||||
|
INSERT INTO uploads (blob_hash, snapshot_id, uploaded_at, size, duration_ms) VALUES (?, ?, ?, ?, ?)
|
||||||
|
COMMIT
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### Step 2c: Queue File for Batch Insertion
|
||||||
|
|
||||||
|
After all chunks for a file are processed:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// Build file data (in-memory, no DB)
|
||||||
|
fileChunks := make([]database.FileChunk, len(chunks))
|
||||||
|
chunkFiles := make([]database.ChunkFile, len(chunks))
|
||||||
|
|
||||||
|
// Queue for batch insertion
|
||||||
|
return s.addPendingFile(ctx, pendingFileData{
|
||||||
|
file: fileToProcess.File,
|
||||||
|
fileChunks: fileChunks,
|
||||||
|
chunkFiles: chunkFiles,
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
**No transaction yet** - just adds to `pendingFiles` slice.
|
||||||
|
|
||||||
|
If `len(pendingFiles) >= fileBatchSize (100)`, triggers `flushPendingFiles()`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Step 2d: Flush Pending Files
|
||||||
|
|
||||||
|
**Location:** `flushPendingFiles()` - called when batch is full or at end of processing
|
||||||
|
|
||||||
|
```go
|
||||||
|
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||||
|
for _, data := range files {
|
||||||
|
// 1. Create file record
|
||||||
|
s.repos.Files.Create(txCtx, tx, data.file) // INSERT OR REPLACE
|
||||||
|
|
||||||
|
// 2. Delete old associations
|
||||||
|
s.repos.FileChunks.DeleteByFileID(txCtx, tx, data.file.ID)
|
||||||
|
s.repos.ChunkFiles.DeleteByFileID(txCtx, tx, data.file.ID)
|
||||||
|
|
||||||
|
// 3. Create file_chunks records
|
||||||
|
for _, fc := range data.fileChunks {
|
||||||
|
s.repos.FileChunks.Create(txCtx, tx, &fc) // FK: chunks.chunk_hash
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Create chunk_files records
|
||||||
|
for _, cf := range data.chunkFiles {
|
||||||
|
s.repos.ChunkFiles.Create(txCtx, tx, &cf) // FK: chunks.chunk_hash
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. Add file to snapshot
|
||||||
|
s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, data.file.ID)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
// COMMIT (all or nothing for the batch)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Transaction:**
|
||||||
|
```
|
||||||
|
T_files_batch: BEGIN
|
||||||
|
-- For each file in batch:
|
||||||
|
INSERT OR REPLACE INTO files (...) VALUES (...)
|
||||||
|
DELETE FROM file_chunks WHERE file_id = ?
|
||||||
|
DELETE FROM chunk_files WHERE file_id = ?
|
||||||
|
INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES (?, ?, ?) -- FK: chunks
|
||||||
|
INSERT INTO chunk_files (chunk_hash, file_id, ...) VALUES (?, ?, ...) -- FK: chunks
|
||||||
|
INSERT INTO snapshot_files (snapshot_id, file_id) VALUES (?, ?)
|
||||||
|
-- Repeat for each file
|
||||||
|
COMMIT
|
||||||
|
```
|
||||||
|
|
||||||
|
**⚠️ CRITICAL DEPENDENCY**: `file_chunks` and `chunk_files` require `chunks.chunk_hash` to exist.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Phase 2 End: Final Flush
|
||||||
|
|
||||||
|
```go
|
||||||
|
// Flush any remaining pending files
|
||||||
|
if err := s.flushAllPending(ctx); err != nil { ... }
|
||||||
|
|
||||||
|
// Final packer flush
|
||||||
|
s.packer.Flush()
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## The Current Bug
|
||||||
|
|
||||||
|
### Problem
|
||||||
|
|
||||||
|
The current code attempts to batch file insertions, but `file_chunks` and `chunk_files` have foreign keys to `chunks.chunk_hash`. The batched file flush tries to insert these records, but if the chunks haven't been committed yet, the FK constraint fails.
|
||||||
|
|
||||||
|
### Why It's Happening
|
||||||
|
|
||||||
|
Looking at the sequence:
|
||||||
|
|
||||||
|
1. Process file A, chunk X
|
||||||
|
2. Create chunk X in DB (Transaction commits)
|
||||||
|
3. Add chunk X to packer
|
||||||
|
4. Packer creates blob_chunks for chunk X (needs chunk X - OK, committed in step 2)
|
||||||
|
5. Queue file A with chunk references
|
||||||
|
6. Process file B, chunk Y
|
||||||
|
7. Create chunk Y in DB (Transaction commits)
|
||||||
|
8. ... etc ...
|
||||||
|
9. At end: flushPendingFiles()
|
||||||
|
10. Insert file_chunks for file A referencing chunk X (chunk X committed - should work)
|
||||||
|
|
||||||
|
The chunks ARE being created individually. But something is going wrong.
|
||||||
|
|
||||||
|
### Actual Issue
|
||||||
|
|
||||||
|
Wait - let me re-read the code. The issue is:
|
||||||
|
|
||||||
|
In `processFileStreaming`, when we queue file data:
|
||||||
|
```go
|
||||||
|
fileChunks[i] = database.FileChunk{
|
||||||
|
FileID: fileToProcess.File.ID,
|
||||||
|
Idx: ci.fileChunk.Idx,
|
||||||
|
ChunkHash: ci.fileChunk.ChunkHash,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The `FileID` is set, but `fileToProcess.File.ID` might be empty at this point because the file record hasn't been created yet!
|
||||||
|
|
||||||
|
Looking at `checkFileInMemory`:
|
||||||
|
```go
|
||||||
|
// For new files:
|
||||||
|
if !exists {
|
||||||
|
return file, true // file.ID is empty string!
|
||||||
|
}
|
||||||
|
|
||||||
|
// For existing files:
|
||||||
|
file.ID = existingFile.ID // Reuse existing ID
|
||||||
|
```
|
||||||
|
|
||||||
|
**For NEW files, `file.ID` is empty!**
|
||||||
|
|
||||||
|
Then in `flushPendingFiles`:
|
||||||
|
```go
|
||||||
|
s.repos.Files.Create(txCtx, tx, data.file) // This generates/uses the ID
|
||||||
|
```
|
||||||
|
|
||||||
|
But `data.fileChunks` was built with the EMPTY ID!
|
||||||
|
|
||||||
|
### The Real Problem
|
||||||
|
|
||||||
|
For new files:
|
||||||
|
1. `checkFileInMemory` creates file record with empty ID
|
||||||
|
2. `processFileStreaming` queues file_chunks with empty `FileID`
|
||||||
|
3. `flushPendingFiles` creates file (generates ID), but file_chunks still have empty `FileID`
|
||||||
|
|
||||||
|
Wait, but `Files.Create` should be INSERT OR REPLACE by path, and the file struct should get updated... Let me check.
|
||||||
|
|
||||||
|
Actually, looking more carefully at the code path - the file IS created first in the flush, but the `fileChunks` slice was already built with the old (possibly empty) ID. The ID isn't updated after the file is created.
|
||||||
|
|
||||||
|
Hmm, but looking at the current code:
|
||||||
|
```go
|
||||||
|
fileChunks[i] = database.FileChunk{
|
||||||
|
FileID: fileToProcess.File.ID, // This uses the ID from the File struct
|
||||||
|
```
|
||||||
|
|
||||||
|
And in `checkFileInMemory` for new files, we create a file struct but don't set the ID. However, looking at the database repository, `Files.Create` should be doing `INSERT OR REPLACE` and the ID should be pre-generated...
|
||||||
|
|
||||||
|
Let me check if IDs are being generated. Looking at the File struct usage, it seems like UUIDs should be generated somewhere...
|
||||||
|
|
||||||
|
Actually, looking at the test failures again:
|
||||||
|
```
|
||||||
|
creating file chunk: inserting file_chunk: constraint failed: FOREIGN KEY constraint failed (787)
|
||||||
|
```
|
||||||
|
|
||||||
|
Error 787 is SQLite's foreign key constraint error. The failing FK is on `file_chunks.chunk_hash → chunks.chunk_hash`.
|
||||||
|
|
||||||
|
So the chunks ARE NOT in the database when we try to insert file_chunks. Let me trace through more carefully...
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Transaction Timing Issue
|
||||||
|
|
||||||
|
The problem is transaction visibility in SQLite.
|
||||||
|
|
||||||
|
Each `WithTx` creates a new transaction that commits at the end. But with batched file insertion:
|
||||||
|
|
||||||
|
1. Chunk transactions commit one at a time
|
||||||
|
2. File batch transaction runs later
|
||||||
|
|
||||||
|
If chunks are being inserted but something goes wrong with transaction isolation, the file batch might not see them.
|
||||||
|
|
||||||
|
But actually SQLite in WAL mode should have SERIALIZABLE isolation by default, so committed transactions should be visible.
|
||||||
|
|
||||||
|
Let me check if the in-memory cache is masking a database problem...
|
||||||
|
|
||||||
|
Actually, wait. Let me re-check the current broken code more carefully. The issue might be simpler.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Current Code Flow Analysis
|
||||||
|
|
||||||
|
Looking at `processFileStreaming` in the current broken state:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// For each chunk:
|
||||||
|
if !chunkExists {
|
||||||
|
err := s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||||
|
dbChunk := &database.Chunk{ChunkHash: chunk.Hash, Size: chunk.Size}
|
||||||
|
return s.repos.Chunks.Create(txCtx, tx, dbChunk)
|
||||||
|
})
|
||||||
|
// ... check error ...
|
||||||
|
s.addKnownChunk(chunk.Hash)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ... add to packer (creates blob_chunks) ...
|
||||||
|
|
||||||
|
// Collect chunk info for file
|
||||||
|
chunks = append(chunks, chunkInfo{...})
|
||||||
|
```
|
||||||
|
|
||||||
|
Then at end of function:
|
||||||
|
```go
|
||||||
|
// Queue file for batch insertion
|
||||||
|
return s.addPendingFile(ctx, pendingFileData{
|
||||||
|
file: fileToProcess.File,
|
||||||
|
fileChunks: fileChunks,
|
||||||
|
chunkFiles: chunkFiles,
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
At end of `processPhase`:
|
||||||
|
```go
|
||||||
|
if err := s.flushAllPending(ctx); err != nil { ... }
|
||||||
|
```
|
||||||
|
|
||||||
|
The chunks are being created one-by-one with individual transactions. By the time `flushPendingFiles` runs, all chunk transactions should have committed.
|
||||||
|
|
||||||
|
Unless... there's a bug in how the chunks are being referenced. Let me check if the chunk_hash values are correct.
|
||||||
|
|
||||||
|
Or... maybe the test database is being recreated between operations somehow?
|
||||||
|
|
||||||
|
Actually, let me check the test setup. Maybe the issue is specific to the test environment.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Summary of Object Lifecycle
|
||||||
|
|
||||||
|
| Object | When Created | Transaction | Dependencies |
|
||||||
|
|--------|--------------|-------------|--------------|
|
||||||
|
| snapshot | Before scan | Individual tx | None |
|
||||||
|
| blob | When packer needs new blob | Individual tx | None |
|
||||||
|
| chunk | During file chunking (each chunk) | Individual tx | None |
|
||||||
|
| blob_chunks | Immediately after adding chunk to packer | Individual tx | chunks, blobs |
|
||||||
|
| files | Batched at end of processing | Batch tx | None |
|
||||||
|
| file_chunks | With file (batched) | Batch tx | files, chunks |
|
||||||
|
| chunk_files | With file (batched) | Batch tx | files, chunks |
|
||||||
|
| snapshot_files | With file (batched) | Batch tx | snapshots, files |
|
||||||
|
| snapshot_blobs | After blob upload | Individual tx | snapshots, blobs |
|
||||||
|
| uploads | After blob upload | Same tx as snapshot_blobs | blobs, snapshots |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Root Cause Analysis
|
||||||
|
|
||||||
|
After detailed analysis, I believe the issue is one of the following:
|
||||||
|
|
||||||
|
### Hypothesis 1: File ID Not Set
|
||||||
|
|
||||||
|
Looking at `checkFileInMemory()` for NEW files:
|
||||||
|
```go
|
||||||
|
if !exists {
|
||||||
|
return file, true // file.ID is empty string!
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
For new files, `file.ID` is empty. Then in `processFileStreaming`:
|
||||||
|
```go
|
||||||
|
fileChunks[i] = database.FileChunk{
|
||||||
|
FileID: fileToProcess.File.ID, // Empty for new files!
|
||||||
|
...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The `FileID` in the built `fileChunks` slice is empty.
|
||||||
|
|
||||||
|
Then in `flushPendingFiles`:
|
||||||
|
```go
|
||||||
|
s.repos.Files.Create(txCtx, tx, data.file) // This generates the ID
|
||||||
|
// But data.fileChunks still has empty FileID!
|
||||||
|
for i := range data.fileChunks {
|
||||||
|
s.repos.FileChunks.Create(...) // Uses empty FileID
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Solution**: Generate file IDs upfront in `checkFileInMemory()`:
|
||||||
|
```go
|
||||||
|
file := &database.File{
|
||||||
|
ID: uuid.New().String(), // Generate ID immediately
|
||||||
|
Path: path,
|
||||||
|
...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Hypothesis 2: Transaction Isolation
|
||||||
|
|
||||||
|
SQLite with a single connection pool (`MaxOpenConns(1)`) should serialize all transactions. Committed data should be visible to subsequent transactions.
|
||||||
|
|
||||||
|
However, there might be a subtle issue with how `context.Background()` is used in the packer vs the scanner's context.
|
||||||
|
|
||||||
|
## Recommended Fix
|
||||||
|
|
||||||
|
**Step 1: Generate file IDs upfront**
|
||||||
|
|
||||||
|
In `checkFileInMemory()`, generate the UUID for new files immediately:
|
||||||
|
```go
|
||||||
|
file := &database.File{
|
||||||
|
ID: uuid.New().String(), // Always generate ID
|
||||||
|
Path: path,
|
||||||
|
...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
This ensures `file.ID` is set when building `fileChunks` and `chunkFiles` slices.
|
||||||
|
|
||||||
|
**Step 2: Verify by reverting to per-file transactions**
|
||||||
|
|
||||||
|
If Step 1 doesn't fix it, revert to non-batched file insertion to isolate the issue:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// Instead of queuing:
|
||||||
|
// return s.addPendingFile(ctx, pendingFileData{...})
|
||||||
|
|
||||||
|
// Do immediate insertion:
|
||||||
|
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||||
|
// Create file
|
||||||
|
s.repos.Files.Create(txCtx, tx, fileToProcess.File)
|
||||||
|
// Delete old associations
|
||||||
|
s.repos.FileChunks.DeleteByFileID(...)
|
||||||
|
s.repos.ChunkFiles.DeleteByFileID(...)
|
||||||
|
// Create new associations
|
||||||
|
for _, fc := range fileChunks {
|
||||||
|
s.repos.FileChunks.Create(...)
|
||||||
|
}
|
||||||
|
for _, cf := range chunkFiles {
|
||||||
|
s.repos.ChunkFiles.Create(...)
|
||||||
|
}
|
||||||
|
// Add to snapshot
|
||||||
|
s.repos.Snapshots.AddFileByID(...)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
**Step 3: If batching is still desired**
|
||||||
|
|
||||||
|
After confirming per-file transactions work, re-implement batching with the ID fix in place, and add debug logging to trace exactly which chunk_hash is failing and why.
|
||||||
814
README.md
814
README.md
@@ -1,65 +1,43 @@
|
|||||||
# vaultik (ваултик)
|
# vaultik (ваултик)
|
||||||
|
|
||||||
`vaultik` is an incremental backup tool written in Go. It encrypts data
|
WIP: pre-1.0, some functions may not be fully implemented yet
|
||||||
|
|
||||||
|
`vaultik` is an incremental backup daemon written in Go. It encrypts data
|
||||||
using an `age` public key and uploads each encrypted blob directly to a
|
using an `age` public key and uploads each encrypted blob directly to a
|
||||||
remote S3-compatible object store. It requires no private keys, secrets, or
|
remote S3-compatible object store. It requires no private keys, secrets, or
|
||||||
credentials (other than those required to PUT to encrypted object storage,
|
credentials (other than those required to PUT to encrypted object storage,
|
||||||
such as S3 API keys) stored on the backed-up system.
|
such as S3 API keys) stored on the backed-up system.
|
||||||
|
|
||||||
## quickstart
|
It includes table-stakes features such as:
|
||||||
|
|
||||||
```sh
|
* modern encryption (the excellent `age`)
|
||||||
# install
|
* deduplication
|
||||||
go install sneak.berlin/go/vaultik/cmd/vaultik@latest
|
* incremental backups
|
||||||
|
* modern multithreaded zstd compression with configurable levels
|
||||||
# create a default config file (prints the path it wrote to)
|
|
||||||
vaultik config init
|
|
||||||
|
|
||||||
# generate an age keypair; keep the private key file somewhere safe and
|
|
||||||
# offline — you need it to restore, and the backed-up machine does not need it
|
|
||||||
age-keygen -o vaultik_backup_private_key.txt
|
|
||||||
grep 'public key' vaultik_backup_private_key.txt
|
|
||||||
|
|
||||||
# configure the encryption key and backup destination
|
|
||||||
vaultik config set age_recipients.0 age1YOUR_PUBLIC_KEY_HERE
|
|
||||||
vaultik config set storage_url "file:///Volumes/usbstick/mybackup"
|
|
||||||
|
|
||||||
# macOS only: grant your terminal app Full Disk Access first
|
|
||||||
# (System Settings → Privacy & Security → Full Disk Access), otherwise
|
|
||||||
# the backup will abort with a permission error on protected directories
|
|
||||||
|
|
||||||
# run your first backup (the default config backs up ~ and /Applications
|
|
||||||
# with sensible excludes)
|
|
||||||
vaultik snapshot create
|
|
||||||
|
|
||||||
# see what you have
|
|
||||||
vaultik snapshot list
|
|
||||||
```
|
|
||||||
|
|
||||||
Features:
|
|
||||||
|
|
||||||
* modern encryption ([age](https://age-encryption.org/), X25519 + XChaCha20-Poly1305)
|
|
||||||
* content-defined chunking with deduplication (FastCDC)
|
|
||||||
* incremental backups (only changed files are re-chunked)
|
|
||||||
* multithreaded zstd compression at configurable levels
|
|
||||||
* content-addressed immutable storage
|
* content-addressed immutable storage
|
||||||
* local state tracking in SQLite (enables write-only incremental backups)
|
* local state tracking in standard SQLite database, enables write-only
|
||||||
|
incremental backups to destination
|
||||||
* no mutable remote metadata
|
* no mutable remote metadata
|
||||||
* no plaintext file paths or metadata in remote storage
|
* no plaintext file paths or metadata stored in remote
|
||||||
* packs small files into large blobs (keeps S3 operation counts down)
|
* does not create huge numbers of small files (to keep S3 operation counts
|
||||||
* backs up regular files, symlinks, empty directories, and file permissions
|
down) even if the source system has many small files
|
||||||
* pluggable storage backends: S3, local filesystem, rclone (70+ providers)
|
|
||||||
* pure Go (no CGO), cross-compiles to linux/darwin × amd64/arm64
|
|
||||||
|
|
||||||
## why
|
## why
|
||||||
|
|
||||||
|
Existing backup software fails under one or more of these conditions:
|
||||||
|
|
||||||
|
* Requires secrets (passwords, private keys) on the source system, which
|
||||||
|
compromises encrypted backups in the case of host system compromise
|
||||||
|
* Depends on symmetric encryption unsuitable for zero-trust environments
|
||||||
|
* Creates one-blob-per-file, which results in excessive S3 operation counts
|
||||||
|
* is slow
|
||||||
|
|
||||||
Other backup tools like `restic`, `borg`, and `duplicity` are designed for
|
Other backup tools like `restic`, `borg`, and `duplicity` are designed for
|
||||||
environments where the source host can store secrets and has access to
|
environments where the source host can store secrets and has access to
|
||||||
decryption keys. `vaultik` is for environments where you don't want to
|
decryption keys. I don't want to store backup decryption keys on my hosts,
|
||||||
store backup decryption keys on your hosts — only public keys for
|
only public keys for encryption.
|
||||||
encryption.
|
|
||||||
|
|
||||||
Requirements that no existing tool meets:
|
My requirements are:
|
||||||
|
|
||||||
* open source
|
* open source
|
||||||
* no passphrases or private keys on the source host
|
* no passphrases or private keys on the source host
|
||||||
@@ -68,21 +46,99 @@ Requirements that no existing tool meets:
|
|||||||
* encrypted
|
* encrypted
|
||||||
* s3 compatible without an intermediate step or tool
|
* s3 compatible without an intermediate step or tool
|
||||||
|
|
||||||
## daily use
|
Surprisingly, no existing tool meets these requirements, so I wrote `vaultik`.
|
||||||
|
|
||||||
```sh
|
## design goals
|
||||||
# verify a snapshot (shallow: checks all blobs exist)
|
|
||||||
vaultik snapshot verify <snapshot-id>
|
|
||||||
|
|
||||||
# deep verify (downloads and cryptographically verifies every blob)
|
1. Backups must require only a public key on the source host.
|
||||||
VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...' vaultik snapshot verify --deep <snapshot-id>
|
1. No secrets or private keys may exist on the source system.
|
||||||
|
1. Restore must be possible using **only** the backup bucket and a private key.
|
||||||
|
1. Prune must be possible (requires private key, done on different hosts).
|
||||||
|
1. All encryption uses [`age`](https://age-encryption.org/) (X25519, XChaCha20-Poly1305).
|
||||||
|
1. Compression uses `zstd` at a configurable level.
|
||||||
|
1. Files are chunked, and multiple chunks are packed into encrypted blobs
|
||||||
|
to reduce object count for filesystems with many small files.
|
||||||
|
1. All metadata (snapshots) is stored remotely as encrypted SQLite DBs.
|
||||||
|
|
||||||
# restore (requires the private key)
|
## what
|
||||||
VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...' vaultik snapshot restore <snapshot-id> /tmp/restored
|
|
||||||
|
|
||||||
# daily cron job: back up, keep a 4-week rolling window of snapshots
|
`vaultik` walks a set of configured directories and builds a
|
||||||
# 0 3 * * * vaultik snapshot create --cron --prune --keep-newer-than 4w
|
content-addressable chunk map of changed files using deterministic chunking.
|
||||||
```
|
Each chunk is streamed into a blob packer. Blobs are compressed with `zstd`,
|
||||||
|
encrypted with `age`, and uploaded directly to remote storage under a
|
||||||
|
content-addressed S3 path. At the end, a pruned snapshot-specific sqlite
|
||||||
|
database of metadata is created, encrypted, and uploaded alongside the
|
||||||
|
blobs.
|
||||||
|
|
||||||
|
No plaintext file contents ever hit disk. No private key or secret
|
||||||
|
passphrase is needed or stored locally.
|
||||||
|
|
||||||
|
## how
|
||||||
|
|
||||||
|
1. **install**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go install git.eeqj.de/sneak/vaultik@latest
|
||||||
|
```
|
||||||
|
|
||||||
|
1. **generate keypair**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
age-keygen -o agekey.txt
|
||||||
|
grep 'public key:' agekey.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
1. **write config**
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# Named snapshots - each snapshot can contain multiple paths
|
||||||
|
snapshots:
|
||||||
|
system:
|
||||||
|
paths:
|
||||||
|
- /etc
|
||||||
|
- /var/lib
|
||||||
|
exclude:
|
||||||
|
- '*.cache' # Snapshot-specific exclusions
|
||||||
|
home:
|
||||||
|
paths:
|
||||||
|
- /home/user/documents
|
||||||
|
- /home/user/photos
|
||||||
|
|
||||||
|
# Global exclusions (apply to all snapshots)
|
||||||
|
exclude:
|
||||||
|
- '*.log'
|
||||||
|
- '*.tmp'
|
||||||
|
- '.git'
|
||||||
|
- 'node_modules'
|
||||||
|
|
||||||
|
age_recipients:
|
||||||
|
- age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj
|
||||||
|
s3:
|
||||||
|
endpoint: https://s3.example.com
|
||||||
|
bucket: vaultik-data
|
||||||
|
prefix: host1/
|
||||||
|
access_key_id: ...
|
||||||
|
secret_access_key: ...
|
||||||
|
region: us-east-1
|
||||||
|
backup_interval: 1h
|
||||||
|
full_scan_interval: 24h
|
||||||
|
min_time_between_run: 15m
|
||||||
|
chunk_size: 10MB
|
||||||
|
blob_size_limit: 1GB
|
||||||
|
```
|
||||||
|
|
||||||
|
1. **run**
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# Create all configured snapshots
|
||||||
|
vaultik --config /etc/vaultik.yaml snapshot create
|
||||||
|
|
||||||
|
# Create specific snapshots by name
|
||||||
|
vaultik --config /etc/vaultik.yaml snapshot create home system
|
||||||
|
|
||||||
|
# Silent mode for cron
|
||||||
|
vaultik --config /etc/vaultik.yaml snapshot create --cron
|
||||||
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -91,500 +147,302 @@ VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...' vaultik snapshot restore <snapshot-i
|
|||||||
### commands
|
### commands
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
vaultik [--config <path>] config init
|
vaultik [--config <path>] snapshot create [snapshot-names...] [--cron] [--daemon] [--prune]
|
||||||
vaultik [--config <path>] config edit
|
|
||||||
vaultik [--config <path>] config get <key>
|
|
||||||
vaultik [--config <path>] config set <key> <value>
|
|
||||||
vaultik [--config <path>] snapshot create [snapshot-names...] [--cron] [--prune] [--keep-newer-than <duration>]
|
|
||||||
vaultik [--config <path>] snapshot list [--json]
|
vaultik [--config <path>] snapshot list [--json]
|
||||||
vaultik [--config <path>] snapshot verify <snapshot-id> [--deep] [--json]
|
vaultik [--config <path>] snapshot verify <snapshot-id> [--deep]
|
||||||
vaultik [--config <path>] snapshot purge [--keep-latest | --older-than <duration>] [--snapshot <name>...] [--force]
|
vaultik [--config <path>] snapshot purge [--keep-latest | --older-than <duration>] [--name <name>] [--force]
|
||||||
vaultik [--config <path>] snapshot remove <snapshot-id> [--dry-run] [--force] [--local-only] [--json]
|
vaultik [--config <path>] snapshot remove <snapshot-id> [--dry-run] [--force]
|
||||||
vaultik [--config <path>] snapshot restore <snapshot-id> <target-dir> [paths...] [--verify]
|
vaultik [--config <path>] snapshot prune
|
||||||
vaultik [--config <path>] prune [--force] [--json]
|
vaultik [--config <path>] restore <snapshot-id> <target-dir> [paths...]
|
||||||
|
vaultik [--config <path>] prune [--dry-run] [--force]
|
||||||
vaultik [--config <path>] info
|
vaultik [--config <path>] info
|
||||||
vaultik [--config <path>] remote info [--json]
|
vaultik [--config <path>] store info
|
||||||
vaultik [--config <path>] remote nuke --force
|
|
||||||
vaultik [--config <path>] database delete [--force]
|
|
||||||
vaultik completion <bash|zsh|fish|powershell>
|
|
||||||
vaultik version
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### global flags
|
### environment
|
||||||
|
|
||||||
* `--config <path>`: Path to config file (default: `$VAULTIK_CONFIG`, then platform config dir, then `/etc/vaultik/config.yml`)
|
* `VAULTIK_AGE_SECRET_KEY`: Required for `restore` and deep `verify`. Contains the age private key for decryption.
|
||||||
* `--verbose`, `-v`: Enable verbose output
|
* `VAULTIK_CONFIG`: Optional path to config file.
|
||||||
* `--debug`: Enable debug output
|
|
||||||
* `--quiet`, `-q`: Suppress non-error output (also suppresses startup banner)
|
|
||||||
* `--skip-errors`: Continue past per-file errors instead of aborting (applies to `snapshot create` and `restore`)
|
|
||||||
|
|
||||||
### environment variables
|
|
||||||
|
|
||||||
* `VAULTIK_AGE_SECRET_KEY`: Age private key for decryption (required for `snapshot restore` and `snapshot verify --deep`)
|
|
||||||
* `VAULTIK_CONFIG`: Path to config file (overridden by `--config`)
|
|
||||||
* `VAULTIK_INDEX_PATH`: Override local SQLite index path
|
|
||||||
|
|
||||||
### shell completion
|
|
||||||
|
|
||||||
```sh
|
|
||||||
# zsh: load for the current session
|
|
||||||
source <(vaultik completion zsh)
|
|
||||||
|
|
||||||
# zsh: install permanently
|
|
||||||
vaultik completion zsh > "${fpath[1]}/_vaultik"
|
|
||||||
|
|
||||||
# bash: load for the current session
|
|
||||||
source <(vaultik completion bash)
|
|
||||||
|
|
||||||
# bash: install permanently (Linux)
|
|
||||||
vaultik completion bash > /etc/bash_completion.d/vaultik
|
|
||||||
|
|
||||||
# fish
|
|
||||||
vaultik completion fish > ~/.config/fish/completions/vaultik.fish
|
|
||||||
```
|
|
||||||
|
|
||||||
### command details
|
### command details
|
||||||
|
|
||||||
**`config init`**: Write a default config file with commented explanations for
|
**snapshot create**: Perform incremental backup of configured snapshots
|
||||||
every setting. Writes to the path from `--config`, `$VAULTIK_CONFIG`, or the
|
* Config is located at `/etc/vaultik/config.yml` by default
|
||||||
platform config directory (`~/Library/Application Support/vaultik/` on macOS,
|
|
||||||
`~/.config/vaultik/` on Linux, `/etc/vaultik/` as root). Refuses to overwrite an
|
|
||||||
existing file. Created with mode `0600` since it will contain credentials.
|
|
||||||
|
|
||||||
**`config edit`**: Open the config file in `$EDITOR` (falls back to `vi`).
|
|
||||||
|
|
||||||
**`config get`**: Print a config value addressed by dotted YAML path
|
|
||||||
(e.g. `vaultik config get storage_url`). Non-scalar values print as YAML.
|
|
||||||
|
|
||||||
**`config set`**: Set a scalar config value by dotted YAML path
|
|
||||||
(e.g. `vaultik config set compression_level 9`,
|
|
||||||
`vaultik config set storage_url "file:///mnt/backups"`). Comments and
|
|
||||||
formatting in the file are preserved; intermediate maps are created as
|
|
||||||
needed.
|
|
||||||
|
|
||||||
**`snapshot create`**: Perform incremental backup of configured snapshots.
|
|
||||||
* Optional snapshot names argument to create specific snapshots (default: all)
|
* Optional snapshot names argument to create specific snapshots (default: all)
|
||||||
* On macOS, the terminal application running vaultik needs Full Disk Access
|
|
||||||
(System Settings → Privacy & Security → Full Disk Access) to read
|
|
||||||
TCC-protected directories; without it the backup aborts with a permission
|
|
||||||
error that explains how to fix it
|
|
||||||
* `--cron`: Silent unless error (for crontab)
|
* `--cron`: Silent unless error (for crontab)
|
||||||
* `--prune`: After backup, drop older snapshots of each backed-up name and
|
* `--daemon`: Run continuously with filesystem monitoring and periodic scans (see [daemon mode](#daemon-mode))
|
||||||
remove orphaned blobs from remote storage. By default keeps only the latest
|
* `--prune`: Delete old snapshots and orphaned blobs after backup
|
||||||
snapshot per name; use `--keep-newer-than` for a rolling window.
|
* `--skip-errors`: Skip file read errors (log them loudly but continue)
|
||||||
* `--keep-newer-than <duration>`: With `--prune`, keep snapshots newer than
|
|
||||||
this duration instead of only the latest (e.g. `4w`, `30d`, `6mo`, `1y`)
|
|
||||||
|
|
||||||
**`snapshot list`**: Show every snapshot known to the destination
|
**snapshot list**: List all snapshots with their timestamps and sizes
|
||||||
store with timestamps and three sizes per snapshot (compressed
|
|
||||||
remote size; total uncompressed chunk size; size of chunks newly
|
|
||||||
referenced by that snapshot). The uncompressed and "new chunk"
|
|
||||||
columns show `<remote only>` for snapshots not in the local index.
|
|
||||||
* `--json`: Output in JSON format
|
* `--json`: Output in JSON format
|
||||||
|
|
||||||
**`snapshot verify`**: Verify snapshot integrity.
|
**snapshot verify**: Verify snapshot integrity
|
||||||
* Default (shallow): checks that all blobs referenced in the manifest exist in storage
|
* `--deep`: Download and verify blob contents (not just existence)
|
||||||
* `--deep`: Downloads and decrypts each blob, verifies chunk hashes against the
|
|
||||||
encrypted metadata database
|
|
||||||
* `--json`: Output results as JSON
|
|
||||||
|
|
||||||
**`snapshot purge`**: Remove old snapshots based on criteria. Retention is
|
**snapshot purge**: Remove old snapshots based on criteria
|
||||||
per-snapshot-name (`--keep-latest` keeps the latest of each name, not the
|
* `--keep-latest`: Keep the most recent snapshot per snapshot name
|
||||||
latest globally).
|
* `--older-than`: Remove snapshots older than duration (e.g., 30d, 6mo, 1y)
|
||||||
* `--keep-latest`: Keep only the most recent snapshot of each name
|
* `--name`: Filter purge to a specific snapshot name
|
||||||
* `--older-than <duration>`: Remove snapshots older than duration (e.g. `30d`, `6m`, `1y`)
|
|
||||||
* `--snapshot <name>`: Restrict to specific snapshot names (repeat for multiple)
|
|
||||||
* `--force`: Skip confirmation prompt
|
* `--force`: Skip confirmation prompt
|
||||||
|
|
||||||
**`snapshot remove`**: Remove one snapshot. By default this removes the
|
**snapshot remove**: Remove a specific snapshot
|
||||||
snapshot from the local index and strips the snapshot's metadata from
|
|
||||||
the backup destination store. Blobs are NOT touched — deleting blobs
|
|
||||||
requires reading every remaining remote manifest (the destination store
|
|
||||||
may hold snapshots this host doesn't know about), which is what
|
|
||||||
`vaultik prune` does. On success the command prints the exact `vaultik
|
|
||||||
prune` invocation to run as a follow-up. Local row cleanup (files,
|
|
||||||
chunks, blobs the snapshot was the last referrer for) runs
|
|
||||||
automatically. If the destination store is unreachable, the local-DB
|
|
||||||
removal still completes and a warning is emitted; rerun `vaultik prune`
|
|
||||||
once the store is reachable to finish remote cleanup. To wipe everything
|
|
||||||
on the destination in one go, use `vaultik remote nuke --force`.
|
|
||||||
* `--local-only`: Skip remote cleanup; only touch the local index
|
|
||||||
* `--dry-run`: Show what would be deleted without deleting
|
* `--dry-run`: Show what would be deleted without deleting
|
||||||
* `--force`: Skip confirmation prompt
|
* `--force`: Skip confirmation prompt
|
||||||
* `--json`: Output result as JSON
|
|
||||||
|
|
||||||
**`snapshot restore`**: Restore files from a backup snapshot.
|
**snapshot prune**: Clean orphaned data from local database
|
||||||
* Requires `VAULTIK_AGE_SECRET_KEY` environment variable
|
|
||||||
|
**restore**: Restore snapshot to target directory
|
||||||
|
* Requires `VAULTIK_AGE_SECRET_KEY` environment variable with age private key
|
||||||
* Optional path arguments to restore specific files/directories (default: all)
|
* Optional path arguments to restore specific files/directories (default: all)
|
||||||
* Preserves file permissions, timestamps, ownership (ownership requires root),
|
* Downloads and decrypts metadata, fetches required blobs, reconstructs files
|
||||||
symlinks, and empty directories
|
* Preserves file permissions, timestamps, and ownership (ownership requires root)
|
||||||
* `--verify`: After restoring, verify every file's chunk hashes match
|
* Handles symlinks and directories
|
||||||
|
|
||||||
**`prune`**: Tidy up everything that isn't needed. Runs three passes:
|
**prune**: Remove unreferenced blobs from remote storage
|
||||||
(1) reconcile the local index against the destination store — any
|
* Scans all snapshots for referenced blobs
|
||||||
local snapshot whose remote metadata is missing is dropped from the
|
* Deletes orphaned blobs
|
||||||
local index; (2) delete orphaned local rows (files, chunks, blobs no
|
|
||||||
longer referenced by any completed snapshot); (3) list every remote
|
|
||||||
manifest on the destination store to compute the still-referenced blob
|
|
||||||
set and delete any blob not in that set. Step (3) reads all remote
|
|
||||||
manifests — network cost scales with the number of snapshots. `snapshot
|
|
||||||
create --prune` runs the same cleanup automatically; this is the
|
|
||||||
manual entry point for the same work.
|
|
||||||
* `--force`: Skip confirmation prompt
|
|
||||||
* `--json`: Output stats as JSON
|
|
||||||
|
|
||||||
**`info`**: Display system configuration, storage settings, encryption
|
**info**: Display system and configuration information
|
||||||
recipients, and local database statistics.
|
|
||||||
|
|
||||||
**`remote info`**: Show storage backend type and location plus detailed
|
**store info**: Display S3 bucket configuration and storage statistics
|
||||||
remote storage inventory: per-snapshot metadata sizes, blob counts, and
|
|
||||||
orphaned blob detection.
|
|
||||||
* `--json`: Output as JSON
|
|
||||||
|
|
||||||
**`remote nuke`**: Delete every snapshot's metadata and every blob from the
|
|
||||||
backup destination store, leaving the bucket prefix empty. Destructive and
|
|
||||||
irreversible. This is the single supported way to wipe the entire
|
|
||||||
destination store.
|
|
||||||
* `--force`: Required to confirm destruction.
|
|
||||||
|
|
||||||
**`database delete`**: Delete the local SQLite state database file
|
|
||||||
entirely. Remote storage is unaffected; the next backup will do a full
|
|
||||||
scan and re-deduplicate against existing remote blobs, and the local
|
|
||||||
index will re-bind to the currently configured storage destination.
|
|
||||||
Use this after changing `storage_url` to a different destination.
|
|
||||||
* `--force`: Skip confirmation prompt
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## storage backends
|
## daemon mode
|
||||||
|
|
||||||
vaultik supports three storage backends, selected via the `storage_url` config field:
|
When `--daemon` is passed to `snapshot create`, vaultik runs as a
|
||||||
|
long-running process that continuously monitors configured directories for
|
||||||
|
changes and creates backups automatically.
|
||||||
|
|
||||||
**S3** (`s3://bucket/prefix?endpoint=host®ion=us-east-1`): Any S3-compatible
|
```sh
|
||||||
object store. Credentials are read from `s3.access_key_id` and
|
vaultik --config /etc/vaultik.yaml snapshot create --daemon
|
||||||
`s3.secret_access_key` in the config file.
|
```
|
||||||
|
|
||||||
**Local filesystem** (`file:///path/to/backup`): Stores blobs and metadata on
|
### how it works
|
||||||
a local or mounted filesystem. Useful for testing or backing up to a NAS.
|
|
||||||
|
|
||||||
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
|
1. **Initial backup**: On startup, a full backup of all configured snapshots
|
||||||
providers. Requires rclone to be configured separately (`rclone config`).
|
runs immediately.
|
||||||
|
2. **Filesystem watching**: All configured snapshot paths are monitored for
|
||||||
|
file changes using OS-native filesystem notifications (inotify on Linux,
|
||||||
|
FSEvents on macOS, ReadDirectoryChangesW on Windows) via the
|
||||||
|
[fsnotify](https://github.com/fsnotify/fsnotify) library.
|
||||||
|
3. **Periodic backups**: At each `backup_interval` tick, if filesystem
|
||||||
|
changes have been detected and `min_time_between_run` has elapsed since
|
||||||
|
the last backup, a backup runs for only the affected snapshots.
|
||||||
|
4. **Full scans**: At each `full_scan_interval` tick, a full backup of all
|
||||||
|
snapshots runs regardless of detected changes. This catches any changes
|
||||||
|
that filesystem notifications may have missed.
|
||||||
|
5. **Graceful shutdown**: On SIGTERM or SIGINT, the daemon completes any
|
||||||
|
in-progress backup before exiting.
|
||||||
|
|
||||||
Legacy S3 configuration via `s3.*` fields (endpoint, bucket, prefix, etc.) is
|
### configuration
|
||||||
still supported for backward compatibility. `storage_url` takes precedence if
|
|
||||||
both are set.
|
These config fields control daemon behavior:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
backup_interval: 1h # How often to check for changes and run backups
|
||||||
|
full_scan_interval: 24h # How often to do a complete scan of all paths
|
||||||
|
min_time_between_run: 15m # Minimum gap between consecutive backup runs
|
||||||
|
```
|
||||||
|
|
||||||
|
### notes
|
||||||
|
|
||||||
|
* New directories created under watched paths are automatically picked up.
|
||||||
|
* The daemon uses the same `CreateSnapshot` logic as one-shot mode — each
|
||||||
|
backup run is a standard incremental snapshot.
|
||||||
|
* The `--prune`, `--cron`, and `--skip-errors` flags work in daemon mode
|
||||||
|
and apply to each individual backup run.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## architecture
|
## architecture
|
||||||
|
|
||||||
### remote storage layout
|
### s3 bucket layout
|
||||||
|
|
||||||
```
|
```
|
||||||
<bucket>/<prefix>/
|
s3://<bucket>/<prefix>/
|
||||||
├── blobs/
|
├── blobs/
|
||||||
│ └── <aa>/<bb>/<full_blob_hash>
|
│ └── <aa>/<bb>/<full_blob_hash>
|
||||||
└── metadata/
|
└── metadata/
|
||||||
└── <snapshot_id>/
|
├── <snapshot_id>/
|
||||||
├── db.zst.age # Encrypted binary SQLite database
|
│ ├── db.zst.age
|
||||||
└── manifest.json.zst # Unencrypted blob list (for pruning)
|
│ └── manifest.json.zst
|
||||||
```
|
```
|
||||||
|
|
||||||
* Blobs are two-level directory sharded using the first 4 hex chars of the blob hash
|
* `blobs/<aa>/<bb>/...`: Two-level directory sharding using first 4 hex chars of blob hash
|
||||||
* `db.zst.age` is a binary SQLite database (zstd compressed, age encrypted)
|
* `metadata/<snapshot_id>/db.zst.age`: Encrypted, compressed SQLite database
|
||||||
containing all file metadata, chunk mappings, and relationships for the snapshot
|
* `metadata/<snapshot_id>/manifest.json.zst`: Unencrypted blob list for pruning
|
||||||
* `manifest.json.zst` is an unencrypted compressed JSON blob list, enabling
|
|
||||||
pruning without the private key
|
|
||||||
|
|
||||||
Snapshot IDs follow the format `<hostname>_<snapshot-name>_<RFC3339-timestamp>`
|
### blob manifest format
|
||||||
(e.g. `server1_home_2025-06-01T12:00:00Z`).
|
|
||||||
|
The `manifest.json.zst` file is unencrypted (compressed JSON) to enable pruning without decryption:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"snapshot_id": "hostname_snapshotname_2025-01-01T12:00:00Z",
|
||||||
|
"blob_hashes": [
|
||||||
|
"aa1234567890abcdef...",
|
||||||
|
"bb2345678901bcdef0..."
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Snapshot IDs follow the format `<hostname>_<snapshot-name>_<timestamp>` (e.g., `server1_home_2025-01-01T12:00:00Z`).
|
||||||
|
|
||||||
|
### local sqlite schema
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE TABLE files (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
path TEXT NOT NULL UNIQUE,
|
||||||
|
mtime INTEGER NOT NULL,
|
||||||
|
size INTEGER NOT NULL,
|
||||||
|
mode INTEGER NOT NULL,
|
||||||
|
uid INTEGER NOT NULL,
|
||||||
|
gid INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE file_chunks (
|
||||||
|
file_id TEXT NOT NULL,
|
||||||
|
idx INTEGER NOT NULL,
|
||||||
|
chunk_hash TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (file_id, idx),
|
||||||
|
FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE chunks (
|
||||||
|
chunk_hash TEXT PRIMARY KEY,
|
||||||
|
size INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE blobs (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
blob_hash TEXT NOT NULL UNIQUE,
|
||||||
|
uncompressed INTEGER NOT NULL,
|
||||||
|
compressed INTEGER NOT NULL,
|
||||||
|
uploaded_at INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE blob_chunks (
|
||||||
|
blob_hash TEXT NOT NULL,
|
||||||
|
chunk_hash TEXT NOT NULL,
|
||||||
|
offset INTEGER NOT NULL,
|
||||||
|
length INTEGER NOT NULL,
|
||||||
|
PRIMARY KEY (blob_hash, chunk_hash)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE chunk_files (
|
||||||
|
chunk_hash TEXT NOT NULL,
|
||||||
|
file_id TEXT NOT NULL,
|
||||||
|
file_offset INTEGER NOT NULL,
|
||||||
|
length INTEGER NOT NULL,
|
||||||
|
PRIMARY KEY (chunk_hash, file_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE snapshots (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
hostname TEXT NOT NULL,
|
||||||
|
vaultik_version TEXT NOT NULL,
|
||||||
|
started_at INTEGER NOT NULL,
|
||||||
|
completed_at INTEGER,
|
||||||
|
file_count INTEGER NOT NULL,
|
||||||
|
chunk_count INTEGER NOT NULL,
|
||||||
|
blob_count INTEGER NOT NULL,
|
||||||
|
total_size INTEGER NOT NULL,
|
||||||
|
blob_size INTEGER NOT NULL,
|
||||||
|
compression_ratio REAL NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE snapshot_files (
|
||||||
|
snapshot_id TEXT NOT NULL,
|
||||||
|
file_id TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (snapshot_id, file_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE snapshot_blobs (
|
||||||
|
snapshot_id TEXT NOT NULL,
|
||||||
|
blob_id TEXT NOT NULL,
|
||||||
|
blob_hash TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (snapshot_id, blob_id)
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
### data flow
|
### data flow
|
||||||
|
|
||||||
**backup:**
|
#### backup
|
||||||
|
|
||||||
1. Open local SQLite index, load known files and chunks into memory
|
1. Load config, open local SQLite index
|
||||||
2. Walk source directories, compare mtime/size/mode against index
|
1. Walk source directories, check mtime/size against index
|
||||||
3. For changed/new files: chunk using content-defined chunking (FastCDC)
|
1. For changed/new files: chunk using content-defined chunking
|
||||||
4. For symlinks and directories: record metadata (no chunking)
|
1. For each chunk: hash, check if already uploaded, add to blob packer
|
||||||
5. For each chunk: hash, check dedup, add to blob packer
|
1. When blob reaches threshold: compress, encrypt, upload to S3
|
||||||
6. When blob reaches size threshold: compress (zstd), encrypt (age), upload
|
1. Build snapshot metadata, compress, encrypt, upload
|
||||||
7. Build snapshot metadata database, compress, encrypt, upload
|
1. Create blob manifest (unencrypted) for pruning support
|
||||||
8. Create unencrypted blob manifest for pruning support
|
|
||||||
|
|
||||||
**restore:**
|
#### restore
|
||||||
|
|
||||||
1. Download and decrypt `metadata/<snapshot_id>/db.zst.age`
|
1. Download `metadata/<snapshot_id>/db.zst.age`
|
||||||
2. Open the binary SQLite database
|
1. Decrypt and decompress SQLite database
|
||||||
3. Query files (optionally filtered by paths)
|
1. Query files table (optionally filtered by paths)
|
||||||
4. Download and decrypt required blobs
|
1. For each file, get ordered chunk list from file_chunks
|
||||||
5. Extract chunks, reconstruct files
|
1. Download required blobs, decrypt, decompress
|
||||||
6. Restore permissions, timestamps, ownership, symlinks
|
1. Extract chunks and reconstruct files
|
||||||
|
1. Restore permissions, mtime, uid/gid
|
||||||
|
|
||||||
**prune:**
|
#### prune
|
||||||
|
|
||||||
1. List all snapshot manifests
|
1. List all snapshot manifests
|
||||||
2. Build set of all referenced blob hashes
|
1. Build set of all referenced blob hashes
|
||||||
3. List all blobs in storage
|
1. List all blobs in storage
|
||||||
4. Delete any blob not in the referenced set
|
1. Delete any blob not in referenced set
|
||||||
|
|
||||||
### chunking and deduplication
|
### chunking
|
||||||
|
|
||||||
* Content-defined chunking using the FastCDC algorithm
|
* Content-defined chunking using FastCDC algorithm
|
||||||
* Average chunk size: configurable (default 10MB)
|
* Average chunk size: configurable (default 10MB)
|
||||||
* Deduplication at file level (unchanged files skipped) and chunk level
|
* Deduplication at chunk level
|
||||||
(identical chunks across files stored once)
|
* Multiple chunks packed into blobs for efficiency
|
||||||
* Multiple chunks packed into blobs to reduce object count
|
|
||||||
|
|
||||||
### encryption
|
### encryption
|
||||||
|
|
||||||
* Asymmetric encryption using age (X25519 + XChaCha20-Poly1305)
|
* Asymmetric encryption using age (X25519 + XChaCha20-Poly1305)
|
||||||
* Only the public key is needed on the source host
|
* Only public key needed on source host
|
||||||
* Each blob and each metadata database is encrypted independently
|
* Each blob encrypted independently
|
||||||
* Multiple recipients supported (encrypt to multiple keys)
|
* Metadata databases also encrypted
|
||||||
|
|
||||||
### compression
|
### compression
|
||||||
|
|
||||||
* zstd compression at configurable level (1-19, default 3)
|
* zstd compression at configurable level
|
||||||
* Applied before encryption at the blob level
|
* Applied before encryption
|
||||||
|
* Blob-level compression for efficiency
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## configuration reference
|
## does not
|
||||||
|
|
||||||
Run `vaultik config init` to generate a fully commented config file.
|
* Store any secrets on the backed-up machine
|
||||||
Key fields:
|
* Require mutable remote metadata
|
||||||
|
* Use tarballs, restic, rsync, or ssh
|
||||||
|
* Require a symmetric passphrase or password
|
||||||
|
* Trust the source system with anything
|
||||||
|
|
||||||
| Field | Default | Description |
|
## does
|
||||||
|-------|---------|-------------|
|
|
||||||
| `age_recipients` | (required) | Age public keys for encryption |
|
* Incremental deduplicated backup
|
||||||
| `snapshots` | (required) | Named snapshot definitions with paths and excludes |
|
* Blob-packed chunk encryption
|
||||||
| `storage_url` | | Storage backend URL (`s3://`, `file://`, `rclone://`) |
|
* Content-addressed immutable blobs
|
||||||
| `s3.*` | | Legacy S3 configuration (endpoint, bucket, credentials) |
|
* Public-key encryption only
|
||||||
| `exclude` | | Global exclude patterns (applied to all snapshots) |
|
* SQLite-based local and snapshot metadata
|
||||||
| `chunk_size` | `10MB` | Average chunk size for content-defined chunking |
|
* Fully stream-processed storage
|
||||||
| `blob_size_limit` | `10GB` | Maximum blob size before splitting |
|
|
||||||
| `compression_level` | `3` | zstd compression level (1-19) |
|
|
||||||
| `hostname` | system hostname | Hostname used in snapshot IDs |
|
|
||||||
| `index_path` | platform data dir | Local SQLite index path |
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## limitations
|
|
||||||
|
|
||||||
* **No extended attributes (xattrs).** ACLs, macOS Finder metadata,
|
|
||||||
quarantine flags, SELinux labels, and other extended attributes are not
|
|
||||||
backed up or restored.
|
|
||||||
* **No hard link detection.** Two hard links to the same inode are backed
|
|
||||||
up as independent files. Content deduplication means the data is stored
|
|
||||||
once, but the hard link relationship is lost on restore.
|
|
||||||
* **No sparse file support.** Sparse files are fully materialized during
|
|
||||||
backup. A 100 GB sparse VM disk that is mostly zeros will consume the
|
|
||||||
full (compressed) size in storage.
|
|
||||||
* **No bandwidth limiting.** Uploads and downloads use whatever bandwidth
|
|
||||||
is available. There is no `--bwlimit` flag yet.
|
|
||||||
* **No parallel blob downloads during restore.** Blobs are fetched
|
|
||||||
sequentially. Restore speed is bound by single-stream throughput.
|
|
||||||
* **Device nodes, named pipes, and sockets are silently skipped.** Only
|
|
||||||
regular files, directories, and symlinks are backed up.
|
|
||||||
* **No database migrations.** If the local SQLite schema changes between
|
|
||||||
versions, delete the local database (`vaultik database delete`) and run
|
|
||||||
a full backup. Remote storage is unaffected.
|
|
||||||
* **Files that change during backup may be inconsistent.** There is no
|
|
||||||
filesystem snapshot or freeze. If a file is modified between the scan
|
|
||||||
and chunk phases, the backed-up copy may reflect a partial write.
|
|
||||||
* **Ownership restoration requires root.** File uid/gid are recorded
|
|
||||||
and restored, but `chown` requires elevated privileges. Without root,
|
|
||||||
files are restored with the current user's ownership.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## roadmap
|
|
||||||
|
|
||||||
Items still to do before / shortly after 1.0. Loosely ordered by
|
|
||||||
priority.
|
|
||||||
|
|
||||||
### correctness and operability
|
|
||||||
|
|
||||||
* **Security audit of the encryption implementation.** Pre-1.0
|
|
||||||
blocker if we're advertising "secure" at the top of this README.
|
|
||||||
age + zstd + content-defined chunking is mostly off-the-shelf
|
|
||||||
pieces, but the seams (key handling, recipient parsing, manifest
|
|
||||||
trust boundary, restore-time identity validation) need an outside
|
|
||||||
read.
|
|
||||||
* **Error-condition tests.** Today's coverage is the happy path
|
|
||||||
plus a few specific regressions. Need fault-injection coverage:
|
|
||||||
network failures mid-blob, disk-full during restore, corrupted /
|
|
||||||
truncated / missing blobs, partial uploads, kill -9 between
|
|
||||||
manifest and db.zst.age writes.
|
|
||||||
* **Verify restored content end-to-end in CI.** The current
|
|
||||||
integration test does this for a small synthetic snapshot but
|
|
||||||
not at scale. A nightly job against a multi-GB representative
|
|
||||||
snapshot would catch silent regressions in the chunker, packer,
|
|
||||||
or restore planner.
|
|
||||||
|
|
||||||
### performance
|
|
||||||
|
|
||||||
* **Parallel blob downloads during restore.** Single-stream right
|
|
||||||
now. With a fast S3 endpoint and a multi-core machine restore is
|
|
||||||
bound by per-blob fetch + decrypt + decompress; running N of
|
|
||||||
those in parallel against the disk cache would close most of the
|
|
||||||
remaining gap. Needs to interact correctly with the locality
|
|
||||||
planner and sweeper.
|
|
||||||
* **Bandwidth limiting (`--bwlimit`).** Both upload and download.
|
|
||||||
Useful for backing up over a shared link. Tricky to make work
|
|
||||||
correctly with the parallel-download story.
|
|
||||||
* **Restart of interrupted restore.** Today restore is restartable
|
|
||||||
in the sense that re-running it overwrites partial output; it
|
|
||||||
doesn't resume from where it stopped or skip already-present
|
|
||||||
files. A `--resume` mode that checks targets before fetching
|
|
||||||
blobs would matter for very large restores.
|
|
||||||
|
|
||||||
### usability
|
|
||||||
|
|
||||||
* **Man pages and richer `--help` examples.** Cobra generates
|
|
||||||
basic help; man pages would be a separate target.
|
|
||||||
* **`--bwlimit` style human-readable size flags** across the
|
|
||||||
command surface where they're currently raw integers.
|
|
||||||
* **`vaultik snapshot diff <a> <b>`** — show which files changed
|
|
||||||
between two snapshots without restoring either.
|
|
||||||
* **Status reporting hook for `--cron`.** When a backup fails
|
|
||||||
silently in cron, the user has no idea. A configurable
|
|
||||||
webhook / email / `notify-send` hook on completion (success and
|
|
||||||
failure) would close the loop.
|
|
||||||
|
|
||||||
### infrastructure
|
|
||||||
|
|
||||||
* **Cross-machine restore documentation.** The "restore from
|
|
||||||
another host" workflow works but isn't documented as a
|
|
||||||
first-class operation in this README. Worth a dedicated section
|
|
||||||
once it's settled.
|
|
||||||
* **Schema migrations.** Currently nonexistent — pre-1.0 schema
|
|
||||||
changes are handled by `vaultik database delete` plus a full
|
|
||||||
re-scan. Post-1.0 we'll need a migration story to keep existing
|
|
||||||
index databases usable across upgrades.
|
|
||||||
* **Storage backend coverage tests.** S3, file://, and rclone://
|
|
||||||
all share the Storer interface but the rclone path is the least
|
|
||||||
exercised in CI.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## output style
|
|
||||||
|
|
||||||
All user-facing output goes through helpers in `internal/ui` and conforms
|
|
||||||
to a uniform style. Color is enabled when stdout is a TTY and the
|
|
||||||
`NO_COLOR` environment variable is unset (https://no-color.org/).
|
|
||||||
|
|
||||||
Message classes:
|
|
||||||
|
|
||||||
| Class | Marker | Alignment | Use for |
|
|
||||||
|-------|--------|-----------|---------|
|
|
||||||
| Banner | none | column 0 | The startup line printed once per invocation |
|
|
||||||
| Begin | `》` (white) | column 0 | An operation is about to start (present-continuous verb) |
|
|
||||||
| Complete | `》` (green) | column 0 | An operation just finished (past-tense verb) |
|
|
||||||
| Info | `》` (white) | column 0 | Neutral status update |
|
|
||||||
| Notice | `》` (cyan) | column 0 | Important note that is not a warning |
|
|
||||||
| Warning | `⚠️ Warning:` (orange/yellow) | column 0 | Recoverable problem |
|
|
||||||
| Error | `🛑 ERROR:` (red) | column 0 | Operation aborted |
|
|
||||||
| Progress | ` 》` (white) | column 2 | Heartbeat or per-item status during a long-running operation |
|
|
||||||
| Detail | ` 》` (white) | column 2 | Continuation/sub-line of a preceding Complete (visually identical to Progress) |
|
|
||||||
|
|
||||||
Conventions:
|
|
||||||
|
|
||||||
* Messages are complete English sentences ending with a period.
|
|
||||||
* Fully qualify terms — say "backup destination store" instead of
|
|
||||||
"storage", "snapshot source files enumeration" instead of "scan",
|
|
||||||
"local index database" instead of "database".
|
|
||||||
* Every operation that emits a Complete also emits a corresponding
|
|
||||||
Begin. Operations that print only a Begin (because completion is
|
|
||||||
obvious from a later Begin) should be rare and intentional.
|
|
||||||
* Use natural verb tense to signal state: "Uploading" for Begin,
|
|
||||||
"Uploaded" for Complete. Never write the words "begin" or "complete"
|
|
||||||
in the body — the marker color already conveys that.
|
|
||||||
* All elapsed and remaining-time fields are explicitly scoped to their
|
|
||||||
subject: write "blob upload elapsed: 30s, blob upload ETA: 03:15:00
|
|
||||||
(est remain 14s)", never just "elapsed 30s, ETA 14s".
|
|
||||||
* "ETA" means an absolute clock time (when the operation will finish),
|
|
||||||
not a remaining-duration. Use `ui.Time()` for the former and
|
|
||||||
`ui.Duration()` for the latter, and label both.
|
|
||||||
* `ui.Time` formats same-day times as `HH:MM:SS` and other-day times as
|
|
||||||
`YYYY-MM-DD HH:MM:SS`. No timezone — local time is implied.
|
|
||||||
|
|
||||||
Value colorizers in `internal/ui` colorize specific value types
|
|
||||||
consistently. Compose messages from these helpers rather than embedding
|
|
||||||
ANSI escapes inline:
|
|
||||||
|
|
||||||
| Helper | Color | Use for |
|
|
||||||
|--------|-------|---------|
|
|
||||||
| `Hex` | cyan | Blob hashes, chunk hashes (truncated to 12 chars + `...`) |
|
|
||||||
| `Snapshot` | bold cyan | Snapshot IDs (untruncated) |
|
|
||||||
| `Path` | blue | Filesystem paths |
|
|
||||||
| `Size` | magenta | Byte counts (human-readable) |
|
|
||||||
| `Speed` | magenta | Bytes-per-second rates |
|
|
||||||
| `Duration` | yellow | Elapsed or remaining time |
|
|
||||||
| `Time` | yellow | Absolute clock times |
|
|
||||||
| `Count` | magenta | Integer counts with thousands separators |
|
|
||||||
| `Percent` | magenta | Percentages |
|
|
||||||
|
|
||||||
When `NO_COLOR` is set or output is not a TTY, all helpers return plain
|
|
||||||
text and the marker prefixes (`》`, `Warning:`, `ERROR:`) emit without
|
|
||||||
ANSI escapes. The emoji prefixes on Warning and Error are always emitted
|
|
||||||
regardless of color setting (emoji are not color).
|
|
||||||
|
|
||||||
## requirements
|
## requirements
|
||||||
|
|
||||||
* Go 1.26 or later
|
* Go 1.24 or later
|
||||||
* S3-compatible object storage (or local filesystem, or rclone remote)
|
* S3-compatible object storage
|
||||||
|
* Sufficient disk space for local index (typically <1GB)
|
||||||
## development workflow
|
|
||||||
|
|
||||||
All changes follow this workflow. No exceptions.
|
|
||||||
|
|
||||||
1. Create a feature branch off `main`.
|
|
||||||
2. Write tests.
|
|
||||||
3. Write the implementation.
|
|
||||||
4. Fix implementation errors until it compiles and tests pass.
|
|
||||||
5. Fix linting errors (`make lint`).
|
|
||||||
6. Update documentation and README as required by the change.
|
|
||||||
7. Format code (`make fmt`).
|
|
||||||
8. Run `make check` (lint + fmt-check + test). Fix any issues. Repeat until clean.
|
|
||||||
9. Commit on the branch.
|
|
||||||
10. Merge to `main`.
|
|
||||||
11. Push.
|
|
||||||
|
|
||||||
Do not commit directly to `main`. Do not skip steps.
|
|
||||||
|
|
||||||
Repository policies for AI agents are in [`AGENTS.md`](AGENTS.md).
|
|
||||||
|
|
||||||
## Entrypoints
|
|
||||||
|
|
||||||
This repository adheres to the
|
|
||||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
|
||||||
standard: normalized scripts in `script/` are the entrypoints for the
|
|
||||||
development workflow, and the Makefile targets are thin shims that call
|
|
||||||
them. We provide:
|
|
||||||
|
|
||||||
* `script/bootstrap` — install all development dependencies (go,
|
|
||||||
golangci-lint, Go module download)
|
|
||||||
* `script/setup` — make a fresh clone ready for development: runs
|
|
||||||
`script/bootstrap`, then `script/install-precommit`
|
|
||||||
* `script/projectname` — print the project name (used for the Docker
|
|
||||||
image tag)
|
|
||||||
* `script/test` — run the test suite (verbose rerun on failure)
|
|
||||||
* `script/lint` — run `golangci-lint run ./...`
|
|
||||||
* `script/lint-fix` — apply the linter's autofixes (rewrites files)
|
|
||||||
* `script/fmt` — format all code (writes)
|
|
||||||
* `script/fmt-check` — check formatting (read-only)
|
|
||||||
* `script/check` — run `script/test`, `script/lint`, and
|
|
||||||
`script/fmt-check`
|
|
||||||
* `script/docker` — build the Docker image tagged via
|
|
||||||
`script/projectname`
|
|
||||||
* `script/cibuild` — CI entrypoint: `docker build .` (the Dockerfile
|
|
||||||
runs the checks)
|
|
||||||
* `script/precommit` — pre-commit gate: `go mod tidy` + `go fmt` (must
|
|
||||||
not change files), then `script/check`
|
|
||||||
* `script/install-precommit` — install the git pre-commit hook that
|
|
||||||
runs `script/precommit`
|
|
||||||
|
|
||||||
## license
|
## license
|
||||||
|
|
||||||
|
|||||||
408
REPO_POLICIES.md
408
REPO_POLICIES.md
@@ -1,408 +0,0 @@
|
|||||||
---
|
|
||||||
title: Repository Policies
|
|
||||||
last_modified: 2026-07-06
|
|
||||||
---
|
|
||||||
|
|
||||||
This document covers repository structure, tooling, and workflow standards. Code
|
|
||||||
style conventions are in separate documents:
|
|
||||||
|
|
||||||
- [Code Styleguide](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE.md)
|
|
||||||
(general, bash, Docker)
|
|
||||||
- [Go](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_GO.md)
|
|
||||||
- [JavaScript](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_JS.md)
|
|
||||||
- [Python](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_PYTHON.md)
|
|
||||||
- [Go HTTP Server Conventions](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/GO_HTTP_SERVER_CONVENTIONS.md)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
- Cross-project documentation (such as this file) must include
|
|
||||||
`last_modified: YYYY-MM-DD` in the YAML front matter so it can be kept in sync
|
|
||||||
with the authoritative source as policies evolve.
|
|
||||||
|
|
||||||
- **ALL external references must be pinned by cryptographic hash.** This
|
|
||||||
includes Docker base images, Go modules, npm packages, GitHub Actions, and
|
|
||||||
anything else fetched from a remote source. Version tags (`@v4`, `@latest`,
|
|
||||||
`:3.21`, etc.) are server-mutable and therefore remote code execution
|
|
||||||
vulnerabilities. The ONLY acceptable way to reference an external dependency
|
|
||||||
is by its content hash (Docker `@sha256:...`, Go module hash in `go.sum`, npm
|
|
||||||
integrity hash in lockfile, GitHub Actions `@<commit-sha>`). No exceptions.
|
|
||||||
This also means never `curl | bash` to install tools like pyenv, nvm, rustup,
|
|
||||||
etc. Instead, download a specific release archive from GitHub, verify its hash
|
|
||||||
(hardcoded in the Dockerfile or script), and only then install. Unverified
|
|
||||||
install scripts are arbitrary remote code execution. This is the single most
|
|
||||||
important rule in this document. Double-check every external reference in
|
|
||||||
every file before committing. There are zero exceptions to this rule.
|
|
||||||
|
|
||||||
- Every repo with software must have a root `Makefile` with these targets:
|
|
||||||
`make bootstrap`, `make setup`, `make test`, `make lint`, `make fmt` (writes),
|
|
||||||
`make fmt-check` (read-only), `make check` (runs `test`, `lint`, `fmt-check`),
|
|
||||||
`make docker`, and `make hooks` (installs pre-commit hook). A model Makefile
|
|
||||||
is at `https://git.eeqj.de/sneak/prompts/raw/branch/main/Makefile`.
|
|
||||||
|
|
||||||
- Repos follow the
|
|
||||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
|
||||||
pattern: the implementation of each Makefile target lives in an executable
|
|
||||||
script in `script/` (`script/bootstrap`, `script/setup`, `script/test`,
|
|
||||||
`script/lint`, `script/fmt`, `script/fmt-check`, `script/check`,
|
|
||||||
`script/docker`), and the Makefile targets are thin shims that call them. The
|
|
||||||
scripts must be POSIX sh (`#!/bin/sh`, `set -eu`, no bashisms) so they run in
|
|
||||||
minimal containers (e.g. alpine images have no bash); locate the repo root
|
|
||||||
with `$(cd "$(dirname "$0")/.." && pwd -P)` and `cd` there before acting. From
|
|
||||||
the standard's canonical set we use `bootstrap`, `setup` (make the repo ready
|
|
||||||
for development after a fresh clone: runs `bootstrap`, then
|
|
||||||
`install-precommit`, plus any repo-specific initialization), `test`, and
|
|
||||||
`cibuild`. `script/bootstrap` installs all dependencies idempotently and
|
|
||||||
assumes nothing is present: base tools come from nix, apt, brew, or apk
|
|
||||||
(detected in that order; apt runs noninteractive). For node it uses the
|
|
||||||
installed node if present; otherwise it installs a PINNED node version via
|
|
||||||
nvm, first installing nvm itself if missing — from a hash-verified GitHub
|
|
||||||
release archive (never `curl | sh`), with bash installed as an explicit
|
|
||||||
prerequisite since nvm requires bash. yarn is then pinned via
|
|
||||||
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
|
|
||||||
always exact versions. `script/cibuild` runs the CI build: it changes to the
|
|
||||||
repo root and runs `docker build .`; the Gitea workflow calls it. Four further
|
|
||||||
scripts are our own extensions to the standard: `script/check` runs
|
|
||||||
`script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
|
|
||||||
what the git pre-commit hook runs, and it calls `script/check`;
|
|
||||||
`script/install-precommit` installs the git pre-commit hook (the `make hooks`
|
|
||||||
target shims to it); and `script/projectname` (literally that filename) simply
|
|
||||||
outputs the project's name. Scripts that need the name call
|
|
||||||
`script/projectname` — e.g. `script/docker` assembles its image tag from it —
|
|
||||||
so those scripts stay byte-identical across all repos. Repo-type-specific
|
|
||||||
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
|
|
||||||
`script/precommit`, not in the hook itself. Model scripts are at
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
|
||||||
must document the provided scripts in an **Entrypoints** section (see the
|
|
||||||
README requirements below).
|
|
||||||
|
|
||||||
- Always use Makefile targets (`make fmt`, `make test`, `make lint`, etc.)
|
|
||||||
instead of invoking the underlying tools directly. The Makefile is the single
|
|
||||||
source of truth for how these operations are run.
|
|
||||||
|
|
||||||
- The Makefile is authoritative documentation for how the repo is used. Beyond
|
|
||||||
the required targets above, it should have targets for every common operation:
|
|
||||||
running a local development server (`make run`, `make dev`), re-initializing
|
|
||||||
or migrating the database (`make db-reset`, `make migrate`), building
|
|
||||||
artifacts (`make build`), generating code, seeding data, or anything else a
|
|
||||||
developer would do regularly. If someone checks out the repo and types
|
|
||||||
`make<tab>`, they should see every meaningful operation available. A new
|
|
||||||
contributor should be able to understand the entire development workflow by
|
|
||||||
reading the Makefile.
|
|
||||||
|
|
||||||
- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
|
|
||||||
as a build step so the build fails if the branch is not green. For non-server
|
|
||||||
repos, the Dockerfile should bring up a development environment and run
|
|
||||||
`make check`. For server repos, `make check` should run as an early build
|
|
||||||
stage before the final image is assembled. Dockerfiles install development
|
|
||||||
prerequisites by running `script/bootstrap` rather than duplicating installs
|
|
||||||
inline; COPY `script/` and the dependency manifests (`package.json` +
|
|
||||||
`yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
|
|
||||||
layer stays cached until dependencies change.
|
|
||||||
|
|
||||||
- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
|
|
||||||
repos use a multistage build where linting runs in an independent stage based
|
|
||||||
on the `golangci/golangci-lint` image (pinned by hash). This stage runs
|
|
||||||
`make fmt-check` and `make lint` before the full build begins. The build stage
|
|
||||||
then declares an explicit dependency on the lint stage via
|
|
||||||
`COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
|
|
||||||
linting before proceeding to compilation and tests. This ensures lint failures
|
|
||||||
surface in seconds rather than minutes, without blocking on dependency
|
|
||||||
download or compilation in the build stage.
|
|
||||||
|
|
||||||
The standard pattern for a Go repo Dockerfile is:
|
|
||||||
|
|
||||||
```dockerfile
|
|
||||||
# Lint stage — fast feedback on formatting and lint issues
|
|
||||||
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
|
||||||
FROM golangci/golangci-lint@sha256:... AS lint
|
|
||||||
WORKDIR /src
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
COPY . .
|
|
||||||
RUN make fmt-check
|
|
||||||
RUN make lint
|
|
||||||
|
|
||||||
# Build stage
|
|
||||||
# golang:1.x-alpine, YYYY-MM-DD
|
|
||||||
FROM golang@sha256:... AS builder
|
|
||||||
WORKDIR /src
|
|
||||||
|
|
||||||
# Force BuildKit to run the lint stage before proceeding
|
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
|
||||||
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
COPY . .
|
|
||||||
RUN make test
|
|
||||||
|
|
||||||
ARG VERSION=dev
|
|
||||||
RUN CGO_ENABLED=0 go build -trimpath \
|
|
||||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
|
||||||
-o /app ./cmd/app/
|
|
||||||
|
|
||||||
# Runtime stage
|
|
||||||
FROM alpine@sha256:...
|
|
||||||
COPY --from=builder /app /usr/local/bin/app
|
|
||||||
ENTRYPOINT ["app"]
|
|
||||||
```
|
|
||||||
|
|
||||||
Key points:
|
|
||||||
- The lint stage uses the `golangci/golangci-lint` image directly (it
|
|
||||||
includes both Go and the linter), so there is no need to install the
|
|
||||||
linter separately.
|
|
||||||
- `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
|
|
||||||
a stage dependency. BuildKit runs stages in parallel by default; without
|
|
||||||
this line, the build stage would not wait for lint to finish and a lint
|
|
||||||
failure might not fail the overall build.
|
|
||||||
- If the project uses `//go:embed` directives that reference build artifacts
|
|
||||||
(e.g. a web frontend compiled in a separate stage), the lint stage must
|
|
||||||
create placeholder files so the embed directives resolve. Example:
|
|
||||||
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
|
|
||||||
The lint stage should not depend on the actual build output — it exists to
|
|
||||||
fail fast.
|
|
||||||
- If the project requires CGO or system libraries for linting (e.g.
|
|
||||||
`vips-dev`), install them in the lint stage with `apk add`.
|
|
||||||
- The build stage runs `make test` after compilation setup. Tests run in the
|
|
||||||
build stage, not the lint stage, because they may require compiled
|
|
||||||
artifacts or heavier dependencies.
|
|
||||||
|
|
||||||
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
|
|
||||||
runs `script/cibuild` (which runs `docker build .`) on push. Since the
|
|
||||||
Dockerfile already runs `make check`, a successful build implies all checks
|
|
||||||
pass.
|
|
||||||
|
|
||||||
- Use platform-standard formatters: `black` for Python, `prettier` for
|
|
||||||
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
|
||||||
two exceptions: four-space indents (except Go), and `proseWrap: always` for
|
|
||||||
Markdown (hard-wrap at 80 columns). Documentation and writing repos (Markdown,
|
|
||||||
HTML, CSS) should also have `.prettierrc` and `.prettierignore`.
|
|
||||||
|
|
||||||
- Pre-commit hook: runs `script/precommit`, which calls `script/check`. If local
|
|
||||||
testing is not possible in the repo, `script/precommit` may skip `script/test`
|
|
||||||
and run only `script/lint` and `script/fmt-check`. The hook is installed by
|
|
||||||
`script/install-precommit`; the Makefile must provide a `make hooks` target
|
|
||||||
that shims to it.
|
|
||||||
|
|
||||||
- All repos with software must have tests that run via the platform-standard
|
|
||||||
test framework (`go test`, `pytest`, `jest`/`vitest`, etc.). If no meaningful
|
|
||||||
tests exist yet, add the most minimal test possible — e.g. importing the
|
|
||||||
module under test to verify it compiles/parses. There is no excuse for
|
|
||||||
`make test` to be a no-op.
|
|
||||||
|
|
||||||
- `make test` must complete in under 20 seconds. Add a 30-second timeout in the
|
|
||||||
Makefile.
|
|
||||||
|
|
||||||
- **`make test` should use the conditional verbose rerun pattern.** Run tests
|
|
||||||
without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
|
|
||||||
show full output. This keeps CI logs and `docker build` output clean on
|
|
||||||
success (just package/suite summaries) while providing full diagnostic detail
|
|
||||||
on failure (every test case, every assertion). The general shell pattern:
|
|
||||||
|
|
||||||
```makefile
|
|
||||||
test:
|
|
||||||
@<test-command> || \
|
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
|
||||||
<test-command-with-v>; exit 1; }
|
|
||||||
```
|
|
||||||
|
|
||||||
Go example:
|
|
||||||
|
|
||||||
```makefile
|
|
||||||
test:
|
|
||||||
@go test -timeout 30s -race -cover ./... || \
|
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
|
||||||
go test -timeout 30s -race -v ./...; exit 1; }
|
|
||||||
```
|
|
||||||
|
|
||||||
Python example:
|
|
||||||
|
|
||||||
```makefile
|
|
||||||
test:
|
|
||||||
@python -m pytest || \
|
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
|
||||||
python -m pytest -v; exit 1; }
|
|
||||||
```
|
|
||||||
|
|
||||||
The `exit 1` ensures the target always fails after a rerun — the first run
|
|
||||||
already proved the tests are broken, so the build must not pass even if a
|
|
||||||
flaky test happens to succeed on the second attempt. The rerun exists solely
|
|
||||||
for diagnostic output.
|
|
||||||
|
|
||||||
- Docker builds must complete in under 5 minutes.
|
|
||||||
|
|
||||||
- `make check` must not modify any files in the repo. Tests may use temporary
|
|
||||||
directories.
|
|
||||||
|
|
||||||
- `main` must always pass `make check`, no exceptions.
|
|
||||||
|
|
||||||
- Never commit secrets. `.env` files, credentials, API keys, and private keys
|
|
||||||
must be in `.gitignore`. No exceptions.
|
|
||||||
|
|
||||||
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
|
|
||||||
editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
|
|
||||||
Fetch the standard `.gitignore` from
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
|
|
||||||
a new repo.
|
|
||||||
|
|
||||||
- **No build artifacts in version control.** Code-derived data (compiled
|
|
||||||
bundles, minified output, generated assets) must never be committed to the
|
|
||||||
repository if it can be avoided. The build process (e.g. Dockerfile, Makefile)
|
|
||||||
should generate these at build time. Notable exception: Go protobuf generated
|
|
||||||
files (`.pb.go`) ARE committed because repos need to work with `go get`, which
|
|
||||||
downloads code but does not execute code generation.
|
|
||||||
|
|
||||||
- Never use `git add -A` or `git add .`. Always stage files explicitly by name.
|
|
||||||
|
|
||||||
- Never force-push to `main`.
|
|
||||||
|
|
||||||
- Make all changes on a feature branch. You can do whatever you want on a
|
|
||||||
feature branch.
|
|
||||||
|
|
||||||
- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
|
|
||||||
manually by the user. Fetch from
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
|
|
||||||
|
|
||||||
- When pinning images or packages by hash, add a comment above the reference
|
|
||||||
with the version and date (YYYY-MM-DD).
|
|
||||||
|
|
||||||
- Use `yarn`, not `npm`.
|
|
||||||
|
|
||||||
- Write all dates as YYYY-MM-DD (ISO 8601).
|
|
||||||
|
|
||||||
- Simple projects should be configured with environment variables.
|
|
||||||
|
|
||||||
- Dockerized web services listen on port 8080 by default, overridable with
|
|
||||||
`PORT`.
|
|
||||||
|
|
||||||
- **HTTP/web services must be hardened for production internet exposure before
|
|
||||||
tagging 1.0.** This means full compliance with security best practices
|
|
||||||
including, without limitation, all of the following:
|
|
||||||
- **Security headers** on every response:
|
|
||||||
- `Strict-Transport-Security` (HSTS) with `max-age` of at least one year
|
|
||||||
and `includeSubDomains`.
|
|
||||||
- `Content-Security-Policy` (CSP) with a restrictive default policy
|
|
||||||
(`default-src 'self'` as a baseline, tightened per-resource as
|
|
||||||
needed). Never use `unsafe-inline` or `unsafe-eval` unless
|
|
||||||
unavoidable, and document the reason.
|
|
||||||
- `X-Frame-Options: DENY` (or `SAMEORIGIN` if framing is required).
|
|
||||||
Prefer the `frame-ancestors` CSP directive as the primary control.
|
|
||||||
- `X-Content-Type-Options: nosniff`.
|
|
||||||
- `Referrer-Policy: strict-origin-when-cross-origin` (or stricter).
|
|
||||||
- `Permissions-Policy` restricting access to browser features the
|
|
||||||
application does not use (camera, microphone, geolocation, etc.).
|
|
||||||
- **Request and response limits:**
|
|
||||||
- Maximum request body size enforced on all endpoints (e.g. Go
|
|
||||||
`http.MaxBytesReader`). Choose a sane default per-route; never accept
|
|
||||||
unbounded input.
|
|
||||||
- Maximum response body size where applicable (e.g. paginated APIs).
|
|
||||||
- `ReadTimeout` and `ReadHeaderTimeout` on the `http.Server` to defend
|
|
||||||
against slowloris attacks.
|
|
||||||
- `WriteTimeout` on the `http.Server`.
|
|
||||||
- `IdleTimeout` on the `http.Server`.
|
|
||||||
- Per-handler execution time limits via `context.WithTimeout` or
|
|
||||||
chi/stdlib `middleware.Timeout`.
|
|
||||||
- **Authentication and session security:**
|
|
||||||
- Rate limiting on password-based authentication endpoints. API keys are
|
|
||||||
high-entropy and not susceptible to brute force, so they are exempt.
|
|
||||||
- CSRF tokens on all state-mutating HTML forms. API endpoints
|
|
||||||
authenticated via `Authorization` header (Bearer token, API key) are
|
|
||||||
exempt because the browser does not attach these automatically.
|
|
||||||
- Passwords stored using bcrypt, scrypt, or argon2 — never plain-text,
|
|
||||||
MD5, or SHA.
|
|
||||||
- Session cookies set with `HttpOnly`, `Secure`, and `SameSite=Lax` (or
|
|
||||||
`Strict`) attributes.
|
|
||||||
- **Reverse proxy awareness:**
|
|
||||||
- True client IP detection when behind a reverse proxy
|
|
||||||
(`X-Forwarded-For`, `X-Real-IP`). The application must accept
|
|
||||||
forwarded headers only from a configured set of trusted proxy
|
|
||||||
addresses — never trust `X-Forwarded-For` unconditionally.
|
|
||||||
- **CORS:**
|
|
||||||
- Authenticated endpoints must restrict `Access-Control-Allow-Origin` to
|
|
||||||
an explicit allowlist of known origins. Wildcard (`*`) is acceptable
|
|
||||||
only for public, unauthenticated read-only APIs.
|
|
||||||
- **Error handling:**
|
|
||||||
- Internal errors must never leak stack traces, SQL queries, file paths,
|
|
||||||
or other implementation details to the client. Return generic error
|
|
||||||
messages in production; detailed errors only when `DEBUG` is enabled.
|
|
||||||
- **TLS:**
|
|
||||||
- Services never terminate TLS directly. They are always deployed behind
|
|
||||||
a TLS-terminating reverse proxy. The service itself listens on plain
|
|
||||||
HTTP. However, HSTS headers and `Secure` cookie flags must still be
|
|
||||||
set by the application so that the browser enforces HTTPS end-to-end.
|
|
||||||
|
|
||||||
This list is non-exhaustive. Apply defense-in-depth: if a standard security
|
|
||||||
hardening measure exists for HTTP services and is not listed here, it is
|
|
||||||
still expected. When in doubt, harden.
|
|
||||||
|
|
||||||
- `README.md` is the primary documentation. Required sections:
|
|
||||||
- **Description**: First line must include the project name, purpose,
|
|
||||||
category (web server, SPA, CLI tool, etc.), license, and author. Example:
|
|
||||||
"µPaaS is an MIT-licensed Go web application by @sneak that receives
|
|
||||||
git-frontend webhooks and deploys applications via Docker in realtime."
|
|
||||||
- **Getting Started**: Copy-pasteable install/usage code block.
|
|
||||||
- **Entrypoints**: Opens by stating that the repo adheres to the
|
|
||||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
|
||||||
standard (with that link), then documents each provided `script/`
|
|
||||||
entrypoint and its purpose.
|
|
||||||
- **Rationale**: Why does this exist?
|
|
||||||
- **Design**: How is the program structured?
|
|
||||||
- **TODO**: Update meticulously, even between commits. When planning, put
|
|
||||||
the todo list in the README so a new agent can pick up where the last one
|
|
||||||
left off.
|
|
||||||
- **License**: MIT, GPL, or WTFPL. Ask the user for new projects. Include a
|
|
||||||
`LICENSE` file in the repo root and a License section in the README.
|
|
||||||
- **Author**: [@sneak](https://sneak.berlin).
|
|
||||||
|
|
||||||
- First commit of a new repo should contain only `README.md`.
|
|
||||||
|
|
||||||
- Go module root: `sneak.berlin/go/<name>`. Always run `go mod tidy` before
|
|
||||||
committing.
|
|
||||||
|
|
||||||
- Use SemVer.
|
|
||||||
|
|
||||||
- Database migrations live in `internal/db/migrations/` and must be embedded in
|
|
||||||
the binary.
|
|
||||||
- `000_migration.sql` — contains ONLY the creation of the migrations
|
|
||||||
tracking table itself. Nothing else.
|
|
||||||
- `001_schema.sql` — the full application schema.
|
|
||||||
- **Pre-1.0.0:** never add additional migration files (002, 003, etc.).
|
|
||||||
There is no installed base to migrate. Edit `001_schema.sql` directly.
|
|
||||||
- **Post-1.0.0:** add new numbered migration files for each schema change.
|
|
||||||
Never edit existing migrations after release.
|
|
||||||
|
|
||||||
- All repos should have an `.editorconfig` enforcing the project's indentation
|
|
||||||
settings.
|
|
||||||
|
|
||||||
- Avoid putting files in the repo root unless necessary. Root should contain
|
|
||||||
only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
|
|
||||||
`LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
|
|
||||||
language-specific config). Everything else goes in a subdirectory. Canonical
|
|
||||||
subdirectory names:
|
|
||||||
- `bin/` — executable scripts and tools
|
|
||||||
- `cmd/` — Go command entrypoints
|
|
||||||
- `configs/` — configuration templates and examples
|
|
||||||
- `deploy/` — deployment manifests (k8s, compose, terraform)
|
|
||||||
- `docs/` — documentation and markdown (README.md stays in root)
|
|
||||||
- `internal/` — Go internal packages
|
|
||||||
- `internal/db/migrations/` — database migrations
|
|
||||||
- `pkg/` — Go library packages
|
|
||||||
- `share/` — systemd units, data files
|
|
||||||
- `static/` — static assets (images, fonts, etc.)
|
|
||||||
- `web/` — web frontend source
|
|
||||||
|
|
||||||
- When setting up a new repo, files from the `prompts` repo may be used as
|
|
||||||
templates. Fetch them from
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/<path>`.
|
|
||||||
|
|
||||||
- New repos must contain at minimum:
|
|
||||||
- `README.md`, `.git`, `.gitignore`, `.editorconfig`
|
|
||||||
- `LICENSE`, `REPO_POLICIES.md` (copy from the `prompts` repo)
|
|
||||||
- `Makefile`
|
|
||||||
- `script/` entrypoints (`bootstrap`, `setup`, `projectname`, `test`,
|
|
||||||
`lint`, `fmt`, `fmt-check`, `check`, `docker`, `cibuild`, `precommit`,
|
|
||||||
`install-precommit`)
|
|
||||||
- `Dockerfile`, `.dockerignore`
|
|
||||||
- `.gitea/workflows/check.yml`
|
|
||||||
- Go: `go.mod`, `go.sum`, `.golangci.yml`
|
|
||||||
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
|
|
||||||
- Python: `pyproject.toml`
|
|
||||||
166
TODO.md
166
TODO.md
@@ -1,58 +1,126 @@
|
|||||||
# Workflow
|
# Vaultik 1.0 TODO
|
||||||
|
|
||||||
* branch (from `main`)
|
Linear list of tasks to complete before 1.0 release.
|
||||||
* do the work in Next Step
|
|
||||||
* move Next Step to the top of Completed Steps
|
|
||||||
* move the top item of Future Steps into Next Step
|
|
||||||
* commit (`TODO.md` changes in the same commit as the work)
|
|
||||||
* merge to `main` if the branch is not protected, otherwise open a PR
|
|
||||||
* push
|
|
||||||
|
|
||||||
# Status
|
## Rclone Storage Backend (Complete)
|
||||||
|
|
||||||
pre-1.0
|
Add rclone as a storage backend via Go library import, allowing vaultik to use any of rclone's 70+ supported cloud storage providers.
|
||||||
|
|
||||||
# Next Step
|
**Configuration:**
|
||||||
|
```yaml
|
||||||
|
storage_url: "rclone://myremote/path/to/backups"
|
||||||
|
```
|
||||||
|
User must have rclone configured separately (via `rclone config`).
|
||||||
|
|
||||||
Continue the lint remediation (issue #61): 1,077 findings remain after
|
**Implementation Steps:**
|
||||||
the mechanical chunk. Next chunk candidates: `paralleltest` (137),
|
1. [x] Add rclone dependency to go.mod
|
||||||
`funcorder` (68), `testpackage` (34), `lll` (79) — still mostly
|
2. [x] Create `internal/storage/rclone.go` implementing `Storer` interface
|
||||||
mechanical — before the judgment-heavy linters (`revive` 142, `err113`
|
- `NewRcloneStorer(remote, path)` - init with `configfile.Install()` and `fs.NewFs()`
|
||||||
96, `mnd` 93, `gosec` 78, `goconst` 55, `cyclop` 52).
|
- `Put` / `PutWithProgress` - use `operations.Rcat()`
|
||||||
|
- `Get` - use `fs.NewObject()` then `obj.Open()`
|
||||||
|
- `Stat` - use `fs.NewObject()` for size/metadata
|
||||||
|
- `Delete` - use `obj.Remove()`
|
||||||
|
- `List` / `ListStream` - use `operations.ListFn()`
|
||||||
|
- `Info` - return remote name
|
||||||
|
3. [x] Update `internal/storage/url.go` - parse `rclone://remote/path` URLs
|
||||||
|
4. [x] Update `internal/storage/module.go` - add rclone case to `storerFromURL()`
|
||||||
|
5. [x] Test with real rclone remote
|
||||||
|
|
||||||
# Completed Steps
|
**Error Mapping:**
|
||||||
|
- `fs.ErrorObjectNotFound` → `ErrNotFound`
|
||||||
|
- `fs.ErrorDirNotFound` → `ErrNotFound`
|
||||||
|
- `fs.ErrorNotFoundInConfigFile` → `ErrRemoteNotFound` (new)
|
||||||
|
|
||||||
- 2026-08-07: Lint remediation chunk 1 (issue #61): `wsl_v5` (1050),
|
---
|
||||||
`nlreturn` (378), and `noinlineerr` (373) all fixed to zero via a new
|
|
||||||
`make lint-fix` autofix entrypoint plus hand-fixes; total findings
|
|
||||||
2,990 → 1,077. Full test suite green.
|
|
||||||
- 2026-08-07: Added the standard `.golangci.yml` and `.editorconfig`
|
|
||||||
(issue #59); lint findings under the new config are tracked in issue
|
|
||||||
#61. `script/bootstrap` now installs sqlite3 (needed by tests).
|
|
||||||
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
|
|
||||||
Makefile shims, README Entrypoints section
|
|
||||||
- 2026-07-02: Consolidated CLI verbs, retired overlapping commands; bound
|
|
||||||
the local index to its backup destination URL.
|
|
||||||
- 2026-06-28: snapshot rm now removes metadata only and prints the prune
|
|
||||||
command; restore skips chown when running as non-root.
|
|
||||||
- 2026-06-26: Snapshot IDs hashed at the storage boundary; snapshot list
|
|
||||||
made resilient to bad remote entries.
|
|
||||||
- 2026-06-24: Collapsed snapshot prune into vaultik prune; restore streams
|
|
||||||
blobs to disk and restores files in blob-locality order; cron output
|
|
||||||
fixes.
|
|
||||||
- 2026-06-17: Restore overhaul: ReadAt chunk reads from cached blobs,
|
|
||||||
reference-counted blob sweeper, integration tests; new internal/ui
|
|
||||||
output layer, banner, and progress lines.
|
|
||||||
- 2025-12-18: Added ARCHITECTURE.md and godoc coverage for exported API.
|
|
||||||
- 2025-07-26: End-to-end integration tests; manifest format refactor;
|
|
||||||
renamed backup to snapshot; afero filesystem abstraction.
|
|
||||||
- 2025-07-20: Initial design and implementation: cobra + fx CLI skeleton,
|
|
||||||
SQLite index database, UUID blob storage with streaming chunking.
|
|
||||||
|
|
||||||
# Future Steps
|
## CLI Polish (Priority)
|
||||||
|
|
||||||
- Reconcile the uncommitted ARCHITECTURE.md edits on main: finish and
|
1. Improve error messages throughout
|
||||||
commit, or revert.
|
- Ensure all errors include actionable context
|
||||||
- Review stale local branches (add-godoc-to-cli-package,
|
- Add suggestions for common issues (e.g., "did you set VAULTIK_AGE_SECRET_KEY?")
|
||||||
feature/pluggable-storage-backend) and merge or delete them.
|
|
||||||
- Define remaining scope for a first tagged release and cut v0.1.0.
|
## Security (Priority)
|
||||||
|
|
||||||
|
1. Audit encryption implementation
|
||||||
|
- Verify age encryption is used correctly
|
||||||
|
- Ensure no plaintext leaks in logs or errors
|
||||||
|
- Verify blob hashes are computed correctly
|
||||||
|
|
||||||
|
1. Secure memory handling for secrets
|
||||||
|
- Clear S3 credentials from memory after client init
|
||||||
|
- Document that age_secret_key is env-var only (already implemented)
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
1. Write integration tests for restore command
|
||||||
|
|
||||||
|
1. Write end-to-end integration test
|
||||||
|
- Create backup
|
||||||
|
- Verify backup
|
||||||
|
- Restore backup
|
||||||
|
- Compare restored files to originals
|
||||||
|
|
||||||
|
1. Add tests for edge cases
|
||||||
|
- Empty directories
|
||||||
|
- Symlinks
|
||||||
|
- Special characters in filenames
|
||||||
|
- Very large files (multi-GB)
|
||||||
|
- Many small files (100k+)
|
||||||
|
|
||||||
|
1. Add tests for error conditions
|
||||||
|
- Network failures during upload
|
||||||
|
- Disk full during restore
|
||||||
|
- Corrupted blobs
|
||||||
|
- Missing blobs
|
||||||
|
|
||||||
|
## Performance
|
||||||
|
|
||||||
|
1. Profile and optimize restore performance
|
||||||
|
- Parallel blob downloads
|
||||||
|
- Streaming decompression/decryption
|
||||||
|
- Efficient chunk reassembly
|
||||||
|
|
||||||
|
1. Add bandwidth limiting option
|
||||||
|
- `--bwlimit` flag for upload/download speed limiting
|
||||||
|
|
||||||
|
## Documentation
|
||||||
|
|
||||||
|
1. Add man page or --help improvements
|
||||||
|
- Detailed help for each command
|
||||||
|
- Examples in help output
|
||||||
|
|
||||||
|
## Final Polish
|
||||||
|
|
||||||
|
1. Ensure version is set correctly in releases
|
||||||
|
|
||||||
|
1. Create release process
|
||||||
|
- Binary releases for supported platforms
|
||||||
|
- Checksums for binaries
|
||||||
|
- Release notes template
|
||||||
|
|
||||||
|
1. Final code review
|
||||||
|
- Remove debug statements
|
||||||
|
- Ensure consistent code style
|
||||||
|
|
||||||
|
1. Tag and release v1.0.0
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Daemon Mode (Complete)
|
||||||
|
|
||||||
|
1. [x] Implement cross-platform filesystem watcher (via fsnotify)
|
||||||
|
- Watches source directories for changes
|
||||||
|
- Tracks dirty paths in memory
|
||||||
|
- Automatically watches new directories
|
||||||
|
|
||||||
|
1. [x] Implement backup scheduler in daemon mode
|
||||||
|
- Respects backup_interval config
|
||||||
|
- Triggers backup when dirty paths exist and interval elapsed
|
||||||
|
- Implements full_scan_interval for periodic full scans
|
||||||
|
- Respects min_time_between_run to prevent excessive runs
|
||||||
|
|
||||||
|
1. [x] Add proper signal handling for daemon
|
||||||
|
- Graceful shutdown on SIGTERM/SIGINT
|
||||||
|
- Completes in-progress backup before exit
|
||||||
|
|
||||||
|
1. [x] Write tests for daemon mode
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
"runtime"
|
"runtime"
|
||||||
"runtime/pprof"
|
"runtime/pprof"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/cli"
|
"git.eeqj.de/sneak/vaultik/internal/cli"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -16,12 +16,9 @@ func main() {
|
|||||||
panic("could not create CPU profile: " + err.Error())
|
panic("could not create CPU profile: " + err.Error())
|
||||||
}
|
}
|
||||||
defer func() { _ = f.Close() }()
|
defer func() { _ = f.Close() }()
|
||||||
|
if err := pprof.StartCPUProfile(f); err != nil {
|
||||||
err = pprof.StartCPUProfile(f)
|
|
||||||
if err != nil {
|
|
||||||
panic("could not start CPU profile: " + err.Error())
|
panic("could not start CPU profile: " + err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
defer pprof.StopCPUProfile()
|
defer pprof.StopCPUProfile()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -33,11 +30,8 @@ func main() {
|
|||||||
panic("could not create memory profile: " + err.Error())
|
panic("could not create memory profile: " + err.Error())
|
||||||
}
|
}
|
||||||
defer func() { _ = f.Close() }()
|
defer func() { _ = f.Close() }()
|
||||||
|
|
||||||
runtime.GC() // get up-to-date statistics
|
runtime.GC() // get up-to-date statistics
|
||||||
|
if err := pprof.WriteHeapProfile(f); err != nil {
|
||||||
err = pprof.WriteHeapProfile(f)
|
|
||||||
if err != nil {
|
|
||||||
panic("could not write memory profile: " + err.Error())
|
panic("could not write memory profile: " + err.Error())
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|||||||
@@ -291,6 +291,21 @@ storage_url: "rclone://las1stor1//srv/pool.2024.04/backups/heraklion"
|
|||||||
# # Default: 5MB
|
# # Default: 5MB
|
||||||
# #part_size: 5MB
|
# #part_size: 5MB
|
||||||
|
|
||||||
|
# How often to run backups in daemon mode
|
||||||
|
# Format: 1h, 30m, 24h, etc
|
||||||
|
# Default: 1h
|
||||||
|
#backup_interval: 1h
|
||||||
|
|
||||||
|
# How often to do a full filesystem scan in daemon mode
|
||||||
|
# Between full scans, inotify is used to detect changes
|
||||||
|
# Default: 24h
|
||||||
|
#full_scan_interval: 24h
|
||||||
|
|
||||||
|
# Minimum time between backup runs in daemon mode
|
||||||
|
# Prevents backups from running too frequently
|
||||||
|
# Default: 15m
|
||||||
|
#min_time_between_run: 15m
|
||||||
|
|
||||||
# Path to local SQLite index database
|
# Path to local SQLite index database
|
||||||
# This database tracks file state for incremental backups
|
# This database tracks file state for incremental backups
|
||||||
# Default: /var/lib/vaultik/index.sqlite
|
# Default: /var/lib/vaultik/index.sqlite
|
||||||
|
|||||||
@@ -5,14 +5,8 @@
|
|||||||
Vaultik uses a local SQLite database to track file metadata, chunk mappings, and blob associations during the backup process. This database serves as an index for incremental backups and enables efficient deduplication.
|
Vaultik uses a local SQLite database to track file metadata, chunk mappings, and blob associations during the backup process. This database serves as an index for incremental backups and enables efficient deduplication.
|
||||||
|
|
||||||
**Important Notes:**
|
**Important Notes:**
|
||||||
- **No Migration Support (pre-1.0)**: Vaultik does not support database schema
|
- **No Migration Support**: Vaultik does not support database schema migrations. If the schema changes, the local database must be deleted and recreated by performing a full backup.
|
||||||
migrations. The local index is treated as disposable — if the schema changes,
|
- **Version Compatibility**: In rare cases, you may need to use the same version of Vaultik to restore a backup as was used to create it. This ensures compatibility with the metadata format stored in S3.
|
||||||
delete the local SQLite database (`vaultik database delete`) and run a full
|
|
||||||
backup. The remote storage is unaffected; the new index will re-deduplicate
|
|
||||||
against existing remote blobs.
|
|
||||||
- **Version Compatibility**: In rare cases, you may need to use the same version
|
|
||||||
of Vaultik to restore a backup as was used to create it. This ensures
|
|
||||||
compatibility with the metadata format stored in S3.
|
|
||||||
|
|
||||||
## Database Tables
|
## Database Tables
|
||||||
|
|
||||||
|
|||||||
@@ -43,19 +43,18 @@ Blobs contain the actual file data from backups and must be encrypted for securi
|
|||||||
Each snapshot has its own subdirectory named with the snapshot ID.
|
Each snapshot has its own subdirectory named with the snapshot ID.
|
||||||
|
|
||||||
### Snapshot ID Format
|
### Snapshot ID Format
|
||||||
- **Format**: `<hostname>_<snapshot-name>_<RFC3339>` (or `<hostname>_<RFC3339>` if no
|
- **Format**: `<hostname>-<YYYYMMDD>-<HHMMSSZ>`
|
||||||
name was specified)
|
- **Example**: `laptop-20240115-143052Z`
|
||||||
- **Example**: `laptop_home_2024-01-15T14:30:52Z`
|
|
||||||
- **Components**:
|
- **Components**:
|
||||||
- Short hostname (everything before the first dot is stripped from the FQDN)
|
- Hostname (may contain hyphens)
|
||||||
- Snapshot name from the configured `snapshots:` map (optional)
|
- Date in YYYYMMDD format
|
||||||
- RFC3339 UTC timestamp
|
- Time in HHMMSSZ format (Z indicates UTC)
|
||||||
|
|
||||||
### Files in Each Snapshot Directory
|
### Files in Each Snapshot Directory
|
||||||
|
|
||||||
#### `db.zst.age` - Encrypted Database
|
#### `db.zst.age` - Encrypted Database Dump
|
||||||
- **What it contains**: Pruned binary SQLite database for this snapshot
|
- **What it contains**: Complete SQLite database dump for this snapshot
|
||||||
- **Format**: Binary SQLite → Zstandard compressed → Age encrypted
|
- **Format**: SQL dump → Zstandard compressed → Age encrypted
|
||||||
- **Encryption**: Encrypted with Age
|
- **Encryption**: Encrypted with Age
|
||||||
- **Purpose**: Contains full file metadata, chunk mappings, and all relationships
|
- **Purpose**: Contains full file metadata, chunk mappings, and all relationships
|
||||||
- **Why encrypted**: Contains sensitive metadata like file paths, permissions, and ownership
|
- **Why encrypted**: Contains sensitive metadata like file paths, permissions, and ownership
|
||||||
@@ -68,7 +67,7 @@ Each snapshot has its own subdirectory named with the snapshot ID.
|
|||||||
- **Structure**:
|
- **Structure**:
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"snapshot_id": "laptop_home_2024-01-15T14:30:52Z",
|
"snapshot_id": "laptop-20240115-143052Z",
|
||||||
"timestamp": "2024-01-15T14:30:52Z",
|
"timestamp": "2024-01-15T14:30:52Z",
|
||||||
"blob_count": 42,
|
"blob_count": 42,
|
||||||
"blobs": [
|
"blobs": [
|
||||||
|
|||||||
7
go.mod
7
go.mod
@@ -1,4 +1,4 @@
|
|||||||
module sneak.berlin/go/vaultik
|
module git.eeqj.de/sneak/vaultik
|
||||||
|
|
||||||
go 1.26.1
|
go 1.26.1
|
||||||
|
|
||||||
@@ -13,11 +13,14 @@ require (
|
|||||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0
|
github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0
|
||||||
github.com/aws/smithy-go v1.23.2
|
github.com/aws/smithy-go v1.23.2
|
||||||
github.com/dustin/go-humanize v1.0.1
|
github.com/dustin/go-humanize v1.0.1
|
||||||
|
github.com/fsnotify/fsnotify v1.9.0
|
||||||
github.com/gobwas/glob v0.2.3
|
github.com/gobwas/glob v0.2.3
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
github.com/johannesboyne/gofakes3 v0.0.0-20250603205740-ed9094be7668
|
github.com/johannesboyne/gofakes3 v0.0.0-20250603205740-ed9094be7668
|
||||||
github.com/klauspost/compress v1.18.1
|
github.com/klauspost/compress v1.18.1
|
||||||
|
github.com/mattn/go-sqlite3 v1.14.29
|
||||||
github.com/rclone/rclone v1.72.1
|
github.com/rclone/rclone v1.72.1
|
||||||
|
github.com/schollz/progressbar/v3 v3.19.0
|
||||||
github.com/spf13/afero v1.15.0
|
github.com/spf13/afero v1.15.0
|
||||||
github.com/spf13/cobra v1.10.1
|
github.com/spf13/cobra v1.10.1
|
||||||
github.com/stretchr/testify v1.11.1
|
github.com/stretchr/testify v1.11.1
|
||||||
@@ -185,6 +188,7 @@ require (
|
|||||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||||
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||||
@@ -215,6 +219,7 @@ require (
|
|||||||
github.com/relvacode/iso8601 v1.7.0 // indirect
|
github.com/relvacode/iso8601 v1.7.0 // indirect
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||||
github.com/rfjakob/eme v1.1.2 // indirect
|
github.com/rfjakob/eme v1.1.2 // indirect
|
||||||
|
github.com/rivo/uniseg v0.4.7 // indirect
|
||||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect
|
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect
|
||||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
|
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
|
||||||
|
|||||||
14
go.sum
14
go.sum
@@ -202,6 +202,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
|
|||||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc=
|
github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc=
|
||||||
github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts=
|
github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts=
|
||||||
|
github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
|
||||||
|
github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
|
||||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 h1:z0uK8UQqjMVYzvk4tiiu3obv2B44+XBsvgEJREQfnO8=
|
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 h1:z0uK8UQqjMVYzvk4tiiu3obv2B44+XBsvgEJREQfnO8=
|
||||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9/go.mod h1:Jl2neWsQaDanWORdqZ4emBl50J4/aRBBS4FyyG9/PFo=
|
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9/go.mod h1:Jl2neWsQaDanWORdqZ4emBl50J4/aRBBS4FyyG9/PFo=
|
||||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||||
@@ -284,8 +286,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
|
|||||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||||
github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
|
github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
|
||||||
github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
|
github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
|
||||||
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
|
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||||
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
|
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||||
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
|
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
|
||||||
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
|
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
|
||||||
github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj5DR5DYFik=
|
github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj5DR5DYFik=
|
||||||
@@ -591,12 +593,16 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
|
|||||||
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||||
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
||||||
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||||
|
github.com/mattn/go-sqlite3 v1.14.29 h1:1O6nRLJKvsi1H2Sj0Hzdfojwt8GiGKm+LOfLaBFaouQ=
|
||||||
|
github.com/mattn/go-sqlite3 v1.14.29/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||||
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
|
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
|
||||||
github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
|
github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
|
||||||
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
|
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
|
||||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||||
github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
|
github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
|
||||||
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||||
|
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||||
github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
|
github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
|
||||||
@@ -701,6 +707,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94
|
|||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||||
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
|
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
|
||||||
github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
|
github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
|
||||||
|
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||||
|
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||||
@@ -715,6 +723,8 @@ github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDj
|
|||||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
|
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
|
||||||
github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw=
|
github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw=
|
||||||
github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
|
github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
|
||||||
|
github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc=
|
||||||
|
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
||||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
||||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||||
|
|||||||
@@ -18,18 +18,17 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/blobgen"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// BlobHandler is a callback function invoked when a blob is finalized and ready for upload.
|
// BlobHandler is a callback function invoked when a blob is finalized and ready for upload.
|
||||||
@@ -125,7 +124,6 @@ type BlobChunkRef struct {
|
|||||||
// BlobWithReader wraps a FinishedBlob with its data reader
|
// BlobWithReader wraps a FinishedBlob with its data reader
|
||||||
type BlobWithReader struct {
|
type BlobWithReader struct {
|
||||||
*FinishedBlob
|
*FinishedBlob
|
||||||
|
|
||||||
Reader io.ReadSeeker
|
Reader io.ReadSeeker
|
||||||
TempFile afero.File // Optional, only set for disk-based blobs
|
TempFile afero.File // Optional, only set for disk-based blobs
|
||||||
InsertedChunkHashes []string // Chunk hashes that were inserted to DB with this blob
|
InsertedChunkHashes []string // Chunk hashes that were inserted to DB with this blob
|
||||||
@@ -136,17 +134,14 @@ type BlobWithReader struct {
|
|||||||
// Returns an error if required configuration fields are missing or invalid.
|
// Returns an error if required configuration fields are missing or invalid.
|
||||||
func NewPacker(cfg PackerConfig) (*Packer, error) {
|
func NewPacker(cfg PackerConfig) (*Packer, error) {
|
||||||
if len(cfg.Recipients) == 0 {
|
if len(cfg.Recipients) == 0 {
|
||||||
return nil, errors.New("recipients are required - blobs must be encrypted")
|
return nil, fmt.Errorf("recipients are required - blobs must be encrypted")
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.MaxBlobSize <= 0 {
|
if cfg.MaxBlobSize <= 0 {
|
||||||
return nil, errors.New("max blob size must be positive")
|
return nil, fmt.Errorf("max blob size must be positive")
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.Fs == nil {
|
if cfg.Fs == nil {
|
||||||
return nil, errors.New("filesystem is required")
|
return nil, fmt.Errorf("filesystem is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
return &Packer{
|
return &Packer{
|
||||||
maxBlobSize: cfg.MaxBlobSize,
|
maxBlobSize: cfg.MaxBlobSize,
|
||||||
compressionLevel: cfg.CompressionLevel,
|
compressionLevel: cfg.CompressionLevel,
|
||||||
@@ -165,7 +160,6 @@ func NewPacker(cfg PackerConfig) (*Packer, error) {
|
|||||||
func (p *Packer) SetBlobHandler(handler BlobHandler) {
|
func (p *Packer) SetBlobHandler(handler BlobHandler) {
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
defer p.mu.Unlock()
|
defer p.mu.Unlock()
|
||||||
|
|
||||||
p.blobHandler = handler
|
p.blobHandler = handler
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -175,7 +169,6 @@ func (p *Packer) SetBlobHandler(handler BlobHandler) {
|
|||||||
func (p *Packer) AddPendingChunk(hash string, size int64) {
|
func (p *Packer) AddPendingChunk(hash string, size int64) {
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
defer p.mu.Unlock()
|
defer p.mu.Unlock()
|
||||||
|
|
||||||
p.pendingChunks = append(p.pendingChunks, PendingChunk{Hash: hash, Size: size})
|
p.pendingChunks = append(p.pendingChunks, PendingChunk{Hash: hash, Size: size})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -190,8 +183,7 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error {
|
|||||||
|
|
||||||
// Initialize new blob if needed
|
// Initialize new blob if needed
|
||||||
if p.currentBlob == nil {
|
if p.currentBlob == nil {
|
||||||
err := p.startNewBlob()
|
if err := p.startNewBlob(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("starting new blob: %w", err)
|
return fmt.Errorf("starting new blob: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -210,8 +202,7 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Add chunk to current blob
|
// Add chunk to current blob
|
||||||
err := p.addChunkToCurrentBlob(chunk)
|
if err := p.addChunkToCurrentBlob(chunk); err != nil {
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -227,8 +218,7 @@ func (p *Packer) Flush() error {
|
|||||||
defer p.mu.Unlock()
|
defer p.mu.Unlock()
|
||||||
|
|
||||||
if p.currentBlob != nil && len(p.currentBlob.chunks) > 0 {
|
if p.currentBlob != nil && len(p.currentBlob.chunks) > 0 {
|
||||||
err := p.finalizeCurrentBlob()
|
if err := p.finalizeCurrentBlob(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("finalizing blob: %w", err)
|
return fmt.Errorf("finalizing blob: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -263,7 +253,6 @@ func (p *Packer) GetFinishedBlobs() []*FinishedBlob {
|
|||||||
|
|
||||||
blobs := p.finishedBlobs
|
blobs := p.finishedBlobs
|
||||||
p.finishedBlobs = make([]*FinishedBlob, 0)
|
p.finishedBlobs = make([]*FinishedBlob, 0)
|
||||||
|
|
||||||
return blobs
|
return blobs
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -278,7 +267,6 @@ func (p *Packer) startNewBlob() error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("parsing blob ID: %w", err)
|
return fmt.Errorf("parsing blob ID: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
blob := &database.Blob{
|
blob := &database.Blob{
|
||||||
ID: blobIDTyped,
|
ID: blobIDTyped,
|
||||||
Hash: types.BlobHash("temp-placeholder-" + blobID), // Temporary placeholder until finalized
|
Hash: types.BlobHash("temp-placeholder-" + blobID), // Temporary placeholder until finalized
|
||||||
@@ -288,11 +276,9 @@ func (p *Packer) startNewBlob() error {
|
|||||||
CompressedSize: 0,
|
CompressedSize: 0,
|
||||||
UploadedTS: nil,
|
UploadedTS: nil,
|
||||||
}
|
}
|
||||||
|
if err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||||
err = p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
|
||||||
return p.repos.Blobs.Create(ctx, tx, blob)
|
return p.repos.Blobs.Create(ctx, tx, blob)
|
||||||
})
|
}); err != nil {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("creating blob record: %w", err)
|
return fmt.Errorf("creating blob record: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -308,7 +294,6 @@ func (p *Packer) startNewBlob() error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
_ = tempFile.Close()
|
_ = tempFile.Close()
|
||||||
_ = p.fs.Remove(tempFile.Name())
|
_ = p.fs.Remove(tempFile.Name())
|
||||||
|
|
||||||
return fmt.Errorf("creating blobgen writer: %w", err)
|
return fmt.Errorf("creating blobgen writer: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -323,7 +308,6 @@ func (p *Packer) startNewBlob() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("Created new blob container", "blob_id", blobID, "temp_file", tempFile.Name())
|
log.Debug("Created new blob container", "blob_id", blobID, "temp_file", tempFile.Name())
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -332,7 +316,6 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error {
|
|||||||
// Skip if chunk already in current blob
|
// Skip if chunk already in current blob
|
||||||
if p.currentBlob.chunkSet[chunk.Hash] {
|
if p.currentBlob.chunkSet[chunk.Hash] {
|
||||||
log.Debug("Skipping duplicate chunk already in current blob", "chunk_hash", chunk.Hash)
|
log.Debug("Skipping duplicate chunk already in current blob", "chunk_hash", chunk.Hash)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -340,8 +323,7 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error {
|
|||||||
offset := p.currentBlob.size
|
offset := p.currentBlob.size
|
||||||
|
|
||||||
// Write to the blobgen writer (compression -> encryption -> disk)
|
// Write to the blobgen writer (compression -> encryption -> disk)
|
||||||
_, err := p.currentBlob.writer.Write(chunk.Data)
|
if _, err := p.currentBlob.writer.Write(chunk.Data); err != nil {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("writing to blob stream: %w", err)
|
return fmt.Errorf("writing to blob stream: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -389,8 +371,7 @@ func (p *Packer) finalizeCurrentBlob() error {
|
|||||||
chunksToInsert := p.pendingChunks
|
chunksToInsert := p.pendingChunks
|
||||||
p.pendingChunks = nil
|
p.pendingChunks = nil
|
||||||
|
|
||||||
err = p.commitBlobToDatabase(blobHash, finalSize, chunksToInsert)
|
if err := p.commitBlobToDatabase(blobHash, finalSize, chunksToInsert); err != nil {
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -420,36 +401,26 @@ func (p *Packer) finalizeCurrentBlob() error {
|
|||||||
|
|
||||||
// closeBlobWriter closes the writer, syncs to disk, and returns the blob hash and final size
|
// closeBlobWriter closes the writer, syncs to disk, and returns the blob hash and final size
|
||||||
func (p *Packer) closeBlobWriter() (string, int64, error) {
|
func (p *Packer) closeBlobWriter() (string, int64, error) {
|
||||||
err := p.currentBlob.writer.Close()
|
if err := p.currentBlob.writer.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
p.cleanupTempFile()
|
p.cleanupTempFile()
|
||||||
|
|
||||||
return "", 0, fmt.Errorf("closing blobgen writer: %w", err)
|
return "", 0, fmt.Errorf("closing blobgen writer: %w", err)
|
||||||
}
|
}
|
||||||
|
if err := p.currentBlob.tempFile.Sync(); err != nil {
|
||||||
err = p.currentBlob.tempFile.Sync()
|
|
||||||
if err != nil {
|
|
||||||
p.cleanupTempFile()
|
p.cleanupTempFile()
|
||||||
|
|
||||||
return "", 0, fmt.Errorf("syncing temp file: %w", err)
|
return "", 0, fmt.Errorf("syncing temp file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
finalSize, err := p.currentBlob.tempFile.Seek(0, io.SeekCurrent)
|
finalSize, err := p.currentBlob.tempFile.Seek(0, io.SeekCurrent)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
p.cleanupTempFile()
|
p.cleanupTempFile()
|
||||||
|
|
||||||
return "", 0, fmt.Errorf("getting file size: %w", err)
|
return "", 0, fmt.Errorf("getting file size: %w", err)
|
||||||
}
|
}
|
||||||
|
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
|
||||||
_, err = p.currentBlob.tempFile.Seek(0, io.SeekStart)
|
|
||||||
if err != nil {
|
|
||||||
p.cleanupTempFile()
|
p.cleanupTempFile()
|
||||||
|
|
||||||
return "", 0, fmt.Errorf("seeking to start: %w", err)
|
return "", 0, fmt.Errorf("seeking to start: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
finalHash := p.currentBlob.writer.Sum256()
|
finalHash := p.currentBlob.writer.Sum256()
|
||||||
|
|
||||||
return hex.EncodeToString(finalHash), finalSize, nil
|
return hex.EncodeToString(finalHash), finalSize, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -461,7 +432,6 @@ func (p *Packer) buildChunkRefs() []*BlobChunkRef {
|
|||||||
ChunkHash: chunk.Hash, Offset: chunk.Offset, Length: chunk.Size,
|
ChunkHash: chunk.Hash, Offset: chunk.Offset, Length: chunk.Size,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return refs
|
return refs
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -474,16 +444,13 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
|
|||||||
blobIDTyped, parseErr := types.ParseBlobID(p.currentBlob.id)
|
blobIDTyped, parseErr := types.ParseBlobID(p.currentBlob.id)
|
||||||
if parseErr != nil {
|
if parseErr != nil {
|
||||||
p.cleanupTempFile()
|
p.cleanupTempFile()
|
||||||
|
|
||||||
return fmt.Errorf("parsing blob ID: %w", parseErr)
|
return fmt.Errorf("parsing blob ID: %w", parseErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||||
for _, chunk := range chunksToInsert {
|
for _, chunk := range chunksToInsert {
|
||||||
dbChunk := &database.Chunk{ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size}
|
dbChunk := &database.Chunk{ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size}
|
||||||
|
if err := p.repos.Chunks.Create(ctx, tx, dbChunk); err != nil {
|
||||||
err := p.repos.Chunks.Create(ctx, tx, dbChunk)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("creating chunk: %w", err)
|
return fmt.Errorf("creating chunk: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -493,9 +460,7 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
|
|||||||
BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash),
|
BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash),
|
||||||
Offset: chunk.Offset, Length: chunk.Size,
|
Offset: chunk.Offset, Length: chunk.Size,
|
||||||
}
|
}
|
||||||
|
if err := p.repos.BlobChunks.Create(ctx, tx, blobChunk); err != nil {
|
||||||
err := p.repos.BlobChunks.Create(ctx, tx, blobChunk)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("creating blob_chunk: %w", err)
|
return fmt.Errorf("creating blob_chunk: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -504,23 +469,19 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
|
|||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
p.cleanupTempFile()
|
p.cleanupTempFile()
|
||||||
|
|
||||||
return fmt.Errorf("finalizing blob transaction: %w", err)
|
return fmt.Errorf("finalizing blob transaction: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("Committed blob transaction",
|
log.Debug("Committed blob transaction",
|
||||||
"chunks_inserted", len(chunksToInsert), "blob_chunks_inserted", len(p.currentBlob.chunks))
|
"chunks_inserted", len(chunksToInsert), "blob_chunks_inserted", len(p.currentBlob.chunks))
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// deliverFinishedBlob passes the blob to the handler or stores it internally
|
// deliverFinishedBlob passes the blob to the handler or stores it internally
|
||||||
func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes []string) error {
|
func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes []string) error {
|
||||||
if p.blobHandler != nil {
|
if p.blobHandler != nil {
|
||||||
_, err := p.currentBlob.tempFile.Seek(0, io.SeekStart)
|
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
|
||||||
if err != nil {
|
|
||||||
p.cleanupTempFile()
|
p.cleanupTempFile()
|
||||||
|
|
||||||
return fmt.Errorf("seeking for handler: %w", err)
|
return fmt.Errorf("seeking for handler: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -531,40 +492,30 @@ func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes
|
|||||||
InsertedChunkHashes: insertedChunkHashes,
|
InsertedChunkHashes: insertedChunkHashes,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = p.blobHandler(blobWithReader)
|
if err := p.blobHandler(blobWithReader); err != nil {
|
||||||
if err != nil {
|
|
||||||
p.cleanupTempFile()
|
p.cleanupTempFile()
|
||||||
|
|
||||||
return fmt.Errorf("blob handler failed: %w", err)
|
return fmt.Errorf("blob handler failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
p.currentBlob = nil
|
p.currentBlob = nil
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// No handler - read data for legacy behavior
|
// No handler - read data for legacy behavior
|
||||||
log.Debug("No blob handler callback configured", "blob_hash", finished.Hash[:8]+"...")
|
log.Debug("No blob handler callback configured", "blob_hash", finished.Hash[:8]+"...")
|
||||||
|
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
|
||||||
_, err := p.currentBlob.tempFile.Seek(0, io.SeekStart)
|
|
||||||
if err != nil {
|
|
||||||
p.cleanupTempFile()
|
p.cleanupTempFile()
|
||||||
|
|
||||||
return fmt.Errorf("seeking to read data: %w", err)
|
return fmt.Errorf("seeking to read data: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
data, err := io.ReadAll(p.currentBlob.tempFile)
|
data, err := io.ReadAll(p.currentBlob.tempFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
p.cleanupTempFile()
|
p.cleanupTempFile()
|
||||||
|
|
||||||
return fmt.Errorf("reading blob data: %w", err)
|
return fmt.Errorf("reading blob data: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
finished.Data = data
|
finished.Data = data
|
||||||
p.finishedBlobs = append(p.finishedBlobs, finished)
|
p.finishedBlobs = append(p.finishedBlobs, finished)
|
||||||
p.cleanupTempFile()
|
p.cleanupTempFile()
|
||||||
p.currentBlob = nil
|
p.currentBlob = nil
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -581,15 +532,13 @@ func (p *Packer) cleanupTempFile() {
|
|||||||
func (p *Packer) PackChunks(chunks []*ChunkRef) error {
|
func (p *Packer) PackChunks(chunks []*ChunkRef) error {
|
||||||
for _, chunk := range chunks {
|
for _, chunk := range chunks {
|
||||||
err := p.AddChunk(chunk)
|
err := p.AddChunk(chunk)
|
||||||
if errors.Is(err, ErrBlobSizeLimitExceeded) {
|
if err == ErrBlobSizeLimitExceeded {
|
||||||
// Finalize current blob and retry
|
// Finalize current blob and retry
|
||||||
err := p.FinalizeBlob()
|
if err := p.FinalizeBlob(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("finalizing blob before retry: %w", err)
|
return fmt.Errorf("finalizing blob before retry: %w", err)
|
||||||
}
|
}
|
||||||
// Retry the chunk
|
// Retry the chunk
|
||||||
err = p.AddChunk(chunk)
|
if err := p.AddChunk(chunk); err != nil {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("adding chunk %s after finalize: %w", chunk.Hash, err)
|
return fmt.Errorf("adding chunk %s after finalize: %w", chunk.Hash, err)
|
||||||
}
|
}
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
|
|||||||
@@ -6,16 +6,15 @@ import (
|
|||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"errors"
|
|
||||||
"io"
|
"io"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"filippo.io/age"
|
"filippo.io/age"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
"github.com/klauspost/compress/zstd"
|
"github.com/klauspost/compress/zstd"
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -41,7 +40,6 @@ func TestPacker(t *testing.T) {
|
|||||||
t.Fatalf("failed to create test db: %v", err)
|
t.Fatalf("failed to create test db: %v", err)
|
||||||
}
|
}
|
||||||
defer func() { _ = db.Close() }()
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
repos := database.NewRepositories(db)
|
||||||
|
|
||||||
cfg := PackerConfig{
|
cfg := PackerConfig{
|
||||||
@@ -51,7 +49,6 @@ func TestPacker(t *testing.T) {
|
|||||||
Repositories: repos,
|
Repositories: repos,
|
||||||
Fs: afero.NewMemMapFs(),
|
Fs: afero.NewMemMapFs(),
|
||||||
}
|
}
|
||||||
|
|
||||||
packer, err := NewPacker(cfg)
|
packer, err := NewPacker(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create packer: %v", err)
|
t.Fatalf("failed to create packer: %v", err)
|
||||||
@@ -67,7 +64,6 @@ func TestPacker(t *testing.T) {
|
|||||||
ChunkHash: types.ChunkHash(hashStr),
|
ChunkHash: types.ChunkHash(hashStr),
|
||||||
Size: int64(len(data)),
|
Size: int64(len(data)),
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||||
})
|
})
|
||||||
@@ -81,14 +77,12 @@ func TestPacker(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Add chunk
|
// Add chunk
|
||||||
err = packer.AddChunk(chunk)
|
if err := packer.AddChunk(chunk); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to add chunk: %v", err)
|
t.Fatalf("failed to add chunk: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Flush
|
// Flush
|
||||||
err = packer.Flush()
|
if err := packer.Flush(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to flush: %v", err)
|
t.Fatalf("failed to flush: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -120,9 +114,7 @@ func TestPacker(t *testing.T) {
|
|||||||
defer reader.Close()
|
defer reader.Close()
|
||||||
|
|
||||||
var decompressed bytes.Buffer
|
var decompressed bytes.Buffer
|
||||||
|
if _, err := io.Copy(&decompressed, reader); err != nil {
|
||||||
_, err = io.Copy(&decompressed, reader)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to decompress: %v", err)
|
t.Fatalf("failed to decompress: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -138,7 +130,6 @@ func TestPacker(t *testing.T) {
|
|||||||
t.Fatalf("failed to create test db: %v", err)
|
t.Fatalf("failed to create test db: %v", err)
|
||||||
}
|
}
|
||||||
defer func() { _ = db.Close() }()
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
repos := database.NewRepositories(db)
|
||||||
|
|
||||||
cfg := PackerConfig{
|
cfg := PackerConfig{
|
||||||
@@ -148,7 +139,6 @@ func TestPacker(t *testing.T) {
|
|||||||
Repositories: repos,
|
Repositories: repos,
|
||||||
Fs: afero.NewMemMapFs(),
|
Fs: afero.NewMemMapFs(),
|
||||||
}
|
}
|
||||||
|
|
||||||
packer, err := NewPacker(cfg)
|
packer, err := NewPacker(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create packer: %v", err)
|
t.Fatalf("failed to create packer: %v", err)
|
||||||
@@ -156,8 +146,7 @@ func TestPacker(t *testing.T) {
|
|||||||
|
|
||||||
// Create multiple small chunks
|
// Create multiple small chunks
|
||||||
chunks := make([]*ChunkRef, 10)
|
chunks := make([]*ChunkRef, 10)
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
for i := range 10 {
|
|
||||||
data := bytes.Repeat([]byte{byte(i)}, 1000)
|
data := bytes.Repeat([]byte{byte(i)}, 1000)
|
||||||
hash := sha256.Sum256(data)
|
hash := sha256.Sum256(data)
|
||||||
hashStr := hex.EncodeToString(hash[:])
|
hashStr := hex.EncodeToString(hash[:])
|
||||||
@@ -167,7 +156,6 @@ func TestPacker(t *testing.T) {
|
|||||||
ChunkHash: types.ChunkHash(hashStr),
|
ChunkHash: types.ChunkHash(hashStr),
|
||||||
Size: int64(len(data)),
|
Size: int64(len(data)),
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||||
})
|
})
|
||||||
@@ -190,8 +178,7 @@ func TestPacker(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Flush
|
// Flush
|
||||||
err = packer.Flush()
|
if err := packer.Flush(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to flush: %v", err)
|
t.Fatalf("failed to flush: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -211,11 +198,9 @@ func TestPacker(t *testing.T) {
|
|||||||
if chunkRef.Offset != expectedOffset {
|
if chunkRef.Offset != expectedOffset {
|
||||||
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunkRef.Offset)
|
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunkRef.Offset)
|
||||||
}
|
}
|
||||||
|
|
||||||
if chunkRef.Length != 1000 {
|
if chunkRef.Length != 1000 {
|
||||||
t.Errorf("chunk %d: expected length 1000, got %d", i, chunkRef.Length)
|
t.Errorf("chunk %d: expected length 1000, got %d", i, chunkRef.Length)
|
||||||
}
|
}
|
||||||
|
|
||||||
expectedOffset += chunkRef.Length
|
expectedOffset += chunkRef.Length
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -227,7 +212,6 @@ func TestPacker(t *testing.T) {
|
|||||||
t.Fatalf("failed to create test db: %v", err)
|
t.Fatalf("failed to create test db: %v", err)
|
||||||
}
|
}
|
||||||
defer func() { _ = db.Close() }()
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
repos := database.NewRepositories(db)
|
||||||
|
|
||||||
// Small blob size limit to force multiple blobs
|
// Small blob size limit to force multiple blobs
|
||||||
@@ -238,7 +222,6 @@ func TestPacker(t *testing.T) {
|
|||||||
Repositories: repos,
|
Repositories: repos,
|
||||||
Fs: afero.NewMemMapFs(),
|
Fs: afero.NewMemMapFs(),
|
||||||
}
|
}
|
||||||
|
|
||||||
packer, err := NewPacker(cfg)
|
packer, err := NewPacker(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create packer: %v", err)
|
t.Fatalf("failed to create packer: %v", err)
|
||||||
@@ -246,8 +229,7 @@ func TestPacker(t *testing.T) {
|
|||||||
|
|
||||||
// Create chunks that will exceed the limit
|
// Create chunks that will exceed the limit
|
||||||
chunks := make([]*ChunkRef, 10)
|
chunks := make([]*ChunkRef, 10)
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
for i := range 10 {
|
|
||||||
data := bytes.Repeat([]byte{byte(i)}, 1000) // 1KB each
|
data := bytes.Repeat([]byte{byte(i)}, 1000) // 1KB each
|
||||||
hash := sha256.Sum256(data)
|
hash := sha256.Sum256(data)
|
||||||
hashStr := hex.EncodeToString(hash[:])
|
hashStr := hex.EncodeToString(hash[:])
|
||||||
@@ -257,7 +239,6 @@ func TestPacker(t *testing.T) {
|
|||||||
ChunkHash: types.ChunkHash(hashStr),
|
ChunkHash: types.ChunkHash(hashStr),
|
||||||
Size: int64(len(data)),
|
Size: int64(len(data)),
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||||
})
|
})
|
||||||
@@ -276,17 +257,14 @@ func TestPacker(t *testing.T) {
|
|||||||
// Add chunks and handle size limit errors
|
// Add chunks and handle size limit errors
|
||||||
for _, chunk := range chunks {
|
for _, chunk := range chunks {
|
||||||
err := packer.AddChunk(chunk)
|
err := packer.AddChunk(chunk)
|
||||||
if errors.Is(err, ErrBlobSizeLimitExceeded) {
|
if err == ErrBlobSizeLimitExceeded {
|
||||||
// Finalize current blob
|
// Finalize current blob
|
||||||
err := packer.FinalizeBlob()
|
if err := packer.FinalizeBlob(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to finalize blob: %v", err)
|
t.Fatalf("failed to finalize blob: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
blobCount++
|
blobCount++
|
||||||
// Retry adding the chunk
|
// Retry adding the chunk
|
||||||
err = packer.AddChunk(chunk)
|
if err := packer.AddChunk(chunk); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to add chunk after finalize: %v", err)
|
t.Fatalf("failed to add chunk after finalize: %v", err)
|
||||||
}
|
}
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
@@ -295,8 +273,7 @@ func TestPacker(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Flush remaining
|
// Flush remaining
|
||||||
err = packer.Flush()
|
if err := packer.Flush(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to flush: %v", err)
|
t.Fatalf("failed to flush: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -324,7 +301,6 @@ func TestPacker(t *testing.T) {
|
|||||||
t.Fatalf("failed to create test db: %v", err)
|
t.Fatalf("failed to create test db: %v", err)
|
||||||
}
|
}
|
||||||
defer func() { _ = db.Close() }()
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
repos := database.NewRepositories(db)
|
||||||
|
|
||||||
// Generate test identity (using the one from parent test)
|
// Generate test identity (using the one from parent test)
|
||||||
@@ -335,7 +311,6 @@ func TestPacker(t *testing.T) {
|
|||||||
Repositories: repos,
|
Repositories: repos,
|
||||||
Fs: afero.NewMemMapFs(),
|
Fs: afero.NewMemMapFs(),
|
||||||
}
|
}
|
||||||
|
|
||||||
packer, err := NewPacker(cfg)
|
packer, err := NewPacker(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create packer: %v", err)
|
t.Fatalf("failed to create packer: %v", err)
|
||||||
@@ -351,7 +326,6 @@ func TestPacker(t *testing.T) {
|
|||||||
ChunkHash: types.ChunkHash(hashStr),
|
ChunkHash: types.ChunkHash(hashStr),
|
||||||
Size: int64(len(data)),
|
Size: int64(len(data)),
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||||
})
|
})
|
||||||
@@ -365,13 +339,10 @@ func TestPacker(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Add chunk and flush
|
// Add chunk and flush
|
||||||
err = packer.AddChunk(chunk)
|
if err := packer.AddChunk(chunk); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to add chunk: %v", err)
|
t.Fatalf("failed to add chunk: %v", err)
|
||||||
}
|
}
|
||||||
|
if err := packer.Flush(); err != nil {
|
||||||
err = packer.Flush()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to flush: %v", err)
|
t.Fatalf("failed to flush: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -390,9 +361,7 @@ func TestPacker(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var decryptedData bytes.Buffer
|
var decryptedData bytes.Buffer
|
||||||
|
if _, err := decryptedData.ReadFrom(decrypted); err != nil {
|
||||||
_, err = decryptedData.ReadFrom(decrypted)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read decrypted data: %v", err)
|
t.Fatalf("failed to read decrypted data: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -404,9 +373,7 @@ func TestPacker(t *testing.T) {
|
|||||||
defer reader.Close()
|
defer reader.Close()
|
||||||
|
|
||||||
var decompressed bytes.Buffer
|
var decompressed bytes.Buffer
|
||||||
|
if _, err := decompressed.ReadFrom(reader); err != nil {
|
||||||
_, err = decompressed.ReadFrom(reader)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to decompress: %v", err)
|
t.Fatalf("failed to decompress: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -26,16 +26,13 @@ func CompressData(data []byte, compressionLevel int, recipients []string) (*Comp
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Write data
|
// Write data
|
||||||
_, err = w.Write(data)
|
if _, err := w.Write(data); err != nil {
|
||||||
if err != nil {
|
|
||||||
_ = w.Close()
|
_ = w.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf("writing data: %w", err)
|
return nil, fmt.Errorf("writing data: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close to flush
|
// Close to flush
|
||||||
err = w.Close()
|
if err := w.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("closing writer: %w", err)
|
return nil, fmt.Errorf("closing writer: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -63,17 +60,14 @@ func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipien
|
|||||||
}()
|
}()
|
||||||
|
|
||||||
// Copy data
|
// Copy data
|
||||||
_, err = io.Copy(w, src)
|
if _, err := io.Copy(w, src); err != nil {
|
||||||
if err != nil {
|
|
||||||
return 0, "", fmt.Errorf("copying data: %w", err)
|
return 0, "", fmt.Errorf("copying data: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close to flush
|
// Close to flush
|
||||||
err = w.Close()
|
if err := w.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return 0, "", fmt.Errorf("closing writer: %w", err)
|
return 0, "", fmt.Errorf("closing writer: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
closed = true
|
closed = true
|
||||||
|
|
||||||
return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil
|
return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil
|
||||||
|
|||||||
@@ -20,14 +20,13 @@ const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s8
|
|||||||
// cause a double close.
|
// cause a double close.
|
||||||
func TestCompressStreamNoDoubleClose(t *testing.T) {
|
func TestCompressStreamNoDoubleClose(t *testing.T) {
|
||||||
input := []byte("regression test data for issue #28 double-close fix")
|
input := []byte("regression test data for issue #28 double-close fix")
|
||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
|
|
||||||
written, hash, err := CompressStream(&buf, bytes.NewReader(input), 3, []string{testRecipient})
|
written, hash, err := CompressStream(&buf, bytes.NewReader(input), 3, []string{testRecipient})
|
||||||
require.NoError(t, err, "CompressStream should not return an error")
|
require.NoError(t, err, "CompressStream should not return an error")
|
||||||
assert.Positive(t, written, "expected bytes written > 0")
|
assert.True(t, written > 0, "expected bytes written > 0")
|
||||||
assert.NotEmpty(t, hash, "expected non-empty hash")
|
assert.NotEmpty(t, hash, "expected non-empty hash")
|
||||||
assert.Positive(t, buf.Len(), "expected non-empty output")
|
assert.True(t, buf.Len() > 0, "expected non-empty output")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestCompressStreamLargeInput exercises CompressStream with a larger payload
|
// TestCompressStreamLargeInput exercises CompressStream with a larger payload
|
||||||
@@ -38,10 +37,9 @@ func TestCompressStreamLargeInput(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
|
|
||||||
written, hash, err := CompressStream(&buf, bytes.NewReader(data), 3, []string{testRecipient})
|
written, hash, err := CompressStream(&buf, bytes.NewReader(data), 3, []string{testRecipient})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Positive(t, written)
|
assert.True(t, written > 0)
|
||||||
assert.NotEmpty(t, hash)
|
assert.NotEmpty(t, hash)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -49,7 +47,6 @@ func TestCompressStreamLargeInput(t *testing.T) {
|
|||||||
// without double-close issues.
|
// without double-close issues.
|
||||||
func TestCompressStreamEmptyInput(t *testing.T) {
|
func TestCompressStreamEmptyInput(t *testing.T) {
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
|
|
||||||
_, hash, err := CompressStream(&buf, strings.NewReader(""), 3, []string{testRecipient})
|
_, hash, err := CompressStream(&buf, strings.NewReader(""), 3, []string{testRecipient})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.NotEmpty(t, hash)
|
assert.NotEmpty(t, hash)
|
||||||
@@ -61,7 +58,7 @@ func TestCompressDataNoDoubleClose(t *testing.T) {
|
|||||||
input := []byte("CompressData regression test for double-close")
|
input := []byte("CompressData regression test for double-close")
|
||||||
result, err := CompressData(input, 3, []string{testRecipient})
|
result, err := CompressData(input, 3, []string{testRecipient})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Positive(t, result.CompressedSize)
|
assert.True(t, result.CompressedSize > 0)
|
||||||
assert.Equal(t, result.UncompressedSize, int64(len(input)))
|
assert.True(t, result.UncompressedSize == int64(len(input)))
|
||||||
assert.NotEmpty(t, result.SHA256)
|
assert.NotEmpty(t, result.SHA256)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -53,14 +53,12 @@ func NewReader(r io.Reader, identity age.Identity) (*Reader, error) {
|
|||||||
func (r *Reader) Read(p []byte) (n int, err error) {
|
func (r *Reader) Read(p []byte) (n int, err error) {
|
||||||
n, err = r.teeReader.Read(p)
|
n, err = r.teeReader.Read(p)
|
||||||
r.bytesRead += int64(n)
|
r.bytesRead += int64(n)
|
||||||
|
|
||||||
return n, err
|
return n, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close closes the decompressor
|
// Close closes the decompressor
|
||||||
func (r *Reader) Close() error {
|
func (r *Reader) Close() error {
|
||||||
r.decompressor.Close()
|
r.decompressor.Close()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -27,8 +27,7 @@ type Writer struct {
|
|||||||
// The hash is computed on the uncompressed input for deterministic content-addressing.
|
// The hash is computed on the uncompressed input for deterministic content-addressing.
|
||||||
func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, error) {
|
func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, error) {
|
||||||
// Validate compression level
|
// Validate compression level
|
||||||
err := validateCompressionLevel(compressionLevel)
|
if err := validateCompressionLevel(compressionLevel); err != nil {
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -37,13 +36,11 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
|||||||
|
|
||||||
// Parse recipients
|
// Parse recipients
|
||||||
var ageRecipients []age.Recipient
|
var ageRecipients []age.Recipient
|
||||||
|
|
||||||
for _, recipient := range recipients {
|
for _, recipient := range recipients {
|
||||||
r, err := age.ParseX25519Recipient(recipient)
|
r, err := age.ParseX25519Recipient(recipient)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("parsing recipient %s: %w", recipient, err)
|
return nil, fmt.Errorf("parsing recipient %s: %w", recipient, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ageRecipients = append(ageRecipients, r)
|
ageRecipients = append(ageRecipients, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,7 +51,10 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Calculate compression concurrency: CPUs - 2, minimum 1
|
// Calculate compression concurrency: CPUs - 2, minimum 1
|
||||||
concurrency := max(runtime.NumCPU()-2, 1)
|
concurrency := runtime.NumCPU() - 2
|
||||||
|
if concurrency < 1 {
|
||||||
|
concurrency = 1
|
||||||
|
}
|
||||||
|
|
||||||
// Create compression writer with encryption as destination
|
// Create compression writer with encryption as destination
|
||||||
compressor, err := zstd.NewWriter(encWriter,
|
compressor, err := zstd.NewWriter(encWriter,
|
||||||
@@ -63,7 +63,6 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
|||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = encWriter.Close()
|
_ = encWriter.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf("creating compression writer: %w", err)
|
return nil, fmt.Errorf("creating compression writer: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -83,21 +82,18 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
|||||||
func (w *Writer) Write(p []byte) (n int, err error) {
|
func (w *Writer) Write(p []byte) (n int, err error) {
|
||||||
n, err = w.teeWriter.Write(p)
|
n, err = w.teeWriter.Write(p)
|
||||||
w.bytesWritten += int64(n)
|
w.bytesWritten += int64(n)
|
||||||
|
|
||||||
return n, err
|
return n, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close closes all layers and returns any errors
|
// Close closes all layers and returns any errors
|
||||||
func (w *Writer) Close() error {
|
func (w *Writer) Close() error {
|
||||||
// Close compressor first
|
// Close compressor first
|
||||||
err := w.compressor.Close()
|
if err := w.compressor.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("closing compressor: %w", err)
|
return fmt.Errorf("closing compressor: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Then close encryptor
|
// Then close encryptor
|
||||||
err = w.encryptor.Close()
|
if err := w.encryptor.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("closing encryptor: %w", err)
|
return fmt.Errorf("closing encryptor: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -113,7 +109,6 @@ func (w *Writer) Sum256() []byte {
|
|||||||
firstHash := w.hasher.Sum(nil)
|
firstHash := w.hasher.Sum(nil)
|
||||||
// Second hash: SHA256(firstHash) - this is the blob ID
|
// Second hash: SHA256(firstHash) - this is the blob ID
|
||||||
secondHash := sha256.Sum256(firstHash)
|
secondHash := sha256.Sum256(firstHash)
|
||||||
|
|
||||||
return secondHash[:]
|
return secondHash[:]
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,6 +123,5 @@ func validateCompressionLevel(level int) error {
|
|||||||
if level < 1 || level > 19 {
|
if level < 1 || level > 19 {
|
||||||
return fmt.Errorf("invalid compression level %d: must be between 1 and 19", level)
|
return fmt.Errorf("invalid compression level %d: must be between 1 and 19", level)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package chunker
|
|||||||
import (
|
import (
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
@@ -51,15 +50,13 @@ func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) {
|
|||||||
defer chunker.Release()
|
defer chunker.Release()
|
||||||
|
|
||||||
var chunks []Chunk
|
var chunks []Chunk
|
||||||
|
|
||||||
offset := int64(0)
|
offset := int64(0)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
chunk, err := chunker.Next()
|
chunk, err := chunker.Next()
|
||||||
if errors.Is(err, io.EOF) {
|
if err == io.EOF {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("reading chunk: %w", err)
|
return nil, fmt.Errorf("reading chunk: %w", err)
|
||||||
}
|
}
|
||||||
@@ -107,10 +104,9 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str
|
|||||||
|
|
||||||
for {
|
for {
|
||||||
chunk, err := chunker.Next()
|
chunk, err := chunker.Next()
|
||||||
if errors.Is(err, io.EOF) {
|
if err == io.EOF {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("reading chunk: %w", err)
|
return "", fmt.Errorf("reading chunk: %w", err)
|
||||||
}
|
}
|
||||||
@@ -121,13 +117,12 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str
|
|||||||
// Pass the data directly - caller must process it before we call Next() again
|
// Pass the data directly - caller must process it before we call Next() again
|
||||||
// (chunker reuses its internal buffer, but since we process synchronously
|
// (chunker reuses its internal buffer, but since we process synchronously
|
||||||
// and completely before continuing, no copy is needed)
|
// and completely before continuing, no copy is needed)
|
||||||
err = callback(Chunk{
|
if err := callback(Chunk{
|
||||||
Hash: hex.EncodeToString(hash[:]),
|
Hash: hex.EncodeToString(hash[:]),
|
||||||
Data: chunk.Data,
|
Data: chunk.Data,
|
||||||
Offset: offset,
|
Offset: offset,
|
||||||
Size: int64(len(chunk.Data)),
|
Size: int64(len(chunk.Data)),
|
||||||
})
|
}); err != nil {
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("callback error: %w", err)
|
return "", fmt.Errorf("callback error: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,8 +143,7 @@ func (c *Chunker) ChunkFile(path string) ([]Chunk, error) {
|
|||||||
return nil, fmt.Errorf("opening file: %w", err)
|
return nil, fmt.Errorf("opening file: %w", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
err := file.Close()
|
if err := file.Close(); err != nil && err.Error() != "invalid argument" {
|
||||||
if err != nil && err.Error() != "invalid argument" {
|
|
||||||
// Log error or handle as needed
|
// Log error or handle as needed
|
||||||
_ = err
|
_ = err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
|
|||||||
|
|
||||||
// Create data with some variation to trigger chunk boundaries
|
// Create data with some variation to trigger chunk boundaries
|
||||||
data := make([]byte, tt.fileSize)
|
data := make([]byte, tt.fileSize)
|
||||||
for i := range data {
|
for i := 0; i < len(data); i++ {
|
||||||
// Use a pattern that should create boundaries
|
// Use a pattern that should create boundaries
|
||||||
data[i] = byte((i * 17) ^ (i >> 5))
|
data[i] = byte((i * 17) ^ (i >> 5))
|
||||||
}
|
}
|
||||||
@@ -59,7 +59,6 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
|
|||||||
t.Errorf("too few chunks: got %d, expected at least %d",
|
t.Errorf("too few chunks: got %d, expected at least %d",
|
||||||
len(chunks), tt.minExpected)
|
len(chunks), tt.minExpected)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(chunks) > tt.maxExpected {
|
if len(chunks) > tt.maxExpected {
|
||||||
t.Errorf("too many chunks: got %d, expected at most %d",
|
t.Errorf("too many chunks: got %d, expected at most %d",
|
||||||
len(chunks), tt.maxExpected)
|
len(chunks), tt.maxExpected)
|
||||||
@@ -70,7 +69,6 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
|
|||||||
for _, chunk := range chunks {
|
for _, chunk := range chunks {
|
||||||
reconstructed = append(reconstructed, chunk.Data...)
|
reconstructed = append(reconstructed, chunk.Data...)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !bytes.Equal(data, reconstructed) {
|
if !bytes.Equal(data, reconstructed) {
|
||||||
t.Error("reconstructed data doesn't match original")
|
t.Error("reconstructed data doesn't match original")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,9 +30,7 @@ func TestChunker(t *testing.T) {
|
|||||||
|
|
||||||
// Generate 2MB of random data
|
// Generate 2MB of random data
|
||||||
data := make([]byte, 2*1024*1024)
|
data := make([]byte, 2*1024*1024)
|
||||||
|
if _, err := rand.Read(data); err != nil {
|
||||||
_, err := rand.Read(data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to generate random data: %v", err)
|
t.Fatalf("failed to generate random data: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -62,7 +60,6 @@ func TestChunker(t *testing.T) {
|
|||||||
if chunk.Offset != expectedOffset {
|
if chunk.Offset != expectedOffset {
|
||||||
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset)
|
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset)
|
||||||
}
|
}
|
||||||
|
|
||||||
expectedOffset += chunk.Size
|
expectedOffset += chunk.Size
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -93,7 +90,6 @@ func TestChunker(t *testing.T) {
|
|||||||
if chunks1[i].Hash != chunks2[i].Hash {
|
if chunks1[i].Hash != chunks2[i].Hash {
|
||||||
t.Errorf("chunk %d: different hashes", i)
|
t.Errorf("chunk %d: different hashes", i)
|
||||||
}
|
}
|
||||||
|
|
||||||
if chunks1[i].Size != chunks2[i].Size {
|
if chunks1[i].Size != chunks2[i].Size {
|
||||||
t.Errorf("chunk %d: different sizes", i)
|
t.Errorf("chunk %d: different sizes", i)
|
||||||
}
|
}
|
||||||
@@ -111,9 +107,7 @@ func TestChunkBoundaries(t *testing.T) {
|
|||||||
|
|
||||||
// Test that minimum chunk size is respected
|
// Test that minimum chunk size is respected
|
||||||
data := make([]byte, minSize+1024)
|
data := make([]byte, minSize+1024)
|
||||||
|
if _, err := rand.Read(data); err != nil {
|
||||||
_, err := rand.Read(data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to generate random data: %v", err)
|
t.Fatalf("failed to generate random data: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -127,7 +121,6 @@ func TestChunkBoundaries(t *testing.T) {
|
|||||||
if i < len(chunks)-1 && chunk.Size < minSize {
|
if i < len(chunks)-1 && chunk.Size < minSize {
|
||||||
t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize)
|
t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize)
|
||||||
}
|
}
|
||||||
|
|
||||||
if chunk.Size > maxSize {
|
if chunk.Size > maxSize {
|
||||||
t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize)
|
t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package chunker
|
package chunker
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"io"
|
"io"
|
||||||
"math"
|
"math"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -29,7 +28,7 @@ type ReusableChunker struct {
|
|||||||
|
|
||||||
// reusableChunkerPool pools ReusableChunker instances to avoid allocations.
|
// reusableChunkerPool pools ReusableChunker instances to avoid allocations.
|
||||||
var reusableChunkerPool = sync.Pool{
|
var reusableChunkerPool = sync.Pool{
|
||||||
New: func() any {
|
New: func() interface{} {
|
||||||
return &ReusableChunker{}
|
return &ReusableChunker{}
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -40,20 +39,17 @@ var bufferPools = sync.Map{}
|
|||||||
|
|
||||||
func getBuffer(size int) []byte {
|
func getBuffer(size int) []byte {
|
||||||
poolI, _ := bufferPools.LoadOrStore(size, &sync.Pool{
|
poolI, _ := bufferPools.LoadOrStore(size, &sync.Pool{
|
||||||
New: func() any {
|
New: func() interface{} {
|
||||||
buf := make([]byte, size)
|
buf := make([]byte, size)
|
||||||
|
|
||||||
return &buf
|
return &buf
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
pool := poolI.(*sync.Pool)
|
pool := poolI.(*sync.Pool)
|
||||||
|
|
||||||
return *pool.Get().(*[]byte)
|
return *pool.Get().(*[]byte)
|
||||||
}
|
}
|
||||||
|
|
||||||
func putBuffer(buf []byte) {
|
func putBuffer(buf []byte) {
|
||||||
size := cap(buf)
|
size := cap(buf)
|
||||||
|
|
||||||
poolI, ok := bufferPools.Load(size)
|
poolI, ok := bufferPools.Load(size)
|
||||||
if ok {
|
if ok {
|
||||||
pool := poolI.(*sync.Pool)
|
pool := poolI.(*sync.Pool)
|
||||||
@@ -81,7 +77,6 @@ func AcquireReusableChunker(rd io.Reader, minSize, avgSize, maxSize int) *Reusab
|
|||||||
if c.buf != nil {
|
if c.buf != nil {
|
||||||
putBuffer(c.buf)
|
putBuffer(c.buf)
|
||||||
}
|
}
|
||||||
|
|
||||||
c.buf = getBuffer(bufSize)
|
c.buf = getBuffer(bufSize)
|
||||||
} else {
|
} else {
|
||||||
// Restore buffer to full capacity (may have been truncated by previous EOF)
|
// Restore buffer to full capacity (may have been truncated by previous EOF)
|
||||||
@@ -125,7 +120,6 @@ func (c *ReusableChunker) fillBuffer() error {
|
|||||||
|
|
||||||
if c.eof {
|
if c.eof {
|
||||||
c.buf = c.buf[:n]
|
c.buf = c.buf[:n]
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -134,24 +128,21 @@ func (c *ReusableChunker) fillBuffer() error {
|
|||||||
|
|
||||||
// Fill the rest of the buffer
|
// Fill the rest of the buffer
|
||||||
m, err := io.ReadFull(c.rd, c.buf[n:])
|
m, err := io.ReadFull(c.rd, c.buf[n:])
|
||||||
if err == io.EOF || errors.Is(err, io.ErrUnexpectedEOF) {
|
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||||
c.buf = c.buf[:n+m]
|
c.buf = c.buf[:n+m]
|
||||||
c.eof = true
|
c.eof = true
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Next returns the next chunk or io.EOF when done.
|
// Next returns the next chunk or io.EOF when done.
|
||||||
// The returned Data slice is only valid until the next call to Next.
|
// The returned Data slice is only valid until the next call to Next.
|
||||||
func (c *ReusableChunker) Next() (FastCDCChunk, error) {
|
func (c *ReusableChunker) Next() (FastCDCChunk, error) {
|
||||||
err := c.fillBuffer()
|
if err := c.fillBuffer(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return FastCDCChunk{}, err
|
return FastCDCChunk{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(c.buf) == 0 {
|
if len(c.buf) == 0 {
|
||||||
return FastCDCChunk{}, io.EOF
|
return FastCDCChunk{}, io.EOF
|
||||||
}
|
}
|
||||||
@@ -198,6 +189,13 @@ func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) {
|
|||||||
return i, fp
|
return i, fp
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func min(a, b int) int {
|
||||||
|
if a < b {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
// 256 random uint64s for the rolling hash function (from FastCDC paper)
|
// 256 random uint64s for the rolling hash function (from FastCDC paper)
|
||||||
var table = [256]uint64{
|
var table = [256]uint64{
|
||||||
0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c,
|
0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c,
|
||||||
|
|||||||
@@ -7,21 +7,19 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/pidlock"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||||
"github.com/adrg/xdg"
|
"github.com/adrg/xdg"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
|
||||||
"sneak.berlin/go/vaultik/internal/globals"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/pidlock"
|
|
||||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
|
||||||
"sneak.berlin/go/vaultik/internal/ui"
|
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// AppOptions contains common options for creating the fx application.
|
// AppOptions contains common options for creating the fx application.
|
||||||
@@ -34,38 +32,16 @@ type AppOptions struct {
|
|||||||
Invokes []fx.Option
|
Invokes []fx.Option
|
||||||
}
|
}
|
||||||
|
|
||||||
// setupGlobals records the startup time and, when an output-suppression
|
// setupGlobals sets up the globals with application startup time
|
||||||
// flag is active, marks the UI writer quiet so that Begin/Complete/
|
func setupGlobals(lc fx.Lifecycle, g *globals.Globals) {
|
||||||
// Info/Notice/Detail/Progress are silenced. Warning and Error are NOT
|
|
||||||
// silenced — per the documented convention that --quiet suppresses
|
|
||||||
// non-error output only. The startup banner is printed by CLIEntry
|
|
||||||
// before cobra parses arguments, gated by the same arg-level check.
|
|
||||||
func setupGlobals(lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts log.LogOptions) {
|
|
||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
OnStart: func(ctx context.Context) error {
|
OnStart: func(ctx context.Context) error {
|
||||||
g.StartTime = time.Now().UTC()
|
g.StartTime = time.Now().UTC()
|
||||||
|
|
||||||
if opts.Cron || opts.Quiet {
|
|
||||||
v.UI.SetQuiet(true)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeStartupBanner prints the two-line application banner followed by a
|
|
||||||
// blank line. Used both from the fx hook (for subcommand invocations) and
|
|
||||||
// from the root cobra Run handler (for `vaultik` with no subcommand).
|
|
||||||
func writeStartupBanner(w *ui.Writer, startTime time.Time, shortCommit string) {
|
|
||||||
w.Banner("%s %s by %s (commit %s, built on %s) starting up at %s.",
|
|
||||||
globals.Appname, globals.Version, globals.Author,
|
|
||||||
shortCommit, globals.CommitDate,
|
|
||||||
startTime.Format(time.RFC3339))
|
|
||||||
w.Banner("%s", globals.Homepage)
|
|
||||||
w.Banner("")
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewApp creates a new fx application with common modules.
|
// NewApp creates a new fx application with common modules.
|
||||||
// It sets up the base modules (config, database, logging, globals) and
|
// It sets up the base modules (config, database, logging, globals) and
|
||||||
// combines them with any additional modules specified in the options.
|
// combines them with any additional modules specified in the options.
|
||||||
@@ -92,25 +68,6 @@ func NewApp(opts AppOptions) *fx.App {
|
|||||||
return fx.New(allOptions...)
|
return fx.New(allOptions...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// cleanStartupError strips fx's dependency-injection call-chain noise from
|
|
||||||
// startup errors. fx wraps the underlying error with messages like
|
|
||||||
//
|
|
||||||
// could not build arguments for function "X" (file:line): failed to build T:
|
|
||||||
// could not build arguments for function "Y" (file:line): failed to build U:
|
|
||||||
// received non-nil error from function "Z" (file:line): <real error>
|
|
||||||
//
|
|
||||||
// Users care about the real error, not the DI plumbing. We strip everything
|
|
||||||
// up through the last "): " (which is always the close-paren of an fx
|
|
||||||
// function-location annotation followed by the wrapped error).
|
|
||||||
func cleanStartupError(err error) error {
|
|
||||||
msg := err.Error()
|
|
||||||
if idx := strings.LastIndex(msg, "): "); idx >= 0 {
|
|
||||||
msg = msg[idx+3:]
|
|
||||||
}
|
|
||||||
|
|
||||||
return errors.New(msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
// RunApp starts and stops the fx application within the given context.
|
// RunApp starts and stops the fx application within the given context.
|
||||||
// It handles graceful shutdown on interrupt signals (SIGINT, SIGTERM) and
|
// It handles graceful shutdown on interrupt signals (SIGINT, SIGTERM) and
|
||||||
// ensures the application stops cleanly. The function blocks until the
|
// ensures the application stops cleanly. The function blocks until the
|
||||||
@@ -125,16 +82,14 @@ func RunApp(ctx context.Context, app *fx.App) error {
|
|||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
// Start the app
|
// Start the app
|
||||||
err := app.Start(ctx)
|
if err := app.Start(ctx); err != nil {
|
||||||
if err != nil {
|
return fmt.Errorf("failed to start app: %w", err)
|
||||||
return cleanStartupError(err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle shutdown
|
// Handle shutdown
|
||||||
shutdownComplete := make(chan struct{})
|
shutdownComplete := make(chan struct{})
|
||||||
go func() {
|
go func() {
|
||||||
defer close(shutdownComplete)
|
defer close(shutdownComplete)
|
||||||
|
|
||||||
<-sigChan
|
<-sigChan
|
||||||
log.Notice("Received interrupt signal, shutting down gracefully...")
|
log.Notice("Received interrupt signal, shutting down gracefully...")
|
||||||
|
|
||||||
@@ -142,8 +97,7 @@ func RunApp(ctx context.Context, app *fx.App) error {
|
|||||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
defer shutdownCancel()
|
defer shutdownCancel()
|
||||||
|
|
||||||
err := app.Stop(shutdownCtx)
|
if err := app.Stop(shutdownCtx); err != nil {
|
||||||
if err != nil {
|
|
||||||
log.Error("Error during shutdown", "error", err)
|
log.Error("Error during shutdown", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -155,11 +109,9 @@ func RunApp(ctx context.Context, app *fx.App) error {
|
|||||||
return nil
|
return nil
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
// Context cancelled (shouldn't happen in normal operation)
|
// Context cancelled (shouldn't happen in normal operation)
|
||||||
err := app.Stop(context.Background())
|
if err := app.Stop(context.Background()); err != nil {
|
||||||
if err != nil {
|
|
||||||
log.Error("Error stopping app", "error", err)
|
log.Error("Error stopping app", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
case <-app.Done():
|
case <-app.Done():
|
||||||
// App finished running (e.g., backup completed)
|
// App finished running (e.g., backup completed)
|
||||||
@@ -173,25 +125,20 @@ func RunApp(ctx context.Context, app *fx.App) error {
|
|||||||
// It acquires a PID lock before starting to prevent concurrent instances.
|
// It acquires a PID lock before starting to prevent concurrent instances.
|
||||||
func RunWithApp(ctx context.Context, opts AppOptions) error {
|
func RunWithApp(ctx context.Context, opts AppOptions) error {
|
||||||
// Acquire PID lock to prevent concurrent instances
|
// Acquire PID lock to prevent concurrent instances
|
||||||
lockDir := filepath.Join(xdg.DataHome, "vaultik")
|
lockDir := filepath.Join(xdg.DataHome, "berlin.sneak.app.vaultik")
|
||||||
|
|
||||||
lock, err := pidlock.Acquire(lockDir)
|
lock, err := pidlock.Acquire(lockDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, pidlock.ErrAlreadyRunning) {
|
if errors.Is(err, pidlock.ErrAlreadyRunning) {
|
||||||
return fmt.Errorf("cannot start: %w", err)
|
return fmt.Errorf("cannot start: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Errorf("failed to acquire lock: %w", err)
|
return fmt.Errorf("failed to acquire lock: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
err := lock.Release()
|
if err := lock.Release(); err != nil {
|
||||||
if err != nil {
|
|
||||||
log.Warn("Failed to release PID lock", "error", err)
|
log.Warn("Failed to release PID lock", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
app := NewApp(opts)
|
app := NewApp(opts)
|
||||||
|
|
||||||
return RunApp(ctx, app)
|
return RunApp(ctx, app)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,39 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCleanStartupError(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
in string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "real fx error chain",
|
|
||||||
in: `could not build arguments for function "sneak.berlin/go/vaultik/internal/cli".newSnapshotCreateCommand.func1.1 (/Users/user/dev/vaultik/internal/cli/snapshot.go:71): failed to build *vaultik.Vaultik: could not build arguments for function "sneak.berlin/go/vaultik/internal/vaultik".New (/Users/user/dev/vaultik/internal/vaultik/vaultik.go:59): failed to build storage.Storer: received non-nil error from function "sneak.berlin/go/vaultik/internal/storage".NewStorer (/Users/user/dev/vaultik/internal/storage/module.go:23): creating base path: mkdir /Volumes/BACKUPS: permission denied`,
|
|
||||||
want: `creating base path: mkdir /Volumes/BACKUPS: permission denied`,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "no fx wrapping",
|
|
||||||
in: "plain error",
|
|
||||||
want: "plain error",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "single fx wrapping",
|
|
||||||
in: `received non-nil error from function "foo" (file.go:1): underlying problem`,
|
|
||||||
want: "underlying problem",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
got := cleanStartupError(errors.New(tt.in)).Error()
|
|
||||||
if got != tt.want {
|
|
||||||
t.Errorf("got %q, want %q", got, tt.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,557 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
)
|
|
||||||
|
|
||||||
const defaultConfigTemplate = `# vaultik configuration
|
|
||||||
# Documentation: https://sneak.berlin/go/vaultik
|
|
||||||
|
|
||||||
# ─── REQUIRED ────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
# Age recipient public keys for encryption.
|
|
||||||
# Backups are encrypted to ALL listed recipients. Any one of the corresponding
|
|
||||||
# private keys can decrypt. Generate a keypair with:
|
|
||||||
# age-keygen -o vaultik_backup_private_key.txt
|
|
||||||
# grep 'public key' vaultik_backup_private_key.txt
|
|
||||||
age_recipients:
|
|
||||||
- age1REPLACE_WITH_YOUR_PUBLIC_KEY
|
|
||||||
|
|
||||||
# Named snapshots. Each snapshot backs up one or more paths and can have its
|
|
||||||
# own exclude patterns in addition to the global excludes below.
|
|
||||||
#
|
|
||||||
# Exclude pattern semantics:
|
|
||||||
# - Patterns starting with / are anchored to the snapshot path root
|
|
||||||
# (e.g. "/Library/Caches" matches only ~/Library/Caches in a ~ snapshot)
|
|
||||||
# - Patterns without a leading / match anywhere in the tree
|
|
||||||
# (e.g. ".cache" matches any directory named .cache at any depth)
|
|
||||||
# - Globs are supported: *, **, ?
|
|
||||||
snapshots:
|
|
||||||
home:
|
|
||||||
paths:
|
|
||||||
- "~"
|
|
||||||
exclude:
|
|
||||||
# Trash, temp, and filesystem metadata
|
|
||||||
- "/.Trash"
|
|
||||||
- "/.Trashes"
|
|
||||||
- "/.fseventsd"
|
|
||||||
- "/.Spotlight-V100"
|
|
||||||
- "/.TemporaryItems"
|
|
||||||
- "/tmp"
|
|
||||||
- "/.rnd"
|
|
||||||
- ".DS_Store"
|
|
||||||
# Caches and package manager state (rebuildable)
|
|
||||||
- ".cache"
|
|
||||||
- ".bundle"
|
|
||||||
- "/.cpan/build"
|
|
||||||
- "/.cpan/sources"
|
|
||||||
- "/.gradle/caches"
|
|
||||||
- "/.dropbox"
|
|
||||||
- "/.minikube/cache"
|
|
||||||
- "/.local/share/containers/podman/machine"
|
|
||||||
- "/.persepolis"
|
|
||||||
- "/Library/Caches"
|
|
||||||
- "/Library/Logs"
|
|
||||||
- "/Library/Cookies"
|
|
||||||
- "/Library/Metadata"
|
|
||||||
- "/Library/Suggestions"
|
|
||||||
- "/Library/PubSub"
|
|
||||||
- "/Library/Homebrew"
|
|
||||||
- "/Library/Developer"
|
|
||||||
- "/Library/Google/GoogleSoftwareUpdate"
|
|
||||||
- "/Library/Preferences/Macromedia/Flash Player"
|
|
||||||
- "/Library/Preferences/SDMHelpData"
|
|
||||||
- "/Library/VoiceTrigger/SAT"
|
|
||||||
# Language/toolchain package caches (rebuildable from registries)
|
|
||||||
- "/.npm"
|
|
||||||
- "/.cargo/registry"
|
|
||||||
- "/.cargo/git"
|
|
||||||
- "/.rustup/toolchains"
|
|
||||||
- "/go/pkg/mod"
|
|
||||||
- "/.m2/repository"
|
|
||||||
- "/.vagrant.d/boxes"
|
|
||||||
- "node_modules"
|
|
||||||
- "__pycache__"
|
|
||||||
- ".venv"
|
|
||||||
# Virtual machine disk images (huge; remove these lines to back them up)
|
|
||||||
- "/Parallels"
|
|
||||||
- "/Virtual Machines.localized"
|
|
||||||
- "/VirtualBox VMs"
|
|
||||||
- "/.orbstack"
|
|
||||||
- "/Library/Containers/com.utmapp.UTM"
|
|
||||||
# Downloaded LLM models (huge, re-downloadable)
|
|
||||||
- "/.ollama/models"
|
|
||||||
- "/.lmstudio/models"
|
|
||||||
# Cloud-synced storage. These are synced to a provider already, and on
|
|
||||||
# modern macOS may contain dataless placeholder files that the backup
|
|
||||||
# would force-download in full.
|
|
||||||
- "/Library/CloudStorage"
|
|
||||||
- "/Library/Mobile Documents"
|
|
||||||
# Android SDK and emulator images (re-downloadable)
|
|
||||||
- "/Library/Android/sdk"
|
|
||||||
- "/.android/avd"
|
|
||||||
# Cloud-synced or restorable-from-server data
|
|
||||||
- "/Library/Mail"
|
|
||||||
- "/Library/Mail Downloads"
|
|
||||||
- "/Library/Safari"
|
|
||||||
- "/Library/Application Support/Evernote"
|
|
||||||
- "/Library/Application Support/MobileSync"
|
|
||||||
- "/Library/Application Support/SyncServices"
|
|
||||||
- "/Library/Application Support/protonmail/bridge/cache"
|
|
||||||
- "/Library/Application Support/Syncthing/index-*"
|
|
||||||
- "/Library/Syncthing/folders"
|
|
||||||
- "/Documents/Dropbox/.dropbox.cache"
|
|
||||||
# Large rebuildable app data (games, media caches, device backups)
|
|
||||||
- "/Applications/Fortnite"
|
|
||||||
- "/Documents/Steam Content"
|
|
||||||
- "/Library/Application Support/Ableton"
|
|
||||||
- "/Library/Application Support/CrossOver Games"
|
|
||||||
- "/Library/Application Support/SecondLife/cache"
|
|
||||||
- "/Library/Application Support/Steam/SteamApps"
|
|
||||||
- "/Library/Containers/com.docker.docker"
|
|
||||||
- "/Library/Group Containers/group.com.apple.secure-control-center-preferences"
|
|
||||||
- "/Library/iTunes/iPad Software Updates"
|
|
||||||
- "/Library/iTunes/iPhone Software Updates"
|
|
||||||
- "/Movies/CacheClip"
|
|
||||||
- "/Movies/ProxyMedia"
|
|
||||||
- "/Music/iTunes/Album Artwork"
|
|
||||||
- "/Pictures/iPod Photo Cache"
|
|
||||||
|
|
||||||
# Third-party applications. OS-provided apps live in /System/Applications
|
|
||||||
# on modern macOS and are never in /Applications, but Apple-installed
|
|
||||||
# App Store apps (Safari, GarageBand, iWork, iMovie) are excluded since
|
|
||||||
# they are re-downloadable.
|
|
||||||
apps:
|
|
||||||
paths:
|
|
||||||
- /Applications
|
|
||||||
exclude:
|
|
||||||
- ".DS_Store"
|
|
||||||
- "/Safari.app"
|
|
||||||
- "/GarageBand.app"
|
|
||||||
- "/iMovie.app"
|
|
||||||
- "/Keynote.app"
|
|
||||||
- "/Numbers.app"
|
|
||||||
- "/Pages.app"
|
|
||||||
- "/Xcode.app"
|
|
||||||
- "/Spotify.app"
|
|
||||||
- "/Steam.app"
|
|
||||||
- "/VirtualBox.app"
|
|
||||||
- "/Utilities/Adobe Installers"
|
|
||||||
|
|
||||||
# Storage backend (pick ONE of the three forms below).
|
|
||||||
#
|
|
||||||
# S3-compatible:
|
|
||||||
# storage_url: "s3://mybucket/backups?endpoint=s3.example.com®ion=us-east-1"
|
|
||||||
# (also set s3.access_key_id and s3.secret_access_key below)
|
|
||||||
#
|
|
||||||
# Local filesystem:
|
|
||||||
# storage_url: "file:///mnt/backups/vaultik"
|
|
||||||
#
|
|
||||||
# Rclone (requires rclone configured separately):
|
|
||||||
# storage_url: "rclone://myremote/path/to/backups"
|
|
||||||
storage_url: ""
|
|
||||||
|
|
||||||
# ─── S3 CREDENTIALS (required for s3:// storage_url) ────────────────────────
|
|
||||||
|
|
||||||
# s3:
|
|
||||||
# access_key_id: YOUR_ACCESS_KEY
|
|
||||||
# secret_access_key: YOUR_SECRET_KEY
|
|
||||||
# # region: us-east-1 # Default: us-east-1
|
|
||||||
# # use_ssl: true # Default: true
|
|
||||||
# # part_size: 5MB # Multipart upload part size. Default: 5MB
|
|
||||||
|
|
||||||
# ─── OPTIONAL ────────────────────────────────────────────────────────────────
|
|
||||||
|
|
||||||
# Global exclude patterns applied to ALL snapshots.
|
|
||||||
# Snapshot-specific excludes are additive.
|
|
||||||
# exclude:
|
|
||||||
# - "*.log"
|
|
||||||
# - "*.tmp"
|
|
||||||
# - ".git"
|
|
||||||
# - "node_modules"
|
|
||||||
|
|
||||||
# Average chunk size for content-defined chunking (FastCDC).
|
|
||||||
# Smaller = better deduplication but more metadata overhead.
|
|
||||||
# Accepts: 1MB, 10M, 64KB, etc.
|
|
||||||
# Default: 10MB
|
|
||||||
# chunk_size: 10MB
|
|
||||||
|
|
||||||
# Maximum blob size before splitting into a new blob.
|
|
||||||
# Accepts: 1GB, 10G, 500MB, etc.
|
|
||||||
# Default: 10GB
|
|
||||||
# blob_size_limit: 10GB
|
|
||||||
|
|
||||||
# Zstd compression level (1-19). Higher = better ratio but slower.
|
|
||||||
# Default: 3
|
|
||||||
# compression_level: 3
|
|
||||||
|
|
||||||
# Hostname used in snapshot IDs. Default: system hostname.
|
|
||||||
# hostname: myserver
|
|
||||||
|
|
||||||
# Path to the local SQLite index database.
|
|
||||||
# Default: the platform data directory, e.g.
|
|
||||||
# macOS: ~/Library/Application Support/vaultik/index.sqlite
|
|
||||||
# Linux: ~/.local/share/vaultik/index.sqlite
|
|
||||||
# index_path: /path/to/index.sqlite
|
|
||||||
`
|
|
||||||
|
|
||||||
// NewConfigCommand creates the config command group.
|
|
||||||
func NewConfigCommand() *cobra.Command {
|
|
||||||
cmd := &cobra.Command{
|
|
||||||
Use: "config",
|
|
||||||
Short: "Manage the configuration file",
|
|
||||||
Long: "Commands for creating, editing, and querying the vaultik config file.",
|
|
||||||
}
|
|
||||||
|
|
||||||
cmd.AddCommand(newConfigInitCommand())
|
|
||||||
cmd.AddCommand(newConfigEditCommand())
|
|
||||||
cmd.AddCommand(newConfigGetCommand())
|
|
||||||
cmd.AddCommand(newConfigSetCommand())
|
|
||||||
|
|
||||||
return cmd
|
|
||||||
}
|
|
||||||
|
|
||||||
// newConfigInitCommand creates the 'config init' subcommand.
|
|
||||||
func newConfigInitCommand() *cobra.Command {
|
|
||||||
return &cobra.Command{
|
|
||||||
Use: "init",
|
|
||||||
Short: "Write a default config file",
|
|
||||||
Long: `Creates a default configuration file with commented explanations
|
|
||||||
for every setting. If a config file already exists at the target path,
|
|
||||||
the command refuses to overwrite it.
|
|
||||||
|
|
||||||
The config is written to the path from --config, $VAULTIK_CONFIG, or
|
|
||||||
the platform default config directory (e.g. ~/Library/Application Support/
|
|
||||||
on macOS, ~/.config/ on Linux, /etc/vaultik/ as root).`,
|
|
||||||
Args: cobra.NoArgs,
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
path := configPathForInit()
|
|
||||||
|
|
||||||
_, err := os.Stat(path)
|
|
||||||
if err == nil {
|
|
||||||
return fmt.Errorf("config file already exists: %s", path)
|
|
||||||
}
|
|
||||||
|
|
||||||
dir := filepath.Dir(path)
|
|
||||||
|
|
||||||
err = os.MkdirAll(dir, 0o755)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("creating config directory %s: %w", dir, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = os.WriteFile(path, []byte(defaultConfigTemplate), 0o600)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("writing config file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Printf("Config written to %s\n", path)
|
|
||||||
fmt.Println("Edit it to set your age_recipients, snapshots, and storage_url.")
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// newConfigEditCommand creates the 'config edit' subcommand.
|
|
||||||
func newConfigEditCommand() *cobra.Command {
|
|
||||||
return &cobra.Command{
|
|
||||||
Use: "edit",
|
|
||||||
Short: "Open the config file in $EDITOR",
|
|
||||||
Args: cobra.NoArgs,
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
path, err := ResolveConfigPath()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
editor := os.Getenv("EDITOR")
|
|
||||||
if editor == "" {
|
|
||||||
editor = "vi"
|
|
||||||
}
|
|
||||||
|
|
||||||
ed := exec.Command(editor, path)
|
|
||||||
ed.Stdin = os.Stdin
|
|
||||||
ed.Stdout = os.Stdout
|
|
||||||
ed.Stderr = os.Stderr
|
|
||||||
|
|
||||||
return ed.Run()
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// newConfigGetCommand creates the 'config get' subcommand.
|
|
||||||
func newConfigGetCommand() *cobra.Command {
|
|
||||||
return &cobra.Command{
|
|
||||||
Use: "get <key>",
|
|
||||||
Short: "Print a config value by dotted path (e.g. storage_url, compression_level)",
|
|
||||||
Args: cobra.ExactArgs(1),
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
path, err := ResolveConfigPath()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
root, err := loadYAMLFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
node, err := yamlPathGet(root, strings.Split(args[0], "."))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if node.Kind == yaml.ScalarNode {
|
|
||||||
fmt.Println(node.Value)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
out, err := yaml.Marshal(node)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("marshaling value: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Print(string(out))
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// newConfigSetCommand creates the 'config set' subcommand.
|
|
||||||
func newConfigSetCommand() *cobra.Command {
|
|
||||||
return &cobra.Command{
|
|
||||||
Use: "set <key> <value>",
|
|
||||||
Short: "Set a config value by dotted path (e.g. compression_level 5)",
|
|
||||||
Long: `Sets a scalar config value addressed by dotted YAML path and writes
|
|
||||||
the file back, preserving comments and formatting. Intermediate maps
|
|
||||||
are created as needed.
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
vaultik config set storage_url "file:///mnt/backups"
|
|
||||||
vaultik config set storage_url "s3://bucket/prefix?endpoint=host®ion=us-east-1"
|
|
||||||
vaultik config set compression_level 9
|
|
||||||
vaultik config set s3.bucket mybucket # legacy S3 fields still supported`,
|
|
||||||
Args: cobra.ExactArgs(2),
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
path, err := ResolveConfigPath()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
root, err := loadYAMLFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = yamlPathSet(root, strings.Split(args[0], "."), args[1])
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
out, err := yaml.Marshal(root)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("marshaling config: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
mode := os.FileMode(0o600)
|
|
||||||
|
|
||||||
info, err := os.Stat(path)
|
|
||||||
if err == nil {
|
|
||||||
mode = info.Mode().Perm()
|
|
||||||
}
|
|
||||||
|
|
||||||
err = os.WriteFile(path, out, mode)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("writing config file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Printf("%s = %s\n", args[0], args[1])
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadYAMLFile parses a YAML file into a yaml.Node document tree,
|
|
||||||
// which preserves comments and ordering for round-tripping.
|
|
||||||
func loadYAMLFile(path string) (*yaml.Node, error) {
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("reading config file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var root yaml.Node
|
|
||||||
|
|
||||||
err = yaml.Unmarshal(data, &root)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("parsing config file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// An empty file yields a zero node; normalize to an empty mapping document.
|
|
||||||
if root.Kind == 0 {
|
|
||||||
root = yaml.Node{
|
|
||||||
Kind: yaml.DocumentNode,
|
|
||||||
Content: []*yaml.Node{{Kind: yaml.MappingNode}},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return &root, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// yamlPathGet navigates a dotted key path through mapping and sequence
|
|
||||||
// nodes and returns the value node. Numeric path components index into
|
|
||||||
// sequences (e.g. "age_recipients.0").
|
|
||||||
func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
|
|
||||||
node := root
|
|
||||||
if node.Kind == yaml.DocumentNode {
|
|
||||||
if len(node.Content) == 0 {
|
|
||||||
return nil, errors.New("empty config file")
|
|
||||||
}
|
|
||||||
|
|
||||||
node = node.Content[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, key := range keys {
|
|
||||||
switch node.Kind {
|
|
||||||
case yaml.MappingNode:
|
|
||||||
found := false
|
|
||||||
|
|
||||||
for j := 0; j+1 < len(node.Content); j += 2 {
|
|
||||||
if node.Content[j].Value == key {
|
|
||||||
node = node.Content[j+1]
|
|
||||||
found = true
|
|
||||||
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !found {
|
|
||||||
return nil, fmt.Errorf("key not found: %s", strings.Join(keys[:i+1], "."))
|
|
||||||
}
|
|
||||||
case yaml.SequenceNode:
|
|
||||||
idx, err := strconv.Atoi(key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], "."))
|
|
||||||
}
|
|
||||||
|
|
||||||
if idx < 0 || idx >= len(node.Content) {
|
|
||||||
return nil, fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content))
|
|
||||||
}
|
|
||||||
|
|
||||||
node = node.Content[idx]
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], "."))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return node, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// yamlPathSet navigates a dotted key path, creating intermediate maps as
|
|
||||||
// needed, and sets the final key to the given scalar value. Numeric path
|
|
||||||
// components index into sequences; an index equal to the sequence length
|
|
||||||
// appends a new element (e.g. "age_recipients.1" on a 1-element list).
|
|
||||||
func yamlPathSet(root *yaml.Node, keys []string, value string) error {
|
|
||||||
node := root
|
|
||||||
if node.Kind == yaml.DocumentNode {
|
|
||||||
if len(node.Content) == 0 {
|
|
||||||
node.Content = []*yaml.Node{{Kind: yaml.MappingNode}}
|
|
||||||
}
|
|
||||||
|
|
||||||
node = node.Content[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, key := range keys {
|
|
||||||
last := i == len(keys)-1
|
|
||||||
|
|
||||||
switch node.Kind {
|
|
||||||
case yaml.MappingNode:
|
|
||||||
var valueNode *yaml.Node
|
|
||||||
|
|
||||||
for j := 0; j+1 < len(node.Content); j += 2 {
|
|
||||||
if node.Content[j].Value == key {
|
|
||||||
valueNode = node.Content[j+1]
|
|
||||||
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if valueNode == nil {
|
|
||||||
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
|
|
||||||
|
|
||||||
valueNode = &yaml.Node{Kind: yaml.MappingNode}
|
|
||||||
if last {
|
|
||||||
valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
|
|
||||||
}
|
|
||||||
|
|
||||||
node.Content = append(node.Content, keyNode, valueNode)
|
|
||||||
} else if last {
|
|
||||||
setScalar(valueNode, value)
|
|
||||||
}
|
|
||||||
|
|
||||||
node = valueNode
|
|
||||||
|
|
||||||
case yaml.SequenceNode:
|
|
||||||
idx, err := strconv.Atoi(key)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], "."))
|
|
||||||
}
|
|
||||||
|
|
||||||
if idx < 0 || idx > len(node.Content) {
|
|
||||||
return fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content))
|
|
||||||
}
|
|
||||||
|
|
||||||
if idx == len(node.Content) {
|
|
||||||
newNode := &yaml.Node{Kind: yaml.MappingNode}
|
|
||||||
if last {
|
|
||||||
newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
|
|
||||||
}
|
|
||||||
|
|
||||||
node.Content = append(node.Content, newNode)
|
|
||||||
} else if last {
|
|
||||||
setScalar(node.Content[idx], value)
|
|
||||||
}
|
|
||||||
|
|
||||||
node = node.Content[idx]
|
|
||||||
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], "."))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// setScalar overwrites a node in place with a plain scalar value.
|
|
||||||
func setScalar(n *yaml.Node, value string) {
|
|
||||||
n.Kind = yaml.ScalarNode
|
|
||||||
n.Tag = ""
|
|
||||||
n.Value = value
|
|
||||||
n.Content = nil
|
|
||||||
n.Style = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// configPathForInit returns the config path to write, checking --config flag,
|
|
||||||
// VAULTIK_CONFIG env, and the platform default.
|
|
||||||
func configPathForInit() string {
|
|
||||||
if rootFlags.ConfigPath != "" {
|
|
||||||
return rootFlags.ConfigPath
|
|
||||||
}
|
|
||||||
|
|
||||||
if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" {
|
|
||||||
return envPath
|
|
||||||
}
|
|
||||||
|
|
||||||
return DefaultConfigPath()
|
|
||||||
}
|
|
||||||
@@ -1,185 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestDefaultConfigTemplateParses ensures the init template is valid YAML
|
|
||||||
// that unmarshals into the Config struct with the expected snapshots.
|
|
||||||
func TestDefaultConfigTemplateParses(t *testing.T) {
|
|
||||||
var cfg config.Config
|
|
||||||
|
|
||||||
err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("default config template is not valid YAML: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(cfg.AgeRecipients) != 1 {
|
|
||||||
t.Errorf("expected 1 placeholder age recipient, got %d", len(cfg.AgeRecipients))
|
|
||||||
}
|
|
||||||
|
|
||||||
home, ok := cfg.Snapshots["home"]
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("expected 'home' snapshot in default config")
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(home.Paths) == 0 {
|
|
||||||
t.Error("home snapshot should have at least one path")
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(home.Exclude) == 0 {
|
|
||||||
t.Error("home snapshot should have exclude patterns")
|
|
||||||
}
|
|
||||||
|
|
||||||
apps, ok := cfg.Snapshots["apps"]
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("expected 'apps' snapshot in default config")
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(apps.Paths) != 1 || apps.Paths[0] != "/Applications" {
|
|
||||||
t.Errorf("apps snapshot should back up /Applications, got %v", apps.Paths)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(apps.Exclude) == 0 {
|
|
||||||
t.Error("apps snapshot should have exclude patterns")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const testYAML = `# top comment
|
|
||||||
compression_level: 3
|
|
||||||
age_recipients:
|
|
||||||
- age1aaa
|
|
||||||
s3:
|
|
||||||
bucket: oldbucket # inline comment
|
|
||||||
region: us-east-1
|
|
||||||
snapshots:
|
|
||||||
home:
|
|
||||||
paths:
|
|
||||||
- "~"
|
|
||||||
`
|
|
||||||
|
|
||||||
func parseTestYAML(t *testing.T) *yaml.Node {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var root yaml.Node
|
|
||||||
|
|
||||||
err := yaml.Unmarshal([]byte(testYAML), &root)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parsing test yaml: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &root
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestYAMLPathGet(t *testing.T) {
|
|
||||||
root := parseTestYAML(t)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
path string
|
|
||||||
want string
|
|
||||||
err bool
|
|
||||||
}{
|
|
||||||
{"compression_level", "3", false},
|
|
||||||
{"s3.bucket", "oldbucket", false},
|
|
||||||
{"s3.region", "us-east-1", false},
|
|
||||||
{"age_recipients.0", "age1aaa", false},
|
|
||||||
{"age_recipients.5", "", true},
|
|
||||||
{"age_recipients.notanumber", "", true},
|
|
||||||
{"s3.nonexistent", "", true},
|
|
||||||
{"nonexistent", "", true},
|
|
||||||
{"compression_level.sub", "", true},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.path, func(t *testing.T) {
|
|
||||||
node, err := yamlPathGet(root, splitPath(tt.path))
|
|
||||||
if tt.err {
|
|
||||||
if err == nil {
|
|
||||||
t.Fatalf("expected error for %q", tt.path)
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if node.Value != tt.want {
|
|
||||||
t.Errorf("get %q = %q, want %q", tt.path, node.Value, tt.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestYAMLPathSet(t *testing.T) {
|
|
||||||
root := parseTestYAML(t)
|
|
||||||
|
|
||||||
// Overwrite existing nested value
|
|
||||||
err := yamlPathSet(root, splitPath("s3.bucket"), "newbucket")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("set s3.bucket: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create new nested key with intermediate map
|
|
||||||
err = yamlPathSet(root, splitPath("s3.endpoint"), "s3.example.com")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("set s3.endpoint: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = yamlPathSet(root, splitPath("newmap.newkey"), "val")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("set newmap.newkey: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Overwrite a sequence element and append a new one
|
|
||||||
err = yamlPathSet(root, splitPath("age_recipients.0"), "age1bbb")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("set age_recipients.0: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = yamlPathSet(root, splitPath("age_recipients.1"), "age1ccc")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("append age_recipients.1: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = yamlPathSet(root, splitPath("age_recipients.5"), "age1ddd")
|
|
||||||
if err == nil {
|
|
||||||
t.Error("expected out-of-range append to fail")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Round-trip and verify values + comment preservation
|
|
||||||
out, err := yaml.Marshal(root)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
text := string(out)
|
|
||||||
|
|
||||||
for _, want := range []string{"newbucket", "s3.example.com", "newkey: val", "# top comment", "# inline comment", "age1bbb", "age1ccc"} {
|
|
||||||
if !contains(text, want) {
|
|
||||||
t.Errorf("round-tripped YAML missing %q:\n%s", want, text)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := yamlPathGet(root, splitPath("s3.bucket"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("get after set: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got.Value != "newbucket" {
|
|
||||||
t.Errorf("s3.bucket = %q after set, want newbucket", got.Value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func splitPath(s string) []string {
|
|
||||||
return strings.Split(s, ".")
|
|
||||||
}
|
|
||||||
|
|
||||||
func contains(haystack, needle string) bool {
|
|
||||||
return strings.Contains(haystack, needle)
|
|
||||||
}
|
|
||||||
@@ -4,9 +4,9 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewDatabaseCommand creates the database command group
|
// NewDatabaseCommand creates the database command group
|
||||||
@@ -18,33 +18,28 @@ func NewDatabaseCommand() *cobra.Command {
|
|||||||
}
|
}
|
||||||
|
|
||||||
cmd.AddCommand(
|
cmd.AddCommand(
|
||||||
newDatabaseDeleteCommand(),
|
newDatabasePurgeCommand(),
|
||||||
)
|
)
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
// newDatabaseDeleteCommand creates the database delete command.
|
// newDatabasePurgeCommand creates the database purge command
|
||||||
// (Renamed from "purge"; the operation removes the SQLite file
|
func newDatabasePurgeCommand() *cobra.Command {
|
||||||
// entirely, which is a delete, not a purge of content.)
|
|
||||||
func newDatabaseDeleteCommand() *cobra.Command {
|
|
||||||
var force bool
|
var force bool
|
||||||
|
|
||||||
cmd := &cobra.Command{
|
cmd := &cobra.Command{
|
||||||
Use: "delete",
|
Use: "purge",
|
||||||
Short: "Delete the local state database file",
|
Short: "Delete the local state database",
|
||||||
Long: `Completely removes the local SQLite state database.
|
Long: `Completely removes the local SQLite state database.
|
||||||
|
|
||||||
This will erase all local tracking of:
|
This will erase all local tracking of:
|
||||||
- File metadata and change detection state
|
- File metadata and change detection state
|
||||||
- Chunk and blob mappings
|
- Chunk and blob mappings
|
||||||
- Local snapshot records
|
- Local snapshot records
|
||||||
- The storage-binding record
|
|
||||||
|
|
||||||
The remote storage is NOT affected. After deletion, the next backup
|
The remote storage is NOT affected. After purging, the next backup will
|
||||||
will perform a full scan and re-deduplicate against existing remote
|
perform a full scan and re-deduplicate against existing remote blobs.
|
||||||
blobs, and the local index will re-bind to the currently configured
|
|
||||||
storage destination on that run.
|
|
||||||
|
|
||||||
Use --force to skip the confirmation prompt.`,
|
Use --force to skip the confirmation prompt.`,
|
||||||
Args: cobra.NoArgs,
|
Args: cobra.NoArgs,
|
||||||
@@ -64,10 +59,8 @@ Use --force to skip the confirmation prompt.`,
|
|||||||
dbPath := cfg.IndexPath
|
dbPath := cfg.IndexPath
|
||||||
|
|
||||||
// Check if database exists
|
// Check if database exists
|
||||||
_, err = os.Stat(dbPath)
|
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
|
||||||
if os.IsNotExist(err) {
|
|
||||||
fmt.Printf("Database does not exist: %s\n", dbPath)
|
fmt.Printf("Database does not exist: %s\n", dbPath)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -75,20 +68,15 @@ Use --force to skip the confirmation prompt.`,
|
|||||||
if !force {
|
if !force {
|
||||||
fmt.Printf("This will delete the local state database at:\n %s\n\n", dbPath)
|
fmt.Printf("This will delete the local state database at:\n %s\n\n", dbPath)
|
||||||
fmt.Print("Are you sure? Type 'yes' to confirm: ")
|
fmt.Print("Are you sure? Type 'yes' to confirm: ")
|
||||||
|
|
||||||
var confirm string
|
var confirm string
|
||||||
|
if _, err := fmt.Scanln(&confirm); err != nil || confirm != "yes" {
|
||||||
_, err = fmt.Scanln(&confirm)
|
|
||||||
if err != nil || confirm != "yes" {
|
|
||||||
fmt.Println("Aborted.")
|
fmt.Println("Aborted.")
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete the database file
|
// Delete the database file
|
||||||
err = os.Remove(dbPath)
|
if err := os.Remove(dbPath); err != nil {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to delete database: %w", err)
|
return fmt.Errorf("failed to delete database: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -100,11 +88,10 @@ Use --force to skip the confirmation prompt.`,
|
|||||||
|
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
if !rootFlags.Quiet {
|
if !rootFlags.Quiet {
|
||||||
fmt.Printf("Database deleted: %s\n", dbPath)
|
fmt.Printf("Database purged: %s\n", dbPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Local state database deleted", "path", dbPath)
|
log.Info("Local state database purged", "path", dbPath)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -19,15 +18,14 @@ import (
|
|||||||
// Can combine units: "1y6mo", "2w3d", "1d12h30m"
|
// Can combine units: "1y6mo", "2w3d", "1d12h30m"
|
||||||
func parseDuration(s string) (time.Duration, error) {
|
func parseDuration(s string) (time.Duration, error) {
|
||||||
// First try standard Go duration parsing
|
// First try standard Go duration parsing
|
||||||
d, err := time.ParseDuration(s)
|
if d, err := time.ParseDuration(s); err == nil {
|
||||||
if err == nil {
|
|
||||||
return d, nil
|
return d, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extended duration parsing
|
// Extended duration parsing
|
||||||
// Check for negative values
|
// Check for negative values
|
||||||
if strings.HasPrefix(strings.TrimSpace(s), "-") {
|
if strings.HasPrefix(strings.TrimSpace(s), "-") {
|
||||||
return 0, errors.New("negative durations are not supported")
|
return 0, fmt.Errorf("negative durations are not supported")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Pattern matches: number + unit, repeated
|
// Pattern matches: number + unit, repeated
|
||||||
@@ -50,7 +48,6 @@ func parseDuration(s string) (time.Duration, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var d time.Duration
|
var d time.Duration
|
||||||
|
|
||||||
switch unit {
|
switch unit {
|
||||||
// Standard time units
|
// Standard time units
|
||||||
case "ns", "nanosecond", "nanoseconds":
|
case "ns", "nanosecond", "nanoseconds":
|
||||||
@@ -78,15 +75,11 @@ func parseDuration(s string) (time.Duration, error) {
|
|||||||
d = time.Duration(value * float64(365*24*time.Hour))
|
d = time.Duration(value * float64(365*24*time.Hour))
|
||||||
default:
|
default:
|
||||||
// Try parsing as standard Go duration unit
|
// Try parsing as standard Go duration unit
|
||||||
testStr := "1" + unit
|
testStr := fmt.Sprintf("1%s", unit)
|
||||||
|
if _, err := time.ParseDuration(testStr); err == nil {
|
||||||
_, err = time.ParseDuration(testStr)
|
|
||||||
if err == nil {
|
|
||||||
// It's a valid Go duration unit, parse the full value
|
// It's a valid Go duration unit, parse the full value
|
||||||
fullStr := fmt.Sprintf("%g%s", value, unit)
|
fullStr := fmt.Sprintf("%g%s", value, unit)
|
||||||
|
if d, err = time.ParseDuration(fullStr); err != nil {
|
||||||
d, err = time.ParseDuration(fullStr)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("invalid duration %q: %w", fullStr, err)
|
return 0, fmt.Errorf("invalid duration %q: %w", fullStr, err)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -185,7 +185,6 @@ func TestParseDuration(t *testing.T) {
|
|||||||
|
|
||||||
if tt.wantErr {
|
if tt.wantErr {
|
||||||
assert.Error(t, err, "expected error for input %q", tt.input)
|
assert.Error(t, err, "expected error for input %q", tt.input)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,72 +2,14 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/globals"
|
|
||||||
"sneak.berlin/go/vaultik/internal/ui"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// CLIEntry is the main entry point for the CLI application.
|
// CLIEntry is the main entry point for the CLI application.
|
||||||
// It prints the startup banner (unless a quiet flag is present in os.Args),
|
// It creates the root command, executes it, and exits with status 1
|
||||||
// executes the root cobra command, and routes any returned error through
|
// if an error occurs. This function should be called from main().
|
||||||
// the ui.Writer so the user sees a properly formatted "🛑 ERROR:" line.
|
|
||||||
func CLIEntry() {
|
func CLIEntry() {
|
||||||
if !bannerSuppressedInArgs(os.Args[1:]) {
|
|
||||||
short := globals.Commit
|
|
||||||
if len(short) > 12 {
|
|
||||||
short = short[:12]
|
|
||||||
}
|
|
||||||
|
|
||||||
writeStartupBanner(ui.New(os.Stdout), time.Now().UTC(), short)
|
|
||||||
}
|
|
||||||
|
|
||||||
rootCmd := NewRootCommand()
|
rootCmd := NewRootCommand()
|
||||||
rootCmd.SilenceErrors = true
|
if err := rootCmd.Execute(); err != nil {
|
||||||
|
|
||||||
err := rootCmd.Execute()
|
|
||||||
if err != nil {
|
|
||||||
ReportError("%s", err.Error())
|
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReportError emits a user-facing error to stderr in the standard
|
|
||||||
// 🛑 ERROR: format. Use it from goroutine error paths (where returning
|
|
||||||
// an error to cobra isn't an option) and anywhere else a CLI command
|
|
||||||
// must surface a failure outside the normal RunE return path.
|
|
||||||
func ReportError(format string, args ...any) {
|
|
||||||
ui.New(os.Stderr).Error(format, args...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// bannerSuppressedInArgs reports whether any of args is a flag that
|
|
||||||
// should suppress the startup banner (--quiet/-q/--cron). Stops at the
|
|
||||||
// "--" argument terminator. Recognizes both long forms and short -q,
|
|
||||||
// including combined short flags like "-qv".
|
|
||||||
func bannerSuppressedInArgs(args []string) bool {
|
|
||||||
for _, a := range args {
|
|
||||||
if a == "--" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
switch a {
|
|
||||||
case "--quiet", "-q", "--cron":
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.HasPrefix(a, "--quiet=") || strings.HasPrefix(a, "--cron=") {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
// Combined short flags like -qv or -vq.
|
|
||||||
if len(a) > 1 && a[0] == '-' && a[1] != '-' {
|
|
||||||
for _, c := range a[1:] {
|
|
||||||
if c == 'q' {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -18,18 +18,15 @@ func TestCLIEntry(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Verify all subcommands are registered
|
// Verify all subcommands are registered
|
||||||
expectedCommands := []string{"config", "snapshot", "prune", "info", "version", "remote", "database"}
|
expectedCommands := []string{"snapshot", "store", "restore", "prune", "verify", "info", "version"}
|
||||||
for _, expected := range expectedCommands {
|
for _, expected := range expectedCommands {
|
||||||
found := false
|
found := false
|
||||||
|
|
||||||
for _, cmd := range cmd.Commands() {
|
for _, cmd := range cmd.Commands() {
|
||||||
if cmd.Use == expected || cmd.Name() == expected {
|
if cmd.Use == expected || cmd.Name() == expected {
|
||||||
found = true
|
found = true
|
||||||
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !found {
|
if !found {
|
||||||
t.Errorf("Expected command '%s' not found", expected)
|
t.Errorf("Expected command '%s' not found", expected)
|
||||||
}
|
}
|
||||||
@@ -41,18 +38,15 @@ func TestCLIEntry(t *testing.T) {
|
|||||||
t.Errorf("Failed to find snapshot command: %v", err)
|
t.Errorf("Failed to find snapshot command: %v", err)
|
||||||
} else {
|
} else {
|
||||||
// Check snapshot subcommands
|
// Check snapshot subcommands
|
||||||
expectedSubCommands := []string{"create", "list", "purge", "verify", "remove", "restore"}
|
expectedSubCommands := []string{"create", "list", "purge", "verify"}
|
||||||
for _, expected := range expectedSubCommands {
|
for _, expected := range expectedSubCommands {
|
||||||
found := false
|
found := false
|
||||||
|
|
||||||
for _, subcmd := range snapshotCmd.Commands() {
|
for _, subcmd := range snapshotCmd.Commands() {
|
||||||
if subcmd.Use == expected || subcmd.Name() == expected {
|
if subcmd.Use == expected || subcmd.Name() == expected {
|
||||||
found = true
|
found = true
|
||||||
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !found {
|
if !found {
|
||||||
t.Errorf("Expected snapshot subcommand '%s' not found", expected)
|
t.Errorf("Expected snapshot subcommand '%s' not found", expected)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,13 +2,12 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewInfoCommand creates the info command
|
// NewInfoCommand creates the info command
|
||||||
@@ -32,7 +31,6 @@ func NewInfoCommand() *cobra.Command {
|
|||||||
|
|
||||||
// Use the app framework
|
// Use the app framework
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
return RunWithApp(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.LogOptions{
|
LogOptions: log.LogOptions{
|
||||||
@@ -46,26 +44,20 @@ func NewInfoCommand() *cobra.Command {
|
|||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
OnStart: func(ctx context.Context) error {
|
OnStart: func(ctx context.Context) error {
|
||||||
go func() {
|
go func() {
|
||||||
err := v.ShowInfo()
|
if err := v.ShowInfo(); err != nil {
|
||||||
if err != nil {
|
if err != context.Canceled {
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
log.Error("Failed to show info", "error", err)
|
log.Error("Failed to show info", "error", err)
|
||||||
ReportError("Failed to show info: %v", err)
|
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||||
err = v.Shutdowner.Shutdown()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
log.Error("Failed to shutdown", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
OnStop: func(ctx context.Context) error {
|
OnStop: func(ctx context.Context) error {
|
||||||
v.Cancel()
|
v.Cancel()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -2,13 +2,12 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewPruneCommand creates the prune command
|
// NewPruneCommand creates the prune command
|
||||||
@@ -17,19 +16,14 @@ func NewPruneCommand() *cobra.Command {
|
|||||||
|
|
||||||
cmd := &cobra.Command{
|
cmd := &cobra.Command{
|
||||||
Use: "prune",
|
Use: "prune",
|
||||||
Short: "Tidy local database and remote storage",
|
Short: "Remove unreferenced blobs",
|
||||||
Long: `Removes orphaned data from both the local index database and
|
Long: `Removes blobs that are not referenced by any snapshot.
|
||||||
unreferenced blobs from the backup destination store.
|
|
||||||
|
|
||||||
Local cleanup drops incomplete snapshots and any files, chunks, or
|
This command scans all snapshots and their manifests to build a list of
|
||||||
blobs no longer referenced by a completed snapshot. Remote cleanup
|
referenced blobs, then removes any blobs in storage that are not in this list.
|
||||||
scans every snapshot manifest in the destination store, builds the
|
|
||||||
set of still-referenced blob hashes, and deletes any blob not in that
|
|
||||||
set.
|
|
||||||
|
|
||||||
Snapshot create --prune and snapshot remove run the same cleanup
|
Use this command after deleting snapshots with 'vaultik purge' to reclaim
|
||||||
automatically; this command is the manual entry point for the same
|
storage space.`,
|
||||||
work (e.g. after a crashed backup or to reclaim storage).`,
|
|
||||||
Args: cobra.NoArgs,
|
Args: cobra.NoArgs,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
// Use unified config resolution
|
// Use unified config resolution
|
||||||
@@ -40,7 +34,6 @@ work (e.g. after a crashed backup or to reclaim storage).`,
|
|||||||
|
|
||||||
// Use the app framework like other commands
|
// Use the app framework like other commands
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
return RunWithApp(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.LogOptions{
|
LogOptions: log.LogOptions{
|
||||||
@@ -56,31 +49,25 @@ work (e.g. after a crashed backup or to reclaim storage).`,
|
|||||||
// Start the prune operation in a goroutine
|
// Start the prune operation in a goroutine
|
||||||
go func() {
|
go func() {
|
||||||
// Run the prune operation
|
// Run the prune operation
|
||||||
err := v.Prune(opts)
|
if err := v.PruneBlobs(opts); err != nil {
|
||||||
if err != nil {
|
if err != context.Canceled {
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
if !opts.JSON {
|
if !opts.JSON {
|
||||||
log.Error("Prune operation failed", "error", err)
|
log.Error("Prune operation failed", "error", err)
|
||||||
ReportError("Prune failed: %v", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Shutdown the app when prune completes
|
// Shutdown the app when prune completes
|
||||||
err = v.Shutdowner.Shutdown()
|
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
log.Error("Failed to shutdown", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
OnStop: func(ctx context.Context) error {
|
OnStop: func(ctx context.Context) error {
|
||||||
log.Debug("Stopping prune operation")
|
log.Debug("Stopping prune operation")
|
||||||
v.Cancel()
|
v.Cancel()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|||||||
101
internal/cli/purge.go
Normal file
101
internal/cli/purge.go
Normal file
@@ -0,0 +1,101 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.uber.org/fx"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewPurgeCommand creates the purge command
|
||||||
|
func NewPurgeCommand() *cobra.Command {
|
||||||
|
opts := &vaultik.SnapshotPurgeOptions{}
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "purge",
|
||||||
|
Short: "Purge old snapshots",
|
||||||
|
Long: `Removes snapshots based on age or count criteria.
|
||||||
|
|
||||||
|
This command allows you to:
|
||||||
|
- Keep only the latest snapshot per name (--keep-latest)
|
||||||
|
- Remove snapshots older than a specific duration (--older-than)
|
||||||
|
- Filter to a specific snapshot name (--name)
|
||||||
|
|
||||||
|
When --keep-latest is used, retention is applied per snapshot name. For example,
|
||||||
|
if you have snapshots named "home" and "system", --keep-latest keeps the most
|
||||||
|
recent of each.
|
||||||
|
|
||||||
|
Use --name to restrict the purge to a single snapshot name.
|
||||||
|
|
||||||
|
Config is located at /etc/vaultik/config.yml by default, but can be overridden by
|
||||||
|
specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
// Validate flags
|
||||||
|
if !opts.KeepLatest && opts.OlderThan == "" {
|
||||||
|
return fmt.Errorf("must specify either --keep-latest or --older-than")
|
||||||
|
}
|
||||||
|
if opts.KeepLatest && opts.OlderThan != "" {
|
||||||
|
return fmt.Errorf("cannot specify both --keep-latest and --older-than")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use unified config resolution
|
||||||
|
configPath, err := ResolveConfigPath()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use the app framework like other commands
|
||||||
|
rootFlags := GetRootFlags()
|
||||||
|
return RunWithApp(cmd.Context(), AppOptions{
|
||||||
|
ConfigPath: configPath,
|
||||||
|
LogOptions: log.LogOptions{
|
||||||
|
Verbose: rootFlags.Verbose,
|
||||||
|
Debug: rootFlags.Debug,
|
||||||
|
Quiet: rootFlags.Quiet,
|
||||||
|
},
|
||||||
|
Modules: []fx.Option{},
|
||||||
|
Invokes: []fx.Option{
|
||||||
|
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||||
|
lc.Append(fx.Hook{
|
||||||
|
OnStart: func(ctx context.Context) error {
|
||||||
|
// Start the purge operation in a goroutine
|
||||||
|
go func() {
|
||||||
|
// Run the purge operation
|
||||||
|
if err := v.PurgeSnapshotsWithOptions(opts); err != nil {
|
||||||
|
if err != context.Canceled {
|
||||||
|
log.Error("Purge operation failed", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Shutdown the app when purge completes
|
||||||
|
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||||
|
log.Error("Failed to shutdown", "error", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
OnStop: func(ctx context.Context) error {
|
||||||
|
log.Debug("Stopping purge operation")
|
||||||
|
v.Cancel()
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, "Keep only the latest snapshot per name")
|
||||||
|
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "", "Remove snapshots older than duration (e.g. 30d, 6m, 1y)")
|
||||||
|
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompts")
|
||||||
|
cmd.Flags().StringVar(&opts.Name, "name", "", "Filter purge to a specific snapshot name")
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
@@ -2,13 +2,12 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewRemoteCommand creates the remote command and subcommands
|
// NewRemoteCommand creates the remote command and subcommands
|
||||||
@@ -21,79 +20,6 @@ func NewRemoteCommand() *cobra.Command {
|
|||||||
|
|
||||||
// Add subcommands
|
// Add subcommands
|
||||||
cmd.AddCommand(newRemoteInfoCommand())
|
cmd.AddCommand(newRemoteInfoCommand())
|
||||||
cmd.AddCommand(newRemoteNukeCommand())
|
|
||||||
|
|
||||||
return cmd
|
|
||||||
}
|
|
||||||
|
|
||||||
// newRemoteNukeCommand creates the 'remote nuke' subcommand.
|
|
||||||
func newRemoteNukeCommand() *cobra.Command {
|
|
||||||
var force bool
|
|
||||||
|
|
||||||
cmd := &cobra.Command{
|
|
||||||
Use: "nuke",
|
|
||||||
Short: "Delete ALL snapshot metadata and blobs from the backup destination store",
|
|
||||||
Long: `Removes every snapshot's metadata and every blob from remote
|
|
||||||
storage. After this command completes successfully the bucket prefix is
|
|
||||||
empty and the next backup starts from scratch.
|
|
||||||
|
|
||||||
This is destructive and irreversible. Requires --force.`,
|
|
||||||
Args: cobra.NoArgs,
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
if !force {
|
|
||||||
return errors.New("remote nuke requires --force (this deletes ALL remote snapshots and blobs)")
|
|
||||||
}
|
|
||||||
|
|
||||||
configPath, err := ResolveConfigPath()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
rootFlags := GetRootFlags()
|
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
|
||||||
ConfigPath: configPath,
|
|
||||||
LogOptions: log.LogOptions{
|
|
||||||
Verbose: rootFlags.Verbose,
|
|
||||||
Debug: rootFlags.Debug,
|
|
||||||
Quiet: rootFlags.Quiet,
|
|
||||||
},
|
|
||||||
Modules: []fx.Option{},
|
|
||||||
Invokes: []fx.Option{
|
|
||||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
|
||||||
lc.Append(fx.Hook{
|
|
||||||
OnStart: func(ctx context.Context) error {
|
|
||||||
go func() {
|
|
||||||
err := v.NukeRemote(true)
|
|
||||||
if err != nil {
|
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
log.Error("Remote nuke failed", "error", err)
|
|
||||||
ReportError("Remote nuke failed: %v", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err = v.Shutdowner.Shutdown()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
OnStop: func(ctx context.Context) error {
|
|
||||||
v.Cancel()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
cmd.Flags().BoolVar(&force, "force", false, "Required: confirm destruction of ALL remote data")
|
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
@@ -119,7 +45,6 @@ func newRemoteInfoCommand() *cobra.Command {
|
|||||||
}
|
}
|
||||||
|
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
return RunWithApp(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.LogOptions{
|
LogOptions: log.LogOptions{
|
||||||
@@ -133,29 +58,22 @@ func newRemoteInfoCommand() *cobra.Command {
|
|||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
OnStart: func(ctx context.Context) error {
|
OnStart: func(ctx context.Context) error {
|
||||||
go func() {
|
go func() {
|
||||||
err := v.RemoteInfo(jsonOutput)
|
if err := v.RemoteInfo(jsonOutput); err != nil {
|
||||||
if err != nil {
|
if err != context.Canceled {
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
if !jsonOutput {
|
if !jsonOutput {
|
||||||
log.Error("Failed to get remote info", "error", err)
|
log.Error("Failed to get remote info", "error", err)
|
||||||
ReportError("Failed to get remote info: %v", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||||
err = v.Shutdowner.Shutdown()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
log.Error("Failed to shutdown", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
OnStop: func(ctx context.Context) error {
|
OnStop: func(ctx context.Context) error {
|
||||||
v.Cancel()
|
v.Cancel()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -2,16 +2,14 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
|
||||||
"sneak.berlin/go/vaultik/internal/globals"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// RestoreOptions contains options for the restore command
|
// RestoreOptions contains options for the restore command
|
||||||
@@ -30,13 +28,13 @@ type RestoreApp struct {
|
|||||||
Shutdowner fx.Shutdowner
|
Shutdowner fx.Shutdowner
|
||||||
}
|
}
|
||||||
|
|
||||||
// newSnapshotRestoreCommand creates the 'snapshot restore' subcommand
|
// NewRestoreCommand creates the restore command
|
||||||
func newSnapshotRestoreCommand() *cobra.Command {
|
func NewRestoreCommand() *cobra.Command {
|
||||||
opts := &RestoreOptions{}
|
opts := &RestoreOptions{}
|
||||||
|
|
||||||
cmd := &cobra.Command{
|
cmd := &cobra.Command{
|
||||||
Use: "restore <snapshot-id> <target-dir> [paths...]",
|
Use: "restore <snapshot-id> <target-dir> [paths...]",
|
||||||
Short: "Restore files from a snapshot",
|
Short: "Restore files from backup",
|
||||||
Long: `Download and decrypt files from a backup snapshot.
|
Long: `Download and decrypt files from a backup snapshot.
|
||||||
|
|
||||||
This command will restore files from the specified snapshot to the target directory.
|
This command will restore files from the specified snapshot to the target directory.
|
||||||
@@ -47,16 +45,16 @@ Requires the VAULTIK_AGE_SECRET_KEY environment variable to be set with the age
|
|||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
# Restore entire snapshot
|
# Restore entire snapshot
|
||||||
vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore
|
vaultik restore myhost_docs_2025-01-01T12:00:00Z /restore
|
||||||
|
|
||||||
# Restore specific file
|
# Restore specific file
|
||||||
vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore /home/user/important.txt
|
vaultik restore myhost_docs_2025-01-01T12:00:00Z /restore /home/user/important.txt
|
||||||
|
|
||||||
# Restore specific directory
|
# Restore specific directory
|
||||||
vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore /home/user/documents/
|
vaultik restore myhost_docs_2025-01-01T12:00:00Z /restore /home/user/documents/
|
||||||
|
|
||||||
# Restore and verify all files
|
# Restore and verify all files
|
||||||
vaultik snapshot restore --verify myhost_docs_2025-01-01T12:00:00Z /restore`,
|
vaultik restore --verify myhost_docs_2025-01-01T12:00:00Z /restore`,
|
||||||
Args: cobra.MinimumNArgs(2),
|
Args: cobra.MinimumNArgs(2),
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
return runRestore(cmd, args, opts)
|
return runRestore(cmd, args, opts)
|
||||||
@@ -71,7 +69,6 @@ Examples:
|
|||||||
// runRestore parses arguments and runs the restore operation through the app framework
|
// runRestore parses arguments and runs the restore operation through the app framework
|
||||||
func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
|
func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
|
||||||
snapshotID := args[0]
|
snapshotID := args[0]
|
||||||
|
|
||||||
opts.TargetDir = args[1]
|
opts.TargetDir = args[1]
|
||||||
if len(args) > 2 {
|
if len(args) > 2 {
|
||||||
opts.Paths = args[2:]
|
opts.Paths = args[2:]
|
||||||
@@ -85,7 +82,6 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
|
|||||||
|
|
||||||
// Use the app framework like other commands
|
// Use the app framework like other commands
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
return RunWithApp(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.LogOptions{
|
LogOptions: log.LogOptions{
|
||||||
@@ -130,31 +126,23 @@ func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option {
|
|||||||
TargetDir: opts.TargetDir,
|
TargetDir: opts.TargetDir,
|
||||||
Paths: opts.Paths,
|
Paths: opts.Paths,
|
||||||
Verify: opts.Verify,
|
Verify: opts.Verify,
|
||||||
SkipErrors: GetRootFlags().SkipErrors,
|
|
||||||
}
|
}
|
||||||
|
if err := app.Vaultik.Restore(restoreOpts); err != nil {
|
||||||
err := app.Vaultik.Restore(restoreOpts)
|
if err != context.Canceled {
|
||||||
if err != nil {
|
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
log.Error("Restore operation failed", "error", err)
|
log.Error("Restore operation failed", "error", err)
|
||||||
ReportError("Restore failed: %v", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Shutdown the app when restore completes
|
// Shutdown the app when restore completes
|
||||||
err = app.Shutdowner.Shutdown()
|
if err := app.Shutdowner.Shutdown(); err != nil {
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
log.Error("Failed to shutdown", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
OnStop: func(ctx context.Context) error {
|
OnStop: func(ctx context.Context) error {
|
||||||
log.Debug("Stopping restore operation")
|
log.Debug("Stopping restore operation")
|
||||||
app.Vaultik.Cancel()
|
app.Vaultik.Cancel()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -3,10 +3,7 @@ package cli
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/adrg/xdg"
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -17,7 +14,6 @@ type RootFlags struct {
|
|||||||
Verbose bool
|
Verbose bool
|
||||||
Debug bool
|
Debug bool
|
||||||
Quiet bool
|
Quiet bool
|
||||||
SkipErrors bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var rootFlags RootFlags
|
var rootFlags RootFlags
|
||||||
@@ -29,30 +25,24 @@ func NewRootCommand() *cobra.Command {
|
|||||||
cmd := &cobra.Command{
|
cmd := &cobra.Command{
|
||||||
Use: "vaultik",
|
Use: "vaultik",
|
||||||
Short: "Secure incremental backup tool with asymmetric encryption",
|
Short: "Secure incremental backup tool with asymmetric encryption",
|
||||||
Long: `vaultik is a secure incremental backup tool that encrypts data using age
|
Long: `vaultik is a secure incremental backup daemon that encrypts data using age
|
||||||
public keys and uploads to S3-compatible storage. No private keys are needed
|
public keys and uploads to S3-compatible storage. No private keys are needed
|
||||||
on the source system.`,
|
on the source system.`,
|
||||||
SilenceUsage: true,
|
SilenceUsage: true,
|
||||||
// Bare 'vaultik' (no subcommand): print help. The banner is
|
|
||||||
// printed once at process startup by CLIEntry, before cobra
|
|
||||||
// parses arguments, so it appears even when cobra rejects
|
|
||||||
// args (e.g. "requires at least 2 arg(s)") and on --help.
|
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
|
||||||
_ = cmd.Help()
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add global flags
|
// Add global flags
|
||||||
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "", "Path to config file (default: $VAULTIK_CONFIG or platform config dir)")
|
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "", "Path to config file (default: $VAULTIK_CONFIG or /etc/vaultik/config.yml)")
|
||||||
cmd.PersistentFlags().BoolVarP(&rootFlags.Verbose, "verbose", "v", false, "Enable verbose output")
|
cmd.PersistentFlags().BoolVarP(&rootFlags.Verbose, "verbose", "v", false, "Enable verbose output")
|
||||||
cmd.PersistentFlags().BoolVar(&rootFlags.Debug, "debug", false, "Enable debug output")
|
cmd.PersistentFlags().BoolVar(&rootFlags.Debug, "debug", false, "Enable debug output")
|
||||||
cmd.PersistentFlags().BoolVarP(&rootFlags.Quiet, "quiet", "q", false, "Suppress non-error output")
|
cmd.PersistentFlags().BoolVarP(&rootFlags.Quiet, "quiet", "q", false, "Suppress non-error output")
|
||||||
cmd.PersistentFlags().BoolVar(&rootFlags.SkipErrors, "skip-errors", false, "Continue past per-file errors instead of aborting (applies to snapshot create and restore)")
|
|
||||||
|
|
||||||
// Add subcommands
|
// Add subcommands
|
||||||
cmd.AddCommand(
|
cmd.AddCommand(
|
||||||
NewConfigCommand(),
|
NewRestoreCommand(),
|
||||||
NewPruneCommand(),
|
NewPruneCommand(),
|
||||||
|
NewVerifyCommand(),
|
||||||
|
NewStoreCommand(),
|
||||||
NewSnapshotCommand(),
|
NewSnapshotCommand(),
|
||||||
NewInfoCommand(),
|
NewInfoCommand(),
|
||||||
NewVersionCommand(),
|
NewVersionCommand(),
|
||||||
@@ -70,54 +60,25 @@ func GetRootFlags() RootFlags {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ResolveConfigPath resolves the config file path from flags, environment, or default.
|
// ResolveConfigPath resolves the config file path from flags, environment, or default.
|
||||||
// Search order: --config flag, VAULTIK_CONFIG env, XDG config dir, /etc/vaultik/config.yml.
|
// It checks in order: 1) --config flag, 2) VAULTIK_CONFIG environment variable,
|
||||||
// Explicit paths from --config and $VAULTIK_CONFIG are checked for existence
|
// 3) default location /etc/vaultik/config.yml. Returns an error if no valid
|
||||||
// so the user gets a clear error instead of a downstream YAML parser failure.
|
// config file can be found through any of these methods.
|
||||||
func ResolveConfigPath() (string, error) {
|
func ResolveConfigPath() (string, error) {
|
||||||
if path := rootFlags.ConfigPath; path != "" {
|
// First check global flag
|
||||||
_, err := os.Stat(path)
|
if rootFlags.ConfigPath != "" {
|
||||||
if err != nil {
|
return rootFlags.ConfigPath, nil
|
||||||
return "", fmt.Errorf("config file from --config not found: %s (run 'vaultik config init --config %s' to create it)", path, path)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return path, nil
|
// Then check environment variable
|
||||||
|
if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" {
|
||||||
|
return envPath, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if path := os.Getenv("VAULTIK_CONFIG"); path != "" {
|
// Finally check default location
|
||||||
if _, err := os.Stat(path); err != nil {
|
defaultPath := "/etc/vaultik/config.yml"
|
||||||
return "", fmt.Errorf("config file from $VAULTIK_CONFIG not found: %s (unset VAULTIK_CONFIG, point it at an existing file, or run 'vaultik config init')", path)
|
if _, err := os.Stat(defaultPath); err == nil {
|
||||||
|
return defaultPath, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return path, nil
|
return "", fmt.Errorf("no config file specified, VAULTIK_CONFIG not set, and %s not found", defaultPath)
|
||||||
}
|
|
||||||
|
|
||||||
for _, path := range defaultConfigPaths() {
|
|
||||||
_, err := os.Stat(path)
|
|
||||||
if err == nil {
|
|
||||||
return path, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return "", fmt.Errorf("no config file found at %s (run 'vaultik config init' to create the default config, or pass --config <path>)", strings.Join(defaultConfigPaths(), " or "))
|
|
||||||
}
|
|
||||||
|
|
||||||
// defaultConfigPaths returns the ordered list of config paths to search.
|
|
||||||
// On macOS: ~/Library/Application Support/vaultik/config.yml
|
|
||||||
// On Linux: ~/.config/vaultik/config.yml
|
|
||||||
// Fallback: /etc/vaultik/config.yml
|
|
||||||
func defaultConfigPaths() []string {
|
|
||||||
return []string{
|
|
||||||
filepath.Join(xdg.ConfigHome, "vaultik", "config.yml"),
|
|
||||||
"/etc/vaultik/config.yml",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// DefaultConfigPath returns the platform-appropriate default config path.
|
|
||||||
// Used by the init command and in help text.
|
|
||||||
func DefaultConfigPath() string {
|
|
||||||
if os.Getuid() == 0 {
|
|
||||||
return "/etc/vaultik/config.yml"
|
|
||||||
}
|
|
||||||
|
|
||||||
return filepath.Join(xdg.ConfigHome, "vaultik", "config.yml")
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,14 +2,13 @@ package cli
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewSnapshotCommand creates the snapshot command and subcommands
|
// NewSnapshotCommand creates the snapshot command and subcommands
|
||||||
@@ -26,7 +25,7 @@ func NewSnapshotCommand() *cobra.Command {
|
|||||||
cmd.AddCommand(newSnapshotPurgeCommand())
|
cmd.AddCommand(newSnapshotPurgeCommand())
|
||||||
cmd.AddCommand(newSnapshotVerifyCommand())
|
cmd.AddCommand(newSnapshotVerifyCommand())
|
||||||
cmd.AddCommand(newSnapshotRemoveCommand())
|
cmd.AddCommand(newSnapshotRemoveCommand())
|
||||||
cmd.AddCommand(newSnapshotRestoreCommand())
|
cmd.AddCommand(newSnapshotPruneCommand())
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
@@ -49,8 +48,6 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
|||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
// Pass snapshot names from args
|
// Pass snapshot names from args
|
||||||
opts.Snapshots = args
|
opts.Snapshots = args
|
||||||
// --skip-errors is a global flag on the root command.
|
|
||||||
opts.SkipErrors = rootFlags.SkipErrors
|
|
||||||
// Use unified config resolution
|
// Use unified config resolution
|
||||||
configPath, err := ResolveConfigPath()
|
configPath, err := ResolveConfigPath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -59,7 +56,6 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
|||||||
|
|
||||||
// Use the backup functionality from cli package
|
// Use the backup functionality from cli package
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
return RunWithApp(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.LogOptions{
|
LogOptions: log.LogOptions{
|
||||||
@@ -75,30 +71,24 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
|||||||
OnStart: func(ctx context.Context) error {
|
OnStart: func(ctx context.Context) error {
|
||||||
// Start the snapshot creation in a goroutine
|
// Start the snapshot creation in a goroutine
|
||||||
go func() {
|
go func() {
|
||||||
// --cron suppression is wired through v.UI by setupGlobals.
|
// Run the snapshot creation
|
||||||
err := v.CreateSnapshot(opts)
|
if err := v.CreateSnapshot(opts); err != nil {
|
||||||
if err != nil {
|
if err != context.Canceled {
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
log.Error("Snapshot creation failed", "error", err)
|
log.Error("Snapshot creation failed", "error", err)
|
||||||
ReportError("Snapshot creation failed: %v", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Shutdown the app when snapshot completes
|
// Shutdown the app when snapshot completes
|
||||||
err = v.Shutdowner.Shutdown()
|
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
log.Error("Failed to shutdown", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
OnStop: func(ctx context.Context) error {
|
OnStop: func(ctx context.Context) error {
|
||||||
log.Debug("Stopping snapshot creation")
|
log.Debug("Stopping snapshot creation")
|
||||||
// Cancel the Vaultik context
|
// Cancel the Vaultik context
|
||||||
v.Cancel()
|
v.Cancel()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -108,9 +98,10 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cmd.Flags().BoolVar(&opts.Daemon, "daemon", false, "Run in daemon mode with inotify monitoring")
|
||||||
cmd.Flags().BoolVar(&opts.Cron, "cron", false, "Run in cron mode (silent unless error)")
|
cmd.Flags().BoolVar(&opts.Cron, "cron", false, "Run in cron mode (silent unless error)")
|
||||||
cmd.Flags().BoolVar(&opts.Prune, "prune", false, "After backup, drop older snapshots of the same name and remove orphaned blobs")
|
cmd.Flags().BoolVar(&opts.Prune, "prune", false, "Delete all previous snapshots and unreferenced blobs after backup")
|
||||||
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "", "With --prune: keep snapshots newer than this duration (e.g. 4w, 30d, 6mo) instead of only the latest")
|
cmd.Flags().BoolVar(&opts.SkipErrors, "skip-errors", false, "Skip file read errors (log them loudly but continue)")
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
@@ -133,7 +124,6 @@ func newSnapshotListCommand() *cobra.Command {
|
|||||||
}
|
}
|
||||||
|
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
return RunWithApp(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.LogOptions{
|
LogOptions: log.LogOptions{
|
||||||
@@ -147,26 +137,20 @@ func newSnapshotListCommand() *cobra.Command {
|
|||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
OnStart: func(ctx context.Context) error {
|
OnStart: func(ctx context.Context) error {
|
||||||
go func() {
|
go func() {
|
||||||
err := v.ListSnapshots(jsonOutput)
|
if err := v.ListSnapshots(jsonOutput); err != nil {
|
||||||
if err != nil {
|
if err != context.Canceled {
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
log.Error("Failed to list snapshots", "error", err)
|
log.Error("Failed to list snapshots", "error", err)
|
||||||
ReportError("Failed to list snapshots: %v", err)
|
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||||
err = v.Shutdowner.Shutdown()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
log.Error("Failed to shutdown", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
OnStop: func(ctx context.Context) error {
|
OnStop: func(ctx context.Context) error {
|
||||||
v.Cancel()
|
v.Cancel()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -190,18 +174,19 @@ func newSnapshotPurgeCommand() *cobra.Command {
|
|||||||
Short: "Purge old snapshots",
|
Short: "Purge old snapshots",
|
||||||
Long: `Removes snapshots based on age or count criteria.
|
Long: `Removes snapshots based on age or count criteria.
|
||||||
|
|
||||||
Retention is per-snapshot-name: --keep-latest keeps the latest of each
|
When --keep-latest is used, retention is applied per snapshot name. For example,
|
||||||
configured snapshot name, not the latest globally. Use --snapshot to
|
if you have snapshots named "home" and "system", --keep-latest keeps the most
|
||||||
restrict the operation to specific snapshot names.`,
|
recent of each.
|
||||||
|
|
||||||
|
Use --name to restrict the purge to a single snapshot name.`,
|
||||||
Args: cobra.NoArgs,
|
Args: cobra.NoArgs,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
// Validate flags
|
// Validate flags
|
||||||
if !opts.KeepLatest && opts.OlderThan == "" {
|
if !opts.KeepLatest && opts.OlderThan == "" {
|
||||||
return errors.New("must specify either --keep-latest or --older-than")
|
return fmt.Errorf("must specify either --keep-latest or --older-than")
|
||||||
}
|
}
|
||||||
|
|
||||||
if opts.KeepLatest && opts.OlderThan != "" {
|
if opts.KeepLatest && opts.OlderThan != "" {
|
||||||
return errors.New("cannot specify both --keep-latest and --older-than")
|
return fmt.Errorf("cannot specify both --keep-latest and --older-than")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use unified config resolution
|
// Use unified config resolution
|
||||||
@@ -211,7 +196,6 @@ restrict the operation to specific snapshot names.`,
|
|||||||
}
|
}
|
||||||
|
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
return RunWithApp(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.LogOptions{
|
LogOptions: log.LogOptions{
|
||||||
@@ -225,26 +209,20 @@ restrict the operation to specific snapshot names.`,
|
|||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
OnStart: func(ctx context.Context) error {
|
OnStart: func(ctx context.Context) error {
|
||||||
go func() {
|
go func() {
|
||||||
err := v.PurgeSnapshotsWithOptions(opts)
|
if err := v.PurgeSnapshotsWithOptions(opts); err != nil {
|
||||||
if err != nil {
|
if err != context.Canceled {
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
log.Error("Failed to purge snapshots", "error", err)
|
log.Error("Failed to purge snapshots", "error", err)
|
||||||
ReportError("Failed to purge snapshots: %v", err)
|
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||||
err = v.Shutdowner.Shutdown()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
log.Error("Failed to shutdown", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
OnStop: func(ctx context.Context) error {
|
OnStop: func(ctx context.Context) error {
|
||||||
v.Cancel()
|
v.Cancel()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -254,10 +232,10 @@ restrict the operation to specific snapshot names.`,
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, "Keep only the latest snapshot of each name")
|
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, "Keep only the latest snapshot per name")
|
||||||
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "", "Remove snapshots older than duration (e.g., 30d, 6m, 1y)")
|
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "", "Remove snapshots older than duration (e.g., 30d, 6m, 1y)")
|
||||||
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt")
|
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt")
|
||||||
cmd.Flags().StringArrayVar(&opts.Names, "snapshot", nil, "Restrict to snapshots with these names (repeat for multiple)")
|
cmd.Flags().StringVar(&opts.Name, "name", "", "Filter purge to a specific snapshot name")
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
@@ -273,14 +251,11 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
|||||||
Args: func(cmd *cobra.Command, args []string) error {
|
Args: func(cmd *cobra.Command, args []string) error {
|
||||||
if len(args) != 1 {
|
if len(args) != 1 {
|
||||||
_ = cmd.Help()
|
_ = cmd.Help()
|
||||||
|
|
||||||
if len(args) == 0 {
|
if len(args) == 0 {
|
||||||
return errors.New("snapshot ID required")
|
return fmt.Errorf("snapshot ID required")
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Errorf("expected 1 argument, got %d", len(args))
|
return fmt.Errorf("expected 1 argument, got %d", len(args))
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
@@ -293,7 +268,6 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
|||||||
}
|
}
|
||||||
|
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
return RunWithApp(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.LogOptions{
|
LogOptions: log.LogOptions{
|
||||||
@@ -307,29 +281,28 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
|||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
OnStart: func(ctx context.Context) error {
|
OnStart: func(ctx context.Context) error {
|
||||||
go func() {
|
go func() {
|
||||||
err := v.VerifySnapshotWithOptions(snapshotID, opts)
|
var err error
|
||||||
|
if opts.Deep {
|
||||||
|
err = v.RunDeepVerify(snapshotID, opts)
|
||||||
|
} else {
|
||||||
|
err = v.VerifySnapshotWithOptions(snapshotID, opts)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, context.Canceled) {
|
if err != context.Canceled {
|
||||||
if !opts.JSON {
|
if !opts.JSON {
|
||||||
log.Error("Verification failed", "error", err)
|
log.Error("Verification failed", "error", err)
|
||||||
ReportError("Verification failed: %v", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||||
err = v.Shutdowner.Shutdown()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
log.Error("Failed to shutdown", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
OnStop: func(ctx context.Context) error {
|
OnStop: func(ctx context.Context) error {
|
||||||
v.Cancel()
|
v.Cancel()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -350,38 +323,34 @@ func newSnapshotRemoveCommand() *cobra.Command {
|
|||||||
opts := &vaultik.RemoveOptions{}
|
opts := &vaultik.RemoveOptions{}
|
||||||
|
|
||||||
cmd := &cobra.Command{
|
cmd := &cobra.Command{
|
||||||
Use: "remove <snapshot-id>",
|
Use: "remove [snapshot-id]",
|
||||||
Aliases: []string{"rm"},
|
Aliases: []string{"rm"},
|
||||||
Short: "Remove a snapshot from local index and remote metadata",
|
Short: "Remove a snapshot from the local database",
|
||||||
Long: `Removes a snapshot.
|
Long: `Removes a snapshot from the local database.
|
||||||
|
|
||||||
By default, this removes the snapshot from the local index database and
|
By default, only removes from the local database. Use --remote to also remove
|
||||||
strips the snapshot's metadata from the backup destination store. Blobs
|
the snapshot metadata from remote storage.
|
||||||
are NOT touched: deleting them requires reading every remaining remote
|
|
||||||
manifest (the destination store may hold snapshots this host doesn't
|
|
||||||
know about), which is what 'vaultik prune' does. On success the command
|
|
||||||
prints the exact 'vaultik prune' invocation to run as a follow-up.
|
|
||||||
|
|
||||||
Use --local-only to skip the remote half (e.g. when you want to forget a
|
Note: This does NOT remove blobs. Use 'vaultik prune' to remove orphaned blobs
|
||||||
snapshot locally without touching the destination store).
|
after removing snapshots.
|
||||||
|
|
||||||
If the remote is unreachable, the local-database removal still completes
|
Use --all --force to remove all snapshots.`,
|
||||||
and a warning is emitted; rerun 'vaultik prune' once the destination store
|
|
||||||
is reachable to finish remote cleanup.
|
|
||||||
|
|
||||||
To wipe the entire destination store and start over, use 'vaultik remote
|
|
||||||
nuke --force' — it is the single supported entry point for that.`,
|
|
||||||
Args: func(cmd *cobra.Command, args []string) error {
|
Args: func(cmd *cobra.Command, args []string) error {
|
||||||
|
all, _ := cmd.Flags().GetBool("all")
|
||||||
|
if all {
|
||||||
|
if len(args) > 0 {
|
||||||
|
_ = cmd.Help()
|
||||||
|
return fmt.Errorf("--all cannot be used with a snapshot ID")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
if len(args) != 1 {
|
if len(args) != 1 {
|
||||||
_ = cmd.Help()
|
_ = cmd.Help()
|
||||||
|
|
||||||
if len(args) == 0 {
|
if len(args) == 0 {
|
||||||
return errors.New("snapshot ID required")
|
return fmt.Errorf("snapshot ID required (or use --all --force)")
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Errorf("expected 1 argument, got %d", len(args))
|
return fmt.Errorf("expected 1 argument, got %d", len(args))
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
@@ -392,7 +361,6 @@ nuke --force' — it is the single supported entry point for that.`,
|
|||||||
}
|
}
|
||||||
|
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
return RunWithApp(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.LogOptions{
|
LogOptions: log.LogOptions{
|
||||||
@@ -406,29 +374,28 @@ nuke --force' — it is the single supported entry point for that.`,
|
|||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
OnStart: func(ctx context.Context) error {
|
OnStart: func(ctx context.Context) error {
|
||||||
go func() {
|
go func() {
|
||||||
_, err := v.RemoveSnapshot(args[0], opts)
|
var err error
|
||||||
|
if opts.All {
|
||||||
|
_, err = v.RemoveAllSnapshots(opts)
|
||||||
|
} else {
|
||||||
|
_, err = v.RemoveSnapshot(args[0], opts)
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, context.Canceled) {
|
if err != context.Canceled {
|
||||||
if !opts.JSON {
|
if !opts.JSON {
|
||||||
log.Error("Failed to remove snapshot", "error", err)
|
log.Error("Failed to remove snapshot", "error", err)
|
||||||
ReportError("Failed to remove snapshot: %v", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||||
err = v.Shutdowner.Shutdown()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
log.Error("Failed to shutdown", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
OnStop: func(ctx context.Context) error {
|
OnStop: func(ctx context.Context) error {
|
||||||
v.Cancel()
|
v.Cancel()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -441,7 +408,65 @@ nuke --force' — it is the single supported entry point for that.`,
|
|||||||
cmd.Flags().BoolVarP(&opts.Force, "force", "f", false, "Skip confirmation prompt")
|
cmd.Flags().BoolVarP(&opts.Force, "force", "f", false, "Skip confirmation prompt")
|
||||||
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Show what would be removed without removing")
|
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Show what would be removed without removing")
|
||||||
cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output result as JSON")
|
cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output result as JSON")
|
||||||
cmd.Flags().BoolVar(&opts.LocalOnly, "local-only", false, "Skip remote cleanup; only touch the local index")
|
cmd.Flags().BoolVar(&opts.Remote, "remote", false, "Also remove snapshot metadata from remote storage")
|
||||||
|
cmd.Flags().BoolVar(&opts.All, "all", false, "Remove all snapshots (requires --force)")
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
// newSnapshotPruneCommand creates the 'snapshot prune' subcommand
|
||||||
|
func newSnapshotPruneCommand() *cobra.Command {
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "prune",
|
||||||
|
Short: "Remove orphaned data from local database",
|
||||||
|
Long: `Removes orphaned files, chunks, and blobs from the local database.
|
||||||
|
|
||||||
|
This cleans up data that is no longer referenced by any snapshot, which can
|
||||||
|
accumulate from incomplete backups or deleted snapshots.`,
|
||||||
|
Args: cobra.NoArgs,
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
// Use unified config resolution
|
||||||
|
configPath, err := ResolveConfigPath()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
rootFlags := GetRootFlags()
|
||||||
|
return RunWithApp(cmd.Context(), AppOptions{
|
||||||
|
ConfigPath: configPath,
|
||||||
|
LogOptions: log.LogOptions{
|
||||||
|
Verbose: rootFlags.Verbose,
|
||||||
|
Debug: rootFlags.Debug,
|
||||||
|
Quiet: rootFlags.Quiet,
|
||||||
|
},
|
||||||
|
Modules: []fx.Option{},
|
||||||
|
Invokes: []fx.Option{
|
||||||
|
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||||
|
lc.Append(fx.Hook{
|
||||||
|
OnStart: func(ctx context.Context) error {
|
||||||
|
go func() {
|
||||||
|
if _, err := v.PruneDatabase(); err != nil {
|
||||||
|
if err != context.Canceled {
|
||||||
|
log.Error("Failed to prune database", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||||
|
log.Error("Failed to shutdown", "error", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
OnStop: func(ctx context.Context) error {
|
||||||
|
v.Cancel()
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|||||||
158
internal/cli/store.go
Normal file
158
internal/cli/store.go
Normal file
@@ -0,0 +1,158 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.uber.org/fx"
|
||||||
|
)
|
||||||
|
|
||||||
|
// StoreApp contains dependencies for store commands
|
||||||
|
type StoreApp struct {
|
||||||
|
Storage storage.Storer
|
||||||
|
Shutdowner fx.Shutdowner
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewStoreCommand creates the store command and subcommands
|
||||||
|
func NewStoreCommand() *cobra.Command {
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "store",
|
||||||
|
Short: "Storage information commands",
|
||||||
|
Long: "Commands for viewing information about the storage backend",
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add subcommands
|
||||||
|
cmd.AddCommand(newStoreInfoCommand())
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
// newStoreInfoCommand creates the 'store info' subcommand
|
||||||
|
func newStoreInfoCommand() *cobra.Command {
|
||||||
|
return &cobra.Command{
|
||||||
|
Use: "info",
|
||||||
|
Short: "Display storage information",
|
||||||
|
Long: "Shows storage configuration and statistics including snapshots and blobs",
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
return runWithApp(cmd.Context(), func(app *StoreApp) error {
|
||||||
|
return app.Info(cmd.Context())
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Info displays storage information
|
||||||
|
func (app *StoreApp) Info(ctx context.Context) error {
|
||||||
|
// Get storage info
|
||||||
|
storageInfo := app.Storage.Info()
|
||||||
|
|
||||||
|
fmt.Printf("Storage Information\n")
|
||||||
|
fmt.Printf("==================\n\n")
|
||||||
|
fmt.Printf("Storage Configuration:\n")
|
||||||
|
fmt.Printf(" Type: %s\n", storageInfo.Type)
|
||||||
|
fmt.Printf(" Location: %s\n\n", storageInfo.Location)
|
||||||
|
|
||||||
|
// Count snapshots by listing metadata/ prefix
|
||||||
|
snapshotCount := 0
|
||||||
|
snapshotCh := app.Storage.ListStream(ctx, "metadata/")
|
||||||
|
snapshotDirs := make(map[string]bool)
|
||||||
|
|
||||||
|
for object := range snapshotCh {
|
||||||
|
if object.Err != nil {
|
||||||
|
return fmt.Errorf("listing snapshots: %w", object.Err)
|
||||||
|
}
|
||||||
|
// Extract snapshot ID from path like metadata/2024-01-15-143052-hostname/
|
||||||
|
parts := strings.Split(object.Key, "/")
|
||||||
|
if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" {
|
||||||
|
snapshotDirs[parts[1]] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
snapshotCount = len(snapshotDirs)
|
||||||
|
|
||||||
|
// Count blobs and calculate total size by listing blobs/ prefix
|
||||||
|
blobCount := 0
|
||||||
|
var totalSize int64
|
||||||
|
|
||||||
|
blobCh := app.Storage.ListStream(ctx, "blobs/")
|
||||||
|
for object := range blobCh {
|
||||||
|
if object.Err != nil {
|
||||||
|
return fmt.Errorf("listing blobs: %w", object.Err)
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(object.Key, "/") { // Skip directories
|
||||||
|
blobCount++
|
||||||
|
totalSize += object.Size
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Storage Statistics:\n")
|
||||||
|
fmt.Printf(" Snapshots: %d\n", snapshotCount)
|
||||||
|
fmt.Printf(" Blobs: %d\n", blobCount)
|
||||||
|
fmt.Printf(" Total Size: %s\n", formatBytes(totalSize))
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatBytes formats bytes into human-readable format
|
||||||
|
func formatBytes(bytes int64) string {
|
||||||
|
const unit = 1024
|
||||||
|
if bytes < unit {
|
||||||
|
return fmt.Sprintf("%d B", bytes)
|
||||||
|
}
|
||||||
|
div, exp := int64(unit), 0
|
||||||
|
for n := bytes / unit; n >= unit; n /= unit {
|
||||||
|
div *= unit
|
||||||
|
exp++
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
|
||||||
|
}
|
||||||
|
|
||||||
|
// runWithApp creates the FX app and runs the given function
|
||||||
|
func runWithApp(ctx context.Context, fn func(*StoreApp) error) error {
|
||||||
|
var result error
|
||||||
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
|
// Use unified config resolution
|
||||||
|
configPath, err := ResolveConfigPath()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = RunWithApp(ctx, AppOptions{
|
||||||
|
ConfigPath: configPath,
|
||||||
|
LogOptions: log.LogOptions{
|
||||||
|
Verbose: rootFlags.Verbose,
|
||||||
|
Debug: rootFlags.Debug,
|
||||||
|
Quiet: rootFlags.Quiet,
|
||||||
|
},
|
||||||
|
Modules: []fx.Option{
|
||||||
|
fx.Provide(func(storer storage.Storer, shutdowner fx.Shutdowner) *StoreApp {
|
||||||
|
return &StoreApp{
|
||||||
|
Storage: storer,
|
||||||
|
Shutdowner: shutdowner,
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
Invokes: []fx.Option{
|
||||||
|
fx.Invoke(func(app *StoreApp, shutdowner fx.Shutdowner) {
|
||||||
|
result = fn(app)
|
||||||
|
// Shutdown after command completes
|
||||||
|
go func() {
|
||||||
|
time.Sleep(100 * time.Millisecond) // Brief delay to ensure clean shutdown
|
||||||
|
if err := shutdowner.Shutdown(); err != nil {
|
||||||
|
log.Error("Failed to shutdown", "error", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
98
internal/cli/verify.go
Normal file
98
internal/cli/verify.go
Normal file
@@ -0,0 +1,98 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"go.uber.org/fx"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewVerifyCommand creates the verify command
|
||||||
|
func NewVerifyCommand() *cobra.Command {
|
||||||
|
opts := &vaultik.VerifyOptions{}
|
||||||
|
|
||||||
|
cmd := &cobra.Command{
|
||||||
|
Use: "verify <snapshot-id>",
|
||||||
|
Short: "Verify snapshot integrity",
|
||||||
|
Long: `Verifies that all blobs referenced in a snapshot exist and optionally verifies their contents.
|
||||||
|
|
||||||
|
Shallow verification (default):
|
||||||
|
- Downloads and decompresses manifest
|
||||||
|
- Checks existence of all blobs in S3
|
||||||
|
- Reports missing blobs
|
||||||
|
|
||||||
|
Deep verification (--deep):
|
||||||
|
- Downloads and decrypts database
|
||||||
|
- Verifies blob lists match between manifest and database
|
||||||
|
- Downloads, decrypts, and decompresses each blob
|
||||||
|
- Verifies SHA256 hash of each chunk matches database
|
||||||
|
- Ensures chunks are ordered correctly
|
||||||
|
|
||||||
|
The command will fail immediately on any verification error and exit with non-zero status.`,
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
snapshotID := args[0]
|
||||||
|
|
||||||
|
// Use unified config resolution
|
||||||
|
configPath, err := ResolveConfigPath()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use the app framework for all verification
|
||||||
|
rootFlags := GetRootFlags()
|
||||||
|
return RunWithApp(cmd.Context(), AppOptions{
|
||||||
|
ConfigPath: configPath,
|
||||||
|
LogOptions: log.LogOptions{
|
||||||
|
Verbose: rootFlags.Verbose,
|
||||||
|
Debug: rootFlags.Debug,
|
||||||
|
Quiet: rootFlags.Quiet || opts.JSON, // Suppress log output in JSON mode
|
||||||
|
},
|
||||||
|
Modules: []fx.Option{},
|
||||||
|
Invokes: []fx.Option{
|
||||||
|
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||||
|
lc.Append(fx.Hook{
|
||||||
|
OnStart: func(ctx context.Context) error {
|
||||||
|
// Run the verify operation directly
|
||||||
|
go func() {
|
||||||
|
var err error
|
||||||
|
if opts.Deep {
|
||||||
|
err = v.RunDeepVerify(snapshotID, opts)
|
||||||
|
} else {
|
||||||
|
err = v.VerifySnapshotWithOptions(snapshotID, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
if err != context.Canceled {
|
||||||
|
if !opts.JSON {
|
||||||
|
log.Error("Verification failed", "error", err)
|
||||||
|
}
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||||
|
log.Error("Failed to shutdown", "error", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
OnStop: func(ctx context.Context) error {
|
||||||
|
log.Debug("Stopping verify operation")
|
||||||
|
v.Cancel()
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.Flags().BoolVar(&opts.Deep, "deep", false, "Perform deep verification by downloading and verifying all blob contents")
|
||||||
|
cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output verification results as JSON")
|
||||||
|
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
@@ -4,8 +4,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"runtime"
|
"runtime"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"sneak.berlin/go/vaultik/internal/globals"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewVersionCommand creates the version command
|
// NewVersionCommand creates the version command
|
||||||
@@ -18,19 +18,8 @@ func NewVersionCommand() *cobra.Command {
|
|||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
fmt.Printf("vaultik %s\n", globals.Version)
|
fmt.Printf("vaultik %s\n", globals.Version)
|
||||||
fmt.Printf(" commit: %s\n", globals.Commit)
|
fmt.Printf(" commit: %s\n", globals.Commit)
|
||||||
fmt.Printf(" build date: %s\n", globals.CommitDate)
|
|
||||||
fmt.Printf(" go: %s\n", runtime.Version())
|
fmt.Printf(" go: %s\n", runtime.Version())
|
||||||
fmt.Printf(" os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
|
fmt.Printf(" os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
|
||||||
fmt.Printf(" author: %s\n", globals.Author)
|
|
||||||
fmt.Printf(" homepage: %s\n", globals.Homepage)
|
|
||||||
fmt.Printf(" license: %s\n", globals.License)
|
|
||||||
|
|
||||||
if globals.Version == "dev" {
|
|
||||||
fmt.Println()
|
|
||||||
fmt.Println("This is a development build (no version information embedded).")
|
|
||||||
fmt.Println("Build a release binary with 'make vaultik' or download from")
|
|
||||||
fmt.Println("https://sneak.berlin/go/vaultik for embedded version metadata.")
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,37 +1,33 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"filippo.io/age"
|
"filippo.io/age"
|
||||||
"git.eeqj.de/sneak/smartconfig"
|
"git.eeqj.de/sneak/smartconfig"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
"github.com/adrg/xdg"
|
"github.com/adrg/xdg"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const appName = "vaultik"
|
const appName = "berlin.sneak.app.vaultik"
|
||||||
|
|
||||||
// expandTilde expands ~ at the start of a path to the user's home directory.
|
// expandTilde expands ~ at the start of a path to the user's home directory.
|
||||||
func expandTilde(path string) string {
|
func expandTilde(path string) string {
|
||||||
if path == "~" {
|
if path == "~" {
|
||||||
home, _ := os.UserHomeDir()
|
home, _ := os.UserHomeDir()
|
||||||
|
|
||||||
return home
|
return home
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.HasPrefix(path, "~/") {
|
if strings.HasPrefix(path, "~/") {
|
||||||
home, _ := os.UserHomeDir()
|
home, _ := os.UserHomeDir()
|
||||||
|
|
||||||
return filepath.Join(home, path[2:])
|
return filepath.Join(home, path[2:])
|
||||||
}
|
}
|
||||||
|
|
||||||
return path
|
return path
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -39,10 +35,8 @@ func expandTilde(path string) string {
|
|||||||
func expandTildeInURL(url string) string {
|
func expandTildeInURL(url string) string {
|
||||||
if strings.HasPrefix(url, "file://~/") {
|
if strings.HasPrefix(url, "file://~/") {
|
||||||
home, _ := os.UserHomeDir()
|
home, _ := os.UserHomeDir()
|
||||||
|
|
||||||
return "file://" + filepath.Join(home, url[9:])
|
return "file://" + filepath.Join(home, url[9:])
|
||||||
}
|
}
|
||||||
|
|
||||||
return url
|
return url
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -70,7 +64,6 @@ func (c *Config) GetExcludes(snapshotName string) []string {
|
|||||||
combined := make([]string, 0, len(c.Exclude)+len(snap.Exclude))
|
combined := make([]string, 0, len(c.Exclude)+len(snap.Exclude))
|
||||||
combined = append(combined, c.Exclude...)
|
combined = append(combined, c.Exclude...)
|
||||||
combined = append(combined, snap.Exclude...)
|
combined = append(combined, snap.Exclude...)
|
||||||
|
|
||||||
return combined
|
return combined
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -82,7 +75,6 @@ func (c *Config) SnapshotNames() []string {
|
|||||||
}
|
}
|
||||||
// Sort for deterministic order
|
// Sort for deterministic order
|
||||||
sort.Strings(names)
|
sort.Strings(names)
|
||||||
|
|
||||||
return names
|
return names
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -93,11 +85,14 @@ func (c *Config) SnapshotNames() []string {
|
|||||||
type Config struct {
|
type Config struct {
|
||||||
AgeRecipients []string `yaml:"age_recipients"`
|
AgeRecipients []string `yaml:"age_recipients"`
|
||||||
AgeSecretKey string `yaml:"age_secret_key"`
|
AgeSecretKey string `yaml:"age_secret_key"`
|
||||||
|
BackupInterval time.Duration `yaml:"backup_interval"`
|
||||||
BlobSizeLimit Size `yaml:"blob_size_limit"`
|
BlobSizeLimit Size `yaml:"blob_size_limit"`
|
||||||
ChunkSize Size `yaml:"chunk_size"`
|
ChunkSize Size `yaml:"chunk_size"`
|
||||||
Exclude []string `yaml:"exclude"` // Global excludes applied to all snapshots
|
Exclude []string `yaml:"exclude"` // Global excludes applied to all snapshots
|
||||||
|
FullScanInterval time.Duration `yaml:"full_scan_interval"`
|
||||||
Hostname string `yaml:"hostname"`
|
Hostname string `yaml:"hostname"`
|
||||||
IndexPath string `yaml:"index_path"`
|
IndexPath string `yaml:"index_path"`
|
||||||
|
MinTimeBetweenRun time.Duration `yaml:"min_time_between_run"`
|
||||||
S3 S3Config `yaml:"s3"`
|
S3 S3Config `yaml:"s3"`
|
||||||
Snapshots map[string]SnapshotConfig `yaml:"snapshots"`
|
Snapshots map[string]SnapshotConfig `yaml:"snapshots"`
|
||||||
CompressionLevel int `yaml:"compression_level"`
|
CompressionLevel int `yaml:"compression_level"`
|
||||||
@@ -135,7 +130,7 @@ type ConfigPath string
|
|||||||
// Returns an error if the path is empty or if loading fails.
|
// Returns an error if the path is empty or if loading fails.
|
||||||
func New(path ConfigPath) (*Config, error) {
|
func New(path ConfigPath) (*Config, error) {
|
||||||
if path == "" {
|
if path == "" {
|
||||||
return nil, errors.New("config path not provided")
|
return nil, fmt.Errorf("config path not provided")
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg, err := Load(string(path))
|
cfg, err := Load(string(path))
|
||||||
@@ -162,20 +157,21 @@ func Load(path string) (*Config, error) {
|
|||||||
// Set defaults
|
// Set defaults
|
||||||
BlobSizeLimit: Size(10 * 1024 * 1024 * 1024), // 10GB
|
BlobSizeLimit: Size(10 * 1024 * 1024 * 1024), // 10GB
|
||||||
ChunkSize: Size(10 * 1024 * 1024), // 10MB
|
ChunkSize: Size(10 * 1024 * 1024), // 10MB
|
||||||
|
BackupInterval: 1 * time.Hour,
|
||||||
|
FullScanInterval: 24 * time.Hour,
|
||||||
|
MinTimeBetweenRun: 15 * time.Minute,
|
||||||
IndexPath: filepath.Join(xdg.DataHome, appName, "index.sqlite"),
|
IndexPath: filepath.Join(xdg.DataHome, appName, "index.sqlite"),
|
||||||
CompressionLevel: 3,
|
CompressionLevel: 3,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Convert smartconfig data to YAML then unmarshal
|
// Convert smartconfig data to YAML then unmarshal
|
||||||
configData := sc.Data()
|
configData := sc.Data()
|
||||||
|
|
||||||
yamlBytes, err := yaml.Marshal(configData)
|
yamlBytes, err := yaml.Marshal(configData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to marshal config data: %w", err)
|
return nil, fmt.Errorf("failed to marshal config data: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = yaml.Unmarshal(yamlBytes, cfg)
|
if err := yaml.Unmarshal(yamlBytes, cfg); err != nil {
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -188,7 +184,6 @@ func Load(path string) (*Config, error) {
|
|||||||
for i, path := range snap.Paths {
|
for i, path := range snap.Paths {
|
||||||
snap.Paths[i] = expandTilde(path)
|
snap.Paths[i] = expandTilde(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg.Snapshots[name] = snap
|
cfg.Snapshots[name] = snap
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -208,7 +203,6 @@ func Load(path string) (*Config, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to get hostname: %w", err)
|
return nil, fmt.Errorf("failed to get hostname: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg.Hostname = hostname
|
cfg.Hostname = hostname
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -216,14 +210,12 @@ func Load(path string) (*Config, error) {
|
|||||||
if cfg.S3.Region == "" {
|
if cfg.S3.Region == "" {
|
||||||
cfg.S3.Region = "us-east-1"
|
cfg.S3.Region = "us-east-1"
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.S3.PartSize == 0 {
|
if cfg.S3.PartSize == 0 {
|
||||||
cfg.S3.PartSize = Size(5 * 1024 * 1024) // 5MB
|
cfg.S3.PartSize = Size(5 * 1024 * 1024) // 5MB
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check config file permissions (warn if world or group readable)
|
// Check config file permissions (warn if world or group readable)
|
||||||
info, err := os.Stat(path)
|
if info, err := os.Stat(path); err == nil {
|
||||||
if err == nil {
|
|
||||||
mode := info.Mode().Perm()
|
mode := info.Mode().Perm()
|
||||||
if mode&0044 != 0 { // group or world readable
|
if mode&0044 != 0 { // group or world readable
|
||||||
log.Warn("Config file has insecure permissions (contains S3 credentials)",
|
log.Warn("Config file has insecure permissions (contains S3 credentials)",
|
||||||
@@ -233,8 +225,7 @@ func Load(path string) (*Config, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err = cfg.Validate()
|
if err := cfg.Validate(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("invalid config: %w", err)
|
return nil, fmt.Errorf("invalid config: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -252,11 +243,11 @@ func Load(path string) (*Config, error) {
|
|||||||
// Returns an error describing the first validation failure encountered.
|
// Returns an error describing the first validation failure encountered.
|
||||||
func (c *Config) Validate() error {
|
func (c *Config) Validate() error {
|
||||||
if len(c.AgeRecipients) == 0 {
|
if len(c.AgeRecipients) == 0 {
|
||||||
return errors.New("at least one age_recipient is required (generate with: age-keygen)")
|
return fmt.Errorf("at least one age_recipient is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(c.Snapshots) == 0 {
|
if len(c.Snapshots) == 0 {
|
||||||
return errors.New("at least one snapshot must be configured (see config.example.yml)")
|
return fmt.Errorf("at least one snapshot must be configured")
|
||||||
}
|
}
|
||||||
|
|
||||||
for name, snap := range c.Snapshots {
|
for name, snap := range c.Snapshots {
|
||||||
@@ -266,21 +257,20 @@ func (c *Config) Validate() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Validate storage configuration
|
// Validate storage configuration
|
||||||
err := c.validateStorage()
|
if err := c.validateStorage(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.ChunkSize.Int64() < 1024*1024 { // 1MB minimum
|
if c.ChunkSize.Int64() < 1024*1024 { // 1MB minimum
|
||||||
return errors.New("chunk_size must be at least 1MB")
|
return fmt.Errorf("chunk_size must be at least 1MB")
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() {
|
if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() {
|
||||||
return errors.New("blob_size_limit must be at least chunk_size")
|
return fmt.Errorf("blob_size_limit must be at least chunk_size")
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.CompressionLevel < 1 || c.CompressionLevel > 19 {
|
if c.CompressionLevel < 1 || c.CompressionLevel > 19 {
|
||||||
return errors.New("compression_level must be between 1 and 19")
|
return fmt.Errorf("compression_level must be between 1 and 19")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -297,43 +287,38 @@ func (c *Config) validateStorage() error {
|
|||||||
// File storage doesn't need S3 credentials
|
// File storage doesn't need S3 credentials
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.HasPrefix(c.StorageURL, "s3://") {
|
if strings.HasPrefix(c.StorageURL, "s3://") {
|
||||||
// S3 storage needs credentials
|
// S3 storage needs credentials
|
||||||
if c.S3.AccessKeyID == "" {
|
if c.S3.AccessKeyID == "" {
|
||||||
return errors.New("s3.access_key_id is required for s3:// URLs")
|
return fmt.Errorf("s3.access_key_id is required for s3:// URLs")
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.S3.SecretAccessKey == "" {
|
if c.S3.SecretAccessKey == "" {
|
||||||
return errors.New("s3.secret_access_key is required for s3:// URLs")
|
return fmt.Errorf("s3.secret_access_key is required for s3:// URLs")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.HasPrefix(c.StorageURL, "rclone://") {
|
if strings.HasPrefix(c.StorageURL, "rclone://") {
|
||||||
// Rclone storage uses rclone's own config
|
// Rclone storage uses rclone's own config
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
return fmt.Errorf("storage_url must start with s3://, file://, or rclone://")
|
||||||
return errors.New("storage_url must start with s3://, file://, or rclone://")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Legacy S3 configuration
|
// Legacy S3 configuration
|
||||||
if c.S3.Endpoint == "" {
|
if c.S3.Endpoint == "" {
|
||||||
return errors.New("storage not configured; set storage_url or provide s3.endpoint + s3.bucket + credentials")
|
return fmt.Errorf("s3.endpoint is required (or set storage_url)")
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.S3.Bucket == "" {
|
if c.S3.Bucket == "" {
|
||||||
return errors.New("s3.bucket is required (or set storage_url)")
|
return fmt.Errorf("s3.bucket is required (or set storage_url)")
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.S3.AccessKeyID == "" {
|
if c.S3.AccessKeyID == "" {
|
||||||
return errors.New("s3.access_key_id is required")
|
return fmt.Errorf("s3.access_key_id is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.S3.SecretAccessKey == "" {
|
if c.S3.SecretAccessKey == "" {
|
||||||
return errors.New("s3.secret_access_key is required")
|
return fmt.Errorf("s3.secret_access_key is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -351,7 +336,6 @@ func extractAgeSecretKey(input string) string {
|
|||||||
if id, ok := identities[0].(*age.X25519Identity); ok {
|
if id, ok := identities[0].(*age.X25519Identity); ok {
|
||||||
return id.String()
|
return id.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
return strings.TrimSpace(input)
|
return strings.TrimSpace(input)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -15,9 +15,7 @@ const (
|
|||||||
func TestMain(m *testing.M) {
|
func TestMain(m *testing.M) {
|
||||||
// Set up test environment
|
// Set up test environment
|
||||||
testConfigPath := filepath.Join("..", "..", "test", "config.yaml")
|
testConfigPath := filepath.Join("..", "..", "test", "config.yaml")
|
||||||
|
if absPath, err := filepath.Abs(testConfigPath); err == nil {
|
||||||
absPath, err := filepath.Abs(testConfigPath)
|
|
||||||
if err == nil {
|
|
||||||
_ = os.Setenv("VAULTIK_CONFIG", absPath)
|
_ = os.Setenv("VAULTIK_CONFIG", absPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -43,7 +41,6 @@ func TestConfigLoad(t *testing.T) {
|
|||||||
if len(cfg.AgeRecipients) != 2 {
|
if len(cfg.AgeRecipients) != 2 {
|
||||||
t.Errorf("Expected 2 age recipients, got %d", len(cfg.AgeRecipients))
|
t.Errorf("Expected 2 age recipients, got %d", len(cfg.AgeRecipients))
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.AgeRecipients[0] != TEST_SNEAK_AGE_PUBLIC_KEY {
|
if cfg.AgeRecipients[0] != TEST_SNEAK_AGE_PUBLIC_KEY {
|
||||||
t.Errorf("Expected first age recipient to be %s, got '%s'", TEST_SNEAK_AGE_PUBLIC_KEY, cfg.AgeRecipients[0])
|
t.Errorf("Expected first age recipient to be %s, got '%s'", TEST_SNEAK_AGE_PUBLIC_KEY, cfg.AgeRecipients[0])
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
@@ -15,23 +14,18 @@ type Size int64
|
|||||||
// UnmarshalYAML implements yaml.Unmarshaler for Size, allowing it to be
|
// UnmarshalYAML implements yaml.Unmarshaler for Size, allowing it to be
|
||||||
// parsed from YAML configuration files. It accepts both numeric values
|
// parsed from YAML configuration files. It accepts both numeric values
|
||||||
// (interpreted as bytes) and string values with units (e.g., "10MB").
|
// (interpreted as bytes) and string values with units (e.g., "10MB").
|
||||||
func (s *Size) UnmarshalYAML(unmarshal func(any) error) error {
|
func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||||
// Try to unmarshal as int64 first
|
// Try to unmarshal as int64 first
|
||||||
var intVal int64
|
var intVal int64
|
||||||
|
if err := unmarshal(&intVal); err == nil {
|
||||||
err := unmarshal(&intVal)
|
|
||||||
if err == nil {
|
|
||||||
*s = Size(intVal)
|
*s = Size(intVal)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try to unmarshal as string
|
// Try to unmarshal as string
|
||||||
var strVal string
|
var strVal string
|
||||||
|
if err := unmarshal(&strVal); err != nil {
|
||||||
err = unmarshal(&strVal)
|
return fmt.Errorf("size must be a number or string")
|
||||||
if err != nil {
|
|
||||||
return errors.New("size must be a number or string")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse the string using go-humanize
|
// Parse the string using go-humanize
|
||||||
@@ -41,7 +35,6 @@ func (s *Size) UnmarshalYAML(unmarshal func(any) error) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
*s = Size(bytes)
|
*s = Size(bytes)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -65,6 +58,5 @@ func ParseSize(s string) (Size, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, fmt.Errorf("invalid size format: %w", err)
|
return 0, fmt.Errorf("invalid size format: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return Size(bytes), nil
|
return Size(bytes), nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package crypto
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -26,7 +25,7 @@ type Encryptor struct {
|
|||||||
// public keys are invalid or if no recipients are specified.
|
// public keys are invalid or if no recipients are specified.
|
||||||
func NewEncryptor(publicKeys []string) (*Encryptor, error) {
|
func NewEncryptor(publicKeys []string) (*Encryptor, error) {
|
||||||
if len(publicKeys) == 0 {
|
if len(publicKeys) == 0 {
|
||||||
return nil, errors.New("at least one recipient is required")
|
return nil, fmt.Errorf("at least one recipient is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
recipients := make([]age.Recipient, 0, len(publicKeys))
|
recipients := make([]age.Recipient, 0, len(publicKeys))
|
||||||
@@ -35,7 +34,6 @@ func NewEncryptor(publicKeys []string) (*Encryptor, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("parsing age recipient %s: %w", key, err)
|
return nil, fmt.Errorf("parsing age recipient %s: %w", key, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
recipients = append(recipients, recipient)
|
recipients = append(recipients, recipient)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -62,14 +60,12 @@ func (e *Encryptor) Encrypt(data []byte) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Write data
|
// Write data
|
||||||
_, err = w.Write(data)
|
if _, err := w.Write(data); err != nil {
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("writing encrypted data: %w", err)
|
return nil, fmt.Errorf("writing encrypted data: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close to flush
|
// Close to flush
|
||||||
err = w.Close()
|
if err := w.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("closing encrypted writer: %w", err)
|
return nil, fmt.Errorf("closing encrypted writer: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -92,14 +88,12 @@ func (e *Encryptor) EncryptStream(dst io.Writer, src io.Reader) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Copy data
|
// Copy data
|
||||||
_, err = io.Copy(w, src)
|
if _, err := io.Copy(w, src); err != nil {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("copying encrypted data: %w", err)
|
return fmt.Errorf("copying encrypted data: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close to flush
|
// Close to flush
|
||||||
err = w.Close()
|
if err := w.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("closing encrypted writer: %w", err)
|
return fmt.Errorf("closing encrypted writer: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -132,7 +126,7 @@ func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) {
|
|||||||
// of the public keys are invalid or if no recipients are specified.
|
// of the public keys are invalid or if no recipients are specified.
|
||||||
func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
|
func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
|
||||||
if len(publicKeys) == 0 {
|
if len(publicKeys) == 0 {
|
||||||
return errors.New("at least one recipient is required")
|
return fmt.Errorf("at least one recipient is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
recipients := make([]age.Recipient, 0, len(publicKeys))
|
recipients := make([]age.Recipient, 0, len(publicKeys))
|
||||||
@@ -141,7 +135,6 @@ func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("parsing age recipient %s: %w", key, err)
|
return fmt.Errorf("parsing age recipient %s: %w", key, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
recipients = append(recipients, recipient)
|
recipients = append(recipients, recipient)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -43,9 +43,7 @@ func TestEncryptor(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var decrypted bytes.Buffer
|
var decrypted bytes.Buffer
|
||||||
|
if _, err := decrypted.ReadFrom(r); err != nil {
|
||||||
_, err = decrypted.ReadFrom(r)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read decrypted data: %v", err)
|
t.Fatalf("failed to read decrypted data: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -60,12 +58,10 @@ func TestEncryptorMultipleRecipients(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to generate identity1: %v", err)
|
t.Fatalf("failed to generate identity1: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
identity2, err := age.GenerateX25519Identity()
|
identity2, err := age.GenerateX25519Identity()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to generate identity2: %v", err)
|
t.Fatalf("failed to generate identity2: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
identity3, err := age.GenerateX25519Identity()
|
identity3, err := age.GenerateX25519Identity()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to generate identity3: %v", err)
|
t.Fatalf("failed to generate identity3: %v", err)
|
||||||
@@ -101,9 +97,7 @@ func TestEncryptorMultipleRecipients(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var decrypted bytes.Buffer
|
var decrypted bytes.Buffer
|
||||||
|
if _, err := decrypted.ReadFrom(r); err != nil {
|
||||||
_, err = decrypted.ReadFrom(r)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("recipient %d failed to read decrypted data: %v", i+1, err)
|
t.Fatalf("recipient %d failed to read decrypted data: %v", i+1, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -129,15 +123,13 @@ func TestEncryptorUpdateRecipients(t *testing.T) {
|
|||||||
|
|
||||||
// Encrypt with first key
|
// Encrypt with first key
|
||||||
plaintext := []byte("test data")
|
plaintext := []byte("test data")
|
||||||
|
|
||||||
ciphertext1, err := enc.Encrypt(plaintext)
|
ciphertext1, err := enc.Encrypt(plaintext)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to encrypt: %v", err)
|
t.Fatalf("failed to encrypt: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update to second key
|
// Update to second key
|
||||||
err = enc.UpdateRecipients([]string{publicKey2})
|
if err := enc.UpdateRecipients([]string{publicKey2}); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to update recipients: %v", err)
|
t.Fatalf("failed to update recipients: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,24 +140,18 @@ func TestEncryptorUpdateRecipients(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// First ciphertext should only decrypt with first identity
|
// First ciphertext should only decrypt with first identity
|
||||||
_, err = age.Decrypt(bytes.NewReader(ciphertext1), identity1)
|
if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity1); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Error("failed to decrypt with identity1")
|
t.Error("failed to decrypt with identity1")
|
||||||
}
|
}
|
||||||
|
if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity2); err == nil {
|
||||||
_, err = age.Decrypt(bytes.NewReader(ciphertext1), identity2)
|
|
||||||
if err == nil {
|
|
||||||
t.Error("should not decrypt with identity2")
|
t.Error("should not decrypt with identity2")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Second ciphertext should only decrypt with second identity
|
// Second ciphertext should only decrypt with second identity
|
||||||
_, err = age.Decrypt(bytes.NewReader(ciphertext2), identity2)
|
if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity2); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Error("failed to decrypt with identity2")
|
t.Error("failed to decrypt with identity2")
|
||||||
}
|
}
|
||||||
|
if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity1); err == nil {
|
||||||
_, err = age.Decrypt(bytes.NewReader(ciphertext2), identity1)
|
|
||||||
if err == nil {
|
|
||||||
t.Error("should not decrypt with identity1")
|
t.Error("should not decrypt with identity1")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package database
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -50,15 +49,12 @@ func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([
|
|||||||
defer CloseRows(rows)
|
defer CloseRows(rows)
|
||||||
|
|
||||||
var blobChunks []*BlobChunk
|
var blobChunks []*BlobChunk
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var bc BlobChunk
|
var bc BlobChunk
|
||||||
|
|
||||||
err := rows.Scan(&bc.BlobID, &bc.ChunkHash, &bc.Offset, &bc.Length)
|
err := rows.Scan(&bc.BlobID, &bc.ChunkHash, &bc.Offset, &bc.Length)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("scanning blob chunk: %w", err)
|
return nil, fmt.Errorf("scanning blob chunk: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
blobChunks = append(blobChunks, &bc)
|
blobChunks = append(blobChunks, &bc)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,9 +70,7 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
|
|||||||
`
|
`
|
||||||
|
|
||||||
LogSQL("GetByChunkHash", query, chunkHash)
|
LogSQL("GetByChunkHash", query, chunkHash)
|
||||||
|
|
||||||
var bc BlobChunk
|
var bc BlobChunk
|
||||||
|
|
||||||
err := r.db.conn.QueryRowContext(ctx, query, chunkHash).Scan(
|
err := r.db.conn.QueryRowContext(ctx, query, chunkHash).Scan(
|
||||||
&bc.BlobID,
|
&bc.BlobID,
|
||||||
&bc.ChunkHash,
|
&bc.ChunkHash,
|
||||||
@@ -84,20 +78,16 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
|
|||||||
&bc.Length,
|
&bc.Length,
|
||||||
)
|
)
|
||||||
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if err == sql.ErrNoRows {
|
||||||
LogSQL("GetByChunkHash", "No rows found", chunkHash)
|
LogSQL("GetByChunkHash", "No rows found", chunkHash)
|
||||||
|
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
LogSQL("GetByChunkHash", "Error", chunkHash, err)
|
LogSQL("GetByChunkHash", "Error", chunkHash, err)
|
||||||
|
|
||||||
return nil, fmt.Errorf("querying blob chunk: %w", err)
|
return nil, fmt.Errorf("querying blob chunk: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
LogSQL("GetByChunkHash", "Found blob", chunkHash, "blob", bc.BlobID)
|
LogSQL("GetByChunkHash", "Found blob", chunkHash, "blob", bc.BlobID)
|
||||||
|
|
||||||
return &bc, nil
|
return &bc, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -111,9 +101,7 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
|
|||||||
`
|
`
|
||||||
|
|
||||||
LogSQL("GetByChunkHashTx", query, chunkHash)
|
LogSQL("GetByChunkHashTx", query, chunkHash)
|
||||||
|
|
||||||
var bc BlobChunk
|
var bc BlobChunk
|
||||||
|
|
||||||
err := tx.QueryRowContext(ctx, query, chunkHash).Scan(
|
err := tx.QueryRowContext(ctx, query, chunkHash).Scan(
|
||||||
&bc.BlobID,
|
&bc.BlobID,
|
||||||
&bc.ChunkHash,
|
&bc.ChunkHash,
|
||||||
@@ -121,20 +109,16 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
|
|||||||
&bc.Length,
|
&bc.Length,
|
||||||
)
|
)
|
||||||
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if err == sql.ErrNoRows {
|
||||||
LogSQL("GetByChunkHashTx", "No rows found", chunkHash)
|
LogSQL("GetByChunkHashTx", "No rows found", chunkHash)
|
||||||
|
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
LogSQL("GetByChunkHashTx", "Error", chunkHash, err)
|
LogSQL("GetByChunkHashTx", "Error", chunkHash, err)
|
||||||
|
|
||||||
return nil, fmt.Errorf("querying blob chunk: %w", err)
|
return nil, fmt.Errorf("querying blob chunk: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
LogSQL("GetByChunkHashTx", "Found blob", chunkHash, "blob", bc.BlobID)
|
LogSQL("GetByChunkHashTx", "Found blob", chunkHash, "blob", bc.BlobID)
|
||||||
|
|
||||||
return &bc, nil
|
return &bc, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,9 +132,7 @@ func (r *BlobChunkRepository) DeleteOrphaned(ctx context.Context) error {
|
|||||||
WHERE blobs.id = blob_chunks.blob_id
|
WHERE blobs.id = blob_chunks.blob_id
|
||||||
)
|
)
|
||||||
`
|
`
|
||||||
|
if _, err := r.db.ExecWithLog(ctx, query1); err != nil {
|
||||||
_, err := r.db.ExecWithLog(ctx, query1)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("deleting blob_chunks with missing blobs: %w", err)
|
return fmt.Errorf("deleting blob_chunks with missing blobs: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -162,9 +144,7 @@ func (r *BlobChunkRepository) DeleteOrphaned(ctx context.Context) error {
|
|||||||
WHERE chunks.chunk_hash = blob_chunks.chunk_hash
|
WHERE chunks.chunk_hash = blob_chunks.chunk_hash
|
||||||
)
|
)
|
||||||
`
|
`
|
||||||
|
if _, err := r.db.ExecWithLog(ctx, query2); err != nil {
|
||||||
_, err = r.db.ExecWithLog(ctx, query2)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("deleting blob_chunks with missing chunks: %w", err)
|
return fmt.Errorf("deleting blob_chunks with missing chunks: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBlobChunkRepository(t *testing.T) {
|
func TestBlobChunkRepository(t *testing.T) {
|
||||||
@@ -22,7 +22,6 @@ func TestBlobChunkRepository(t *testing.T) {
|
|||||||
Hash: types.BlobHash("blob1-hash"),
|
Hash: types.BlobHash("blob1-hash"),
|
||||||
CreatedTS: time.Now(),
|
CreatedTS: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
err := repos.Blobs.Create(ctx, nil, blob)
|
err := repos.Blobs.Create(ctx, nil, blob)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create blob: %v", err)
|
t.Fatalf("failed to create blob: %v", err)
|
||||||
@@ -35,7 +34,6 @@ func TestBlobChunkRepository(t *testing.T) {
|
|||||||
ChunkHash: chunkHash,
|
ChunkHash: chunkHash,
|
||||||
Size: 1024,
|
Size: 1024,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||||
@@ -62,7 +60,6 @@ func TestBlobChunkRepository(t *testing.T) {
|
|||||||
Offset: 1024,
|
Offset: 1024,
|
||||||
Length: 2048,
|
Length: 2048,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.BlobChunks.Create(ctx, nil, bc2)
|
err = repos.BlobChunks.Create(ctx, nil, bc2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create second blob chunk: %v", err)
|
t.Fatalf("failed to create second blob chunk: %v", err)
|
||||||
@@ -74,7 +71,6 @@ func TestBlobChunkRepository(t *testing.T) {
|
|||||||
Offset: 3072,
|
Offset: 3072,
|
||||||
Length: 512,
|
Length: 512,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.BlobChunks.Create(ctx, nil, bc3)
|
err = repos.BlobChunks.Create(ctx, nil, bc3)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create third blob chunk: %v", err)
|
t.Fatalf("failed to create third blob chunk: %v", err)
|
||||||
@@ -85,7 +81,6 @@ func TestBlobChunkRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get blob chunks: %v", err)
|
t.Fatalf("failed to get blob chunks: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(blobChunks) != 3 {
|
if len(blobChunks) != 3 {
|
||||||
t.Errorf("expected 3 chunks, got %d", len(blobChunks))
|
t.Errorf("expected 3 chunks, got %d", len(blobChunks))
|
||||||
}
|
}
|
||||||
@@ -103,15 +98,12 @@ func TestBlobChunkRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get blob chunk by chunk hash: %v", err)
|
t.Fatalf("failed to get blob chunk by chunk hash: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if bc == nil {
|
if bc == nil {
|
||||||
t.Fatal("expected blob chunk, got nil")
|
t.Fatal("expected blob chunk, got nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
if bc.BlobID != blob.ID {
|
if bc.BlobID != blob.ID {
|
||||||
t.Errorf("wrong blob ID: expected %s, got %s", blob.ID, bc.BlobID)
|
t.Errorf("wrong blob ID: expected %s, got %s", blob.ID, bc.BlobID)
|
||||||
}
|
}
|
||||||
|
|
||||||
if bc.Offset != 1024 {
|
if bc.Offset != 1024 {
|
||||||
t.Errorf("wrong offset: expected 1024, got %d", bc.Offset)
|
t.Errorf("wrong offset: expected 1024, got %d", bc.Offset)
|
||||||
}
|
}
|
||||||
@@ -121,7 +113,6 @@ func TestBlobChunkRepository(t *testing.T) {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint")
|
t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint")
|
||||||
}
|
}
|
||||||
|
|
||||||
if !strings.Contains(err.Error(), "UNIQUE") && !strings.Contains(err.Error(), "constraint") {
|
if !strings.Contains(err.Error(), "UNIQUE") && !strings.Contains(err.Error(), "constraint") {
|
||||||
t.Fatalf("expected constraint error, got: %v", err)
|
t.Fatalf("expected constraint error, got: %v", err)
|
||||||
}
|
}
|
||||||
@@ -131,7 +122,6 @@ func TestBlobChunkRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if bc != nil {
|
if bc != nil {
|
||||||
t.Error("expected nil for non-existent chunk")
|
t.Error("expected nil for non-existent chunk")
|
||||||
}
|
}
|
||||||
@@ -160,7 +150,6 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create blob1: %v", err)
|
t.Fatalf("failed to create blob1: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Blobs.Create(ctx, nil, blob2)
|
err = repos.Blobs.Create(ctx, nil, blob2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create blob2: %v", err)
|
t.Fatalf("failed to create blob2: %v", err)
|
||||||
@@ -173,7 +162,6 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
|||||||
ChunkHash: chunkHash,
|
ChunkHash: chunkHash,
|
||||||
Size: 1024,
|
Size: 1024,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||||
@@ -201,7 +189,6 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get blob1 chunks: %v", err)
|
t.Fatalf("failed to get blob1 chunks: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(chunks) != 2 {
|
if len(chunks) != 2 {
|
||||||
t.Errorf("expected 2 chunks for blob1, got %d", len(chunks))
|
t.Errorf("expected 2 chunks for blob1, got %d", len(chunks))
|
||||||
}
|
}
|
||||||
@@ -211,7 +198,6 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get blob2 chunks: %v", err)
|
t.Fatalf("failed to get blob2 chunks: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(chunks) != 2 {
|
if len(chunks) != 2 {
|
||||||
t.Errorf("expected 2 chunks for blob2, got %d", len(chunks))
|
t.Errorf("expected 2 chunks for blob2, got %d", len(chunks))
|
||||||
}
|
}
|
||||||
@@ -221,7 +207,6 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get shared chunk: %v", err)
|
t.Fatalf("failed to get shared chunk: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if bc == nil {
|
if bc == nil {
|
||||||
t.Fatal("expected shared chunk, got nil")
|
t.Fatal("expected shared chunk, got nil")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,11 +3,10 @@ package database
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
type BlobRepository struct {
|
type BlobRepository struct {
|
||||||
@@ -25,12 +24,10 @@ func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) err
|
|||||||
`
|
`
|
||||||
|
|
||||||
var finishedTS, uploadedTS *int64
|
var finishedTS, uploadedTS *int64
|
||||||
|
|
||||||
if blob.FinishedTS != nil {
|
if blob.FinishedTS != nil {
|
||||||
ts := blob.FinishedTS.Unix()
|
ts := blob.FinishedTS.Unix()
|
||||||
finishedTS = &ts
|
finishedTS = &ts
|
||||||
}
|
}
|
||||||
|
|
||||||
if blob.UploadedTS != nil {
|
if blob.UploadedTS != nil {
|
||||||
ts := blob.UploadedTS.Unix()
|
ts := blob.UploadedTS.Unix()
|
||||||
uploadedTS = &ts
|
uploadedTS = &ts
|
||||||
@@ -59,11 +56,9 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err
|
|||||||
WHERE blob_hash = ?
|
WHERE blob_hash = ?
|
||||||
`
|
`
|
||||||
|
|
||||||
var (
|
var blob Blob
|
||||||
blob Blob
|
var createdTSUnix int64
|
||||||
createdTSUnix int64
|
var finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||||
finishedTSUnix, uploadedTSUnix sql.NullInt64
|
|
||||||
)
|
|
||||||
|
|
||||||
err := r.db.conn.QueryRowContext(ctx, query, hash).Scan(
|
err := r.db.conn.QueryRowContext(ctx, query, hash).Scan(
|
||||||
&blob.ID,
|
&blob.ID,
|
||||||
@@ -75,10 +70,9 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err
|
|||||||
&uploadedTSUnix,
|
&uploadedTSUnix,
|
||||||
)
|
)
|
||||||
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if err == sql.ErrNoRows {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("querying blob: %w", err)
|
return nil, fmt.Errorf("querying blob: %w", err)
|
||||||
}
|
}
|
||||||
@@ -88,12 +82,10 @@ func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, err
|
|||||||
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
|
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
|
||||||
blob.FinishedTS = &ts
|
blob.FinishedTS = &ts
|
||||||
}
|
}
|
||||||
|
|
||||||
if uploadedTSUnix.Valid {
|
if uploadedTSUnix.Valid {
|
||||||
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
|
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
|
||||||
blob.UploadedTS = &ts
|
blob.UploadedTS = &ts
|
||||||
}
|
}
|
||||||
|
|
||||||
return &blob, nil
|
return &blob, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -105,11 +97,9 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
|
|||||||
WHERE id = ?
|
WHERE id = ?
|
||||||
`
|
`
|
||||||
|
|
||||||
var (
|
var blob Blob
|
||||||
blob Blob
|
var createdTSUnix int64
|
||||||
createdTSUnix int64
|
var finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||||
finishedTSUnix, uploadedTSUnix sql.NullInt64
|
|
||||||
)
|
|
||||||
|
|
||||||
err := r.db.conn.QueryRowContext(ctx, query, id).Scan(
|
err := r.db.conn.QueryRowContext(ctx, query, id).Scan(
|
||||||
&blob.ID,
|
&blob.ID,
|
||||||
@@ -121,10 +111,9 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
|
|||||||
&uploadedTSUnix,
|
&uploadedTSUnix,
|
||||||
)
|
)
|
||||||
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if err == sql.ErrNoRows {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("querying blob: %w", err)
|
return nil, fmt.Errorf("querying blob: %w", err)
|
||||||
}
|
}
|
||||||
@@ -134,69 +123,13 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
|
|||||||
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
|
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
|
||||||
blob.FinishedTS = &ts
|
blob.FinishedTS = &ts
|
||||||
}
|
}
|
||||||
|
|
||||||
if uploadedTSUnix.Valid {
|
if uploadedTSUnix.Valid {
|
||||||
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
|
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
|
||||||
blob.UploadedTS = &ts
|
blob.UploadedTS = &ts
|
||||||
}
|
}
|
||||||
|
|
||||||
return &blob, nil
|
return &blob, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetAll returns every blob row keyed by blob ID. Useful at restore
|
|
||||||
// start to translate the per-chunk blob_id references in chunkToBlobMap
|
|
||||||
// into blob hashes without doing one GetByID query per chunk.
|
|
||||||
func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
|
|
||||||
query := `
|
|
||||||
SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts
|
|
||||||
FROM blobs
|
|
||||||
`
|
|
||||||
|
|
||||||
rows, err := r.db.conn.QueryContext(ctx, query)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("querying blobs: %w", err)
|
|
||||||
}
|
|
||||||
defer CloseRows(rows)
|
|
||||||
|
|
||||||
out := make(map[string]*Blob)
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
var (
|
|
||||||
blob Blob
|
|
||||||
createdTSUnix int64
|
|
||||||
finishedTSUnix, uploadedTSUnix sql.NullInt64
|
|
||||||
)
|
|
||||||
|
|
||||||
err := rows.Scan(
|
|
||||||
&blob.ID,
|
|
||||||
&blob.Hash,
|
|
||||||
&createdTSUnix,
|
|
||||||
&finishedTSUnix,
|
|
||||||
&blob.UncompressedSize,
|
|
||||||
&blob.CompressedSize,
|
|
||||||
&uploadedTSUnix,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("scanning blob: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
|
|
||||||
if finishedTSUnix.Valid {
|
|
||||||
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
|
|
||||||
blob.FinishedTS = &ts
|
|
||||||
}
|
|
||||||
|
|
||||||
if uploadedTSUnix.Valid {
|
|
||||||
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
|
|
||||||
blob.UploadedTS = &ts
|
|
||||||
}
|
|
||||||
|
|
||||||
out[blob.ID.String()] = &blob
|
|
||||||
}
|
|
||||||
|
|
||||||
return out, rows.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpdateFinished updates a blob when it's finalized
|
// UpdateFinished updates a blob when it's finalized
|
||||||
func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id string, hash string, uncompressedSize, compressedSize int64) error {
|
func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id string, hash string, uncompressedSize, compressedSize int64) error {
|
||||||
query := `
|
query := `
|
||||||
@@ -206,7 +139,6 @@ func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id stri
|
|||||||
`
|
`
|
||||||
|
|
||||||
now := time.Now().UTC().Unix()
|
now := time.Now().UTC().Unix()
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
if tx != nil {
|
if tx != nil {
|
||||||
_, err = tx.ExecContext(ctx, query, hash, now, uncompressedSize, compressedSize, id)
|
_, err = tx.ExecContext(ctx, query, hash, now, uncompressedSize, compressedSize, id)
|
||||||
@@ -230,7 +162,6 @@ func (r *BlobRepository) UpdateUploaded(ctx context.Context, tx *sql.Tx, id stri
|
|||||||
`
|
`
|
||||||
|
|
||||||
now := time.Now().UTC().Unix()
|
now := time.Now().UTC().Unix()
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
if tx != nil {
|
if tx != nil {
|
||||||
_, err = tx.ExecContext(ctx, query, now, id)
|
_, err = tx.ExecContext(ctx, query, now, id)
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBlobRepository(t *testing.T) {
|
func TestBlobRepository(t *testing.T) {
|
||||||
@@ -32,15 +32,12 @@ func TestBlobRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get blob: %v", err)
|
t.Fatalf("failed to get blob: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved == nil {
|
if retrieved == nil {
|
||||||
t.Fatal("expected blob, got nil")
|
t.Fatal("expected blob, got nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.Hash != blob.Hash {
|
if retrieved.Hash != blob.Hash {
|
||||||
t.Errorf("blob hash mismatch: got %s, want %s", retrieved.Hash, blob.Hash)
|
t.Errorf("blob hash mismatch: got %s, want %s", retrieved.Hash, blob.Hash)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !retrieved.CreatedTS.Equal(blob.CreatedTS) {
|
if !retrieved.CreatedTS.Equal(blob.CreatedTS) {
|
||||||
t.Errorf("created timestamp mismatch: got %v, want %v", retrieved.CreatedTS, blob.CreatedTS)
|
t.Errorf("created timestamp mismatch: got %v, want %v", retrieved.CreatedTS, blob.CreatedTS)
|
||||||
}
|
}
|
||||||
@@ -50,11 +47,9 @@ func TestBlobRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get blob by ID: %v", err)
|
t.Fatalf("failed to get blob by ID: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrievedByID == nil {
|
if retrievedByID == nil {
|
||||||
t.Fatal("expected blob, got nil")
|
t.Fatal("expected blob, got nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrievedByID.ID != blob.ID {
|
if retrievedByID.ID != blob.ID {
|
||||||
t.Errorf("blob ID mismatch: got %s, want %s", retrievedByID.ID, blob.ID)
|
t.Errorf("blob ID mismatch: got %s, want %s", retrievedByID.ID, blob.ID)
|
||||||
}
|
}
|
||||||
@@ -65,7 +60,6 @@ func TestBlobRepository(t *testing.T) {
|
|||||||
Hash: types.BlobHash("blobhash456"),
|
Hash: types.BlobHash("blobhash456"),
|
||||||
CreatedTS: time.Now().Truncate(time.Second),
|
CreatedTS: time.Now().Truncate(time.Second),
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repo.Create(ctx, nil, blob2)
|
err = repo.Create(ctx, nil, blob2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create second blob: %v", err)
|
t.Fatalf("failed to create second blob: %v", err)
|
||||||
@@ -73,7 +67,6 @@ func TestBlobRepository(t *testing.T) {
|
|||||||
|
|
||||||
// Test UpdateFinished
|
// Test UpdateFinished
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
|
|
||||||
err = repo.UpdateFinished(ctx, nil, blob.ID.String(), blob.Hash.String(), 1000, 500)
|
err = repo.UpdateFinished(ctx, nil, blob.ID.String(), blob.Hash.String(), 1000, 500)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to update blob as finished: %v", err)
|
t.Fatalf("failed to update blob as finished: %v", err)
|
||||||
@@ -84,15 +77,12 @@ func TestBlobRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get updated blob: %v", err)
|
t.Fatalf("failed to get updated blob: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if updated.FinishedTS == nil {
|
if updated.FinishedTS == nil {
|
||||||
t.Fatal("expected finished timestamp to be set")
|
t.Fatal("expected finished timestamp to be set")
|
||||||
}
|
}
|
||||||
|
|
||||||
if updated.UncompressedSize != 1000 {
|
if updated.UncompressedSize != 1000 {
|
||||||
t.Errorf("expected uncompressed size 1000, got %d", updated.UncompressedSize)
|
t.Errorf("expected uncompressed size 1000, got %d", updated.UncompressedSize)
|
||||||
}
|
}
|
||||||
|
|
||||||
if updated.CompressedSize != 500 {
|
if updated.CompressedSize != 500 {
|
||||||
t.Errorf("expected compressed size 500, got %d", updated.CompressedSize)
|
t.Errorf("expected compressed size 500, got %d", updated.CompressedSize)
|
||||||
}
|
}
|
||||||
@@ -108,7 +98,6 @@ func TestBlobRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get uploaded blob: %v", err)
|
t.Fatalf("failed to get uploaded blob: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if uploaded.UploadedTS == nil {
|
if uploaded.UploadedTS == nil {
|
||||||
t.Fatal("expected uploaded timestamp to be set")
|
t.Fatal("expected uploaded timestamp to be set")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestCascadeDeleteDebug tests cascade delete with debug output
|
// TestCascadeDeleteDebug tests cascade delete with debug output
|
||||||
@@ -19,12 +19,10 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
|||||||
|
|
||||||
// Check if foreign keys are enabled
|
// Check if foreign keys are enabled
|
||||||
var fkEnabled int
|
var fkEnabled int
|
||||||
|
|
||||||
err := db.conn.QueryRow("PRAGMA foreign_keys").Scan(&fkEnabled)
|
err := db.conn.QueryRow("PRAGMA foreign_keys").Scan(&fkEnabled)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("Foreign keys enabled: %d", fkEnabled)
|
t.Logf("Foreign keys enabled: %d", fkEnabled)
|
||||||
|
|
||||||
// Create a file
|
// Create a file
|
||||||
@@ -36,21 +34,18 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
|||||||
UID: 1000,
|
UID: 1000,
|
||||||
GID: 1000,
|
GID: 1000,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Files.Create(ctx, nil, file)
|
err = repos.Files.Create(ctx, nil, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file: %v", err)
|
t.Fatalf("failed to create file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("Created file with ID: %s", file.ID)
|
t.Logf("Created file with ID: %s", file.ID)
|
||||||
|
|
||||||
// Create chunks and file-chunk mappings
|
// Create chunks and file-chunk mappings
|
||||||
for i := range 3 {
|
for i := 0; i < 3; i++ {
|
||||||
chunk := &Chunk{
|
chunk := &Chunk{
|
||||||
ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)),
|
ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)),
|
||||||
Size: 1024,
|
Size: 1024,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create chunk: %v", err)
|
t.Fatalf("failed to create chunk: %v", err)
|
||||||
@@ -61,12 +56,10 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
|||||||
Idx: i,
|
Idx: i,
|
||||||
ChunkHash: chunk.ChunkHash,
|
ChunkHash: chunk.ChunkHash,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.FileChunks.Create(ctx, nil, fc)
|
err = repos.FileChunks.Create(ctx, nil, fc)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file chunk: %v", err)
|
t.Fatalf("failed to create file chunk: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash)
|
t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -75,12 +68,10 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("File chunks before delete: %d", len(fileChunks))
|
t.Logf("File chunks before delete: %d", len(fileChunks))
|
||||||
|
|
||||||
// Check the foreign key constraint
|
// Check the foreign key constraint
|
||||||
var fkInfo string
|
var fkInfo string
|
||||||
|
|
||||||
err = db.conn.QueryRow(`
|
err = db.conn.QueryRow(`
|
||||||
SELECT sql FROM sqlite_master
|
SELECT sql FROM sqlite_master
|
||||||
WHERE type='table' AND name='file_chunks'
|
WHERE type='table' AND name='file_chunks'
|
||||||
@@ -88,12 +79,10 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("file_chunks table definition:\n%s", fkInfo)
|
t.Logf("file_chunks table definition:\n%s", fkInfo)
|
||||||
|
|
||||||
// Delete the file
|
// Delete the file
|
||||||
t.Log("Deleting file...")
|
t.Log("Deleting file...")
|
||||||
|
|
||||||
err = repos.Files.DeleteByID(ctx, nil, file.ID)
|
err = repos.Files.DeleteByID(ctx, nil, file.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to delete file: %v", err)
|
t.Fatalf("failed to delete file: %v", err)
|
||||||
@@ -104,7 +93,6 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if deletedFile != nil {
|
if deletedFile != nil {
|
||||||
t.Error("file should have been deleted")
|
t.Error("file should have been deleted")
|
||||||
} else {
|
} else {
|
||||||
@@ -116,17 +104,14 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("File chunks after delete: %d", len(fileChunks))
|
t.Logf("File chunks after delete: %d", len(fileChunks))
|
||||||
|
|
||||||
// Manually check the database
|
// Manually check the database
|
||||||
var count int
|
var count int
|
||||||
|
|
||||||
err = db.conn.QueryRow("SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID).Scan(&count)
|
err = db.conn.QueryRow("SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID).Scan(&count)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("Manual count of file_chunks for deleted file: %d", count)
|
t.Logf("Manual count of file_chunks for deleted file: %d", count)
|
||||||
|
|
||||||
if len(fileChunks) != 0 {
|
if len(fileChunks) != 0 {
|
||||||
|
|||||||
@@ -4,9 +4,8 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ChunkFileRepository struct {
|
type ChunkFileRepository struct {
|
||||||
@@ -91,25 +90,18 @@ func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.File
|
|||||||
// scanChunkFiles is a helper that scans chunk file rows
|
// scanChunkFiles is a helper that scans chunk file rows
|
||||||
func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) {
|
func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) {
|
||||||
var chunkFiles []*ChunkFile
|
var chunkFiles []*ChunkFile
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var (
|
var cf ChunkFile
|
||||||
cf ChunkFile
|
var chunkHashStr, fileIDStr string
|
||||||
chunkHashStr, fileIDStr string
|
|
||||||
)
|
|
||||||
|
|
||||||
err := rows.Scan(&chunkHashStr, &fileIDStr, &cf.FileOffset, &cf.Length)
|
err := rows.Scan(&chunkHashStr, &fileIDStr, &cf.FileOffset, &cf.Length)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("scanning chunk file: %w", err)
|
return nil, fmt.Errorf("scanning chunk file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
cf.ChunkHash = types.ChunkHash(chunkHashStr)
|
cf.ChunkHash = types.ChunkHash(chunkHashStr)
|
||||||
|
|
||||||
cf.FileID, err = types.ParseFileID(fileIDStr)
|
cf.FileID, err = types.ParseFileID(fileIDStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
chunkFiles = append(chunkFiles, &cf)
|
chunkFiles = append(chunkFiles, &cf)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -144,13 +136,14 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
|||||||
const batchSize = 500
|
const batchSize = 500
|
||||||
|
|
||||||
for i := 0; i < len(fileIDs); i += batchSize {
|
for i := 0; i < len(fileIDs); i += batchSize {
|
||||||
end := min(i+batchSize, len(fileIDs))
|
end := i + batchSize
|
||||||
|
if end > len(fileIDs) {
|
||||||
|
end = len(fileIDs)
|
||||||
|
}
|
||||||
batch := fileIDs[i:end]
|
batch := fileIDs[i:end]
|
||||||
|
|
||||||
query := "DELETE FROM chunk_files WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
|
query := "DELETE FROM chunk_files WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
|
||||||
|
args := make([]interface{}, len(batch))
|
||||||
args := make([]any, len(batch))
|
|
||||||
for j, id := range batch {
|
for j, id := range batch {
|
||||||
args[j] = id.String()
|
args[j] = id.String()
|
||||||
}
|
}
|
||||||
@@ -161,7 +154,6 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
|||||||
} else {
|
} else {
|
||||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("batch deleting chunk_files: %w", err)
|
return fmt.Errorf("batch deleting chunk_files: %w", err)
|
||||||
}
|
}
|
||||||
@@ -180,28 +172,21 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [
|
|||||||
const batchSize = 200
|
const batchSize = 200
|
||||||
|
|
||||||
for i := 0; i < len(cfs); i += batchSize {
|
for i := 0; i < len(cfs); i += batchSize {
|
||||||
end := min(i+batchSize, len(cfs))
|
end := i + batchSize
|
||||||
|
if end > len(cfs) {
|
||||||
|
end = len(cfs)
|
||||||
|
}
|
||||||
batch := cfs[i:end]
|
batch := cfs[i:end]
|
||||||
|
|
||||||
query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES "
|
query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES "
|
||||||
|
args := make([]interface{}, 0, len(batch)*4)
|
||||||
args := make([]any, 0, len(batch)*4)
|
|
||||||
|
|
||||||
var querySb183 strings.Builder
|
|
||||||
|
|
||||||
for j, cf := range batch {
|
for j, cf := range batch {
|
||||||
if j > 0 {
|
if j > 0 {
|
||||||
querySb183.WriteString(", ")
|
query += ", "
|
||||||
}
|
}
|
||||||
|
query += "(?, ?, ?, ?)"
|
||||||
querySb183.WriteString("(?, ?, ?, ?)")
|
|
||||||
|
|
||||||
args = append(args, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
|
args = append(args, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
|
||||||
}
|
}
|
||||||
|
|
||||||
query += querySb183.String()
|
|
||||||
|
|
||||||
query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING"
|
query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING"
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
@@ -210,7 +195,6 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [
|
|||||||
} else {
|
} else {
|
||||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("batch inserting chunk_files: %w", err)
|
return fmt.Errorf("batch inserting chunk_files: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestChunkFileRepository(t *testing.T) {
|
func TestChunkFileRepository(t *testing.T) {
|
||||||
@@ -28,7 +28,6 @@ func TestChunkFileRepository(t *testing.T) {
|
|||||||
GID: 1000,
|
GID: 1000,
|
||||||
LinkTarget: "",
|
LinkTarget: "",
|
||||||
}
|
}
|
||||||
|
|
||||||
err := fileRepo.Create(ctx, nil, file1)
|
err := fileRepo.Create(ctx, nil, file1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file1: %v", err)
|
t.Fatalf("failed to create file1: %v", err)
|
||||||
@@ -43,7 +42,6 @@ func TestChunkFileRepository(t *testing.T) {
|
|||||||
GID: 1000,
|
GID: 1000,
|
||||||
LinkTarget: "",
|
LinkTarget: "",
|
||||||
}
|
}
|
||||||
|
|
||||||
err = fileRepo.Create(ctx, nil, file2)
|
err = fileRepo.Create(ctx, nil, file2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file2: %v", err)
|
t.Fatalf("failed to create file2: %v", err)
|
||||||
@@ -54,7 +52,6 @@ func TestChunkFileRepository(t *testing.T) {
|
|||||||
ChunkHash: types.ChunkHash("chunk1"),
|
ChunkHash: types.ChunkHash("chunk1"),
|
||||||
Size: 1024,
|
Size: 1024,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = chunksRepo.Create(ctx, nil, chunk)
|
err = chunksRepo.Create(ctx, nil, chunk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create chunk: %v", err)
|
t.Fatalf("failed to create chunk: %v", err)
|
||||||
@@ -80,7 +77,6 @@ func TestChunkFileRepository(t *testing.T) {
|
|||||||
FileOffset: 2048,
|
FileOffset: 2048,
|
||||||
Length: 1024,
|
Length: 1024,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repo.Create(ctx, nil, cf2)
|
err = repo.Create(ctx, nil, cf2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create second chunk file: %v", err)
|
t.Fatalf("failed to create second chunk file: %v", err)
|
||||||
@@ -91,7 +87,6 @@ func TestChunkFileRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get chunk files: %v", err)
|
t.Fatalf("failed to get chunk files: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(chunkFiles) != 2 {
|
if len(chunkFiles) != 2 {
|
||||||
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
|
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
|
||||||
}
|
}
|
||||||
@@ -99,17 +94,14 @@ func TestChunkFileRepository(t *testing.T) {
|
|||||||
// Verify both files are returned
|
// Verify both files are returned
|
||||||
foundFile1 := false
|
foundFile1 := false
|
||||||
foundFile2 := false
|
foundFile2 := false
|
||||||
|
|
||||||
for _, cf := range chunkFiles {
|
for _, cf := range chunkFiles {
|
||||||
if cf.FileID == file1.ID && cf.FileOffset == 0 {
|
if cf.FileID == file1.ID && cf.FileOffset == 0 {
|
||||||
foundFile1 = true
|
foundFile1 = true
|
||||||
}
|
}
|
||||||
|
|
||||||
if cf.FileID == file2.ID && cf.FileOffset == 2048 {
|
if cf.FileID == file2.ID && cf.FileOffset == 2048 {
|
||||||
foundFile2 = true
|
foundFile2 = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !foundFile1 || !foundFile2 {
|
if !foundFile1 || !foundFile2 {
|
||||||
t.Error("not all expected files found")
|
t.Error("not all expected files found")
|
||||||
}
|
}
|
||||||
@@ -119,11 +111,9 @@ func TestChunkFileRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get chunks by file ID: %v", err)
|
t.Fatalf("failed to get chunks by file ID: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(chunkFiles) != 1 {
|
if len(chunkFiles) != 1 {
|
||||||
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
|
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
|
||||||
}
|
}
|
||||||
|
|
||||||
if chunkFiles[0].ChunkHash != types.ChunkHash("chunk1") {
|
if chunkFiles[0].ChunkHash != types.ChunkHash("chunk1") {
|
||||||
t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash)
|
t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash)
|
||||||
}
|
}
|
||||||
@@ -150,18 +140,13 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
|||||||
file2 := &File{Path: "/file2.txt", MTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000}
|
file2 := &File{Path: "/file2.txt", MTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000}
|
||||||
file3 := &File{Path: "/file3.txt", MTime: testTime, Size: 2048, Mode: 0644, UID: 1000, GID: 1000}
|
file3 := &File{Path: "/file3.txt", MTime: testTime, Size: 2048, Mode: 0644, UID: 1000, GID: 1000}
|
||||||
|
|
||||||
err := fileRepo.Create(ctx, nil, file1)
|
if err := fileRepo.Create(ctx, nil, file1); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create file1: %v", err)
|
t.Fatalf("failed to create file1: %v", err)
|
||||||
}
|
}
|
||||||
|
if err := fileRepo.Create(ctx, nil, file2); err != nil {
|
||||||
err = fileRepo.Create(ctx, nil, file2)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create file2: %v", err)
|
t.Fatalf("failed to create file2: %v", err)
|
||||||
}
|
}
|
||||||
|
if err := fileRepo.Create(ctx, nil, file3); err != nil {
|
||||||
err = fileRepo.Create(ctx, nil, file3)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create file3: %v", err)
|
t.Fatalf("failed to create file3: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -172,7 +157,6 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
|||||||
ChunkHash: chunkHash,
|
ChunkHash: chunkHash,
|
||||||
Size: 1024,
|
Size: 1024,
|
||||||
}
|
}
|
||||||
|
|
||||||
err := chunksRepo.Create(ctx, nil, chunk)
|
err := chunksRepo.Create(ctx, nil, chunk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||||
@@ -210,7 +194,6 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get files for chunk1: %v", err)
|
t.Fatalf("failed to get files for chunk1: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(files) != 2 {
|
if len(files) != 2 {
|
||||||
t.Errorf("expected 2 files for chunk1, got %d", len(files))
|
t.Errorf("expected 2 files for chunk1, got %d", len(files))
|
||||||
}
|
}
|
||||||
@@ -220,7 +203,6 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get files for chunk2: %v", err)
|
t.Fatalf("failed to get files for chunk2: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(files) != 2 {
|
if len(files) != 2 {
|
||||||
t.Errorf("expected 2 files for chunk2, got %d", len(files))
|
t.Errorf("expected 2 files for chunk2, got %d", len(files))
|
||||||
}
|
}
|
||||||
@@ -230,7 +212,6 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get chunks for file2: %v", err)
|
t.Fatalf("failed to get chunks for file2: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(file2Chunks) != 3 {
|
if len(file2Chunks) != 3 {
|
||||||
t.Errorf("expected 3 chunks for file2, got %d", len(file2Chunks))
|
t.Errorf("expected 3 chunks for file2, got %d", len(file2Chunks))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,11 +3,9 @@ package database
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ChunkRepository struct {
|
type ChunkRepository struct {
|
||||||
@@ -53,10 +51,9 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e
|
|||||||
&chunk.Size,
|
&chunk.Size,
|
||||||
)
|
)
|
||||||
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if err == sql.ErrNoRows {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("querying chunk: %w", err)
|
return nil, fmt.Errorf("querying chunk: %w", err)
|
||||||
}
|
}
|
||||||
@@ -74,22 +71,14 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
|
|||||||
FROM chunks
|
FROM chunks
|
||||||
WHERE chunk_hash IN (`
|
WHERE chunk_hash IN (`
|
||||||
|
|
||||||
args := make([]any, len(hashes))
|
args := make([]interface{}, len(hashes))
|
||||||
|
|
||||||
var querySb75 strings.Builder
|
|
||||||
|
|
||||||
for i, hash := range hashes {
|
for i, hash := range hashes {
|
||||||
if i > 0 {
|
if i > 0 {
|
||||||
querySb75.WriteString(", ")
|
query += ", "
|
||||||
}
|
}
|
||||||
|
query += "?"
|
||||||
querySb75.WriteString("?")
|
|
||||||
|
|
||||||
args[i] = hash
|
args[i] = hash
|
||||||
}
|
}
|
||||||
|
|
||||||
query += querySb75.String()
|
|
||||||
|
|
||||||
query += ") ORDER BY chunk_hash"
|
query += ") ORDER BY chunk_hash"
|
||||||
|
|
||||||
rows, err := r.db.conn.QueryContext(ctx, query, args...)
|
rows, err := r.db.conn.QueryContext(ctx, query, args...)
|
||||||
@@ -99,7 +88,6 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
|
|||||||
defer CloseRows(rows)
|
defer CloseRows(rows)
|
||||||
|
|
||||||
var chunks []*Chunk
|
var chunks []*Chunk
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var chunk Chunk
|
var chunk Chunk
|
||||||
|
|
||||||
@@ -134,7 +122,6 @@ func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk
|
|||||||
defer CloseRows(rows)
|
defer CloseRows(rows)
|
||||||
|
|
||||||
var chunks []*Chunk
|
var chunks []*Chunk
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var chunk Chunk
|
var chunk Chunk
|
||||||
|
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) {
|
|||||||
defer CloseRows(rows)
|
defer CloseRows(rows)
|
||||||
|
|
||||||
var chunks []*Chunk
|
var chunks []*Chunk
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var chunk Chunk
|
var chunk Chunk
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestChunkRepository(t *testing.T) {
|
func TestChunkRepository(t *testing.T) {
|
||||||
@@ -30,15 +30,12 @@ func TestChunkRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get chunk: %v", err)
|
t.Fatalf("failed to get chunk: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved == nil {
|
if retrieved == nil {
|
||||||
t.Fatal("expected chunk, got nil")
|
t.Fatal("expected chunk, got nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.ChunkHash != chunk.ChunkHash {
|
if retrieved.ChunkHash != chunk.ChunkHash {
|
||||||
t.Errorf("chunk hash mismatch: got %s, want %s", retrieved.ChunkHash, chunk.ChunkHash)
|
t.Errorf("chunk hash mismatch: got %s, want %s", retrieved.ChunkHash, chunk.ChunkHash)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.Size != chunk.Size {
|
if retrieved.Size != chunk.Size {
|
||||||
t.Errorf("size mismatch: got %d, want %d", retrieved.Size, chunk.Size)
|
t.Errorf("size mismatch: got %d, want %d", retrieved.Size, chunk.Size)
|
||||||
}
|
}
|
||||||
@@ -54,7 +51,6 @@ func TestChunkRepository(t *testing.T) {
|
|||||||
ChunkHash: types.ChunkHash("chunkhash456"),
|
ChunkHash: types.ChunkHash("chunkhash456"),
|
||||||
Size: 8192,
|
Size: 8192,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repo.Create(ctx, nil, chunk2)
|
err = repo.Create(ctx, nil, chunk2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create second chunk: %v", err)
|
t.Fatalf("failed to create second chunk: %v", err)
|
||||||
@@ -64,7 +60,6 @@ func TestChunkRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get chunks by hashes: %v", err)
|
t.Fatalf("failed to get chunks by hashes: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(chunks) != 2 {
|
if len(chunks) != 2 {
|
||||||
t.Errorf("expected 2 chunks, got %d", len(chunks))
|
t.Errorf("expected 2 chunks, got %d", len(chunks))
|
||||||
}
|
}
|
||||||
@@ -74,7 +69,6 @@ func TestChunkRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to list unpacked chunks: %v", err)
|
t.Fatalf("failed to list unpacked chunks: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(unpacked) != 2 {
|
if len(unpacked) != 2 {
|
||||||
t.Errorf("expected 2 unpacked chunks, got %d", len(unpacked))
|
t.Errorf("expected 2 unpacked chunks, got %d", len(unpacked))
|
||||||
}
|
}
|
||||||
@@ -92,7 +86,6 @@ func TestChunkRepositoryNotFound(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if chunk != nil {
|
if chunk != nil {
|
||||||
t.Error("expected nil for non-existent chunk")
|
t.Error("expected nil for non-existent chunk")
|
||||||
}
|
}
|
||||||
@@ -102,7 +95,6 @@ func TestChunkRepositoryNotFound(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if chunks != nil {
|
if chunks != nil {
|
||||||
t.Error("expected nil for empty hash list")
|
t.Error("expected nil for empty hash list")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,32 +6,24 @@
|
|||||||
// multiple source files. Blobs are content-addressed, meaning their filename
|
// multiple source files. Blobs are content-addressed, meaning their filename
|
||||||
// is derived from their SHA256 hash after compression and encryption.
|
// is derived from their SHA256 hash after compression and encryption.
|
||||||
//
|
//
|
||||||
// Schema is managed via numbered SQL migrations embedded in the schema/
|
// The database does not support migrations. If the schema changes, delete
|
||||||
// directory. Migration 000.sql bootstraps the schema_migrations tracking
|
// the local database and perform a full backup to recreate it.
|
||||||
// table; subsequent migrations (001, 002, …) are applied in order.
|
|
||||||
package database
|
package database
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"embed"
|
_ "embed"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
_ "modernc.org/sqlite"
|
_ "modernc.org/sqlite"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
//go:embed schema/*.sql
|
//go:embed schema.sql
|
||||||
var schemaFS embed.FS
|
var schemaSQL string
|
||||||
|
|
||||||
// bootstrapVersion is the migration that creates the schema_migrations
|
|
||||||
// table itself. It is applied before the normal migration loop.
|
|
||||||
const bootstrapVersion = 0
|
|
||||||
|
|
||||||
// DB represents the Vaultik local index database connection.
|
// DB represents the Vaultik local index database connection.
|
||||||
// It uses SQLite to track file metadata, content-defined chunks, and blob associations.
|
// It uses SQLite to track file metadata, content-defined chunks, and blob associations.
|
||||||
@@ -43,46 +35,6 @@ type DB struct {
|
|||||||
path string
|
path string
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseMigrationVersion extracts the numeric version prefix from a migration
|
|
||||||
// filename. Filenames must follow the pattern "<version>.sql" or
|
|
||||||
// "<version>_<description>.sql", where version is a zero-padded numeric
|
|
||||||
// string (e.g. "001", "002"). Returns the version as an integer and an
|
|
||||||
// error if the filename does not match the expected pattern.
|
|
||||||
func ParseMigrationVersion(filename string) (int, error) {
|
|
||||||
name := strings.TrimSuffix(filename, filepath.Ext(filename))
|
|
||||||
if name == "" {
|
|
||||||
return 0, fmt.Errorf("invalid migration filename %q: empty name", filename)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Split on underscore to separate version from description.
|
|
||||||
// If there's no underscore, the entire stem is the version.
|
|
||||||
versionStr := name
|
|
||||||
if before, _, ok := strings.Cut(name, "_"); ok {
|
|
||||||
versionStr = before
|
|
||||||
}
|
|
||||||
|
|
||||||
if versionStr == "" {
|
|
||||||
return 0, fmt.Errorf("invalid migration filename %q: empty version prefix", filename)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate the version is purely numeric.
|
|
||||||
for _, ch := range versionStr {
|
|
||||||
if ch < '0' || ch > '9' {
|
|
||||||
return 0, fmt.Errorf(
|
|
||||||
"invalid migration filename %q: version %q contains non-numeric character %q",
|
|
||||||
filename, versionStr, string(ch),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
version, err := strconv.Atoi(versionStr)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("invalid migration filename %q: %w", filename, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return version, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// New creates a new database connection at the specified path.
|
// New creates a new database connection at the specified path.
|
||||||
// It creates the schema if needed and configures SQLite with WAL mode for
|
// It creates the schema if needed and configures SQLite with WAL mode for
|
||||||
// better concurrency. SQLite handles crash recovery automatically when
|
// better concurrency. SQLite handles crash recovery automatically when
|
||||||
@@ -98,7 +50,6 @@ func New(ctx context.Context, path string) (*DB, error) {
|
|||||||
|
|
||||||
// First attempt with standard WAL mode
|
// First attempt with standard WAL mode
|
||||||
log.Debug("Attempting to open database with WAL mode", "path", path)
|
log.Debug("Attempting to open database with WAL mode", "path", path)
|
||||||
|
|
||||||
conn, err := sql.Open(
|
conn, err := sql.Open(
|
||||||
"sqlite",
|
"sqlite",
|
||||||
path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000&_locking_mode=NORMAL&_foreign_keys=ON",
|
path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000&_locking_mode=NORMAL&_foreign_keys=ON",
|
||||||
@@ -111,31 +62,23 @@ func New(ctx context.Context, path string) (*DB, error) {
|
|||||||
conn.SetMaxOpenConns(1)
|
conn.SetMaxOpenConns(1)
|
||||||
conn.SetMaxIdleConns(1)
|
conn.SetMaxIdleConns(1)
|
||||||
|
|
||||||
err := conn.PingContext(ctx)
|
if err := conn.PingContext(ctx); err == nil {
|
||||||
if err == nil {
|
|
||||||
// Success on first try
|
// Success on first try
|
||||||
log.Debug("Database opened successfully with WAL mode", "path", path)
|
log.Debug("Database opened successfully with WAL mode", "path", path)
|
||||||
|
|
||||||
// Enable foreign keys explicitly
|
// Enable foreign keys explicitly
|
||||||
_, err = conn.ExecContext(ctx, "PRAGMA foreign_keys = ON")
|
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys = ON"); err != nil {
|
||||||
if err != nil {
|
|
||||||
log.Warn("Failed to enable foreign keys", "error", err)
|
log.Warn("Failed to enable foreign keys", "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
db := &DB{conn: conn, path: path}
|
db := &DB{conn: conn, path: path}
|
||||||
|
if err := db.createSchema(ctx); err != nil {
|
||||||
err := applyMigrations(ctx, conn)
|
|
||||||
if err != nil {
|
|
||||||
_ = conn.Close()
|
_ = conn.Close()
|
||||||
|
return nil, fmt.Errorf("creating schema: %w", err)
|
||||||
return nil, fmt.Errorf("applying migrations: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return db, nil
|
return db, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("Failed to ping database, closing connection", "path", path, "error", err)
|
log.Debug("Failed to ping database, closing connection", "path", path, "error", err)
|
||||||
|
|
||||||
_ = conn.Close()
|
_ = conn.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -144,7 +87,6 @@ func New(ctx context.Context, path string) (*DB, error) {
|
|||||||
"Database appears locked, attempting recovery with TRUNCATE mode",
|
"Database appears locked, attempting recovery with TRUNCATE mode",
|
||||||
"path", path,
|
"path", path,
|
||||||
)
|
)
|
||||||
|
|
||||||
conn, err = sql.Open(
|
conn, err = sql.Open(
|
||||||
"sqlite",
|
"sqlite",
|
||||||
path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000&_foreign_keys=ON",
|
path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000&_foreign_keys=ON",
|
||||||
@@ -160,12 +102,9 @@ func New(ctx context.Context, path string) (*DB, error) {
|
|||||||
conn.SetMaxOpenConns(1)
|
conn.SetMaxOpenConns(1)
|
||||||
conn.SetMaxIdleConns(1)
|
conn.SetMaxIdleConns(1)
|
||||||
|
|
||||||
err = conn.PingContext(ctx)
|
if err := conn.PingContext(ctx); err != nil {
|
||||||
if err != nil {
|
|
||||||
log.Debug("Failed to ping database in recovery mode, closing", "path", path, "error", err)
|
log.Debug("Failed to ping database in recovery mode, closing", "path", path, "error", err)
|
||||||
|
|
||||||
_ = conn.Close()
|
_ = conn.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"database still locked after recovery attempt: %w",
|
"database still locked after recovery attempt: %w",
|
||||||
err,
|
err,
|
||||||
@@ -176,29 +115,22 @@ func New(ctx context.Context, path string) (*DB, error) {
|
|||||||
|
|
||||||
// Switch back to WAL mode
|
// Switch back to WAL mode
|
||||||
log.Debug("Switching database back to WAL mode", "path", path)
|
log.Debug("Switching database back to WAL mode", "path", path)
|
||||||
|
if _, err := conn.ExecContext(ctx, "PRAGMA journal_mode=WAL"); err != nil {
|
||||||
_, err = conn.ExecContext(ctx, "PRAGMA journal_mode=WAL")
|
|
||||||
if err != nil {
|
|
||||||
log.Warn("Failed to switch back to WAL mode", "path", path, "error", err)
|
log.Warn("Failed to switch back to WAL mode", "path", path, "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure foreign keys are enabled
|
// Ensure foreign keys are enabled
|
||||||
_, err = conn.ExecContext(ctx, "PRAGMA foreign_keys=ON")
|
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=ON"); err != nil {
|
||||||
if err != nil {
|
|
||||||
log.Warn("Failed to enable foreign keys", "path", path, "error", err)
|
log.Warn("Failed to enable foreign keys", "path", path, "error", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
db := &DB{conn: conn, path: path}
|
db := &DB{conn: conn, path: path}
|
||||||
|
if err := db.createSchema(ctx); err != nil {
|
||||||
err = applyMigrations(ctx, conn)
|
|
||||||
if err != nil {
|
|
||||||
_ = conn.Close()
|
_ = conn.Close()
|
||||||
|
return nil, fmt.Errorf("creating schema: %w", err)
|
||||||
return nil, fmt.Errorf("applying migrations: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("Database connection established successfully", "path", path)
|
log.Debug("Database connection established successfully", "path", path)
|
||||||
|
|
||||||
return db, nil
|
return db, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -207,16 +139,11 @@ func New(ctx context.Context, path string) (*DB, error) {
|
|||||||
// Returns an error if the database connection cannot be closed properly.
|
// Returns an error if the database connection cannot be closed properly.
|
||||||
func (db *DB) Close() error {
|
func (db *DB) Close() error {
|
||||||
log.Debug("Closing database connection", "path", db.path)
|
log.Debug("Closing database connection", "path", db.path)
|
||||||
|
if err := db.conn.Close(); err != nil {
|
||||||
err := db.conn.Close()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to close database", "path", db.path, "error", err)
|
log.Error("Failed to close database", "path", db.path, "error", err)
|
||||||
|
|
||||||
return fmt.Errorf("failed to close database: %w", err)
|
return fmt.Errorf("failed to close database: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("Database connection closed successfully", "path", db.path)
|
log.Debug("Database connection closed successfully", "path", db.path)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -252,10 +179,9 @@ func (db *DB) BeginTx(
|
|||||||
func (db *DB) ExecWithLog(
|
func (db *DB) ExecWithLog(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
query string,
|
query string,
|
||||||
args ...any,
|
args ...interface{},
|
||||||
) (sql.Result, error) {
|
) (sql.Result, error) {
|
||||||
LogSQL("Execute", query, args...)
|
LogSQL("Execute", query, args...)
|
||||||
|
|
||||||
return db.conn.ExecContext(ctx, query, args...)
|
return db.conn.ExecContext(ctx, query, args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -266,128 +192,15 @@ func (db *DB) ExecWithLog(
|
|||||||
func (db *DB) QueryRowWithLog(
|
func (db *DB) QueryRowWithLog(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
query string,
|
query string,
|
||||||
args ...any,
|
args ...interface{},
|
||||||
) *sql.Row {
|
) *sql.Row {
|
||||||
LogSQL("QueryRow", query, args...)
|
LogSQL("QueryRow", query, args...)
|
||||||
|
|
||||||
return db.conn.QueryRowContext(ctx, query, args...)
|
return db.conn.QueryRowContext(ctx, query, args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// collectMigrations reads the embedded schema directory and returns
|
func (db *DB) createSchema(ctx context.Context) error {
|
||||||
// migration filenames sorted lexicographically.
|
_, err := db.conn.ExecContext(ctx, schemaSQL)
|
||||||
func collectMigrations() ([]string, error) {
|
|
||||||
entries, err := schemaFS.ReadDir("schema")
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read schema directory: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var migrations []string
|
|
||||||
|
|
||||||
for _, entry := range entries {
|
|
||||||
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".sql") {
|
|
||||||
migrations = append(migrations, entry.Name())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.Strings(migrations)
|
|
||||||
|
|
||||||
return migrations, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// bootstrapMigrationsTable ensures the schema_migrations table exists
|
|
||||||
// by applying 000.sql if the table is missing.
|
|
||||||
func bootstrapMigrationsTable(ctx context.Context, db *sql.DB) error {
|
|
||||||
var tableExists int
|
|
||||||
|
|
||||||
err := db.QueryRowContext(ctx,
|
|
||||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'",
|
|
||||||
).Scan(&tableExists)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to check for migrations table: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tableExists > 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
content, err := schemaFS.ReadFile("schema/000.sql")
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to read bootstrap migration 000.sql: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Info("applying bootstrap migration", "version", bootstrapVersion)
|
|
||||||
|
|
||||||
_, err = db.ExecContext(ctx, string(content))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to apply bootstrap migration: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// applyMigrations applies all pending migrations to db. It first bootstraps
|
|
||||||
// the schema_migrations table via 000.sql, then iterates through remaining
|
|
||||||
// migration files in order.
|
|
||||||
func applyMigrations(ctx context.Context, db *sql.DB) error {
|
|
||||||
err := bootstrapMigrationsTable(ctx, db)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
|
||||||
|
|
||||||
migrations, err := collectMigrations()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, migration := range migrations {
|
|
||||||
version, parseErr := ParseMigrationVersion(migration)
|
|
||||||
if parseErr != nil {
|
|
||||||
return parseErr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if already applied.
|
|
||||||
var count int
|
|
||||||
|
|
||||||
err := db.QueryRowContext(ctx,
|
|
||||||
"SELECT COUNT(*) FROM schema_migrations WHERE version = ?",
|
|
||||||
version,
|
|
||||||
).Scan(&count)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to check migration status: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if count > 0 {
|
|
||||||
log.Debug("migration already applied", "version", version)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read and apply migration.
|
|
||||||
content, readErr := schemaFS.ReadFile(filepath.Join("schema", migration))
|
|
||||||
if readErr != nil {
|
|
||||||
return fmt.Errorf("failed to read migration %s: %w", migration, readErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Info("applying migration", "version", version)
|
|
||||||
|
|
||||||
_, execErr := db.ExecContext(ctx, string(content))
|
|
||||||
if execErr != nil {
|
|
||||||
return fmt.Errorf("failed to apply migration %s: %w", migration, execErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Record migration as applied.
|
|
||||||
_, recErr := db.ExecContext(ctx,
|
|
||||||
"INSERT INTO schema_migrations (version) VALUES (?)",
|
|
||||||
version,
|
|
||||||
)
|
|
||||||
if recErr != nil {
|
|
||||||
return fmt.Errorf("failed to record migration %s: %w", migration, recErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Info("migration applied successfully", "version", version)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewTestDB creates an in-memory SQLite database for testing purposes.
|
// NewTestDB creates an in-memory SQLite database for testing purposes.
|
||||||
@@ -403,7 +216,6 @@ func repeatPlaceholder(n int) string {
|
|||||||
if n <= 0 {
|
if n <= 0 {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
return strings.Repeat(", ?", n)
|
return strings.Repeat(", ?", n)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -414,7 +226,7 @@ func repeatPlaceholder(n int) string {
|
|||||||
// The operation parameter describes the type of SQL operation (e.g., "Execute", "Query").
|
// The operation parameter describes the type of SQL operation (e.g., "Execute", "Query").
|
||||||
// The query parameter is the SQL statement being executed.
|
// The query parameter is the SQL statement being executed.
|
||||||
// The args parameter contains the query arguments that will be interpolated.
|
// The args parameter contains the query arguments that will be interpolated.
|
||||||
func LogSQL(operation, query string, args ...any) {
|
func LogSQL(operation, query string, args ...interface{}) {
|
||||||
if strings.Contains(os.Getenv("GODEBUG"), "vaultik") {
|
if strings.Contains(os.Getenv("GODEBUG"), "vaultik") {
|
||||||
log.Debug(
|
log.Debug(
|
||||||
"SQL "+operation,
|
"SQL "+operation,
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package database
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -17,8 +16,7 @@ func TestDatabase(t *testing.T) {
|
|||||||
t.Fatalf("failed to create database: %v", err)
|
t.Fatalf("failed to create database: %v", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
err := db.Close()
|
if err := db.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Errorf("failed to close database: %v", err)
|
t.Errorf("failed to close database: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -28,17 +26,15 @@ func TestDatabase(t *testing.T) {
|
|||||||
t.Fatal("database connection is nil")
|
t.Fatal("database connection is nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test schema creation (already done in New via migrations)
|
// Test schema creation (already done in New)
|
||||||
// Verify tables exist
|
// Verify tables exist
|
||||||
tables := []string{
|
tables := []string{
|
||||||
"schema_migrations",
|
|
||||||
"files", "file_chunks", "chunks", "blobs",
|
"files", "file_chunks", "chunks", "blobs",
|
||||||
"blob_chunks", "chunk_files", "snapshots",
|
"blob_chunks", "chunk_files", "snapshots",
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, table := range tables {
|
for _, table := range tables {
|
||||||
var name string
|
var name string
|
||||||
|
|
||||||
err := db.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name)
|
err := db.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("table %s does not exist: %v", table, err)
|
t.Errorf("table %s does not exist: %v", table, err)
|
||||||
@@ -65,8 +61,7 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
|
|||||||
t.Fatalf("failed to create database: %v", err)
|
t.Fatalf("failed to create database: %v", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
err := db.Close()
|
if err := db.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Errorf("failed to close database: %v", err)
|
t.Errorf("failed to close database: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -76,10 +71,9 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
|
|||||||
index int
|
index int
|
||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
results := make(chan result, 10)
|
results := make(chan result, 10)
|
||||||
|
|
||||||
for i := range 10 {
|
for i := 0; i < 10; i++ {
|
||||||
go func(i int) {
|
go func(i int) {
|
||||||
_, err := db.ExecWithLog(ctx, "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)",
|
_, err := db.ExecWithLog(ctx, "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)",
|
||||||
fmt.Sprintf("hash%d", i), i*1024)
|
fmt.Sprintf("hash%d", i), i*1024)
|
||||||
@@ -88,7 +82,7 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Wait for all goroutines and check results
|
// Wait for all goroutines and check results
|
||||||
for range 10 {
|
for i := 0; i < 10; i++ {
|
||||||
r := <-results
|
r := <-results
|
||||||
if r.err != nil {
|
if r.err != nil {
|
||||||
t.Fatalf("concurrent insert %d failed: %v", r.index, r.err)
|
t.Fatalf("concurrent insert %d failed: %v", r.index, r.err)
|
||||||
@@ -97,171 +91,11 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
|
|||||||
|
|
||||||
// Verify all inserts succeeded
|
// Verify all inserts succeeded
|
||||||
var count int
|
var count int
|
||||||
|
|
||||||
err = db.conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM chunks").Scan(&count)
|
err = db.conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM chunks").Scan(&count)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to count chunks: %v", err)
|
t.Fatalf("failed to count chunks: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if count != 10 {
|
if count != 10 {
|
||||||
t.Errorf("expected 10 chunks, got %d", count)
|
t.Errorf("expected 10 chunks, got %d", count)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseMigrationVersion(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
filename string
|
|
||||||
wantVer int
|
|
||||||
wantError bool
|
|
||||||
}{
|
|
||||||
{name: "valid 000.sql", filename: "000.sql", wantVer: 0, wantError: false},
|
|
||||||
{name: "valid 001.sql", filename: "001.sql", wantVer: 1, wantError: false},
|
|
||||||
{name: "valid 099.sql", filename: "099.sql", wantVer: 99, wantError: false},
|
|
||||||
{name: "valid with description", filename: "001_initial_schema.sql", wantVer: 1, wantError: false},
|
|
||||||
{name: "valid large version", filename: "123_big_migration.sql", wantVer: 123, wantError: false},
|
|
||||||
{name: "invalid alpha version", filename: "abc.sql", wantVer: 0, wantError: true},
|
|
||||||
{name: "invalid mixed chars", filename: "12a.sql", wantVer: 0, wantError: true},
|
|
||||||
{name: "invalid no extension", filename: "schema.sql", wantVer: 0, wantError: true},
|
|
||||||
{name: "empty string", filename: "", wantVer: 0, wantError: true},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
got, err := ParseMigrationVersion(tc.filename)
|
|
||||||
if tc.wantError {
|
|
||||||
if err == nil {
|
|
||||||
t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error", tc.filename, got)
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("ParseMigrationVersion(%q) unexpected error: %v", tc.filename, err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if got != tc.wantVer {
|
|
||||||
t.Errorf("ParseMigrationVersion(%q) = %d; want %d", tc.filename, got, tc.wantVer)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestApplyMigrations_Idempotent(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to open database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
err := conn.Close()
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("failed to close database: %v", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
conn.SetMaxOpenConns(1)
|
|
||||||
conn.SetMaxIdleConns(1)
|
|
||||||
|
|
||||||
// First run: apply all migrations.
|
|
||||||
err = applyMigrations(ctx, conn)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("first applyMigrations failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Count rows in schema_migrations after first run.
|
|
||||||
var countBefore int
|
|
||||||
|
|
||||||
err = conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&countBefore)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to count schema_migrations after first run: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Second run: must be a no-op.
|
|
||||||
err = applyMigrations(ctx, conn)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("second applyMigrations failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Count rows in schema_migrations after second run — must be unchanged.
|
|
||||||
var countAfter int
|
|
||||||
|
|
||||||
err = conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&countAfter)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to count schema_migrations after second run: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if countBefore != countAfter {
|
|
||||||
t.Errorf("schema_migrations row count changed: before=%d, after=%d", countBefore, countAfter)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to open database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
err := conn.Close()
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("failed to close database: %v", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
conn.SetMaxOpenConns(1)
|
|
||||||
conn.SetMaxIdleConns(1)
|
|
||||||
|
|
||||||
// Verify schema_migrations does NOT exist yet.
|
|
||||||
var tableBefore int
|
|
||||||
|
|
||||||
err = conn.QueryRowContext(ctx,
|
|
||||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'",
|
|
||||||
).Scan(&tableBefore)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to check for table before bootstrap: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tableBefore != 0 {
|
|
||||||
t.Fatal("schema_migrations table should not exist before bootstrap")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run bootstrap.
|
|
||||||
err = bootstrapMigrationsTable(ctx, conn)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("bootstrapMigrationsTable failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify schema_migrations now exists.
|
|
||||||
var tableAfter int
|
|
||||||
|
|
||||||
err = conn.QueryRowContext(ctx,
|
|
||||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'",
|
|
||||||
).Scan(&tableAfter)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to check for table after bootstrap: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tableAfter != 1 {
|
|
||||||
t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d", tableAfter)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify version 0 row exists.
|
|
||||||
var version int
|
|
||||||
|
|
||||||
err = conn.QueryRowContext(ctx,
|
|
||||||
"SELECT version FROM schema_migrations WHERE version = 0",
|
|
||||||
).Scan(&version)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("version 0 row not found in schema_migrations: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if version != 0 {
|
|
||||||
t.Errorf("expected version 0, got %d", version)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -7,15 +7,14 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Fatal prints an error message to stderr and exits with status 1
|
// Fatal prints an error message to stderr and exits with status 1
|
||||||
func Fatal(format string, args ...any) {
|
func Fatal(format string, args ...interface{}) {
|
||||||
fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...)
|
fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CloseRows closes rows and exits on error
|
// CloseRows closes rows and exits on error
|
||||||
func CloseRows(rows *sql.Rows) {
|
func CloseRows(rows *sql.Rows) {
|
||||||
err := rows.Close()
|
if err := rows.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
Fatal("failed to close rows: %v", err)
|
Fatal("failed to close rows: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,9 +4,8 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
type FileChunkRepository struct {
|
type FileChunkRepository struct {
|
||||||
@@ -85,7 +84,6 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path
|
|||||||
`
|
`
|
||||||
|
|
||||||
LogSQL("GetByPathTx", query, path)
|
LogSQL("GetByPathTx", query, path)
|
||||||
|
|
||||||
rows, err := tx.QueryContext(ctx, query, path)
|
rows, err := tx.QueryContext(ctx, query, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("querying file chunks: %w", err)
|
return nil, fmt.Errorf("querying file chunks: %w", err)
|
||||||
@@ -94,30 +92,23 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path
|
|||||||
|
|
||||||
fileChunks, err := r.scanFileChunks(rows)
|
fileChunks, err := r.scanFileChunks(rows)
|
||||||
LogSQL("GetByPathTx", "Complete", path, "count", len(fileChunks))
|
LogSQL("GetByPathTx", "Complete", path, "count", len(fileChunks))
|
||||||
|
|
||||||
return fileChunks, err
|
return fileChunks, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// scanFileChunks is a helper that scans file chunk rows
|
// scanFileChunks is a helper that scans file chunk rows
|
||||||
func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) {
|
func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) {
|
||||||
var fileChunks []*FileChunk
|
var fileChunks []*FileChunk
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var (
|
var fc FileChunk
|
||||||
fc FileChunk
|
var fileIDStr, chunkHashStr string
|
||||||
fileIDStr, chunkHashStr string
|
|
||||||
)
|
|
||||||
|
|
||||||
err := rows.Scan(&fileIDStr, &fc.Idx, &chunkHashStr)
|
err := rows.Scan(&fileIDStr, &fc.Idx, &chunkHashStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("scanning file chunk: %w", err)
|
return nil, fmt.Errorf("scanning file chunk: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fc.FileID, err = types.ParseFileID(fileIDStr)
|
fc.FileID, err = types.ParseFileID(fileIDStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fc.ChunkHash = types.ChunkHash(chunkHashStr)
|
fc.ChunkHash = types.ChunkHash(chunkHashStr)
|
||||||
fileChunks = append(fileChunks, &fc)
|
fileChunks = append(fileChunks, &fc)
|
||||||
}
|
}
|
||||||
@@ -170,13 +161,14 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
|||||||
const batchSize = 500
|
const batchSize = 500
|
||||||
|
|
||||||
for i := 0; i < len(fileIDs); i += batchSize {
|
for i := 0; i < len(fileIDs); i += batchSize {
|
||||||
end := min(i+batchSize, len(fileIDs))
|
end := i + batchSize
|
||||||
|
if end > len(fileIDs) {
|
||||||
|
end = len(fileIDs)
|
||||||
|
}
|
||||||
batch := fileIDs[i:end]
|
batch := fileIDs[i:end]
|
||||||
|
|
||||||
query := "DELETE FROM file_chunks WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
|
query := "DELETE FROM file_chunks WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
|
||||||
|
args := make([]interface{}, len(batch))
|
||||||
args := make([]any, len(batch))
|
|
||||||
for j, id := range batch {
|
for j, id := range batch {
|
||||||
args[j] = id.String()
|
args[j] = id.String()
|
||||||
}
|
}
|
||||||
@@ -187,7 +179,6 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
|||||||
} else {
|
} else {
|
||||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("batch deleting file_chunks: %w", err)
|
return fmt.Errorf("batch deleting file_chunks: %w", err)
|
||||||
}
|
}
|
||||||
@@ -208,29 +199,22 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [
|
|||||||
const batchSize = 300
|
const batchSize = 300
|
||||||
|
|
||||||
for i := 0; i < len(fcs); i += batchSize {
|
for i := 0; i < len(fcs); i += batchSize {
|
||||||
end := min(i+batchSize, len(fcs))
|
end := i + batchSize
|
||||||
|
if end > len(fcs) {
|
||||||
|
end = len(fcs)
|
||||||
|
}
|
||||||
batch := fcs[i:end]
|
batch := fcs[i:end]
|
||||||
|
|
||||||
// Build the query with multiple value sets
|
// Build the query with multiple value sets
|
||||||
query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES "
|
query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES "
|
||||||
|
args := make([]interface{}, 0, len(batch)*3)
|
||||||
args := make([]any, 0, len(batch)*3)
|
|
||||||
|
|
||||||
var querySb211 strings.Builder
|
|
||||||
|
|
||||||
for j, fc := range batch {
|
for j, fc := range batch {
|
||||||
if j > 0 {
|
if j > 0 {
|
||||||
querySb211.WriteString(", ")
|
query += ", "
|
||||||
}
|
}
|
||||||
|
query += "(?, ?, ?)"
|
||||||
querySb211.WriteString("(?, ?, ?)")
|
|
||||||
|
|
||||||
args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
|
args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
query += querySb211.String()
|
|
||||||
|
|
||||||
query += " ON CONFLICT(file_id, idx) DO NOTHING"
|
query += " ON CONFLICT(file_id, idx) DO NOTHING"
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
@@ -239,7 +223,6 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [
|
|||||||
} else {
|
} else {
|
||||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("batch inserting file_chunks: %w", err)
|
return fmt.Errorf("batch inserting file_chunks: %w", err)
|
||||||
}
|
}
|
||||||
@@ -253,7 +236,6 @@ func (r *FileChunkRepository) GetByFile(ctx context.Context, path string) ([]*Fi
|
|||||||
LogSQL("GetByFile", "Starting", path)
|
LogSQL("GetByFile", "Starting", path)
|
||||||
result, err := r.GetByPath(ctx, path)
|
result, err := r.GetByPath(ctx, path)
|
||||||
LogSQL("GetByFile", "Complete", path, "count", len(result))
|
LogSQL("GetByFile", "Complete", path, "count", len(result))
|
||||||
|
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -262,6 +244,5 @@ func (r *FileChunkRepository) GetByFileTx(ctx context.Context, tx *sql.Tx, path
|
|||||||
LogSQL("GetByFileTx", "Starting", path)
|
LogSQL("GetByFileTx", "Starting", path)
|
||||||
result, err := r.GetByPathTx(ctx, tx, path)
|
result, err := r.GetByPathTx(ctx, tx, path)
|
||||||
LogSQL("GetByFileTx", "Complete", path, "count", len(result))
|
LogSQL("GetByFileTx", "Complete", path, "count", len(result))
|
||||||
|
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestFileChunkRepository(t *testing.T) {
|
func TestFileChunkRepository(t *testing.T) {
|
||||||
@@ -28,7 +28,6 @@ func TestFileChunkRepository(t *testing.T) {
|
|||||||
GID: 1000,
|
GID: 1000,
|
||||||
LinkTarget: "",
|
LinkTarget: "",
|
||||||
}
|
}
|
||||||
|
|
||||||
err := fileRepo.Create(ctx, nil, file)
|
err := fileRepo.Create(ctx, nil, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file: %v", err)
|
t.Fatalf("failed to create file: %v", err)
|
||||||
@@ -37,13 +36,11 @@ func TestFileChunkRepository(t *testing.T) {
|
|||||||
// Create chunks first
|
// Create chunks first
|
||||||
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
|
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
|
||||||
chunkRepo := NewChunkRepository(db)
|
chunkRepo := NewChunkRepository(db)
|
||||||
|
|
||||||
for _, chunkHash := range chunks {
|
for _, chunkHash := range chunks {
|
||||||
chunk := &Chunk{
|
chunk := &Chunk{
|
||||||
ChunkHash: chunkHash,
|
ChunkHash: chunkHash,
|
||||||
Size: 1024,
|
Size: 1024,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = chunkRepo.Create(ctx, nil, chunk)
|
err = chunkRepo.Create(ctx, nil, chunk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||||
@@ -68,7 +65,6 @@ func TestFileChunkRepository(t *testing.T) {
|
|||||||
Idx: 1,
|
Idx: 1,
|
||||||
ChunkHash: types.ChunkHash("chunk2"),
|
ChunkHash: types.ChunkHash("chunk2"),
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repo.Create(ctx, nil, fc2)
|
err = repo.Create(ctx, nil, fc2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create second file chunk: %v", err)
|
t.Fatalf("failed to create second file chunk: %v", err)
|
||||||
@@ -79,7 +75,6 @@ func TestFileChunkRepository(t *testing.T) {
|
|||||||
Idx: 2,
|
Idx: 2,
|
||||||
ChunkHash: types.ChunkHash("chunk3"),
|
ChunkHash: types.ChunkHash("chunk3"),
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repo.Create(ctx, nil, fc3)
|
err = repo.Create(ctx, nil, fc3)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create third file chunk: %v", err)
|
t.Fatalf("failed to create third file chunk: %v", err)
|
||||||
@@ -90,7 +85,6 @@ func TestFileChunkRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get file chunks: %v", err)
|
t.Fatalf("failed to get file chunks: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(fileChunks) != 3 {
|
if len(fileChunks) != 3 {
|
||||||
t.Errorf("expected 3 chunks, got %d", len(fileChunks))
|
t.Errorf("expected 3 chunks, got %d", len(fileChunks))
|
||||||
}
|
}
|
||||||
@@ -118,7 +112,6 @@ func TestFileChunkRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get deleted file chunks: %v", err)
|
t.Fatalf("failed to get deleted file chunks: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(fileChunks) != 0 {
|
if len(fileChunks) != 0 {
|
||||||
t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks))
|
t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks))
|
||||||
}
|
}
|
||||||
@@ -147,26 +140,22 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
|
|||||||
GID: 1000,
|
GID: 1000,
|
||||||
LinkTarget: "",
|
LinkTarget: "",
|
||||||
}
|
}
|
||||||
|
|
||||||
err := fileRepo.Create(ctx, nil, file)
|
err := fileRepo.Create(ctx, nil, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file %s: %v", path, err)
|
t.Fatalf("failed to create file %s: %v", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
files[i] = file
|
files[i] = file
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create all chunks first
|
// Create all chunks first
|
||||||
chunkRepo := NewChunkRepository(db)
|
chunkRepo := NewChunkRepository(db)
|
||||||
|
|
||||||
for i := range files {
|
for i := range files {
|
||||||
for j := range 2 {
|
for j := 0; j < 2; j++ {
|
||||||
chunkHash := types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j))
|
chunkHash := types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j))
|
||||||
chunk := &Chunk{
|
chunk := &Chunk{
|
||||||
ChunkHash: chunkHash,
|
ChunkHash: chunkHash,
|
||||||
Size: 1024,
|
Size: 1024,
|
||||||
}
|
}
|
||||||
|
|
||||||
err := chunkRepo.Create(ctx, nil, chunk)
|
err := chunkRepo.Create(ctx, nil, chunk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||||
@@ -176,13 +165,12 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
|
|||||||
|
|
||||||
// Create chunks for multiple files
|
// Create chunks for multiple files
|
||||||
for i, file := range files {
|
for i, file := range files {
|
||||||
for j := range 2 {
|
for j := 0; j < 2; j++ {
|
||||||
fc := &FileChunk{
|
fc := &FileChunk{
|
||||||
FileID: file.ID,
|
FileID: file.ID,
|
||||||
Idx: j,
|
Idx: j,
|
||||||
ChunkHash: types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)),
|
ChunkHash: types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)),
|
||||||
}
|
}
|
||||||
|
|
||||||
err := repo.Create(ctx, nil, fc)
|
err := repo.Create(ctx, nil, fc)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file chunk: %v", err)
|
t.Fatalf("failed to create file chunk: %v", err)
|
||||||
@@ -196,7 +184,6 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get chunks for file %d: %v", i, err)
|
t.Fatalf("failed to get chunks for file %d: %v", i, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(chunks) != 2 {
|
if len(chunks) != 2 {
|
||||||
t.Errorf("expected 2 chunks for file %d, got %d", i, len(chunks))
|
t.Errorf("expected 2 chunks for file %d, got %d", i, len(chunks))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,13 +3,11 @@ package database
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
type FileRepository struct {
|
type FileRepository struct {
|
||||||
@@ -40,11 +38,8 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
|
|||||||
RETURNING id
|
RETURNING id
|
||||||
`
|
`
|
||||||
|
|
||||||
var (
|
var idStr string
|
||||||
idStr string
|
var err error
|
||||||
err error
|
|
||||||
)
|
|
||||||
|
|
||||||
if tx != nil {
|
if tx != nil {
|
||||||
LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String())
|
LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String())
|
||||||
err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr)
|
err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr)
|
||||||
@@ -73,10 +68,9 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err
|
|||||||
`
|
`
|
||||||
|
|
||||||
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, path))
|
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, path))
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if err == sql.ErrNoRows {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("querying file: %w", err)
|
return nil, fmt.Errorf("querying file: %w", err)
|
||||||
}
|
}
|
||||||
@@ -93,10 +87,9 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e
|
|||||||
`
|
`
|
||||||
|
|
||||||
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, id.String()))
|
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, id.String()))
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if err == sql.ErrNoRows {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("querying file: %w", err)
|
return nil, fmt.Errorf("querying file: %w", err)
|
||||||
}
|
}
|
||||||
@@ -115,10 +108,9 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin
|
|||||||
file, err := r.scanFile(tx.QueryRowContext(ctx, query, path))
|
file, err := r.scanFile(tx.QueryRowContext(ctx, query, path))
|
||||||
LogSQL("GetByPathTx Scan complete", query, path)
|
LogSQL("GetByPathTx Scan complete", query, path)
|
||||||
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if err == sql.ErrNoRows {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("querying file: %w", err)
|
return nil, fmt.Errorf("querying file: %w", err)
|
||||||
}
|
}
|
||||||
@@ -128,12 +120,10 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin
|
|||||||
|
|
||||||
// scanFile is a helper that scans a single file row
|
// scanFile is a helper that scans a single file row
|
||||||
func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
|
func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
|
||||||
var (
|
var file File
|
||||||
file File
|
var idStr, pathStr, sourcePathStr string
|
||||||
idStr, pathStr, sourcePathStr string
|
var mtimeUnix int64
|
||||||
mtimeUnix int64
|
var linkTarget sql.NullString
|
||||||
linkTarget sql.NullString
|
|
||||||
)
|
|
||||||
|
|
||||||
err := row.Scan(
|
err := row.Scan(
|
||||||
&idStr,
|
&idStr,
|
||||||
@@ -154,10 +144,8 @@ func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
file.Path = types.FilePath(pathStr)
|
file.Path = types.FilePath(pathStr)
|
||||||
file.SourcePath = types.SourcePath(sourcePathStr)
|
file.SourcePath = types.SourcePath(sourcePathStr)
|
||||||
|
|
||||||
file.MTime = time.Unix(mtimeUnix, 0).UTC()
|
file.MTime = time.Unix(mtimeUnix, 0).UTC()
|
||||||
if linkTarget.Valid {
|
if linkTarget.Valid {
|
||||||
file.LinkTarget = types.FilePath(linkTarget.String)
|
file.LinkTarget = types.FilePath(linkTarget.String)
|
||||||
@@ -168,12 +156,10 @@ func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
|
|||||||
|
|
||||||
// scanFileRows is a helper that scans a file row from rows iterator
|
// scanFileRows is a helper that scans a file row from rows iterator
|
||||||
func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) {
|
func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) {
|
||||||
var (
|
var file File
|
||||||
file File
|
var idStr, pathStr, sourcePathStr string
|
||||||
idStr, pathStr, sourcePathStr string
|
var mtimeUnix int64
|
||||||
mtimeUnix int64
|
var linkTarget sql.NullString
|
||||||
linkTarget sql.NullString
|
|
||||||
)
|
|
||||||
|
|
||||||
err := rows.Scan(
|
err := rows.Scan(
|
||||||
&idStr,
|
&idStr,
|
||||||
@@ -194,10 +180,8 @@ func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
file.Path = types.FilePath(pathStr)
|
file.Path = types.FilePath(pathStr)
|
||||||
file.SourcePath = types.SourcePath(sourcePathStr)
|
file.SourcePath = types.SourcePath(sourcePathStr)
|
||||||
|
|
||||||
file.MTime = time.Unix(mtimeUnix, 0).UTC()
|
file.MTime = time.Unix(mtimeUnix, 0).UTC()
|
||||||
if linkTarget.Valid {
|
if linkTarget.Valid {
|
||||||
file.LinkTarget = types.FilePath(linkTarget.String)
|
file.LinkTarget = types.FilePath(linkTarget.String)
|
||||||
@@ -221,13 +205,11 @@ func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time)
|
|||||||
defer CloseRows(rows)
|
defer CloseRows(rows)
|
||||||
|
|
||||||
var files []*File
|
var files []*File
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
file, err := r.scanFileRows(rows)
|
file, err := r.scanFileRows(rows)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("scanning file: %w", err)
|
return nil, fmt.Errorf("scanning file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
files = append(files, file)
|
files = append(files, file)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -284,13 +266,11 @@ func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*Fi
|
|||||||
defer CloseRows(rows)
|
defer CloseRows(rows)
|
||||||
|
|
||||||
var files []*File
|
var files []*File
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
file, err := r.scanFileRows(rows)
|
file, err := r.scanFileRows(rows)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("scanning file: %w", err)
|
return nil, fmt.Errorf("scanning file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
files = append(files, file)
|
files = append(files, file)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -312,13 +292,11 @@ func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
|
|||||||
defer CloseRows(rows)
|
defer CloseRows(rows)
|
||||||
|
|
||||||
var files []*File
|
var files []*File
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
file, err := r.scanFileRows(rows)
|
file, err := r.scanFileRows(rows)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("scanning file: %w", err)
|
return nil, fmt.Errorf("scanning file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
files = append(files, file)
|
files = append(files, file)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -336,28 +314,21 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F
|
|||||||
const batchSize = 100
|
const batchSize = 100
|
||||||
|
|
||||||
for i := 0; i < len(files); i += batchSize {
|
for i := 0; i < len(files); i += batchSize {
|
||||||
end := min(i+batchSize, len(files))
|
end := i + batchSize
|
||||||
|
if end > len(files) {
|
||||||
|
end = len(files)
|
||||||
|
}
|
||||||
batch := files[i:end]
|
batch := files[i:end]
|
||||||
|
|
||||||
query := `INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target) VALUES `
|
query := `INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target) VALUES `
|
||||||
|
args := make([]interface{}, 0, len(batch)*9)
|
||||||
args := make([]any, 0, len(batch)*9)
|
|
||||||
|
|
||||||
var querySb325 strings.Builder
|
|
||||||
|
|
||||||
for j, f := range batch {
|
for j, f := range batch {
|
||||||
if j > 0 {
|
if j > 0 {
|
||||||
querySb325.WriteString(", ")
|
query += ", "
|
||||||
}
|
}
|
||||||
|
query += "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
||||||
querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
|
||||||
|
|
||||||
args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String())
|
args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
query += querySb325.String()
|
|
||||||
|
|
||||||
query += ` ON CONFLICT(path) DO UPDATE SET
|
query += ` ON CONFLICT(path) DO UPDATE SET
|
||||||
source_path = excluded.source_path,
|
source_path = excluded.source_path,
|
||||||
mtime = excluded.mtime,
|
mtime = excluded.mtime,
|
||||||
@@ -373,7 +344,6 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F
|
|||||||
} else {
|
} else {
|
||||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("batch inserting files: %w", err)
|
return fmt.Errorf("batch inserting files: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package database
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -20,8 +20,7 @@ func setupTestDB(t *testing.T) (*DB, func()) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
cleanup := func() {
|
cleanup := func() {
|
||||||
err := db.Close()
|
if err := db.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Errorf("failed to close database: %v", err)
|
t.Errorf("failed to close database: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -57,23 +56,18 @@ func TestFileRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get file: %v", err)
|
t.Fatalf("failed to get file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved == nil {
|
if retrieved == nil {
|
||||||
t.Fatal("expected file, got nil")
|
t.Fatal("expected file, got nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.Path != file.Path {
|
if retrieved.Path != file.Path {
|
||||||
t.Errorf("path mismatch: got %s, want %s", retrieved.Path, file.Path)
|
t.Errorf("path mismatch: got %s, want %s", retrieved.Path, file.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !retrieved.MTime.Equal(file.MTime) {
|
if !retrieved.MTime.Equal(file.MTime) {
|
||||||
t.Errorf("mtime mismatch: got %v, want %v", retrieved.MTime, file.MTime)
|
t.Errorf("mtime mismatch: got %v, want %v", retrieved.MTime, file.MTime)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.Size != file.Size {
|
if retrieved.Size != file.Size {
|
||||||
t.Errorf("size mismatch: got %d, want %d", retrieved.Size, file.Size)
|
t.Errorf("size mismatch: got %d, want %d", retrieved.Size, file.Size)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.Mode != file.Mode {
|
if retrieved.Mode != file.Mode {
|
||||||
t.Errorf("mode mismatch: got %o, want %o", retrieved.Mode, file.Mode)
|
t.Errorf("mode mismatch: got %o, want %o", retrieved.Mode, file.Mode)
|
||||||
}
|
}
|
||||||
@@ -81,7 +75,6 @@ func TestFileRepository(t *testing.T) {
|
|||||||
// Test Update (upsert)
|
// Test Update (upsert)
|
||||||
file.Size = 2048
|
file.Size = 2048
|
||||||
file.MTime = time.Now().Truncate(time.Second)
|
file.MTime = time.Now().Truncate(time.Second)
|
||||||
|
|
||||||
err = repo.Create(ctx, nil, file)
|
err = repo.Create(ctx, nil, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to update file: %v", err)
|
t.Fatalf("failed to update file: %v", err)
|
||||||
@@ -91,7 +84,6 @@ func TestFileRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get updated file: %v", err)
|
t.Fatalf("failed to get updated file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.Size != 2048 {
|
if retrieved.Size != 2048 {
|
||||||
t.Errorf("size not updated: got %d, want %d", retrieved.Size, 2048)
|
t.Errorf("size not updated: got %d, want %d", retrieved.Size, 2048)
|
||||||
}
|
}
|
||||||
@@ -101,7 +93,6 @@ func TestFileRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to list files: %v", err)
|
t.Fatalf("failed to list files: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(files) != 1 {
|
if len(files) != 1 {
|
||||||
t.Errorf("expected 1 file, got %d", len(files))
|
t.Errorf("expected 1 file, got %d", len(files))
|
||||||
}
|
}
|
||||||
@@ -116,7 +107,6 @@ func TestFileRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error getting deleted file: %v", err)
|
t.Fatalf("error getting deleted file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved != nil {
|
if retrieved != nil {
|
||||||
t.Error("expected nil for deleted file")
|
t.Error("expected nil for deleted file")
|
||||||
}
|
}
|
||||||
@@ -149,11 +139,9 @@ func TestFileRepositorySymlink(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get symlink: %v", err)
|
t.Fatalf("failed to get symlink: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !retrieved.IsSymlink() {
|
if !retrieved.IsSymlink() {
|
||||||
t.Error("expected IsSymlink() to be true")
|
t.Error("expected IsSymlink() to be true")
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.LinkTarget != symlink.LinkTarget {
|
if retrieved.LinkTarget != symlink.LinkTarget {
|
||||||
t.Errorf("link target mismatch: got %s, want %s", retrieved.LinkTarget, symlink.LinkTarget)
|
t.Errorf("link target mismatch: got %s, want %s", retrieved.LinkTarget, symlink.LinkTarget)
|
||||||
}
|
}
|
||||||
@@ -177,13 +165,12 @@ func TestFileRepositoryTransaction(t *testing.T) {
|
|||||||
GID: 1000,
|
GID: 1000,
|
||||||
}
|
}
|
||||||
|
|
||||||
err := repos.Files.Create(ctx, tx, file)
|
if err := repos.Files.Create(ctx, tx, file); err != nil {
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return error to trigger rollback
|
// Return error to trigger rollback
|
||||||
return errors.New("test rollback")
|
return fmt.Errorf("test rollback")
|
||||||
})
|
})
|
||||||
|
|
||||||
if err == nil || err.Error() != "test rollback" {
|
if err == nil || err.Error() != "test rollback" {
|
||||||
@@ -195,7 +182,6 @@ func TestFileRepositoryTransaction(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error checking for file: %v", err)
|
t.Fatalf("error checking for file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved != nil {
|
if retrieved != nil {
|
||||||
t.Error("file should not exist after rollback")
|
t.Error("file should not exist after rollback")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,57 +0,0 @@
|
|||||||
package database
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
)
|
|
||||||
|
|
||||||
// LocalMetaKeyStorageURL is the key under which the destination store's
|
|
||||||
// URL is recorded when a mutating command first binds the local index
|
|
||||||
// to a specific backup destination.
|
|
||||||
const LocalMetaKeyStorageURL = "storage_url"
|
|
||||||
|
|
||||||
// LocalMetaRepository provides keyed access to host-local settings
|
|
||||||
// stored in the local_meta table.
|
|
||||||
type LocalMetaRepository struct {
|
|
||||||
db *DB
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewLocalMetaRepository(db *DB) *LocalMetaRepository {
|
|
||||||
return &LocalMetaRepository{db: db}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get returns the value stored at key, or the empty string if the key
|
|
||||||
// is not set. A missing key is not an error — the caller distinguishes
|
|
||||||
// "unset" (bind on first use) from "set to something" (compare).
|
|
||||||
func (r *LocalMetaRepository) Get(ctx context.Context, key string) (string, error) {
|
|
||||||
var value string
|
|
||||||
|
|
||||||
err := r.db.conn.QueryRowContext(ctx,
|
|
||||||
"SELECT value FROM local_meta WHERE key = ?", key,
|
|
||||||
).Scan(&value)
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("reading local_meta %q: %w", key, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return value, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set writes key=value, replacing any prior value.
|
|
||||||
func (r *LocalMetaRepository) Set(ctx context.Context, key, value string) error {
|
|
||||||
_, err := r.db.ExecWithLog(ctx,
|
|
||||||
`INSERT INTO local_meta (key, value) VALUES (?, ?)
|
|
||||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value`,
|
|
||||||
key, value,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("writing local_meta %q: %w", key, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
package database_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestLocalMetaEmptyOnFresh(t *testing.T) {
|
|
||||||
db, err := database.NewTestDB()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
|
||||||
|
|
||||||
got, err := repos.LocalMeta.Get(context.Background(), database.LocalMetaKeyStorageURL)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Empty(t, got, "fresh DB must return empty for unset keys, not error")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLocalMetaSetGetRoundTrip(t *testing.T) {
|
|
||||||
db, err := database.NewTestDB()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
require.NoError(t, repos.LocalMeta.Set(ctx, database.LocalMetaKeyStorageURL, "file:///mnt/backups"))
|
|
||||||
|
|
||||||
got, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, "file:///mnt/backups", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLocalMetaSetOverwrites(t *testing.T) {
|
|
||||||
db, err := database.NewTestDB()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
repos := database.NewRepositories(db)
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
require.NoError(t, repos.LocalMeta.Set(ctx, database.LocalMetaKeyStorageURL, "s3://old"))
|
|
||||||
require.NoError(t, repos.LocalMeta.Set(ctx, database.LocalMetaKeyStorageURL, "s3://new"))
|
|
||||||
|
|
||||||
got, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, "s3://new", got)
|
|
||||||
}
|
|
||||||
@@ -5,7 +5,7 @@ package database
|
|||||||
import (
|
import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
// File represents a file or directory in the backup system.
|
// File represents a file or directory in the backup system.
|
||||||
|
|||||||
@@ -6,9 +6,9 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Module provides database dependencies
|
// Module provides database dependencies
|
||||||
@@ -34,16 +34,11 @@ func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) {
|
|||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
OnStop: func(ctx context.Context) error {
|
OnStop: func(ctx context.Context) error {
|
||||||
log.Debug("Database module OnStop hook called")
|
log.Debug("Database module OnStop hook called")
|
||||||
|
if err := db.Close(); err != nil {
|
||||||
err := db.Close()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to close database in OnStop hook", "error", err)
|
log.Error("Failed to close database in OnStop hook", "error", err)
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("Database closed successfully in OnStop hook")
|
log.Debug("Database closed successfully in OnStop hook")
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ type Repositories struct {
|
|||||||
ChunkFiles *ChunkFileRepository
|
ChunkFiles *ChunkFileRepository
|
||||||
Snapshots *SnapshotRepository
|
Snapshots *SnapshotRepository
|
||||||
Uploads *UploadRepository
|
Uploads *UploadRepository
|
||||||
LocalMeta *LocalMetaRepository
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewRepositories creates a new Repositories instance with all repository types.
|
// NewRepositories creates a new Repositories instance with all repository types.
|
||||||
@@ -35,7 +34,6 @@ func NewRepositories(db *DB) *Repositories {
|
|||||||
ChunkFiles: NewChunkFileRepository(db),
|
ChunkFiles: NewChunkFileRepository(db),
|
||||||
Snapshots: NewSnapshotRepository(db),
|
Snapshots: NewSnapshotRepository(db),
|
||||||
Uploads: NewUploadRepository(db.conn),
|
Uploads: NewUploadRepository(db.conn),
|
||||||
LocalMeta: NewLocalMetaRepository(db),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -50,25 +48,20 @@ type TxFunc func(ctx context.Context, tx *sql.Tx) error
|
|||||||
// This method should be used for all write operations to ensure atomicity.
|
// This method should be used for all write operations to ensure atomicity.
|
||||||
func (r *Repositories) WithTx(ctx context.Context, fn TxFunc) error {
|
func (r *Repositories) WithTx(ctx context.Context, fn TxFunc) error {
|
||||||
LogSQL("WithTx", "Beginning transaction", "")
|
LogSQL("WithTx", "Beginning transaction", "")
|
||||||
|
|
||||||
tx, err := r.db.BeginTx(ctx, nil)
|
tx, err := r.db.BeginTx(ctx, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("beginning transaction: %w", err)
|
return fmt.Errorf("beginning transaction: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
LogSQL("WithTx", "Transaction started", "")
|
LogSQL("WithTx", "Transaction started", "")
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
if p := recover(); p != nil {
|
if p := recover(); p != nil {
|
||||||
rollbackErr := tx.Rollback()
|
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||||
if rollbackErr != nil {
|
|
||||||
Fatal("failed to rollback transaction: %v", rollbackErr)
|
Fatal("failed to rollback transaction: %v", rollbackErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
panic(p)
|
panic(p)
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
rollbackErr := tx.Rollback()
|
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||||
if rollbackErr != nil {
|
|
||||||
Fatal("failed to rollback transaction: %v", rollbackErr)
|
Fatal("failed to rollback transaction: %v", rollbackErr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -95,7 +88,6 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error {
|
|||||||
opts := &sql.TxOptions{
|
opts := &sql.TxOptions{
|
||||||
ReadOnly: true,
|
ReadOnly: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
tx, err := r.db.BeginTx(ctx, opts)
|
tx, err := r.db.BeginTx(ctx, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("beginning read transaction: %w", err)
|
return fmt.Errorf("beginning read transaction: %w", err)
|
||||||
@@ -103,15 +95,12 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error {
|
|||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
if p := recover(); p != nil {
|
if p := recover(); p != nil {
|
||||||
rollbackErr := tx.Rollback()
|
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||||
if rollbackErr != nil {
|
|
||||||
Fatal("failed to rollback transaction: %v", rollbackErr)
|
Fatal("failed to rollback transaction: %v", rollbackErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
panic(p)
|
panic(p)
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
rollbackErr := tx.Rollback()
|
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||||
if rollbackErr != nil {
|
|
||||||
Fatal("failed to rollback transaction: %v", rollbackErr)
|
Fatal("failed to rollback transaction: %v", rollbackErr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,11 +3,11 @@ package database
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRepositoriesTransaction(t *testing.T) {
|
func TestRepositoriesTransaction(t *testing.T) {
|
||||||
@@ -28,9 +28,7 @@ func TestRepositoriesTransaction(t *testing.T) {
|
|||||||
UID: 1000,
|
UID: 1000,
|
||||||
GID: 1000,
|
GID: 1000,
|
||||||
}
|
}
|
||||||
|
if err := repos.Files.Create(ctx, tx, file); err != nil {
|
||||||
err := repos.Files.Create(ctx, tx, file)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -39,9 +37,7 @@ func TestRepositoriesTransaction(t *testing.T) {
|
|||||||
ChunkHash: types.ChunkHash("tx_chunk1"),
|
ChunkHash: types.ChunkHash("tx_chunk1"),
|
||||||
Size: 512,
|
Size: 512,
|
||||||
}
|
}
|
||||||
|
if err := repos.Chunks.Create(ctx, tx, chunk1); err != nil {
|
||||||
err = repos.Chunks.Create(ctx, tx, chunk1)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -49,9 +45,7 @@ func TestRepositoriesTransaction(t *testing.T) {
|
|||||||
ChunkHash: types.ChunkHash("tx_chunk2"),
|
ChunkHash: types.ChunkHash("tx_chunk2"),
|
||||||
Size: 512,
|
Size: 512,
|
||||||
}
|
}
|
||||||
|
if err := repos.Chunks.Create(ctx, tx, chunk2); err != nil {
|
||||||
err = repos.Chunks.Create(ctx, tx, chunk2)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -61,9 +55,7 @@ func TestRepositoriesTransaction(t *testing.T) {
|
|||||||
Idx: 0,
|
Idx: 0,
|
||||||
ChunkHash: chunk1.ChunkHash,
|
ChunkHash: chunk1.ChunkHash,
|
||||||
}
|
}
|
||||||
|
if err := repos.FileChunks.Create(ctx, tx, fc1); err != nil {
|
||||||
err = repos.FileChunks.Create(ctx, tx, fc1)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -72,9 +64,7 @@ func TestRepositoriesTransaction(t *testing.T) {
|
|||||||
Idx: 1,
|
Idx: 1,
|
||||||
ChunkHash: chunk2.ChunkHash,
|
ChunkHash: chunk2.ChunkHash,
|
||||||
}
|
}
|
||||||
|
if err := repos.FileChunks.Create(ctx, tx, fc2); err != nil {
|
||||||
err = repos.FileChunks.Create(ctx, tx, fc2)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,9 +74,7 @@ func TestRepositoriesTransaction(t *testing.T) {
|
|||||||
Hash: types.BlobHash("tx_blob1"),
|
Hash: types.BlobHash("tx_blob1"),
|
||||||
CreatedTS: time.Now().Truncate(time.Second),
|
CreatedTS: time.Now().Truncate(time.Second),
|
||||||
}
|
}
|
||||||
|
if err := repos.Blobs.Create(ctx, tx, blob); err != nil {
|
||||||
err = repos.Blobs.Create(ctx, tx, blob)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -97,9 +85,7 @@ func TestRepositoriesTransaction(t *testing.T) {
|
|||||||
Offset: 0,
|
Offset: 0,
|
||||||
Length: 512,
|
Length: 512,
|
||||||
}
|
}
|
||||||
|
if err := repos.BlobChunks.Create(ctx, tx, bc1); err != nil {
|
||||||
err = repos.BlobChunks.Create(ctx, tx, bc1)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -109,14 +95,13 @@ func TestRepositoriesTransaction(t *testing.T) {
|
|||||||
Offset: 512,
|
Offset: 512,
|
||||||
Length: 512,
|
Length: 512,
|
||||||
}
|
}
|
||||||
|
if err := repos.BlobChunks.Create(ctx, tx, bc2); err != nil {
|
||||||
err = repos.BlobChunks.Create(ctx, tx, bc2)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("transaction failed: %v", err)
|
t.Fatalf("transaction failed: %v", err)
|
||||||
}
|
}
|
||||||
@@ -126,7 +111,6 @@ func TestRepositoriesTransaction(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get file: %v", err)
|
t.Fatalf("failed to get file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if file == nil {
|
if file == nil {
|
||||||
t.Error("expected file after transaction")
|
t.Error("expected file after transaction")
|
||||||
}
|
}
|
||||||
@@ -135,7 +119,6 @@ func TestRepositoriesTransaction(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get file chunks: %v", err)
|
t.Fatalf("failed to get file chunks: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(chunks) != 2 {
|
if len(chunks) != 2 {
|
||||||
t.Errorf("expected 2 file chunks, got %d", len(chunks))
|
t.Errorf("expected 2 file chunks, got %d", len(chunks))
|
||||||
}
|
}
|
||||||
@@ -144,7 +127,6 @@ func TestRepositoriesTransaction(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get blob: %v", err)
|
t.Fatalf("failed to get blob: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if blob == nil {
|
if blob == nil {
|
||||||
t.Error("expected blob after transaction")
|
t.Error("expected blob after transaction")
|
||||||
}
|
}
|
||||||
@@ -168,9 +150,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
|
|||||||
UID: 1000,
|
UID: 1000,
|
||||||
GID: 1000,
|
GID: 1000,
|
||||||
}
|
}
|
||||||
|
if err := repos.Files.Create(ctx, tx, file); err != nil {
|
||||||
err := repos.Files.Create(ctx, tx, file)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -179,14 +159,12 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
|
|||||||
ChunkHash: types.ChunkHash("rollback_chunk"),
|
ChunkHash: types.ChunkHash("rollback_chunk"),
|
||||||
Size: 1024,
|
Size: 1024,
|
||||||
}
|
}
|
||||||
|
if err := repos.Chunks.Create(ctx, tx, chunk); err != nil {
|
||||||
err = repos.Chunks.Create(ctx, tx, chunk)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return error to trigger rollback
|
// Return error to trigger rollback
|
||||||
return errors.New("intentional rollback")
|
return fmt.Errorf("intentional rollback")
|
||||||
})
|
})
|
||||||
|
|
||||||
if err == nil || err.Error() != "intentional rollback" {
|
if err == nil || err.Error() != "intentional rollback" {
|
||||||
@@ -198,7 +176,6 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error checking for file: %v", err)
|
t.Fatalf("error checking for file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if file != nil {
|
if file != nil {
|
||||||
t.Error("file should not exist after rollback")
|
t.Error("file should not exist after rollback")
|
||||||
}
|
}
|
||||||
@@ -207,7 +184,6 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error checking for chunk: %v", err)
|
t.Fatalf("error checking for chunk: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if chunk != nil {
|
if chunk != nil {
|
||||||
t.Error("chunk should not exist after rollback")
|
t.Error("chunk should not exist after rollback")
|
||||||
}
|
}
|
||||||
@@ -229,7 +205,6 @@ func TestRepositoriesReadTransaction(t *testing.T) {
|
|||||||
UID: 1000,
|
UID: 1000,
|
||||||
GID: 1000,
|
GID: 1000,
|
||||||
}
|
}
|
||||||
|
|
||||||
err := repos.Files.Create(ctx, nil, file)
|
err := repos.Files.Create(ctx, nil, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file: %v", err)
|
t.Fatalf("failed to create file: %v", err)
|
||||||
@@ -237,10 +212,8 @@ func TestRepositoriesReadTransaction(t *testing.T) {
|
|||||||
|
|
||||||
// Test read-only transaction
|
// Test read-only transaction
|
||||||
var retrievedFile *File
|
var retrievedFile *File
|
||||||
|
|
||||||
err = repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
err = repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
retrievedFile, err = repos.Files.GetByPathTx(ctx, tx, "/test/read_file.txt")
|
retrievedFile, err = repos.Files.GetByPathTx(ctx, tx, "/test/read_file.txt")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -259,6 +232,7 @@ func TestRepositoriesReadTransaction(t *testing.T) {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("read transaction failed: %v", err)
|
t.Fatalf("read transaction failed: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,12 +3,11 @@ package database
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestFileRepositoryUUIDGeneration tests that files get unique UUIDs
|
// TestFileRepositoryUUIDGeneration tests that files get unique UUIDs
|
||||||
@@ -40,7 +39,6 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
uuids := make(map[string]bool)
|
uuids := make(map[string]bool)
|
||||||
|
|
||||||
for _, file := range files {
|
for _, file := range files {
|
||||||
err := repo.Create(ctx, nil, file)
|
err := repo.Create(ctx, nil, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -56,7 +54,6 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
|
|||||||
if uuids[file.ID.String()] {
|
if uuids[file.ID.String()] {
|
||||||
t.Errorf("duplicate UUID generated: %s", file.ID)
|
t.Errorf("duplicate UUID generated: %s", file.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
uuids[file.ID.String()] = true
|
uuids[file.ID.String()] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -93,19 +90,16 @@ func TestFileRepositoryGetByID(t *testing.T) {
|
|||||||
if retrieved.ID != file.ID {
|
if retrieved.ID != file.ID {
|
||||||
t.Errorf("ID mismatch: expected %s, got %s", file.ID, retrieved.ID)
|
t.Errorf("ID mismatch: expected %s, got %s", file.ID, retrieved.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.Path != file.Path {
|
if retrieved.Path != file.Path {
|
||||||
t.Errorf("Path mismatch: expected %s, got %s", file.Path, retrieved.Path)
|
t.Errorf("Path mismatch: expected %s, got %s", file.Path, retrieved.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test non-existent ID
|
// Test non-existent ID
|
||||||
nonExistentID := types.NewFileID() // Generate a new UUID that won't exist in the database
|
nonExistentID := types.NewFileID() // Generate a new UUID that won't exist in the database
|
||||||
|
|
||||||
nonExistent, err := repo.GetByID(ctx, nonExistentID)
|
nonExistent, err := repo.GetByID(ctx, nonExistentID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("GetByID should not return error for non-existent ID: %v", err)
|
t.Fatalf("GetByID should not return error for non-existent ID: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if nonExistent != nil {
|
if nonExistent != nil {
|
||||||
t.Error("expected nil for non-existent ID")
|
t.Error("expected nil for non-existent ID")
|
||||||
}
|
}
|
||||||
@@ -141,7 +135,6 @@ func TestOrphanedFileCleanup(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file1: %v", err)
|
t.Fatalf("failed to create file1: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Files.Create(ctx, nil, file2)
|
err = repos.Files.Create(ctx, nil, file2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file2: %v", err)
|
t.Fatalf("failed to create file2: %v", err)
|
||||||
@@ -153,7 +146,6 @@ func TestOrphanedFileCleanup(t *testing.T) {
|
|||||||
Hostname: "test-host",
|
Hostname: "test-host",
|
||||||
StartedAt: time.Now(),
|
StartedAt: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Snapshots.Create(ctx, nil, snapshot)
|
err = repos.Snapshots.Create(ctx, nil, snapshot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create snapshot: %v", err)
|
t.Fatalf("failed to create snapshot: %v", err)
|
||||||
@@ -176,7 +168,6 @@ func TestOrphanedFileCleanup(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error getting file: %v", err)
|
t.Fatalf("error getting file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if orphanedFile != nil {
|
if orphanedFile != nil {
|
||||||
t.Error("orphaned file should have been deleted")
|
t.Error("orphaned file should have been deleted")
|
||||||
}
|
}
|
||||||
@@ -186,7 +177,6 @@ func TestOrphanedFileCleanup(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error getting file: %v", err)
|
t.Fatalf("error getting file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if referencedFile == nil {
|
if referencedFile == nil {
|
||||||
t.Error("referenced file should not have been deleted")
|
t.Error("referenced file should not have been deleted")
|
||||||
}
|
}
|
||||||
@@ -214,7 +204,6 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create chunk1: %v", err)
|
t.Fatalf("failed to create chunk1: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Chunks.Create(ctx, nil, chunk2)
|
err = repos.Chunks.Create(ctx, nil, chunk2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create chunk2: %v", err)
|
t.Fatalf("failed to create chunk2: %v", err)
|
||||||
@@ -229,7 +218,6 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
|||||||
UID: 1000,
|
UID: 1000,
|
||||||
GID: 1000,
|
GID: 1000,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Files.Create(ctx, nil, file)
|
err = repos.Files.Create(ctx, nil, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file: %v", err)
|
t.Fatalf("failed to create file: %v", err)
|
||||||
@@ -241,7 +229,6 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
|||||||
Idx: 0,
|
Idx: 0,
|
||||||
ChunkHash: chunk2.ChunkHash,
|
ChunkHash: chunk2.ChunkHash,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.FileChunks.Create(ctx, nil, fc)
|
err = repos.FileChunks.Create(ctx, nil, fc)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file chunk: %v", err)
|
t.Fatalf("failed to create file chunk: %v", err)
|
||||||
@@ -258,7 +245,6 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error getting chunk: %v", err)
|
t.Fatalf("error getting chunk: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if orphanedChunk != nil {
|
if orphanedChunk != nil {
|
||||||
t.Error("orphaned chunk should have been deleted")
|
t.Error("orphaned chunk should have been deleted")
|
||||||
}
|
}
|
||||||
@@ -268,7 +254,6 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error getting chunk: %v", err)
|
t.Fatalf("error getting chunk: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if referencedChunk == nil {
|
if referencedChunk == nil {
|
||||||
t.Error("referenced chunk should not have been deleted")
|
t.Error("referenced chunk should not have been deleted")
|
||||||
}
|
}
|
||||||
@@ -298,7 +283,6 @@ func TestOrphanedBlobCleanup(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create blob1: %v", err)
|
t.Fatalf("failed to create blob1: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Blobs.Create(ctx, nil, blob2)
|
err = repos.Blobs.Create(ctx, nil, blob2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create blob2: %v", err)
|
t.Fatalf("failed to create blob2: %v", err)
|
||||||
@@ -310,7 +294,6 @@ func TestOrphanedBlobCleanup(t *testing.T) {
|
|||||||
Hostname: "test-host",
|
Hostname: "test-host",
|
||||||
StartedAt: time.Now(),
|
StartedAt: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Snapshots.Create(ctx, nil, snapshot)
|
err = repos.Snapshots.Create(ctx, nil, snapshot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create snapshot: %v", err)
|
t.Fatalf("failed to create snapshot: %v", err)
|
||||||
@@ -333,7 +316,6 @@ func TestOrphanedBlobCleanup(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error getting blob: %v", err)
|
t.Fatalf("error getting blob: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if orphanedBlob != nil {
|
if orphanedBlob != nil {
|
||||||
t.Error("orphaned blob should have been deleted")
|
t.Error("orphaned blob should have been deleted")
|
||||||
}
|
}
|
||||||
@@ -343,7 +325,6 @@ func TestOrphanedBlobCleanup(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error getting blob: %v", err)
|
t.Fatalf("error getting blob: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if referencedBlob == nil {
|
if referencedBlob == nil {
|
||||||
t.Error("referenced blob should not have been deleted")
|
t.Error("referenced blob should not have been deleted")
|
||||||
}
|
}
|
||||||
@@ -366,7 +347,6 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
|||||||
UID: 1000,
|
UID: 1000,
|
||||||
GID: 1000,
|
GID: 1000,
|
||||||
}
|
}
|
||||||
|
|
||||||
err := repos.Files.Create(ctx, nil, file)
|
err := repos.Files.Create(ctx, nil, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file: %v", err)
|
t.Fatalf("failed to create file: %v", err)
|
||||||
@@ -379,7 +359,6 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
|||||||
ChunkHash: chunkHash,
|
ChunkHash: chunkHash,
|
||||||
Size: 1024,
|
Size: 1024,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create chunk: %v", err)
|
t.Fatalf("failed to create chunk: %v", err)
|
||||||
@@ -391,7 +370,6 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
|||||||
Idx: i,
|
Idx: i,
|
||||||
ChunkHash: chunkHash,
|
ChunkHash: chunkHash,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.FileChunks.Create(ctx, nil, fc)
|
err = repos.FileChunks.Create(ctx, nil, fc)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file chunk: %v", err)
|
t.Fatalf("failed to create file chunk: %v", err)
|
||||||
@@ -403,7 +381,6 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get file chunks: %v", err)
|
t.Fatalf("failed to get file chunks: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(fileChunks) != 3 {
|
if len(fileChunks) != 3 {
|
||||||
t.Errorf("expected 3 chunks, got %d", len(fileChunks))
|
t.Errorf("expected 3 chunks, got %d", len(fileChunks))
|
||||||
}
|
}
|
||||||
@@ -418,7 +395,6 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get file chunks after delete: %v", err)
|
t.Fatalf("failed to get file chunks after delete: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(fileChunks) != 0 {
|
if len(fileChunks) != 0 {
|
||||||
t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks))
|
t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks))
|
||||||
}
|
}
|
||||||
@@ -454,7 +430,6 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file1: %v", err)
|
t.Fatalf("failed to create file1: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Files.Create(ctx, nil, file2)
|
err = repos.Files.Create(ctx, nil, file2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file2: %v", err)
|
t.Fatalf("failed to create file2: %v", err)
|
||||||
@@ -465,7 +440,6 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
|||||||
ChunkHash: types.ChunkHash("shared-chunk"),
|
ChunkHash: types.ChunkHash("shared-chunk"),
|
||||||
Size: 1024,
|
Size: 1024,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create chunk: %v", err)
|
t.Fatalf("failed to create chunk: %v", err)
|
||||||
@@ -489,7 +463,6 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create chunk file 1: %v", err)
|
t.Fatalf("failed to create chunk file 1: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.ChunkFiles.Create(ctx, nil, cf2)
|
err = repos.ChunkFiles.Create(ctx, nil, cf2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create chunk file 2: %v", err)
|
t.Fatalf("failed to create chunk file 2: %v", err)
|
||||||
@@ -500,7 +473,6 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get chunk files: %v", err)
|
t.Fatalf("failed to get chunk files: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(chunkFiles) != 2 {
|
if len(chunkFiles) != 2 {
|
||||||
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
|
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
|
||||||
}
|
}
|
||||||
@@ -510,7 +482,6 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get chunks by file ID: %v", err)
|
t.Fatalf("failed to get chunks by file ID: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(chunkFiles) != 1 {
|
if len(chunkFiles) != 1 {
|
||||||
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
|
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
|
||||||
}
|
}
|
||||||
@@ -557,19 +528,15 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) {
|
|||||||
if retrieved.VaultikVersion != snapshot.VaultikVersion {
|
if retrieved.VaultikVersion != snapshot.VaultikVersion {
|
||||||
t.Errorf("version mismatch: expected %s, got %s", snapshot.VaultikVersion, retrieved.VaultikVersion)
|
t.Errorf("version mismatch: expected %s, got %s", snapshot.VaultikVersion, retrieved.VaultikVersion)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.VaultikGitRevision != snapshot.VaultikGitRevision {
|
if retrieved.VaultikGitRevision != snapshot.VaultikGitRevision {
|
||||||
t.Errorf("git revision mismatch: expected %s, got %s", snapshot.VaultikGitRevision, retrieved.VaultikGitRevision)
|
t.Errorf("git revision mismatch: expected %s, got %s", snapshot.VaultikGitRevision, retrieved.VaultikGitRevision)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.CompressionLevel != snapshot.CompressionLevel {
|
if retrieved.CompressionLevel != snapshot.CompressionLevel {
|
||||||
t.Errorf("compression level mismatch: expected %d, got %d", snapshot.CompressionLevel, retrieved.CompressionLevel)
|
t.Errorf("compression level mismatch: expected %d, got %d", snapshot.CompressionLevel, retrieved.CompressionLevel)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.BlobUncompressedSize != snapshot.BlobUncompressedSize {
|
if retrieved.BlobUncompressedSize != snapshot.BlobUncompressedSize {
|
||||||
t.Errorf("uncompressed size mismatch: expected %d, got %d", snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize)
|
t.Errorf("uncompressed size mismatch: expected %d, got %d", snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.UploadDurationMs != snapshot.UploadDurationMs {
|
if retrieved.UploadDurationMs != snapshot.UploadDurationMs {
|
||||||
t.Errorf("upload duration mismatch: expected %d, got %d", snapshot.UploadDurationMs, retrieved.UploadDurationMs)
|
t.Errorf("upload duration mismatch: expected %d, got %d", snapshot.UploadDurationMs, retrieved.UploadDurationMs)
|
||||||
}
|
}
|
||||||
@@ -599,7 +566,6 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create snapshot1: %v", err)
|
t.Fatalf("failed to create snapshot1: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Snapshots.Create(ctx, nil, snapshot2)
|
err = repos.Snapshots.Create(ctx, nil, snapshot2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create snapshot2: %v", err)
|
t.Fatalf("failed to create snapshot2: %v", err)
|
||||||
@@ -616,7 +582,6 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
|||||||
UID: 1000,
|
UID: 1000,
|
||||||
GID: 1000,
|
GID: 1000,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Files.Create(ctx, nil, files[i])
|
err = repos.Files.Create(ctx, nil, files[i])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file%d: %v", i, err)
|
t.Fatalf("failed to create file%d: %v", i, err)
|
||||||
@@ -633,17 +598,14 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[1].ID)
|
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[1].ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[1].ID)
|
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[1].ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[2].ID)
|
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[2].ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -654,7 +616,6 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Snapshots.Delete(ctx, snapshot1.ID.String())
|
err = repos.Snapshots.Delete(ctx, snapshot1.ID.String())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -672,7 +633,6 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error getting file0: %v", err)
|
t.Fatalf("error getting file0: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if file0 != nil {
|
if file0 != nil {
|
||||||
t.Error("file0 should have been deleted")
|
t.Error("file0 should have been deleted")
|
||||||
}
|
}
|
||||||
@@ -682,7 +642,6 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error getting file1: %v", err)
|
t.Fatalf("error getting file1: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if file1 == nil {
|
if file1 == nil {
|
||||||
t.Error("file1 should still exist")
|
t.Error("file1 should still exist")
|
||||||
}
|
}
|
||||||
@@ -692,7 +651,6 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error getting file2: %v", err)
|
t.Fatalf("error getting file2: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if file2 == nil {
|
if file2 == nil {
|
||||||
t.Error("file2 should still exist")
|
t.Error("file2 should still exist")
|
||||||
}
|
}
|
||||||
@@ -715,19 +673,17 @@ func TestCascadeDelete(t *testing.T) {
|
|||||||
UID: 1000,
|
UID: 1000,
|
||||||
GID: 1000,
|
GID: 1000,
|
||||||
}
|
}
|
||||||
|
|
||||||
err := repos.Files.Create(ctx, nil, file)
|
err := repos.Files.Create(ctx, nil, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file: %v", err)
|
t.Fatalf("failed to create file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create chunks and file-chunk mappings
|
// Create chunks and file-chunk mappings
|
||||||
for i := range 3 {
|
for i := 0; i < 3; i++ {
|
||||||
chunk := &Chunk{
|
chunk := &Chunk{
|
||||||
ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)),
|
ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)),
|
||||||
Size: 1024,
|
Size: 1024,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create chunk: %v", err)
|
t.Fatalf("failed to create chunk: %v", err)
|
||||||
@@ -738,7 +694,6 @@ func TestCascadeDelete(t *testing.T) {
|
|||||||
Idx: i,
|
Idx: i,
|
||||||
ChunkHash: chunk.ChunkHash,
|
ChunkHash: chunk.ChunkHash,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.FileChunks.Create(ctx, nil, fc)
|
err = repos.FileChunks.Create(ctx, nil, fc)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file chunk: %v", err)
|
t.Fatalf("failed to create file chunk: %v", err)
|
||||||
@@ -750,7 +705,6 @@ func TestCascadeDelete(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(fileChunks) != 3 {
|
if len(fileChunks) != 3 {
|
||||||
t.Errorf("expected 3 file chunks, got %d", len(fileChunks))
|
t.Errorf("expected 3 file chunks, got %d", len(fileChunks))
|
||||||
}
|
}
|
||||||
@@ -766,7 +720,6 @@ func TestCascadeDelete(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(fileChunks) != 0 {
|
if len(fileChunks) != 0 {
|
||||||
t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks))
|
t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks))
|
||||||
}
|
}
|
||||||
@@ -791,7 +744,6 @@ func TestTransactionIsolation(t *testing.T) {
|
|||||||
UID: 1000,
|
UID: 1000,
|
||||||
GID: 1000,
|
GID: 1000,
|
||||||
}
|
}
|
||||||
|
|
||||||
err := repos.Files.Create(ctx, tx, file)
|
err := repos.Files.Create(ctx, tx, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -802,8 +754,9 @@ func TestTransactionIsolation(t *testing.T) {
|
|||||||
// For now, we'll just test that rollback works
|
// For now, we'll just test that rollback works
|
||||||
|
|
||||||
// Return an error to trigger rollback
|
// Return an error to trigger rollback
|
||||||
return errors.New("intentional rollback")
|
return fmt.Errorf("intentional rollback")
|
||||||
})
|
})
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error from transaction")
|
t.Fatal("expected error from transaction")
|
||||||
}
|
}
|
||||||
@@ -813,7 +766,6 @@ func TestTransactionIsolation(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(files) != 0 {
|
if len(files) != 0 {
|
||||||
t.Error("file should not exist after rollback")
|
t.Error("file should not exist after rollback")
|
||||||
}
|
}
|
||||||
@@ -838,14 +790,13 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
|
|||||||
Hostname: "test-host",
|
Hostname: "test-host",
|
||||||
StartedAt: time.Now(),
|
StartedAt: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
err := repos.Snapshots.Create(ctx, nil, snapshot)
|
err := repos.Snapshots.Create(ctx, nil, snapshot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create many files, some orphaned
|
// Create many files, some orphaned
|
||||||
for i := range 20 {
|
for i := 0; i < 20; i++ {
|
||||||
file := &File{
|
file := &File{
|
||||||
Path: types.FilePath(fmt.Sprintf("/concurrent-%d.txt", i)),
|
Path: types.FilePath(fmt.Sprintf("/concurrent-%d.txt", i)),
|
||||||
MTime: time.Now().Truncate(time.Second),
|
MTime: time.Now().Truncate(time.Second),
|
||||||
@@ -854,7 +805,6 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
|
|||||||
UID: 1000,
|
UID: 1000,
|
||||||
GID: 1000,
|
GID: 1000,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Files.Create(ctx, nil, file)
|
err = repos.Files.Create(ctx, nil, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -872,15 +822,14 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
|
|||||||
// Run multiple cleanup operations concurrently
|
// Run multiple cleanup operations concurrently
|
||||||
// Note: SQLite has limited support for concurrent writes, so we expect some to fail
|
// Note: SQLite has limited support for concurrent writes, so we expect some to fail
|
||||||
done := make(chan error, 3)
|
done := make(chan error, 3)
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
for range 3 {
|
|
||||||
go func() {
|
go func() {
|
||||||
done <- repos.Files.DeleteOrphaned(ctx)
|
done <- repos.Files.DeleteOrphaned(ctx)
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wait for all to complete
|
// Wait for all to complete
|
||||||
for i := range 3 {
|
for i := 0; i < 3; i++ {
|
||||||
err := <-done
|
err := <-done
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("cleanup %d failed: %v", i, err)
|
t.Errorf("cleanup %d failed: %v", i, err)
|
||||||
@@ -901,12 +850,10 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
|
|||||||
// Verify all remaining files are even-numbered
|
// Verify all remaining files are even-numbered
|
||||||
for _, file := range files {
|
for _, file := range files {
|
||||||
var num int
|
var num int
|
||||||
|
|
||||||
_, err := fmt.Sscanf(file.Path.String(), "/concurrent-%d.txt", &num)
|
_, err := fmt.Sscanf(file.Path.String(), "/concurrent-%d.txt", &num)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Logf("failed to parse file number from %s: %v", file.Path, err)
|
t.Logf("failed to parse file number from %s: %v", file.Path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if num%2 != 0 {
|
if num%2 != 0 {
|
||||||
t.Errorf("odd-numbered file %s should have been deleted", file.Path)
|
t.Errorf("odd-numbered file %s should have been deleted", file.Path)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -36,14 +36,12 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file1: %v", err)
|
t.Fatalf("failed to create file1: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("Created file1 with ID: %s", file1.ID)
|
t.Logf("Created file1 with ID: %s", file1.ID)
|
||||||
|
|
||||||
err = repos.Files.Create(ctx, nil, file2)
|
err = repos.Files.Create(ctx, nil, file2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file2: %v", err)
|
t.Fatalf("failed to create file2: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("Created file2 with ID: %s", file2.ID)
|
t.Logf("Created file2 with ID: %s", file2.ID)
|
||||||
|
|
||||||
// Create a snapshot and reference only file2
|
// Create a snapshot and reference only file2
|
||||||
@@ -52,22 +50,18 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
|||||||
Hostname: "test-host",
|
Hostname: "test-host",
|
||||||
StartedAt: time.Now(),
|
StartedAt: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Snapshots.Create(ctx, nil, snapshot)
|
err = repos.Snapshots.Create(ctx, nil, snapshot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create snapshot: %v", err)
|
t.Fatalf("failed to create snapshot: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("Created snapshot: %s", snapshot.ID)
|
t.Logf("Created snapshot: %s", snapshot.ID)
|
||||||
|
|
||||||
// Check snapshot_files before adding
|
// Check snapshot_files before adding
|
||||||
var count int
|
var count int
|
||||||
|
|
||||||
err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count)
|
err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("snapshot_files count before add: %d", count)
|
t.Logf("snapshot_files count before add: %d", count)
|
||||||
|
|
||||||
// Add file2 to snapshot
|
// Add file2 to snapshot
|
||||||
@@ -75,7 +69,6 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to add file to snapshot: %v", err)
|
t.Fatalf("failed to add file to snapshot: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("Added file2 to snapshot")
|
t.Logf("Added file2 to snapshot")
|
||||||
|
|
||||||
// Check snapshot_files after adding
|
// Check snapshot_files after adding
|
||||||
@@ -83,7 +76,6 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("snapshot_files count after add: %d", count)
|
t.Logf("snapshot_files count after add: %d", count)
|
||||||
|
|
||||||
// Check which files are referenced
|
// Check which files are referenced
|
||||||
@@ -92,22 +84,16 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
err := rows.Close()
|
if err := rows.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Logf("failed to close rows: %v", err)
|
t.Logf("failed to close rows: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
t.Log("Files in snapshot_files:")
|
t.Log("Files in snapshot_files:")
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var fileID string
|
var fileID string
|
||||||
|
if err := rows.Scan(&fileID); err != nil {
|
||||||
err := rows.Scan(&fileID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf(" - %s", fileID)
|
t.Logf(" - %s", fileID)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -116,7 +102,6 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("Files count before cleanup: %d", count)
|
t.Logf("Files count before cleanup: %d", count)
|
||||||
|
|
||||||
// Run orphaned cleanup
|
// Run orphaned cleanup
|
||||||
@@ -124,7 +109,6 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to delete orphaned files: %v", err)
|
t.Fatalf("failed to delete orphaned files: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Log("Ran orphaned cleanup")
|
t.Log("Ran orphaned cleanup")
|
||||||
|
|
||||||
// Check files after cleanup
|
// Check files after cleanup
|
||||||
@@ -132,7 +116,6 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("Files count after cleanup: %d", count)
|
t.Logf("Files count after cleanup: %d", count)
|
||||||
|
|
||||||
// List remaining files
|
// List remaining files
|
||||||
@@ -140,9 +123,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Log("Remaining files:")
|
t.Log("Remaining files:")
|
||||||
|
|
||||||
for _, f := range files {
|
for _, f := range files {
|
||||||
t.Logf(" - ID: %s, Path: %s", f.ID, f.Path)
|
t.Logf(" - ID: %s, Path: %s", f.ID, f.Path)
|
||||||
}
|
}
|
||||||
@@ -152,12 +133,10 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error getting file: %v", err)
|
t.Fatalf("error getting file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if orphanedFile != nil {
|
if orphanedFile != nil {
|
||||||
t.Error("orphaned file should have been deleted")
|
t.Error("orphaned file should have been deleted")
|
||||||
// Let's check why it wasn't deleted
|
// Let's check why it wasn't deleted
|
||||||
var exists bool
|
var exists bool
|
||||||
|
|
||||||
err = db.conn.QueryRow(`
|
err = db.conn.QueryRow(`
|
||||||
SELECT EXISTS(
|
SELECT EXISTS(
|
||||||
SELECT 1 FROM snapshot_files
|
SELECT 1 FROM snapshot_files
|
||||||
@@ -166,7 +145,6 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("File1 exists in snapshot_files: %v", exists)
|
t.Logf("File1 exists in snapshot_files: %v", exists)
|
||||||
} else {
|
} else {
|
||||||
t.Log("Orphaned file was correctly deleted")
|
t.Log("Orphaned file was correctly deleted")
|
||||||
@@ -177,7 +155,6 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error getting file: %v", err)
|
t.Fatalf("error getting file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if referencedFile == nil {
|
if referencedFile == nil {
|
||||||
t.Error("referenced file should not have been deleted")
|
t.Error("referenced file should not have been deleted")
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestFileRepositoryEdgeCases tests edge cases for file repository
|
// TestFileRepositoryEdgeCases tests edge cases for file repository
|
||||||
@@ -98,7 +98,6 @@ func TestFileRepositoryEdgeCases(t *testing.T) {
|
|||||||
if (err != nil) != tt.wantErr {
|
if (err != nil) != tt.wantErr {
|
||||||
t.Errorf("Create() error = %v, wantErr %v", err, tt.wantErr)
|
t.Errorf("Create() error = %v, wantErr %v", err, tt.wantErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil && tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
|
if err != nil && tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
|
||||||
t.Errorf("Create() error = %v, want error containing %q", err, tt.errMsg)
|
t.Errorf("Create() error = %v, want error containing %q", err, tt.errMsg)
|
||||||
}
|
}
|
||||||
@@ -137,7 +136,6 @@ func TestDuplicateHandling(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file1: %v", err)
|
t.Fatalf("failed to create file1: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
originalID := file1.ID
|
originalID := file1.ID
|
||||||
|
|
||||||
// Create with same path should update the existing record (UPSERT behavior)
|
// Create with same path should update the existing record (UPSERT behavior)
|
||||||
@@ -192,7 +190,6 @@ func TestDuplicateHandling(t *testing.T) {
|
|||||||
UID: 1000,
|
UID: 1000,
|
||||||
GID: 1000,
|
GID: 1000,
|
||||||
}
|
}
|
||||||
|
|
||||||
err := repos.Files.Create(ctx, nil, file)
|
err := repos.Files.Create(ctx, nil, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -202,7 +199,6 @@ func TestDuplicateHandling(t *testing.T) {
|
|||||||
ChunkHash: types.ChunkHash("test-chunk-dup"),
|
ChunkHash: types.ChunkHash("test-chunk-dup"),
|
||||||
Size: 1024,
|
Size: 1024,
|
||||||
}
|
}
|
||||||
|
|
||||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -329,7 +325,6 @@ func TestLargeDatasets(t *testing.T) {
|
|||||||
Hostname: "test-host",
|
Hostname: "test-host",
|
||||||
StartedAt: time.Now(),
|
StartedAt: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
err := repos.Snapshots.Create(ctx, nil, snapshot)
|
err := repos.Snapshots.Create(ctx, nil, snapshot)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -337,12 +332,11 @@ func TestLargeDatasets(t *testing.T) {
|
|||||||
|
|
||||||
// Create many files
|
// Create many files
|
||||||
const fileCount = 1000
|
const fileCount = 1000
|
||||||
|
|
||||||
fileIDs := make([]types.FileID, fileCount)
|
fileIDs := make([]types.FileID, fileCount)
|
||||||
|
|
||||||
t.Run("create many files", func(t *testing.T) {
|
t.Run("create many files", func(t *testing.T) {
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
for i := range fileCount {
|
for i := 0; i < fileCount; i++ {
|
||||||
file := &File{
|
file := &File{
|
||||||
Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)),
|
Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)),
|
||||||
MTime: time.Now(),
|
MTime: time.Now(),
|
||||||
@@ -351,12 +345,10 @@ func TestLargeDatasets(t *testing.T) {
|
|||||||
UID: uint32(1000 + (i % 10)),
|
UID: uint32(1000 + (i % 10)),
|
||||||
GID: uint32(1000 + (i % 10)),
|
GID: uint32(1000 + (i % 10)),
|
||||||
}
|
}
|
||||||
|
|
||||||
err := repos.Files.Create(ctx, nil, file)
|
err := repos.Files.Create(ctx, nil, file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create file %d: %v", i, err)
|
t.Fatalf("failed to create file %d: %v", i, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fileIDs[i] = file.ID
|
fileIDs[i] = file.ID
|
||||||
|
|
||||||
// Add half to snapshot
|
// Add half to snapshot
|
||||||
@@ -367,35 +359,29 @@ func TestLargeDatasets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("Created %d files in %v", fileCount, time.Since(start))
|
t.Logf("Created %d files in %v", fileCount, time.Since(start))
|
||||||
})
|
})
|
||||||
|
|
||||||
// Test ListByPrefix performance
|
// Test ListByPrefix performance
|
||||||
t.Run("list by prefix performance", func(t *testing.T) {
|
t.Run("list by prefix performance", func(t *testing.T) {
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
|
|
||||||
files, err := repos.Files.ListByPrefix(ctx, "/large/")
|
files, err := repos.Files.ListByPrefix(ctx, "/large/")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(files) != fileCount {
|
if len(files) != fileCount {
|
||||||
t.Errorf("expected %d files, got %d", fileCount, len(files))
|
t.Errorf("expected %d files, got %d", fileCount, len(files))
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("Listed %d files in %v", len(files), time.Since(start))
|
t.Logf("Listed %d files in %v", len(files), time.Since(start))
|
||||||
})
|
})
|
||||||
|
|
||||||
// Test orphaned cleanup performance
|
// Test orphaned cleanup performance
|
||||||
t.Run("orphaned cleanup performance", func(t *testing.T) {
|
t.Run("orphaned cleanup performance", func(t *testing.T) {
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
|
|
||||||
err := repos.Files.DeleteOrphaned(ctx)
|
err := repos.Files.DeleteOrphaned(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Logf("Cleaned up orphaned files in %v", time.Since(start))
|
t.Logf("Cleaned up orphaned files in %v", time.Since(start))
|
||||||
|
|
||||||
// Verify correct number remain
|
// Verify correct number remain
|
||||||
@@ -403,7 +389,6 @@ func TestLargeDatasets(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(files) != fileCount/2 {
|
if len(files) != fileCount/2 {
|
||||||
t.Errorf("expected %d files after cleanup, got %d", fileCount/2, len(files))
|
t.Errorf("expected %d files after cleanup, got %d", fileCount/2, len(files))
|
||||||
}
|
}
|
||||||
@@ -424,7 +409,6 @@ func TestErrorPropagation(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("GetByID should not return error for non-existent ID, got: %v", err)
|
t.Errorf("GetByID should not return error for non-existent ID, got: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if file != nil {
|
if file != nil {
|
||||||
t.Error("expected nil file for non-existent ID")
|
t.Error("expected nil file for non-existent ID")
|
||||||
}
|
}
|
||||||
@@ -436,7 +420,6 @@ func TestErrorPropagation(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("GetByPath should not return error for non-existent path, got: %v", err)
|
t.Errorf("GetByPath should not return error for non-existent path, got: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if file != nil {
|
if file != nil {
|
||||||
t.Error("expected nil file for non-existent path")
|
t.Error("expected nil file for non-existent path")
|
||||||
}
|
}
|
||||||
@@ -449,12 +432,10 @@ func TestErrorPropagation(t *testing.T) {
|
|||||||
Idx: 0,
|
Idx: 0,
|
||||||
ChunkHash: types.ChunkHash("some-chunk"),
|
ChunkHash: types.ChunkHash("some-chunk"),
|
||||||
}
|
}
|
||||||
|
|
||||||
err := repos.FileChunks.Create(ctx, nil, fc)
|
err := repos.FileChunks.Create(ctx, nil, fc)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Error("expected error for invalid foreign key")
|
t.Error("expected error for invalid foreign key")
|
||||||
}
|
}
|
||||||
|
|
||||||
if !strings.Contains(err.Error(), "FOREIGN KEY") {
|
if !strings.Contains(err.Error(), "FOREIGN KEY") {
|
||||||
t.Errorf("expected foreign key error, got: %v", err)
|
t.Errorf("expected foreign key error, got: %v", err)
|
||||||
}
|
}
|
||||||
@@ -494,7 +475,6 @@ func TestQueryInjection(t *testing.T) {
|
|||||||
|
|
||||||
// Verify tables still exist
|
// Verify tables still exist
|
||||||
var count int
|
var count int
|
||||||
|
|
||||||
err := db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
|
err := db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal("files table was damaged by injection")
|
t.Fatal("files table was damaged by injection")
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
-- Migration 001: Initial Vaultik schema
|
-- Vaultik Database Schema
|
||||||
-- All core tables for tracking files, chunks, blobs, snapshots, and uploads.
|
-- Note: This database does not support migrations. If the schema changes,
|
||||||
|
-- delete the local database and perform a full backup to recreate it.
|
||||||
|
|
||||||
-- Files table: stores metadata about files in the filesystem
|
-- Files table: stores metadata about files in the filesystem
|
||||||
CREATE TABLE IF NOT EXISTS files (
|
CREATE TABLE IF NOT EXISTS files (
|
||||||
@@ -133,17 +134,3 @@ CREATE TABLE IF NOT EXISTS uploads (
|
|||||||
|
|
||||||
-- Index for efficient snapshot lookups
|
-- Index for efficient snapshot lookups
|
||||||
CREATE INDEX IF NOT EXISTS idx_uploads_snapshot_id ON uploads(snapshot_id);
|
CREATE INDEX IF NOT EXISTS idx_uploads_snapshot_id ON uploads(snapshot_id);
|
||||||
|
|
||||||
-- Local metadata: keyed, host-local settings that bind the state of the
|
|
||||||
-- local index database to external context. The primary use is
|
|
||||||
-- storage_url: once a backup writes blobs to a destination, the local
|
|
||||||
-- index is only valid against that destination — if the configured
|
|
||||||
-- storage_url later changes, the scanner would silently think already-
|
|
||||||
-- known chunks are still on the new (empty) destination and skip
|
|
||||||
-- uploading them, corrupting future snapshots. On every mutating
|
|
||||||
-- command startup, we compare the configured storage_url to the stored
|
|
||||||
-- one and refuse to proceed on mismatch.
|
|
||||||
CREATE TABLE IF NOT EXISTS local_meta (
|
|
||||||
key TEXT PRIMARY KEY,
|
|
||||||
value TEXT NOT NULL
|
|
||||||
);
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
-- Migration 000: Schema migrations tracking table
|
|
||||||
-- Applied as a bootstrap step before the normal migration loop.
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
||||||
version INTEGER PRIMARY KEY,
|
|
||||||
applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
||||||
);
|
|
||||||
|
|
||||||
INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);
|
|
||||||
11
internal/database/schema/008_uploads.sql
Normal file
11
internal/database/schema/008_uploads.sql
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
-- Track blob upload metrics
|
||||||
|
CREATE TABLE IF NOT EXISTS uploads (
|
||||||
|
blob_hash TEXT PRIMARY KEY,
|
||||||
|
uploaded_at TIMESTAMP NOT NULL,
|
||||||
|
size INTEGER NOT NULL,
|
||||||
|
duration_ms INTEGER NOT NULL,
|
||||||
|
FOREIGN KEY (blob_hash) REFERENCES blobs(blob_hash)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_uploads_uploaded_at ON uploads(uploaded_at);
|
||||||
|
CREATE INDEX idx_uploads_duration ON uploads(duration_ms);
|
||||||
@@ -3,12 +3,10 @@ package database
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
type SnapshotRepository struct {
|
type SnapshotRepository struct {
|
||||||
@@ -28,7 +26,6 @@ func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *S
|
|||||||
`
|
`
|
||||||
|
|
||||||
var completedAt *int64
|
var completedAt *int64
|
||||||
|
|
||||||
if snapshot.CompletedAt != nil {
|
if snapshot.CompletedAt != nil {
|
||||||
ts := snapshot.CompletedAt.Unix()
|
ts := snapshot.CompletedAt.Unix()
|
||||||
completedAt = &ts
|
completedAt = &ts
|
||||||
@@ -87,11 +84,9 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps
|
|||||||
func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx, snapshotID string, blobUncompressedSize int64, compressionLevel int, uploadDurationMs int64) error {
|
func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx, snapshotID string, blobUncompressedSize int64, compressionLevel int, uploadDurationMs int64) error {
|
||||||
// Calculate compression ratio based on uncompressed vs compressed sizes
|
// Calculate compression ratio based on uncompressed vs compressed sizes
|
||||||
var compressionRatio float64
|
var compressionRatio float64
|
||||||
|
|
||||||
if blobUncompressedSize > 0 {
|
if blobUncompressedSize > 0 {
|
||||||
// Get current blob_size from DB to calculate ratio
|
// Get current blob_size from DB to calculate ratio
|
||||||
var blobSize int64
|
var blobSize int64
|
||||||
|
|
||||||
queryGet := `SELECT blob_size FROM snapshots WHERE id = ?`
|
queryGet := `SELECT blob_size FROM snapshots WHERE id = ?`
|
||||||
if tx != nil {
|
if tx != nil {
|
||||||
err := tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
|
err := tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
|
||||||
@@ -104,7 +99,6 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx
|
|||||||
return fmt.Errorf("getting blob size: %w", err)
|
return fmt.Errorf("getting blob size: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
compressionRatio = float64(blobSize) / float64(blobUncompressedSize)
|
compressionRatio = float64(blobSize) / float64(blobUncompressedSize)
|
||||||
} else {
|
} else {
|
||||||
compressionRatio = 1.0
|
compressionRatio = 1.0
|
||||||
@@ -130,7 +124,6 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("updating extended stats: %w", err)
|
return fmt.Errorf("updating extended stats: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -143,11 +136,9 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
|
|||||||
WHERE id = ?
|
WHERE id = ?
|
||||||
`
|
`
|
||||||
|
|
||||||
var (
|
var snapshot Snapshot
|
||||||
snapshot Snapshot
|
var startedAtUnix int64
|
||||||
startedAtUnix int64
|
var completedAtUnix *int64
|
||||||
completedAtUnix *int64
|
|
||||||
)
|
|
||||||
|
|
||||||
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(
|
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(
|
||||||
&snapshot.ID,
|
&snapshot.ID,
|
||||||
@@ -168,10 +159,9 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
|
|||||||
&snapshot.UploadDurationMs,
|
&snapshot.UploadDurationMs,
|
||||||
)
|
)
|
||||||
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if err == sql.ErrNoRows {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("querying snapshot: %w", err)
|
return nil, fmt.Errorf("querying snapshot: %w", err)
|
||||||
}
|
}
|
||||||
@@ -200,13 +190,10 @@ func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snap
|
|||||||
defer CloseRows(rows)
|
defer CloseRows(rows)
|
||||||
|
|
||||||
var snapshots []*Snapshot
|
var snapshots []*Snapshot
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var (
|
var snapshot Snapshot
|
||||||
snapshot Snapshot
|
var startedAtUnix int64
|
||||||
startedAtUnix int64
|
var completedAtUnix *int64
|
||||||
completedAtUnix *int64
|
|
||||||
)
|
|
||||||
|
|
||||||
err := rows.Scan(
|
err := rows.Scan(
|
||||||
&snapshot.ID,
|
&snapshot.ID,
|
||||||
@@ -314,35 +301,28 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx,
|
|||||||
const batchSize = 400
|
const batchSize = 400
|
||||||
|
|
||||||
for i := 0; i < len(fileIDs); i += batchSize {
|
for i := 0; i < len(fileIDs); i += batchSize {
|
||||||
end := min(i+batchSize, len(fileIDs))
|
end := i + batchSize
|
||||||
|
if end > len(fileIDs) {
|
||||||
|
end = len(fileIDs)
|
||||||
|
}
|
||||||
batch := fileIDs[i:end]
|
batch := fileIDs[i:end]
|
||||||
|
|
||||||
query := "INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) VALUES "
|
query := "INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) VALUES "
|
||||||
|
args := make([]interface{}, 0, len(batch)*2)
|
||||||
args := make([]any, 0, len(batch)*2)
|
|
||||||
|
|
||||||
var querySb312 strings.Builder
|
|
||||||
|
|
||||||
for j, fileID := range batch {
|
for j, fileID := range batch {
|
||||||
if j > 0 {
|
if j > 0 {
|
||||||
querySb312.WriteString(", ")
|
query += ", "
|
||||||
}
|
}
|
||||||
|
query += "(?, ?)"
|
||||||
querySb312.WriteString("(?, ?)")
|
|
||||||
|
|
||||||
args = append(args, snapshotID, fileID.String())
|
args = append(args, snapshotID, fileID.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
query += querySb312.String()
|
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
if tx != nil {
|
if tx != nil {
|
||||||
_, err = tx.ExecContext(ctx, query, args...)
|
_, err = tx.ExecContext(ctx, query, args...)
|
||||||
} else {
|
} else {
|
||||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("batch adding files to snapshot: %w", err)
|
return fmt.Errorf("batch adding files to snapshot: %w", err)
|
||||||
}
|
}
|
||||||
@@ -351,47 +331,6 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PopulateReferencedBlobs ensures snapshot_blobs contains an entry for
|
|
||||||
// every blob that holds a chunk referenced by any file in the snapshot.
|
|
||||||
// This is necessary because the AddBlob hook only runs when a blob is
|
|
||||||
// newly uploaded during a snapshot — fully-deduplicated snapshots (where
|
|
||||||
// every chunk already exists in storage from a prior run) would otherwise
|
|
||||||
// have an empty snapshot_blobs set and be impossible to restore.
|
|
||||||
//
|
|
||||||
// Returns the number of rows inserted (i.e. blobs that were previously
|
|
||||||
// referenced indirectly via file_chunks but not yet recorded in
|
|
||||||
// snapshot_blobs for this snapshot).
|
|
||||||
func (r *SnapshotRepository) PopulateReferencedBlobs(ctx context.Context, tx *sql.Tx, snapshotID string) (int64, error) {
|
|
||||||
query := `
|
|
||||||
INSERT OR IGNORE INTO snapshot_blobs (snapshot_id, blob_id, blob_hash)
|
|
||||||
SELECT DISTINCT ?, blobs.id, blobs.blob_hash
|
|
||||||
FROM blobs
|
|
||||||
JOIN blob_chunks ON blob_chunks.blob_id = blobs.id
|
|
||||||
JOIN file_chunks ON file_chunks.chunk_hash = blob_chunks.chunk_hash
|
|
||||||
JOIN snapshot_files ON snapshot_files.file_id = file_chunks.file_id
|
|
||||||
WHERE snapshot_files.snapshot_id = ?
|
|
||||||
AND blobs.blob_hash IS NOT NULL
|
|
||||||
`
|
|
||||||
|
|
||||||
var (
|
|
||||||
result sql.Result
|
|
||||||
err error
|
|
||||||
)
|
|
||||||
if tx != nil {
|
|
||||||
result, err = tx.ExecContext(ctx, query, snapshotID, snapshotID)
|
|
||||||
} else {
|
|
||||||
result, err = r.db.ExecWithLog(ctx, query, snapshotID, snapshotID)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("populating referenced blobs: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
n, _ := result.RowsAffected()
|
|
||||||
|
|
||||||
return n, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddBlob adds a blob to a snapshot
|
// AddBlob adds a blob to a snapshot
|
||||||
func (r *SnapshotRepository) AddBlob(ctx context.Context, tx *sql.Tx, snapshotID string, blobID types.BlobID, blobHash types.BlobHash) error {
|
func (r *SnapshotRepository) AddBlob(ctx context.Context, tx *sql.Tx, snapshotID string, blobID types.BlobID, blobHash types.BlobHash) error {
|
||||||
query := `
|
query := `
|
||||||
@@ -429,15 +368,11 @@ func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID strin
|
|||||||
defer CloseRows(rows)
|
defer CloseRows(rows)
|
||||||
|
|
||||||
var blobs []string
|
var blobs []string
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var blobHash string
|
var blobHash string
|
||||||
|
if err := rows.Scan(&blobHash); err != nil {
|
||||||
err := rows.Scan(&blobHash)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("scanning blob hash: %w", err)
|
return nil, fmt.Errorf("scanning blob hash: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
blobs = append(blobs, blobHash)
|
blobs = append(blobs, blobHash)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -454,7 +389,6 @@ func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context,
|
|||||||
`
|
`
|
||||||
|
|
||||||
var totalSize int64
|
var totalSize int64
|
||||||
|
|
||||||
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
|
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, fmt.Errorf("querying total compressed size: %w", err)
|
return 0, fmt.Errorf("querying total compressed size: %w", err)
|
||||||
@@ -463,67 +397,6 @@ func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context,
|
|||||||
return totalSize, nil
|
return totalSize, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetSnapshotUncompressedChunkSize returns the sum of plaintext sizes of all unique
|
|
||||||
// chunks referenced by a snapshot (via snapshot_files → file_chunks → chunks).
|
|
||||||
func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(ctx context.Context, snapshotID string) (int64, error) {
|
|
||||||
query := `
|
|
||||||
SELECT COALESCE(SUM(c.size), 0)
|
|
||||||
FROM (
|
|
||||||
SELECT DISTINCT fc.chunk_hash
|
|
||||||
FROM snapshot_files sf
|
|
||||||
JOIN file_chunks fc ON sf.file_id = fc.file_id
|
|
||||||
WHERE sf.snapshot_id = ?
|
|
||||||
) sc
|
|
||||||
JOIN chunks c ON sc.chunk_hash = c.chunk_hash
|
|
||||||
`
|
|
||||||
|
|
||||||
var totalSize int64
|
|
||||||
|
|
||||||
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("querying uncompressed chunk size: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return totalSize, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetSnapshotNewChunkSize returns the sum of plaintext sizes of chunks that are
|
|
||||||
// referenced by this snapshot but not by any earlier completed snapshot known to
|
|
||||||
// the local database. The result is the marginal uncompressed data this snapshot
|
|
||||||
// added to the dedup pool — i.e., the delta from prior snapshots.
|
|
||||||
func (r *SnapshotRepository) GetSnapshotNewChunkSize(ctx context.Context, snapshotID string) (int64, error) {
|
|
||||||
query := `
|
|
||||||
WITH this_snap_chunks AS (
|
|
||||||
SELECT DISTINCT fc.chunk_hash
|
|
||||||
FROM snapshot_files sf
|
|
||||||
JOIN file_chunks fc ON sf.file_id = fc.file_id
|
|
||||||
WHERE sf.snapshot_id = ?
|
|
||||||
),
|
|
||||||
prior_chunks AS (
|
|
||||||
SELECT DISTINCT fc.chunk_hash
|
|
||||||
FROM snapshots s
|
|
||||||
JOIN snapshot_files sf ON sf.snapshot_id = s.id
|
|
||||||
JOIN file_chunks fc ON fc.file_id = sf.file_id
|
|
||||||
WHERE s.completed_at IS NOT NULL
|
|
||||||
AND s.id != ?
|
|
||||||
AND s.started_at < (SELECT started_at FROM snapshots WHERE id = ?)
|
|
||||||
)
|
|
||||||
SELECT COALESCE(SUM(c.size), 0)
|
|
||||||
FROM chunks c
|
|
||||||
JOIN this_snap_chunks t ON c.chunk_hash = t.chunk_hash
|
|
||||||
WHERE c.chunk_hash NOT IN (SELECT chunk_hash FROM prior_chunks)
|
|
||||||
`
|
|
||||||
|
|
||||||
var totalSize int64
|
|
||||||
|
|
||||||
err := r.db.conn.QueryRowContext(ctx, query, snapshotID, snapshotID, snapshotID).Scan(&totalSize)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("querying new chunk size: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return totalSize, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetIncompleteSnapshots returns all snapshots that haven't been completed
|
// GetIncompleteSnapshots returns all snapshots that haven't been completed
|
||||||
func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Snapshot, error) {
|
func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Snapshot, error) {
|
||||||
query := `
|
query := `
|
||||||
@@ -540,13 +413,10 @@ func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Sna
|
|||||||
defer CloseRows(rows)
|
defer CloseRows(rows)
|
||||||
|
|
||||||
var snapshots []*Snapshot
|
var snapshots []*Snapshot
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var (
|
var snapshot Snapshot
|
||||||
snapshot Snapshot
|
var startedAtUnix int64
|
||||||
startedAtUnix int64
|
var completedAtUnix *int64
|
||||||
completedAtUnix *int64
|
|
||||||
)
|
|
||||||
|
|
||||||
err := rows.Scan(
|
err := rows.Scan(
|
||||||
&snapshot.ID,
|
&snapshot.ID,
|
||||||
@@ -594,13 +464,10 @@ func (r *SnapshotRepository) GetIncompleteByHostname(ctx context.Context, hostna
|
|||||||
defer CloseRows(rows)
|
defer CloseRows(rows)
|
||||||
|
|
||||||
var snapshots []*Snapshot
|
var snapshots []*Snapshot
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var (
|
var snapshot Snapshot
|
||||||
snapshot Snapshot
|
var startedAtUnix int64
|
||||||
startedAtUnix int64
|
var completedAtUnix *int64
|
||||||
completedAtUnix *int64
|
|
||||||
)
|
|
||||||
|
|
||||||
err := rows.Scan(
|
err := rows.Scan(
|
||||||
&snapshot.ID,
|
&snapshot.ID,
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -52,19 +52,15 @@ func TestSnapshotRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get snapshot: %v", err)
|
t.Fatalf("failed to get snapshot: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved == nil {
|
if retrieved == nil {
|
||||||
t.Fatal("expected snapshot, got nil")
|
t.Fatal("expected snapshot, got nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.ID != snapshot.ID {
|
if retrieved.ID != snapshot.ID {
|
||||||
t.Errorf("ID mismatch: got %s, want %s", retrieved.ID, snapshot.ID)
|
t.Errorf("ID mismatch: got %s, want %s", retrieved.ID, snapshot.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.Hostname != snapshot.Hostname {
|
if retrieved.Hostname != snapshot.Hostname {
|
||||||
t.Errorf("hostname mismatch: got %s, want %s", retrieved.Hostname, snapshot.Hostname)
|
t.Errorf("hostname mismatch: got %s, want %s", retrieved.Hostname, snapshot.Hostname)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.FileCount != snapshot.FileCount {
|
if retrieved.FileCount != snapshot.FileCount {
|
||||||
t.Errorf("file count mismatch: got %d, want %d", retrieved.FileCount, snapshot.FileCount)
|
t.Errorf("file count mismatch: got %d, want %d", retrieved.FileCount, snapshot.FileCount)
|
||||||
}
|
}
|
||||||
@@ -79,27 +75,21 @@ func TestSnapshotRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get updated snapshot: %v", err)
|
t.Fatalf("failed to get updated snapshot: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.FileCount != 200 {
|
if retrieved.FileCount != 200 {
|
||||||
t.Errorf("file count not updated: got %d, want %d", retrieved.FileCount, 200)
|
t.Errorf("file count not updated: got %d, want %d", retrieved.FileCount, 200)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.ChunkCount != 1000 {
|
if retrieved.ChunkCount != 1000 {
|
||||||
t.Errorf("chunk count not updated: got %d, want %d", retrieved.ChunkCount, 1000)
|
t.Errorf("chunk count not updated: got %d, want %d", retrieved.ChunkCount, 1000)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.BlobCount != 20 {
|
if retrieved.BlobCount != 20 {
|
||||||
t.Errorf("blob count not updated: got %d, want %d", retrieved.BlobCount, 20)
|
t.Errorf("blob count not updated: got %d, want %d", retrieved.BlobCount, 20)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.TotalSize != twoHundredMebibytes {
|
if retrieved.TotalSize != twoHundredMebibytes {
|
||||||
t.Errorf("total size not updated: got %d, want %d", retrieved.TotalSize, twoHundredMebibytes)
|
t.Errorf("total size not updated: got %d, want %d", retrieved.TotalSize, twoHundredMebibytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
if retrieved.BlobSize != sixtyMebibytes {
|
if retrieved.BlobSize != sixtyMebibytes {
|
||||||
t.Errorf("blob size not updated: got %d, want %d", retrieved.BlobSize, sixtyMebibytes)
|
t.Errorf("blob size not updated: got %d, want %d", retrieved.BlobSize, sixtyMebibytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
expectedRatio := compressionRatioPoint3 // 0.3
|
expectedRatio := compressionRatioPoint3 // 0.3
|
||||||
if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 {
|
if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 {
|
||||||
t.Errorf("compression ratio not updated: got %f, want %f", retrieved.CompressionRatio, expectedRatio)
|
t.Errorf("compression ratio not updated: got %f, want %f", retrieved.CompressionRatio, expectedRatio)
|
||||||
@@ -118,7 +108,6 @@ func TestSnapshotRepository(t *testing.T) {
|
|||||||
ChunkCount: int64(500 * i),
|
ChunkCount: int64(500 * i),
|
||||||
BlobCount: int64(10 * i),
|
BlobCount: int64(10 * i),
|
||||||
}
|
}
|
||||||
|
|
||||||
err := repo.Create(ctx, nil, s)
|
err := repo.Create(ctx, nil, s)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create snapshot %d: %v", i, err)
|
t.Fatalf("failed to create snapshot %d: %v", i, err)
|
||||||
@@ -130,13 +119,12 @@ func TestSnapshotRepository(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to list recent snapshots: %v", err)
|
t.Fatalf("failed to list recent snapshots: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(recent) != 3 {
|
if len(recent) != 3 {
|
||||||
t.Errorf("expected 3 recent snapshots, got %d", len(recent))
|
t.Errorf("expected 3 recent snapshots, got %d", len(recent))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify order (most recent first)
|
// Verify order (most recent first)
|
||||||
for i := range len(recent) - 1 {
|
for i := 0; i < len(recent)-1; i++ {
|
||||||
if recent[i].StartedAt.Before(recent[i+1].StartedAt) {
|
if recent[i].StartedAt.Before(recent[i+1].StartedAt) {
|
||||||
t.Error("snapshots not in descending order")
|
t.Error("snapshots not in descending order")
|
||||||
}
|
}
|
||||||
@@ -155,7 +143,6 @@ func TestSnapshotRepositoryNotFound(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if snapshot != nil {
|
if snapshot != nil {
|
||||||
t.Error("expected nil for non-existent snapshot")
|
t.Error("expected nil for non-existent snapshot")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,10 +3,9 @@ package database
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Upload represents a blob upload record
|
// Upload represents a blob upload record
|
||||||
@@ -54,7 +53,6 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
|
|||||||
`
|
`
|
||||||
|
|
||||||
var upload Upload
|
var upload Upload
|
||||||
|
|
||||||
err := r.conn.QueryRowContext(ctx, query, blobHash).Scan(
|
err := r.conn.QueryRowContext(ctx, query, blobHash).Scan(
|
||||||
&upload.BlobHash,
|
&upload.BlobHash,
|
||||||
&upload.UploadedAt,
|
&upload.UploadedAt,
|
||||||
@@ -62,10 +60,9 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
|
|||||||
&upload.DurationMs,
|
&upload.DurationMs,
|
||||||
)
|
)
|
||||||
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
if err == sql.ErrNoRows {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -87,22 +84,17 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]*
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
err := rows.Close()
|
if err := rows.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
log.Error("failed to close rows", "error", err)
|
log.Error("failed to close rows", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
var uploads []*Upload
|
var uploads []*Upload
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var upload Upload
|
var upload Upload
|
||||||
|
if err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs); err != nil {
|
||||||
err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
uploads = append(uploads, &upload)
|
uploads = append(uploads, &upload)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -123,7 +115,6 @@ func (r *UploadRepository) GetUploadStats(ctx context.Context, since time.Time)
|
|||||||
`
|
`
|
||||||
|
|
||||||
var stats UploadStats
|
var stats UploadStats
|
||||||
|
|
||||||
err := r.conn.QueryRowContext(ctx, query, since).Scan(
|
err := r.conn.QueryRowContext(ctx, query, since).Scan(
|
||||||
&stats.Count,
|
&stats.Count,
|
||||||
&stats.TotalSize,
|
&stats.TotalSize,
|
||||||
@@ -147,13 +138,10 @@ type UploadStats struct {
|
|||||||
// GetCountBySnapshot returns the count of uploads for a specific snapshot
|
// GetCountBySnapshot returns the count of uploads for a specific snapshot
|
||||||
func (r *UploadRepository) GetCountBySnapshot(ctx context.Context, snapshotID string) (int64, error) {
|
func (r *UploadRepository) GetCountBySnapshot(ctx context.Context, snapshotID string) (int64, error) {
|
||||||
query := `SELECT COUNT(*) FROM uploads WHERE snapshot_id = ?`
|
query := `SELECT COUNT(*) FROM uploads WHERE snapshot_id = ?`
|
||||||
|
|
||||||
var count int64
|
var count int64
|
||||||
|
|
||||||
err := r.conn.QueryRowContext(ctx, query, snapshotID).Scan(&count)
|
err := r.conn.QueryRowContext(ctx, query, snapshotID).Scan(&count)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return count, nil
|
return count, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,24 +13,11 @@ var Version string = "dev"
|
|||||||
// Commit is the git commit hash, populated from main().
|
// Commit is the git commit hash, populated from main().
|
||||||
var Commit string = "unknown"
|
var Commit string = "unknown"
|
||||||
|
|
||||||
// CommitDate is the ISO-8601 date of the commit, populated from main().
|
|
||||||
var CommitDate string = "unknown"
|
|
||||||
|
|
||||||
// Author identifies the upstream author of vaultik.
|
|
||||||
const Author = "Jeffrey Paul <sneak@sneak.berlin>"
|
|
||||||
|
|
||||||
// Homepage is the canonical URL for vaultik.
|
|
||||||
const Homepage = "https://sneak.berlin/go/vaultik"
|
|
||||||
|
|
||||||
// License is the SPDX identifier for the project license.
|
|
||||||
const License = "MIT"
|
|
||||||
|
|
||||||
// Globals contains application-wide configuration and metadata.
|
// Globals contains application-wide configuration and metadata.
|
||||||
type Globals struct {
|
type Globals struct {
|
||||||
Appname string
|
Appname string
|
||||||
Version string
|
Version string
|
||||||
Commit string
|
Commit string
|
||||||
CommitDate string
|
|
||||||
StartTime time.Time
|
StartTime time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -40,16 +27,5 @@ func New() (*Globals, error) {
|
|||||||
Appname: Appname,
|
Appname: Appname,
|
||||||
Version: Version,
|
Version: Version,
|
||||||
Commit: Commit,
|
Commit: Commit,
|
||||||
CommitDate: CommitDate,
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ShortCommit returns the first 12 chars of the commit hash, or the
|
|
||||||
// whole string if it's shorter (e.g. "unknown").
|
|
||||||
func (g *Globals) ShortCommit() string {
|
|
||||||
if len(g.Commit) > 12 {
|
|
||||||
return g.Commit[:12]
|
|
||||||
}
|
|
||||||
|
|
||||||
return g.Commit
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -46,12 +46,8 @@ func Initialize(cfg Config) {
|
|||||||
var level slog.Level
|
var level slog.Level
|
||||||
|
|
||||||
if cfg.Cron || cfg.Quiet {
|
if cfg.Cron || cfg.Quiet {
|
||||||
// In cron/quiet mode keep warnings and errors visible — the
|
// In quiet/cron mode, only show errors
|
||||||
// whole point of --cron is to stay silent only on total
|
level = slog.LevelError
|
||||||
// success, so that anything cron emails to root is genuinely
|
|
||||||
// "something went wrong, look at it." A backup with stuck
|
|
||||||
// permission errors or skipped files should NOT be silent.
|
|
||||||
level = slog.LevelWarn
|
|
||||||
} else if cfg.Debug || strings.Contains(os.Getenv("GODEBUG"), "vaultik") {
|
} else if cfg.Debug || strings.Contains(os.Getenv("GODEBUG"), "vaultik") {
|
||||||
level = slog.LevelDebug
|
level = slog.LevelDebug
|
||||||
} else if cfg.Verbose {
|
} else if cfg.Verbose {
|
||||||
@@ -84,7 +80,6 @@ func getCaller(skip int) string {
|
|||||||
if !ok {
|
if !ok {
|
||||||
return "unknown"
|
return "unknown"
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf("%s:%d", filepath.Base(file), line)
|
return fmt.Sprintf("%s:%d", filepath.Base(file), line)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -95,7 +90,6 @@ func Fatal(msg string, args ...any) {
|
|||||||
args = append(args, "caller", getCaller(2))
|
args = append(args, "caller", getCaller(2))
|
||||||
logger.Error(msg, args...)
|
logger.Error(msg, args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -174,7 +168,6 @@ func With(args ...any) *slog.Logger {
|
|||||||
if logger != nil {
|
if logger != nil {
|
||||||
return logger.With(args...)
|
return logger.With(args...)
|
||||||
}
|
}
|
||||||
|
|
||||||
return slog.Default()
|
return slog.Default()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -33,7 +33,6 @@ func NewTTYHandler(out io.Writer, opts *slog.HandlerOptions) *TTYHandler {
|
|||||||
if opts == nil {
|
if opts == nil {
|
||||||
opts = &slog.HandlerOptions{}
|
opts = &slog.HandlerOptions{}
|
||||||
}
|
}
|
||||||
|
|
||||||
return &TTYHandler{
|
return &TTYHandler{
|
||||||
out: out,
|
out: out,
|
||||||
opts: *opts,
|
opts: *opts,
|
||||||
@@ -55,9 +54,7 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error {
|
|||||||
|
|
||||||
// Level and color
|
// Level and color
|
||||||
level := r.Level.String()
|
level := r.Level.String()
|
||||||
|
|
||||||
var levelColor string
|
var levelColor string
|
||||||
|
|
||||||
switch r.Level {
|
switch r.Level {
|
||||||
case slog.LevelDebug:
|
case slog.LevelDebug:
|
||||||
levelColor = colorGray
|
levelColor = colorGray
|
||||||
@@ -99,12 +96,10 @@ func (h *TTYHandler) Handle(_ context.Context, r slog.Record) error {
|
|||||||
_, _ = fmt.Fprintf(h.out, " %s%s%s=%s%s%s",
|
_, _ = fmt.Fprintf(h.out, " %s%s%s=%s%s%s",
|
||||||
colorCyan, a.Key, colorReset,
|
colorCyan, a.Key, colorReset,
|
||||||
colorBlue, value, colorReset)
|
colorBlue, value, colorReset)
|
||||||
|
|
||||||
return true
|
return true
|
||||||
})
|
})
|
||||||
|
|
||||||
_, _ = fmt.Fprintln(h.out)
|
_, _ = fmt.Fprintln(h.out)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -127,7 +122,6 @@ func formatDuration(d time.Duration) string {
|
|||||||
} else if d < time.Minute {
|
} else if d < time.Minute {
|
||||||
return fmt.Sprintf("%.1fs", d.Seconds())
|
return fmt.Sprintf("%.1fs", d.Seconds())
|
||||||
}
|
}
|
||||||
|
|
||||||
return d.String()
|
return d.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -137,12 +131,10 @@ func formatBytes(b int64) string {
|
|||||||
if b < unit {
|
if b < unit {
|
||||||
return fmt.Sprintf("%d B", b)
|
return fmt.Sprintf("%d B", b)
|
||||||
}
|
}
|
||||||
|
|
||||||
div, exp := int64(unit), 0
|
div, exp := int64(unit), 0
|
||||||
for n := b / unit; n >= unit; n /= unit {
|
for n := b / unit; n >= unit; n /= unit {
|
||||||
div *= unit
|
div *= unit
|
||||||
exp++
|
exp++
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
|
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -63,3 +63,10 @@ type Chunk struct {
|
|||||||
Offset int64
|
Offset int64
|
||||||
Length int64
|
Length int64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DirtyPath represents a path marked for backup by inotify
|
||||||
|
type DirtyPath struct {
|
||||||
|
Path string
|
||||||
|
MarkedAt time.Time
|
||||||
|
EventType string // "create", "modify", "delete"
|
||||||
|
}
|
||||||
|
|||||||
@@ -72,13 +72,11 @@ func (l *Lock) Release() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
err = os.Remove(l.path)
|
if err := os.Remove(l.path); err != nil && !os.IsNotExist(err) {
|
||||||
if err != nil && !os.IsNotExist(err) {
|
|
||||||
return fmt.Errorf("removing PID file: %w", err)
|
return fmt.Errorf("removing PID file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
l.path = "" // Prevent double-release
|
l.path = "" // Prevent double-release
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -106,6 +104,5 @@ func isProcessRunning(pid int) bool {
|
|||||||
|
|
||||||
// On Unix, FindProcess always succeeds. We need to send signal 0 to check.
|
// On Unix, FindProcess always succeeds. We need to send signal 0 to check.
|
||||||
err = process.Signal(syscall.Signal(0))
|
err = process.Signal(syscall.Signal(0))
|
||||||
|
|
||||||
return err == nil
|
return err == nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,7 +40,6 @@ func TestAcquireBlocksSecondInstance(t *testing.T) {
|
|||||||
// Acquire first lock
|
// Acquire first lock
|
||||||
lock1, err := Acquire(tmpDir)
|
lock1, err := Acquire(tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
require.NotNil(t, lock1)
|
require.NotNil(t, lock1)
|
||||||
defer func() { _ = lock1.Release() }()
|
defer func() { _ = lock1.Release() }()
|
||||||
|
|
||||||
@@ -62,7 +61,6 @@ func TestAcquireWithStaleLock(t *testing.T) {
|
|||||||
// Should be able to acquire lock (stale lock is cleaned up)
|
// Should be able to acquire lock (stale lock is cleaned up)
|
||||||
lock, err := Acquire(tmpDir)
|
lock, err := Acquire(tmpDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
require.NotNil(t, lock)
|
require.NotNil(t, lock)
|
||||||
defer func() { _ = lock.Release() }()
|
defer func() { _ = lock.Release() }()
|
||||||
|
|
||||||
@@ -90,7 +88,6 @@ func TestReleaseIsIdempotent(t *testing.T) {
|
|||||||
|
|
||||||
func TestReleaseNilLock(t *testing.T) {
|
func TestReleaseNilLock(t *testing.T) {
|
||||||
var lock *Lock
|
var lock *Lock
|
||||||
|
|
||||||
err := lock.Release()
|
err := lock.Release()
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
}
|
}
|
||||||
@@ -101,7 +98,6 @@ func TestAcquireCreatesDirectory(t *testing.T) {
|
|||||||
|
|
||||||
lock, err := Acquire(nestedDir)
|
lock, err := Acquire(nestedDir)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
require.NotNil(t, lock)
|
require.NotNil(t, lock)
|
||||||
defer func() { _ = lock.Release() }()
|
defer func() { _ = lock.Release() }()
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package s3
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"io"
|
"io"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
||||||
@@ -11,7 +10,6 @@ import (
|
|||||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||||
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
|
||||||
"github.com/aws/smithy-go/logging"
|
"github.com/aws/smithy-go/logging"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -42,7 +40,7 @@ type Config struct {
|
|||||||
// Used to suppress SDK warnings about checksums.
|
// Used to suppress SDK warnings about checksums.
|
||||||
type nopLogger struct{}
|
type nopLogger struct{}
|
||||||
|
|
||||||
func (nopLogger) Logf(classification logging.Classification, format string, v ...any) {}
|
func (nopLogger) Logf(classification logging.Classification, format string, v ...interface{}) {}
|
||||||
|
|
||||||
// NewClient creates a new S3 client with the provided configuration.
|
// NewClient creates a new S3 client with the provided configuration.
|
||||||
// It establishes a connection to the S3-compatible storage service and
|
// It establishes a connection to the S3-compatible storage service and
|
||||||
@@ -92,7 +90,6 @@ func (c *Client) PutObject(ctx context.Context, key string, data io.Reader) erro
|
|||||||
Key: aws.String(fullKey),
|
Key: aws.String(fullKey),
|
||||||
Body: data,
|
Body: data,
|
||||||
})
|
})
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -138,7 +135,6 @@ func (c *Client) PutObjectWithProgress(ctx context.Context, key string, data io.
|
|||||||
// close the returned reader when done to avoid resource leaks.
|
// close the returned reader when done to avoid resource leaks.
|
||||||
func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, error) {
|
func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||||
fullKey := c.prefix + key
|
fullKey := c.prefix + key
|
||||||
|
|
||||||
result, err := c.s3Client.GetObject(ctx, &s3.GetObjectInput{
|
result, err := c.s3Client.GetObject(ctx, &s3.GetObjectInput{
|
||||||
Bucket: aws.String(c.bucket),
|
Bucket: aws.String(c.bucket),
|
||||||
Key: aws.String(fullKey),
|
Key: aws.String(fullKey),
|
||||||
@@ -146,7 +142,6 @@ func (c *Client) GetObject(ctx context.Context, key string) (io.ReadCloser, erro
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return result.Body, nil
|
return result.Body, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -159,7 +154,6 @@ func (c *Client) DeleteObject(ctx context.Context, key string) error {
|
|||||||
Bucket: aws.String(c.bucket),
|
Bucket: aws.String(c.bucket),
|
||||||
Key: aws.String(fullKey),
|
Key: aws.String(fullKey),
|
||||||
})
|
})
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -172,7 +166,6 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
|
|||||||
fullPrefix := c.prefix + prefix
|
fullPrefix := c.prefix + prefix
|
||||||
|
|
||||||
var keys []string
|
var keys []string
|
||||||
|
|
||||||
paginator := s3.NewListObjectsV2Paginator(c.s3Client, &s3.ListObjectsV2Input{
|
paginator := s3.NewListObjectsV2Paginator(c.s3Client, &s3.ListObjectsV2Input{
|
||||||
Bucket: aws.String(c.bucket),
|
Bucket: aws.String(c.bucket),
|
||||||
Prefix: aws.String(fullPrefix),
|
Prefix: aws.String(fullPrefix),
|
||||||
@@ -191,7 +184,6 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
|
|||||||
if len(key) > len(c.prefix) {
|
if len(key) > len(c.prefix) {
|
||||||
key = key[len(c.prefix):]
|
key = key[len(c.prefix):]
|
||||||
}
|
}
|
||||||
|
|
||||||
keys = append(keys, key)
|
keys = append(keys, key)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -206,23 +198,15 @@ func (c *Client) ListObjects(ctx context.Context, prefix string) ([]string, erro
|
|||||||
// Note: This method returns false for any error, not just "not found".
|
// Note: This method returns false for any error, not just "not found".
|
||||||
func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) {
|
func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) {
|
||||||
fullKey := c.prefix + key
|
fullKey := c.prefix + key
|
||||||
|
|
||||||
_, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
|
_, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||||
Bucket: aws.String(c.bucket),
|
Bucket: aws.String(c.bucket),
|
||||||
Key: aws.String(fullKey),
|
Key: aws.String(fullKey),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
var (
|
// Check if it's a not found error
|
||||||
notFound *s3types.NotFound
|
// TODO: Add proper error type checking
|
||||||
noSuchKey *s3types.NoSuchKey
|
|
||||||
)
|
|
||||||
if errors.As(err, ¬Found) || errors.As(err, &noSuchKey) {
|
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -258,7 +242,6 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive
|
|||||||
page, err := paginator.NextPage(ctx)
|
page, err := paginator.NextPage(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ch <- ObjectInfo{Err: err}
|
ch <- ObjectInfo{Err: err}
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -269,7 +252,6 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive
|
|||||||
if len(key) > len(c.prefix) {
|
if len(key) > len(c.prefix) {
|
||||||
key = key[len(c.prefix):]
|
key = key[len(c.prefix):]
|
||||||
}
|
}
|
||||||
|
|
||||||
ch <- ObjectInfo{
|
ch <- ObjectInfo{
|
||||||
Key: key,
|
Key: key,
|
||||||
Size: *obj.Size,
|
Size: *obj.Size,
|
||||||
@@ -288,7 +270,6 @@ func (c *Client) ListObjectsStream(ctx context.Context, prefix string, recursive
|
|||||||
// Returns an error if the object doesn't exist or if the operation fails.
|
// Returns an error if the object doesn't exist or if the operation fails.
|
||||||
func (c *Client) StatObject(ctx context.Context, key string) (*ObjectInfo, error) {
|
func (c *Client) StatObject(ctx context.Context, key string) (*ObjectInfo, error) {
|
||||||
fullKey := c.prefix + key
|
fullKey := c.prefix + key
|
||||||
|
|
||||||
result, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
|
result, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||||
Bucket: aws.String(c.bucket),
|
Bucket: aws.String(c.bucket),
|
||||||
Key: aws.String(fullKey),
|
Key: aws.String(fullKey),
|
||||||
@@ -327,7 +308,6 @@ func (c *Client) Endpoint() string {
|
|||||||
if c.endpoint == "" {
|
if c.endpoint == "" {
|
||||||
return "s3.amazonaws.com"
|
return "s3.amazonaws.com"
|
||||||
}
|
}
|
||||||
|
|
||||||
return c.endpoint
|
return c.endpoint
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -344,14 +324,11 @@ func (pr *progressReader) Read(p []byte) (int, error) {
|
|||||||
n, err := pr.reader.Read(p)
|
n, err := pr.reader.Read(p)
|
||||||
if n > 0 {
|
if n > 0 {
|
||||||
atomic.AddInt64(&pr.read, int64(n))
|
atomic.AddInt64(&pr.read, int64(n))
|
||||||
|
|
||||||
if pr.callback != nil {
|
if pr.callback != nil {
|
||||||
callbackErr := pr.callback(atomic.LoadInt64(&pr.read))
|
if callbackErr := pr.callback(atomic.LoadInt64(&pr.read)); callbackErr != nil {
|
||||||
if callbackErr != nil {
|
|
||||||
return n, callbackErr
|
return n, callbackErr
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return n, err
|
return n, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,14 +6,13 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/s3"
|
"git.eeqj.de/sneak/vaultik/internal/s3"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestClient(t *testing.T) {
|
func TestClient(t *testing.T) {
|
||||||
ts := NewTestServer(t)
|
ts := NewTestServer(t)
|
||||||
defer func() {
|
defer func() {
|
||||||
err := ts.Cleanup()
|
if err := ts.Cleanup(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Errorf("cleanup failed: %v", err)
|
t.Errorf("cleanup failed: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -36,7 +35,6 @@ func TestClient(t *testing.T) {
|
|||||||
// Test PutObject
|
// Test PutObject
|
||||||
testKey := "foo/bar.txt"
|
testKey := "foo/bar.txt"
|
||||||
testData := []byte("test data")
|
testData := []byte("test data")
|
||||||
|
|
||||||
err = client.PutObject(ctx, testKey, bytes.NewReader(testData))
|
err = client.PutObject(ctx, testKey, bytes.NewReader(testData))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to put object: %v", err)
|
t.Fatalf("failed to put object: %v", err)
|
||||||
@@ -48,8 +46,7 @@ func TestClient(t *testing.T) {
|
|||||||
t.Fatalf("failed to get object: %v", err)
|
t.Fatalf("failed to get object: %v", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
err := reader.Close()
|
if err := reader.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Errorf("failed to close reader: %v", err)
|
t.Errorf("failed to close reader: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -68,7 +65,6 @@ func TestClient(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to head object: %v", err)
|
t.Fatalf("failed to head object: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !exists {
|
if !exists {
|
||||||
t.Error("expected object to exist")
|
t.Error("expected object to exist")
|
||||||
}
|
}
|
||||||
@@ -78,11 +74,9 @@ func TestClient(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to list objects: %v", err)
|
t.Fatalf("failed to list objects: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(keys) != 1 {
|
if len(keys) != 1 {
|
||||||
t.Errorf("expected 1 key, got %d", len(keys))
|
t.Errorf("expected 1 key, got %d", len(keys))
|
||||||
}
|
}
|
||||||
|
|
||||||
if keys[0] != testKey {
|
if keys[0] != testKey {
|
||||||
t.Errorf("unexpected key: got %s, want %s", keys[0], testKey)
|
t.Errorf("unexpected key: got %s, want %s", keys[0], testKey)
|
||||||
}
|
}
|
||||||
@@ -98,7 +92,6 @@ func TestClient(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to head object after deletion: %v", err)
|
t.Fatalf("failed to head object after deletion: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if exists {
|
if exists {
|
||||||
t.Error("expected object to not exist after deletion")
|
t.Error("expected object to not exist after deletion")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,8 +3,8 @@ package s3
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Module exports S3 functionality as an fx module.
|
// Module exports S3 functionality as an fx module.
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package s3_test
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -58,8 +57,7 @@ func NewTestServer(t *testing.T) *TestServer {
|
|||||||
|
|
||||||
// Start server in background
|
// Start server in background
|
||||||
go func() {
|
go func() {
|
||||||
err := server.ListenAndServe()
|
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
|
||||||
t.Logf("test server error: %v", err)
|
t.Logf("test server error: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -79,7 +77,7 @@ func NewTestServer(t *testing.T) *TestServer {
|
|||||||
"",
|
"",
|
||||||
)),
|
)),
|
||||||
config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody),
|
config.WithClientLogMode(aws.LogRetries|aws.LogRequestWithBody|aws.LogResponseWithBody),
|
||||||
config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...any) {
|
config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...interface{}) {
|
||||||
// Capture logs to buffer instead of stdout
|
// Capture logs to buffer instead of stdout
|
||||||
fmt.Fprintf(logBuf, "SDK %s %s %s\n",
|
fmt.Fprintf(logBuf, "SDK %s %s %s\n",
|
||||||
time.Now().Format("2006/01/02 15:04:05"),
|
time.Now().Format("2006/01/02 15:04:05"),
|
||||||
@@ -127,8 +125,7 @@ func (ts *TestServer) Cleanup() error {
|
|||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
err := ts.server.Shutdown(ctx)
|
if err := ts.server.Shutdown(ctx); err != nil {
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -144,8 +141,7 @@ func (ts *TestServer) Client() *s3.Client {
|
|||||||
func TestBasicS3Operations(t *testing.T) {
|
func TestBasicS3Operations(t *testing.T) {
|
||||||
ts := NewTestServer(t)
|
ts := NewTestServer(t)
|
||||||
defer func() {
|
defer func() {
|
||||||
err := ts.Cleanup()
|
if err := ts.Cleanup(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Errorf("cleanup failed: %v", err)
|
t.Errorf("cleanup failed: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -176,8 +172,7 @@ func TestBasicS3Operations(t *testing.T) {
|
|||||||
t.Fatalf("failed to get object: %v", err)
|
t.Fatalf("failed to get object: %v", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
err := result.Body.Close()
|
if err := result.Body.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Errorf("failed to close body: %v", err)
|
t.Errorf("failed to close body: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -197,8 +192,7 @@ func TestBasicS3Operations(t *testing.T) {
|
|||||||
func TestBlobOperations(t *testing.T) {
|
func TestBlobOperations(t *testing.T) {
|
||||||
ts := NewTestServer(t)
|
ts := NewTestServer(t)
|
||||||
defer func() {
|
defer func() {
|
||||||
err := ts.Cleanup()
|
if err := ts.Cleanup(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Errorf("cleanup failed: %v", err)
|
t.Errorf("cleanup failed: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -261,8 +255,7 @@ func TestBlobOperations(t *testing.T) {
|
|||||||
func TestMetadataOperations(t *testing.T) {
|
func TestMetadataOperations(t *testing.T) {
|
||||||
ts := NewTestServer(t)
|
ts := NewTestServer(t)
|
||||||
defer func() {
|
defer func() {
|
||||||
err := ts.Cleanup()
|
if err := ts.Cleanup(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Errorf("cleanup failed: %v", err)
|
t.Errorf("cleanup failed: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|||||||
@@ -4,8 +4,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/hex"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
@@ -15,8 +13,8 @@ import (
|
|||||||
"testing/fstest"
|
"testing/fstest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
// MockS3Client is a mock implementation of S3 operations for testing
|
// MockS3Client is a mock implementation of S3 operations for testing
|
||||||
@@ -32,7 +30,6 @@ func NewMockS3Client() *MockS3Client {
|
|||||||
|
|
||||||
func (m *MockS3Client) PutBlob(ctx context.Context, hash string, data []byte) error {
|
func (m *MockS3Client) PutBlob(ctx context.Context, hash string, data []byte) error {
|
||||||
m.storage[hash] = data
|
m.storage[hash] = data
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -41,13 +38,11 @@ func (m *MockS3Client) GetBlob(ctx context.Context, hash string) ([]byte, error)
|
|||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("blob not found: %s", hash)
|
return nil, fmt.Errorf("blob not found: %s", hash)
|
||||||
}
|
}
|
||||||
|
|
||||||
return data, nil
|
return data, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *MockS3Client) BlobExists(ctx context.Context, hash string) (bool, error) {
|
func (m *MockS3Client) BlobExists(ctx context.Context, hash string) (bool, error) {
|
||||||
_, ok := m.storage[hash]
|
_, ok := m.storage[hash]
|
||||||
|
|
||||||
return ok, nil
|
return ok, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -86,15 +81,12 @@ func TestBackupWithInMemoryFS(t *testing.T) {
|
|||||||
|
|
||||||
// Initialize the database
|
// Initialize the database
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
db, err := database.New(ctx, dbPath)
|
db, err := database.New(ctx, dbPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to create database: %v", err)
|
t.Fatalf("Failed to create database: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
err := db.Close()
|
if err := db.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Logf("Failed to close database: %v", err)
|
t.Logf("Failed to close database: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -150,14 +142,12 @@ func TestBackupWithInMemoryFS(t *testing.T) {
|
|||||||
if !expectedFiles[file.Path.String()] {
|
if !expectedFiles[file.Path.String()] {
|
||||||
t.Errorf("Unexpected file in database: %s", file.Path)
|
t.Errorf("Unexpected file in database: %s", file.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
delete(expectedFiles, file.Path.String())
|
delete(expectedFiles, file.Path.String())
|
||||||
|
|
||||||
// Verify file metadata
|
// Verify file metadata
|
||||||
fsFile := testFS[file.Path.String()]
|
fsFile := testFS[file.Path.String()]
|
||||||
if fsFile == nil {
|
if fsFile == nil {
|
||||||
t.Errorf("File %s not found in test filesystem", file.Path)
|
t.Errorf("File %s not found in test filesystem", file.Path)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -197,7 +187,6 @@ func TestBackupWithInMemoryFS(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to get blob hashes: %v", err)
|
t.Fatalf("Failed to get blob hashes: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(blobHashes) == 0 {
|
if len(blobHashes) == 0 {
|
||||||
t.Error("Expected at least one blob to be created")
|
t.Error("Expected at least one blob to be created")
|
||||||
}
|
}
|
||||||
@@ -208,7 +197,6 @@ func TestBackupWithInMemoryFS(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("Failed to check blob %s: %v", blobHash, err)
|
t.Errorf("Failed to check blob %s: %v", blobHash, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !exists {
|
if !exists {
|
||||||
t.Errorf("Blob %s not found in S3", blobHash)
|
t.Errorf("Blob %s not found in S3", blobHash)
|
||||||
}
|
}
|
||||||
@@ -241,15 +229,12 @@ func TestBackupDeduplication(t *testing.T) {
|
|||||||
|
|
||||||
// Initialize the database
|
// Initialize the database
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
db, err := database.New(ctx, dbPath)
|
db, err := database.New(ctx, dbPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to create database: %v", err)
|
t.Fatalf("Failed to create database: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
err := db.Close()
|
if err := db.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Logf("Failed to close database: %v", err)
|
t.Logf("Failed to close database: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -363,7 +348,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
UID: 1000, // Default UID for test
|
UID: 1000, // Default UID for test
|
||||||
GID: 1000, // Default GID for test
|
GID: 1000, // Default GID for test
|
||||||
}
|
}
|
||||||
|
|
||||||
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
return b.repos.Files.Create(ctx, tx, file)
|
return b.repos.Files.Create(ctx, tx, file)
|
||||||
})
|
})
|
||||||
@@ -380,8 +364,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
err := f.Close()
|
if err := f.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
// Log but don't fail since we're already in an error path potentially
|
// Log but don't fail since we're already in an error path potentially
|
||||||
fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err)
|
fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err)
|
||||||
}
|
}
|
||||||
@@ -393,10 +376,9 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
|
|
||||||
for {
|
for {
|
||||||
n, err := f.Read(buffer)
|
n, err := f.Read(buffer)
|
||||||
if err != nil && !errors.Is(err, io.EOF) {
|
if err != nil && err != io.EOF {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if n == 0 {
|
if n == 0 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@@ -413,13 +395,11 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
ChunkHash: types.ChunkHash(chunkHash),
|
ChunkHash: types.ChunkHash(chunkHash),
|
||||||
Size: int64(n),
|
Size: int64(n),
|
||||||
}
|
}
|
||||||
|
|
||||||
return b.repos.Chunks.Create(ctx, tx, chunk)
|
return b.repos.Chunks.Create(ctx, tx, chunk)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
processedChunks[chunkHash] = true
|
processedChunks[chunkHash] = true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -430,7 +410,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
Idx: chunkIndex,
|
Idx: chunkIndex,
|
||||||
ChunkHash: types.ChunkHash(chunkHash),
|
ChunkHash: types.ChunkHash(chunkHash),
|
||||||
}
|
}
|
||||||
|
|
||||||
return b.repos.FileChunks.Create(ctx, tx, fileChunk)
|
return b.repos.FileChunks.Create(ctx, tx, fileChunk)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -445,7 +424,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
FileOffset: int64(chunkIndex * defaultChunkSize),
|
FileOffset: int64(chunkIndex * defaultChunkSize),
|
||||||
Length: int64(n),
|
Length: int64(n),
|
||||||
}
|
}
|
||||||
|
|
||||||
return b.repos.ChunkFiles.Create(ctx, tx, chunkFile)
|
return b.repos.ChunkFiles.Create(ctx, tx, chunkFile)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -457,6 +435,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -479,21 +458,18 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
dummyData := []byte(chunkHash)
|
dummyData := []byte(chunkHash)
|
||||||
|
|
||||||
// Upload to S3 as a blob
|
// Upload to S3 as a blob
|
||||||
err = b.s3Client.PutBlob(ctx, blobHash, dummyData)
|
if err := b.s3Client.PutBlob(ctx, blobHash, dummyData); err != nil {
|
||||||
if err != nil {
|
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create blob entry in a short transaction
|
// Create blob entry in a short transaction
|
||||||
blobID := types.NewBlobID()
|
blobID := types.NewBlobID()
|
||||||
|
|
||||||
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
blob := &database.Blob{
|
blob := &database.Blob{
|
||||||
ID: blobID,
|
ID: blobID,
|
||||||
Hash: types.BlobHash(blobHash),
|
Hash: types.BlobHash(blobHash),
|
||||||
CreatedTS: time.Now(),
|
CreatedTS: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
return b.repos.Blobs.Create(ctx, tx, blob)
|
return b.repos.Blobs.Create(ctx, tx, blob)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -511,7 +487,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
Offset: 0,
|
Offset: 0,
|
||||||
Length: chunk.Size,
|
Length: chunk.Size,
|
||||||
}
|
}
|
||||||
|
|
||||||
return b.repos.BlobChunks.Create(ctx, tx, blobChunk)
|
return b.repos.BlobChunks.Create(ctx, tx, blobChunk)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -531,6 +506,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, fileCount, chunkCount, blobCount, totalSize, blobSize)
|
return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, fileCount, chunkCount, blobCount, totalSize, blobSize)
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -541,18 +517,16 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
func calculateHash(data []byte) string {
|
func calculateHash(data []byte) string {
|
||||||
h := sha256.New()
|
h := sha256.New()
|
||||||
h.Write(data)
|
h.Write(data)
|
||||||
|
return fmt.Sprintf("%x", h.Sum(nil))
|
||||||
return hex.EncodeToString(h.Sum(nil))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func generateLargeFileContent(size int) []byte {
|
func generateLargeFileContent(size int) []byte {
|
||||||
data := make([]byte, size)
|
data := make([]byte, size)
|
||||||
// Fill with pattern that changes every chunk to avoid deduplication
|
// Fill with pattern that changes every chunk to avoid deduplication
|
||||||
for i := range size {
|
for i := 0; i < size; i++ {
|
||||||
chunkNum := i / defaultChunkSize
|
chunkNum := i / defaultChunkSize
|
||||||
data[i] = byte((i + chunkNum) % 256)
|
data[i] = byte((i + chunkNum) % 256)
|
||||||
}
|
}
|
||||||
|
|
||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,12 +7,12 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func setupExcludeTestFS(t *testing.T) afero.Fs {
|
func setupExcludeTestFS(t *testing.T) afero.Fs {
|
||||||
@@ -63,7 +63,6 @@ func setupExcludeTestFS(t *testing.T) afero.Fs {
|
|||||||
}
|
}
|
||||||
|
|
||||||
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
for path, content := range files {
|
for path, content := range files {
|
||||||
dir := filepath.Dir(path)
|
dir := filepath.Dir(path)
|
||||||
err := fs.MkdirAll(dir, 0755)
|
err := fs.MkdirAll(dir, 0755)
|
||||||
@@ -108,7 +107,6 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn
|
|||||||
|
|
||||||
func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) {
|
func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
snap := &database.Snapshot{
|
snap := &database.Snapshot{
|
||||||
ID: types.SnapshotID(snapshotID),
|
ID: types.SnapshotID(snapshotID),
|
||||||
@@ -123,7 +121,6 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep
|
|||||||
BlobSize: 0,
|
BlobSize: 0,
|
||||||
CompressionRatio: 1.0,
|
CompressionRatio: 1.0,
|
||||||
}
|
}
|
||||||
|
|
||||||
return repos.Snapshots.Create(ctx, tx, snap)
|
return repos.Snapshots.Create(ctx, tx, snap)
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -131,10 +128,8 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep
|
|||||||
|
|
||||||
func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
|
func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -153,10 +148,8 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
|
|||||||
|
|
||||||
func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
|
func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -172,10 +165,8 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
|
|||||||
|
|
||||||
func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
|
func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -191,10 +182,8 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
|
|||||||
|
|
||||||
func TestExcludePatterns_MultiplePatterns(t *testing.T) {
|
func TestExcludePatterns_MultiplePatterns(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -210,10 +199,8 @@ func TestExcludePatterns_MultiplePatterns(t *testing.T) {
|
|||||||
|
|
||||||
func TestExcludePatterns_NoExclusions(t *testing.T) {
|
func TestExcludePatterns_NoExclusions(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -228,10 +215,8 @@ func TestExcludePatterns_NoExclusions(t *testing.T) {
|
|||||||
|
|
||||||
func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
|
func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -247,10 +232,8 @@ func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
|
|||||||
|
|
||||||
func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
|
func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -266,10 +249,8 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
|
|||||||
|
|
||||||
func TestExcludePatterns_ExactFileName(t *testing.T) {
|
func TestExcludePatterns_ExactFileName(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -286,10 +267,8 @@ func TestExcludePatterns_ExactFileName(t *testing.T) {
|
|||||||
func TestExcludePatterns_CaseSensitive(t *testing.T) {
|
func TestExcludePatterns_CaseSensitive(t *testing.T) {
|
||||||
// Pattern matching should be case-sensitive
|
// Pattern matching should be case-sensitive
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"THUMBS.DB"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"THUMBS.DB"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -308,7 +287,6 @@ func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) {
|
|||||||
// Some users might add trailing slashes to directory patterns
|
// Some users might add trailing slashes to directory patterns
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -327,7 +305,6 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) {
|
|||||||
// Exclude .hidden file specifically in src directory
|
// Exclude .hidden file specifically in src directory
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -366,7 +343,6 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs {
|
|||||||
}
|
}
|
||||||
|
|
||||||
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
for path, content := range files {
|
for path, content := range files {
|
||||||
dir := filepath.Dir(path)
|
dir := filepath.Dir(path)
|
||||||
err := fs.MkdirAll(dir, 0755)
|
err := fs.MkdirAll(dir, 0755)
|
||||||
@@ -383,10 +359,8 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs {
|
|||||||
func TestExcludePatterns_AnchoredPattern(t *testing.T) {
|
func TestExcludePatterns_AnchoredPattern(t *testing.T) {
|
||||||
// Pattern starting with / should only match from root of source dir
|
// Pattern starting with / should only match from root of source dir
|
||||||
fs := setupAnchoredTestFS(t)
|
fs := setupAnchoredTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/projectname"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/projectname"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -404,10 +378,8 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) {
|
|||||||
func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
|
func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
|
||||||
// Pattern without leading / should match anywhere in path
|
// Pattern without leading / should match anywhere in path
|
||||||
fs := setupAnchoredTestFS(t)
|
fs := setupAnchoredTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"projectname"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"projectname"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -426,10 +398,8 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
|
|||||||
func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
|
func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
|
||||||
// Anchored pattern with glob
|
// Anchored pattern with glob
|
||||||
fs := setupAnchoredTestFS(t)
|
fs := setupAnchoredTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/src/*.go"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/src/*.go"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -446,10 +416,8 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
|
|||||||
func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
|
func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
|
||||||
// Anchored pattern for exact file at root
|
// Anchored pattern for exact file at root
|
||||||
fs := setupAnchoredTestFS(t)
|
fs := setupAnchoredTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/file.txt"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/file.txt"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -467,10 +435,8 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
|
|||||||
func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
|
func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
|
||||||
// Unanchored pattern for file should match anywhere
|
// Unanchored pattern for file should match anywhere
|
||||||
fs := setupAnchoredTestFS(t)
|
fs := setupAnchoredTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"file.txt"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"file.txt"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|||||||
@@ -6,13 +6,13 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestFileContentChange verifies that when a file's content changes,
|
// TestFileContentChange verifies that when a file's content changes,
|
||||||
@@ -30,11 +30,9 @@ func TestFileContentChange(t *testing.T) {
|
|||||||
|
|
||||||
// Create test database
|
// Create test database
|
||||||
db, err := database.NewTestDB()
|
db, err := database.NewTestDB()
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer func() {
|
defer func() {
|
||||||
err := db.Close()
|
if err := db.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Errorf("failed to close database: %v", err)
|
t.Errorf("failed to close database: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -61,7 +59,6 @@ func TestFileContentChange(t *testing.T) {
|
|||||||
VaultikVersion: "test",
|
VaultikVersion: "test",
|
||||||
StartedAt: time.Now(),
|
StartedAt: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -84,7 +81,6 @@ func TestFileContentChange(t *testing.T) {
|
|||||||
|
|
||||||
// Modify the file
|
// Modify the file
|
||||||
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
|
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
|
||||||
|
|
||||||
err = afero.WriteFile(fs, "/test.txt", []byte("Modified content with different data"), 0644)
|
err = afero.WriteFile(fs, "/test.txt", []byte("Modified content with different data"), 0644)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
@@ -97,7 +93,6 @@ func TestFileContentChange(t *testing.T) {
|
|||||||
VaultikVersion: "test",
|
VaultikVersion: "test",
|
||||||
StartedAt: time.Now(),
|
StartedAt: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -135,7 +130,6 @@ func TestFileContentChange(t *testing.T) {
|
|||||||
// Verify that chunk_files for old chunk no longer references this file
|
// Verify that chunk_files for old chunk no longer references this file
|
||||||
oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash)
|
oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
for _, cf := range oldChunkFiles {
|
for _, cf := range oldChunkFiles {
|
||||||
file, err := repos.Files.GetByID(ctx, cf.FileID)
|
file, err := repos.Files.GetByID(ctx, cf.FileID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -165,11 +159,9 @@ func TestMultipleFileChanges(t *testing.T) {
|
|||||||
|
|
||||||
// Create test database
|
// Create test database
|
||||||
db, err := database.NewTestDB()
|
db, err := database.NewTestDB()
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer func() {
|
defer func() {
|
||||||
err := db.Close()
|
if err := db.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
t.Errorf("failed to close database: %v", err)
|
t.Errorf("failed to close database: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -196,7 +188,6 @@ func TestMultipleFileChanges(t *testing.T) {
|
|||||||
VaultikVersion: "test",
|
VaultikVersion: "test",
|
||||||
StartedAt: time.Now(),
|
StartedAt: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -209,7 +200,6 @@ func TestMultipleFileChanges(t *testing.T) {
|
|||||||
|
|
||||||
// Modify two files
|
// Modify two files
|
||||||
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
|
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
|
||||||
|
|
||||||
err = afero.WriteFile(fs, "/file1.txt", []byte("Modified content 1"), 0644)
|
err = afero.WriteFile(fs, "/file1.txt", []byte("Modified content 1"), 0644)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
err = afero.WriteFile(fs, "/file3.txt", []byte("Modified content 3"), 0644)
|
err = afero.WriteFile(fs, "/file3.txt", []byte("Modified content 3"), 0644)
|
||||||
@@ -224,7 +214,6 @@ func TestMultipleFileChanges(t *testing.T) {
|
|||||||
VaultikVersion: "test",
|
VaultikVersion: "test",
|
||||||
StartedAt: time.Now(),
|
StartedAt: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|||||||
@@ -35,9 +35,7 @@ func DecodeManifest(r io.Reader) (*Manifest, error) {
|
|||||||
|
|
||||||
// Decode JSON manifest
|
// Decode JSON manifest
|
||||||
var manifest Manifest
|
var manifest Manifest
|
||||||
|
if err := json.NewDecoder(zr).Decode(&manifest); err != nil {
|
||||||
err = json.NewDecoder(zr).Decode(&manifest)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("decoding manifest: %w", err)
|
return nil, fmt.Errorf("decoding manifest: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,21 +52,17 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) {
|
|||||||
|
|
||||||
// Compress using zstd
|
// Compress using zstd
|
||||||
var compressedBuf bytes.Buffer
|
var compressedBuf bytes.Buffer
|
||||||
|
|
||||||
writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel)))
|
writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel)))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("creating zstd writer: %w", err)
|
return nil, fmt.Errorf("creating zstd writer: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = writer.Write(jsonData)
|
if _, err := writer.Write(jsonData); err != nil {
|
||||||
if err != nil {
|
|
||||||
_ = writer.Close()
|
_ = writer.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf("writing compressed data: %w", err)
|
return nil, fmt.Errorf("writing compressed data: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = writer.Close()
|
if err := writer.Close(); err != nil {
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("closing zstd writer: %w", err)
|
return nil, fmt.Errorf("closing zstd writer: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,18 +1,16 @@
|
|||||||
package snapshot
|
package snapshot
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||||
|
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
|
||||||
"sneak.berlin/go/vaultik/internal/ui"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// ScannerParams holds parameters for scanner creation
|
// ScannerParams holds parameters for scanner creation
|
||||||
type ScannerParams struct {
|
type ScannerParams struct {
|
||||||
EnableProgress bool
|
EnableProgress bool
|
||||||
UI *ui.Writer // Where user-facing scanner messages go; nil = discard
|
|
||||||
Fs afero.Fs
|
Fs afero.Fs
|
||||||
Exclude []string // Exclude patterns (combined global + snapshot-specific)
|
Exclude []string // Exclude patterns (combined global + snapshot-specific)
|
||||||
SkipErrors bool // Skip file read errors (log loudly but continue)
|
SkipErrors bool // Skip file read errors (log loudly but continue)
|
||||||
@@ -48,7 +46,6 @@ func provideScannerFactory(cfg *config.Config, repos *database.Repositories, sto
|
|||||||
CompressionLevel: cfg.CompressionLevel,
|
CompressionLevel: cfg.CompressionLevel,
|
||||||
AgeRecipients: cfg.AgeRecipients,
|
AgeRecipients: cfg.AgeRecipients,
|
||||||
EnableProgress: params.EnableProgress,
|
EnableProgress: params.EnableProgress,
|
||||||
UI: params.UI,
|
|
||||||
Exclude: excludes,
|
Exclude: excludes,
|
||||||
SkipErrors: params.SkipErrors,
|
SkipErrors: params.SkipErrors,
|
||||||
})
|
})
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user