Compare commits
163
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
311756d452 | ||
|
|
c355ef4d25 | ||
|
|
5927e1aa3d | ||
|
|
9ca962969a | ||
|
|
89ebfc78e2 | ||
|
|
07ef3a1c78 | ||
|
|
c423d13191 | ||
|
|
75a10d3a22 | ||
|
|
3d56dd7eb0 | ||
|
|
bdce350041 | ||
|
|
753bc3ef60 | ||
|
|
d2a0510cb4 | ||
|
|
583f65040a | ||
|
|
d257f8f658 | ||
|
|
696ed9ab4d | ||
|
|
f21e7c9e70 | ||
|
|
c16ef476a9 | ||
|
|
e3f407b440 | ||
|
|
b6e4a218a3 | ||
|
|
c51f693527 | ||
|
|
3f9c2e5033 | ||
|
|
50816b7415 | ||
|
|
c3bb3b5580 | ||
|
|
3bcdbcfd83 | ||
|
|
50e20b460e | ||
|
|
af607e3597 | ||
|
|
e496aa334b | ||
|
|
cc58583130 | ||
|
|
b87b72d4b9 | ||
|
|
e7b49d58ab | ||
|
|
919229f224 | ||
|
|
68cffba35d | ||
|
|
26cbb63749 | ||
|
|
bf1d3c6bad | ||
|
|
dca3c50cd2 | ||
|
|
1ee0d291ca | ||
|
|
2f7e37153c | ||
|
|
26909b058a | ||
|
|
2dfdc5f095 | ||
|
|
217d60eeaa | ||
|
|
f7ba056814 | ||
|
|
82eb352eb5 | ||
|
|
0296e26210 | ||
|
|
1e05fa0dd7 | ||
|
|
bec964fc20 | ||
|
|
b1451bb17e | ||
|
|
ee83f50281 | ||
|
|
76f79af733 | ||
|
|
070a8a5447 | ||
|
|
40516d1263 | ||
|
|
a66e1f9844 | ||
|
|
5e4df7d04f | ||
|
|
d34868a9c4 | ||
|
|
04fce150bc | ||
|
|
23d22a0f19 | ||
|
|
c9c72ef29d | ||
|
|
928c389a5a | ||
|
|
2aaeeb4966 | ||
|
|
43346e62db | ||
|
|
df975bb8f0 | ||
|
|
fc56b0cb30 | ||
|
|
1f32820607 | ||
|
|
34a4d163f2 | ||
|
|
9497a31d0f | ||
|
|
d330f9f031 | ||
|
|
fda6d7a7eb | ||
|
|
1a97a80a81 | ||
|
|
d492f34e8d | ||
|
|
c17426b556 | ||
|
|
c06d53ad45 | ||
|
|
b39d765374 | ||
|
|
017ad7d3a6 | ||
|
|
fd759a921a | ||
|
|
a84b911155 | ||
|
|
5ce1dfa39e | ||
|
|
aa3e8f081b | ||
|
|
1f22b9c603 | ||
|
|
60abeb636a | ||
|
|
7ae49a1b2c | ||
|
|
a92b1a82ad | ||
|
|
39d5d21d48 | ||
|
|
44c9008e7e | ||
|
|
8036d93914 | ||
|
|
b55d5763ad | ||
|
|
53febb48d2 | ||
|
|
d55ddc5914 | ||
|
|
d9319dc0fb | ||
|
|
af330f2777 | ||
|
|
683fb0b103 | ||
|
|
cf8a527d35 | ||
|
|
a63c729fbc | ||
|
|
a1065d4f1f | ||
|
|
0e9c96c8b5 | ||
|
|
cafae65f61 | ||
|
|
7a0d5bfd73 | ||
|
|
8d1c8982d7 | ||
|
|
e75367c594 | ||
|
|
64c69cd8e3 | ||
|
|
132f7149ca | ||
|
|
f1ce085972 | ||
|
|
d8edf90fac | ||
|
|
301ea217e8 | ||
|
|
9f537b9c4c | ||
|
|
cf5b643bee | ||
|
|
3113014b58 | ||
|
|
706284d590 | ||
|
|
75564a504e | ||
|
|
edd3e5fdb2 | ||
|
|
d5796bd6c1 | ||
|
|
90e855ef99 | ||
|
|
2185421c01 | ||
|
|
ce0d7b45a1 | ||
|
|
1266a263fc | ||
|
|
70632e4353 | ||
|
|
77b9d943e4 | ||
|
|
fc4d0d6dc7 | ||
|
|
22227aa0c5 | ||
|
|
9cb14d143d | ||
|
|
00d4b36e35 | ||
|
|
8de8f8e5cc | ||
|
|
6e6e107243 | ||
|
|
6bb6f7c8a8 | ||
|
|
8e55d2f970 | ||
|
|
b0747657e3 | ||
|
|
2a9718855c | ||
|
|
485f3296d9 | ||
|
|
adf73c5413 | ||
|
|
8959741c90 | ||
|
|
e534746cf3 | ||
|
|
5397b37c13 | ||
|
|
2df2792a75 | ||
|
|
4fe568f803 | ||
|
|
27e85f01f2 | ||
|
|
d479bfcd52 | ||
|
|
cb16d6869f | ||
|
|
ff85f1e4f8 | ||
|
|
b2e160944f | ||
|
|
307867f59e | ||
|
|
9d12d500fa | ||
|
|
2e2bf01130 | ||
|
|
e9687c68b7 | ||
|
|
a8970a87fc | ||
|
|
e6ee488d9d | ||
|
|
2e2b02a056 | ||
|
|
0b95cb4308 | ||
|
|
4a3e61f8e1 | ||
|
|
6fbcac0cd8 | ||
|
|
34f73f72d8 | ||
|
|
ee240faa32 | ||
|
|
f719ab3adc | ||
|
|
1a8baf7491 | ||
|
|
7d5d3fa598 | ||
|
|
ac5d2f4a0d | ||
|
|
b250ddfa94 | ||
|
|
fe3ad13a91 | ||
|
|
ebd6619638 | ||
|
|
20d3a9ac8c | ||
|
|
0889cf2804 | ||
|
|
f9ebb4bf25 | ||
|
|
9f2d722734 | ||
|
|
6821215b0e | ||
|
|
f97a1dc2eb | ||
|
|
18c14d1507 |
@@ -3,6 +3,8 @@
|
||||
*.md
|
||||
LICENSE
|
||||
vaultik
|
||||
dist
|
||||
.tool
|
||||
coverage.out
|
||||
coverage.html
|
||||
.DS_Store
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
end_of_line = lf
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
@@ -1,9 +1,9 @@
|
||||
name: check
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
branches: [main, next]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches: [main, next]
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -11,4 +11,4 @@ jobs:
|
||||
# actions/checkout v4, 2024-09-16
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
- name: Build and check
|
||||
run: docker build .
|
||||
run: script/cibuild
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
name: release
|
||||
on:
|
||||
push:
|
||||
tags: ["v*"]
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# actions/checkout v4, 2024-09-16
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
with:
|
||||
# goreleaser needs the tags and the full history: the version
|
||||
# it stamps comes from the tag, and the changelog comes from
|
||||
# the commits since the previous one. A shallow checkout
|
||||
# silently produces a mislabelled release.
|
||||
fetch-depth: 0
|
||||
# goreleaser is not a compiler: it shells out to `go` for the
|
||||
# `before:` hook and for every one of the four cross-compiles.
|
||||
# Nothing else in this repo puts a Go toolchain on the runner --
|
||||
# check.yml runs script/cibuild, which does all of its work inside
|
||||
# the digest-pinned Dockerfile images -- so without this step the
|
||||
# release either fails at the before-hook or, worse, ships binaries
|
||||
# built by whatever Go the runner happens to carry.
|
||||
#
|
||||
# actions/setup-go would pin the action by commit sha, but the Go
|
||||
# tarball it downloads at runtime is verified against no value in
|
||||
# this repo, and the action exposes no checksum input.
|
||||
# REPO_POLICIES.md requires every external reference to be pinned
|
||||
# by hash with no exceptions, and this is the compiler that
|
||||
# produces the published binaries -- the input where a substituted
|
||||
# artifact matters most. So Go is installed the way goreleaser is:
|
||||
# script/install-go downloads the exact archive for go.mod's `go`
|
||||
# directive and refuses it unless its sha256 matches the value
|
||||
# committed in the script, then puts .tool/go/bin on PATH for the
|
||||
# steps below.
|
||||
- name: Install Go
|
||||
run: script/install-go
|
||||
- name: Install goreleaser
|
||||
run: script/install-goreleaser
|
||||
- name: Release
|
||||
run: script/release
|
||||
env:
|
||||
# RELEASE_TOKEN is a repository Actions secret: a Gitea access
|
||||
# token with write access to this repository's releases (scope
|
||||
# write:repository), owned by an account that can publish here.
|
||||
# It is deliberately not the runner's automatic token, which is
|
||||
# not guaranteed to carry that scope.
|
||||
GITEA_TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||
# Build with the toolchain install-go just verified, never a
|
||||
# different one auto-downloaded from a `toolchain` directive:
|
||||
# the point of the hash pin is that this exact compiler makes
|
||||
# the release.
|
||||
GOTOOLCHAIN: local
|
||||
+7
-1
@@ -1,5 +1,11 @@
|
||||
# Binary
|
||||
vaultik
|
||||
/vaultik
|
||||
|
||||
# goreleaser output
|
||||
/dist/
|
||||
|
||||
# Locally installed pinned tools (script/install-goreleaser)
|
||||
/.tool/
|
||||
|
||||
# Test artifacts
|
||||
*.out
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
version: "2"
|
||||
|
||||
# Config schema uses the golangci-lint v2 layout (settings live under
|
||||
# linters.settings, not top-level linters-settings) so that the
|
||||
# thresholds below are actually applied by golangci-lint >= v2.
|
||||
|
||||
run:
|
||||
timeout: 5m
|
||||
modules-download-mode: readonly
|
||||
|
||||
linters:
|
||||
default: all
|
||||
disable:
|
||||
# Genuinely incompatible with project patterns
|
||||
- exhaustruct # Requires all struct fields
|
||||
- depguard # Dependency allow/block lists
|
||||
- godot # Requires comments to end with periods
|
||||
- wsl # Deprecated, replaced by wsl_v5
|
||||
- wrapcheck # Too verbose for internal packages
|
||||
- varnamelen # Short names like db, id are idiomatic Go
|
||||
settings:
|
||||
lll:
|
||||
line-length: 88
|
||||
funlen:
|
||||
lines: 80
|
||||
statements: 50
|
||||
cyclop:
|
||||
max-complexity: 15
|
||||
dupl:
|
||||
threshold: 100
|
||||
|
||||
issues:
|
||||
max-issues-per-linter: 0
|
||||
max-same-issues: 0
|
||||
@@ -0,0 +1,69 @@
|
||||
version: 2
|
||||
|
||||
project_name: vaultik
|
||||
|
||||
# This repo lives on Gitea, not GitHub. Without this block goreleaser
|
||||
# talks to the GitHub API by default and a `goreleaser release` either
|
||||
# fails outright or publishes somewhere nobody is looking.
|
||||
gitea_urls:
|
||||
api: https://git.eeqj.de/api/v1
|
||||
download: https://git.eeqj.de
|
||||
|
||||
before:
|
||||
hooks:
|
||||
- go mod tidy
|
||||
|
||||
builds:
|
||||
- id: vaultik
|
||||
main: ./cmd/vaultik
|
||||
binary: vaultik
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
goos:
|
||||
- linux
|
||||
- darwin
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
ldflags:
|
||||
- -s -w
|
||||
- -X 'sneak.berlin/go/vaultik/internal/globals.Version={{ .Version }}'
|
||||
- -X 'sneak.berlin/go/vaultik/internal/globals.Commit={{ .Commit }}'
|
||||
- -X 'sneak.berlin/go/vaultik/internal/globals.CommitDate={{ slice .CommitDate 0 10 }}'
|
||||
|
||||
archives:
|
||||
- id: default
|
||||
name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
|
||||
formats:
|
||||
- tar.gz
|
||||
files:
|
||||
- LICENSE
|
||||
- README.md
|
||||
|
||||
checksum:
|
||||
name_template: "checksums.txt"
|
||||
algorithm: sha256
|
||||
|
||||
# A snapshot is not a release and must not name itself like one. The
|
||||
# previous `{{ incpatch .Version }}-next` derived a plausible-looking
|
||||
# release number from the last tag -- and with no tags in the repo at
|
||||
# all, from goreleaser's fabricated v0.0.0. This produces the same
|
||||
# string script/version produces for an untagged build, so a snapshot
|
||||
# binary and a `make vaultik` binary of the same clean commit agree.
|
||||
snapshot:
|
||||
version_template: "dev-{{ slice .FullCommit 0 12 }}"
|
||||
|
||||
changelog:
|
||||
sort: asc
|
||||
use: git
|
||||
filters:
|
||||
exclude:
|
||||
- "^docs:"
|
||||
- "^test:"
|
||||
- "^chore:"
|
||||
- "Merge pull request"
|
||||
- "Merge branch"
|
||||
|
||||
release:
|
||||
draft: true
|
||||
prerelease: auto
|
||||
@@ -38,10 +38,9 @@ Version: 2025-06-08
|
||||
1. Before committing, tests must pass (`make test`), linting must pass
|
||||
(`make lint`), and code must be formatted (`make fmt`). For go, those
|
||||
makefile targets should use `go fmt` and `go test -v ./...` and
|
||||
`golangci-lint run`. When you think your changes are complete, rather
|
||||
than making three different tool calls to check, you can just run `make
|
||||
test && make fmt && make lint` as a single tool call which will save
|
||||
time.
|
||||
`golangci-lint run`. Each Makefile target does exactly one thing — to
|
||||
run lint + fmt-check + test together (the standard pre-commit gate),
|
||||
use `make check`.
|
||||
|
||||
2. Always write a `Makefile` with the default target being `test`, and with
|
||||
a `fmt` target that formats the code. The `test` target should run all
|
||||
@@ -84,8 +83,8 @@ Version: 2025-06-08
|
||||
possible to mock or stub these side-effects in tests.
|
||||
|
||||
9. Always use structured logging. Log any relevant state/context with the
|
||||
messages (but do not log secrets). If stdout is not a terminal, output
|
||||
the structured logs in jsonl format.
|
||||
messages (but do not log secrets). If the log stream is not a terminal,
|
||||
output the structured logs in jsonl format.
|
||||
|
||||
10. Avoid using bare strings or numbers in code, especially if they appear
|
||||
anywhere more than once. Always define a constant (usually at the top
|
||||
@@ -103,3 +102,9 @@ Version: 2025-06-08
|
||||
build files are acceptable in the root, but source code and other files
|
||||
should be organized in appropriate subdirectories.
|
||||
|
||||
13. Pre-1.0: NEVER write database migrations. There are no live databases
|
||||
anywhere — every user's local index can be rebuilt from a fresh full
|
||||
backup. When the schema changes, just change `schema.sql` (and any code
|
||||
that touches the affected tables). The local index is disposable until
|
||||
1.0 ships and is tagged.
|
||||
|
||||
|
||||
+18
-13
@@ -53,8 +53,8 @@ The database tracks five primary entities and their relationships:
|
||||
### Entity Descriptions
|
||||
|
||||
#### File (`database.File`)
|
||||
Represents a file or directory in the backup system. Stores metadata needed for restoration:
|
||||
- Path, mtime
|
||||
Represents a file, directory, or symlink in the backup system. Stores metadata needed for restoration:
|
||||
- Path, source_path (for restore path stripping), mtime
|
||||
- Size, mode, ownership (uid, gid)
|
||||
- Symlink target (if applicable)
|
||||
|
||||
@@ -95,7 +95,7 @@ Maps chunks to their position within blobs:
|
||||
|
||||
#### Snapshot (`database.Snapshot`)
|
||||
Represents a point-in-time backup:
|
||||
- `ID`: Format is `{hostname}-{YYYYMMDD}-{HHMMSS}Z`
|
||||
- `ID`: Format is `{hostname}_{snapshot-name}_{RFC3339}` (e.g. `server1_home_2025-06-01T12:00:00Z`)
|
||||
- Tracks file count, chunk count, blob count, sizes, compression ratio
|
||||
- `CompletedAt`: Null until snapshot finishes successfully
|
||||
|
||||
@@ -127,7 +127,7 @@ fx.New(
|
||||
config.Module, // 5. Config
|
||||
database.Module, // 6. Database + Repositories
|
||||
log.Module, // 7. Logger initialization
|
||||
s3.Module, // 8. S3 client
|
||||
storage.Module, // 8. Storage backend (S3/file/rclone)
|
||||
snapshot.Module, // 9. SnapshotManager + ScannerFactory
|
||||
fx.Provide(vaultik.New), // 10. Vaultik orchestrator
|
||||
)
|
||||
@@ -161,7 +161,7 @@ type Vaultik struct {
|
||||
Config *config.Config
|
||||
DB *database.DB
|
||||
Repositories *database.Repositories
|
||||
S3Client *s3.Client
|
||||
Storage storage.Storer
|
||||
ScannerFactory snapshot.ScannerFactory
|
||||
SnapshotManager *snapshot.SnapshotManager
|
||||
Shutdowner fx.Shutdowner
|
||||
@@ -341,12 +341,11 @@ CreateSnapshot(opts)
|
||||
└─► SnapshotManager.ExportSnapshotMetadata()
|
||||
│
|
||||
├─► Copy database to temp file
|
||||
├─► Clean to only current snapshot data
|
||||
├─► Dump to SQL
|
||||
├─► Compress with zstd
|
||||
├─► Clean to only current snapshot data (VACUUM)
|
||||
├─► Compress binary SQLite with zstd
|
||||
├─► Encrypt with age
|
||||
├─► Upload db.zst.age to S3
|
||||
└─► Upload manifest.json.zst to S3
|
||||
├─► Upload db.zst.age to storage
|
||||
└─► Upload manifest.json.zst to storage
|
||||
```
|
||||
|
||||
## Deduplication Strategy
|
||||
@@ -367,11 +366,17 @@ bucket/
|
||||
│ └── {full-hash} # Compressed+encrypted blob
|
||||
│
|
||||
└── metadata/
|
||||
└── {snapshot-id}/
|
||||
├── db.zst.age # Encrypted database dump
|
||||
└── manifest.json.zst # Blob list (for verification)
|
||||
└── {remote-key}/
|
||||
├── db.zst.age # Encrypted binary SQLite database
|
||||
└── manifest.json.zst # Blob list (for pruning/verification)
|
||||
```
|
||||
|
||||
The `{remote-key}` directory name is a one-way double SHA-256 hash of the human
|
||||
snapshot ID, so the human ID (hostname, snapshot name, timestamp) is never
|
||||
written to the store as a directory name. See
|
||||
[docs/REPOSTRUCTURE.md](docs/REPOSTRUCTURE.md#remote-key-derivation) for the
|
||||
derivation and a worked example.
|
||||
|
||||
## Thread Safety
|
||||
|
||||
- `Packer`: Thread-safe via mutex. Multiple goroutines can call `AddChunk()`.
|
||||
|
||||
+58
-33
@@ -1,7 +1,30 @@
|
||||
# Lint stage
|
||||
# golangci/golangci-lint:v2.11.3-alpine, 2026-03-17
|
||||
FROM golangci/golangci-lint:v2.11.3-alpine@sha256:b1c3de5862ad0a95b4e45a993b0f00415835d687e4f12c845c7493b86c13414e AS lint
|
||||
# This file has no lint stage, deliberately.
|
||||
#
|
||||
# Linting lives in Dockerfile.lint, built by script/lint, and
|
||||
# script/cibuild builds both. A lint stage here would have to either
|
||||
# shell out to `make lint` -- which is now `docker build`, so
|
||||
# docker-in-docker inside a BuildKit step with no daemon -- or call
|
||||
# golangci-lint directly, which would mean a second, independently
|
||||
# bumpable digest pin for the linter alongside the one in
|
||||
# Dockerfile.lint. Two pins for one tool is the drift that
|
||||
# https://git.eeqj.de/sneak/vaultik/issues/78 was filed over. See
|
||||
# https://git.eeqj.de/sneak/vaultik/issues/113 for the ruling.
|
||||
#
|
||||
# Consequence, stated rather than left to be discovered: script/docker
|
||||
# builds this file only and therefore does not lint. `make fmt-check`
|
||||
# and `make test` still run here, so what a green build of this file
|
||||
# means is "formatted, tested, and it compiles" -- the lint verdict
|
||||
# comes from script/lint or script/cibuild.
|
||||
|
||||
# Build stage
|
||||
# golang:1.26.1-alpine, 2026-03-17
|
||||
FROM golang:1.26.1-alpine@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f6c5bd6be49ed82039 AS builder
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
# Build tooling: make, plus a C toolchain because `go test -race` needs cgo.
|
||||
# The sqlite driver is pure Go (modernc.org/sqlite), so no sqlite library or
|
||||
# CLI is required.
|
||||
RUN apk add --no-cache make build-base
|
||||
|
||||
WORKDIR /src
|
||||
@@ -13,42 +36,44 @@ RUN go mod download
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Run formatting check and linter
|
||||
RUN make fmt-check
|
||||
RUN make lint
|
||||
# Run the format check and the tests.
|
||||
#
|
||||
# CHECK_EPOCH must stay immediately above these RUNs. These layers are
|
||||
# keyed on its value, so they are cache-eligible only for a value
|
||||
# already built against this same tree. script/cibuild and script/docker
|
||||
# each pass a fresh value on every invocation, which is what makes their
|
||||
# green mean the checks really executed.
|
||||
#
|
||||
# The value is expanded into each check command rather than left to a
|
||||
# bare declaration, so the cache miss does not depend on BuildKit's
|
||||
# unreferenced-ARG handling staying as it is. It also puts the epoch in
|
||||
# the build log, where a reader can see the layer was keyed fresh.
|
||||
#
|
||||
# The guard is what makes a build that omits --build-arg fail instead of
|
||||
# lie. An unset ARG is an empty string, and an empty string is a
|
||||
# perfectly stable cache key: without the guard the first such build
|
||||
# runs the checks and every one after it on an unchanged tree replays
|
||||
# these layers from cache, executes nothing, and still exits 0. Failed
|
||||
# steps are never cached, so the guard fails on EVERY invocation rather
|
||||
# than once -- a bare `docker build .` is a loud error, not a quiet
|
||||
# green. Do not give CHECK_EPOCH a default value; a default would
|
||||
# satisfy the guard with a constant and restore the hole.
|
||||
#
|
||||
# Everything above this line (apk, go.mod, `go mod download`) is
|
||||
# deliberately outside the busted range and keeps caching.
|
||||
ARG CHECK_EPOCH
|
||||
RUN [ -n "$CHECK_EPOCH" ] || exit 1
|
||||
RUN echo "check epoch: ${CHECK_EPOCH}" && make fmt-check
|
||||
RUN echo "check epoch: ${CHECK_EPOCH}" && make test
|
||||
|
||||
# Build stage
|
||||
# golang:1.26.1-alpine, 2026-03-17
|
||||
FROM golang:1.26.1-alpine@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f6c5bd6be49ed82039 AS builder
|
||||
|
||||
# Depend on lint stage passing
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
# Install build dependencies for CGO (mattn/go-sqlite3) and sqlite3 CLI (tests)
|
||||
RUN apk add --no-cache make build-base sqlite
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
# Copy go mod files first for better layer caching
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Run tests
|
||||
RUN make test
|
||||
|
||||
# Build with CGO enabled (required for mattn/go-sqlite3)
|
||||
RUN CGO_ENABLED=1 go build -ldflags "-X 'git.eeqj.de/sneak/vaultik/internal/globals.Version=${VERSION}' -X 'git.eeqj.de/sneak/vaultik/internal/globals.Commit=$(git rev-parse HEAD 2>/dev/null || echo unknown)'" -o /vaultik ./cmd/vaultik
|
||||
# Build (pure Go, no CGO required since we use modernc.org/sqlite)
|
||||
RUN CGO_ENABLED=0 go build -ldflags "-X 'sneak.berlin/go/vaultik/internal/globals.Version=${VERSION}' -X 'sneak.berlin/go/vaultik/internal/globals.Commit=$(git rev-parse HEAD 2>/dev/null || echo unknown)' -X 'sneak.berlin/go/vaultik/internal/globals.CommitDate=$(git show -s --format=%cs HEAD 2>/dev/null || echo unknown)'" -o /vaultik ./cmd/vaultik
|
||||
|
||||
# Runtime stage
|
||||
# alpine:3.21, 2026-02-25
|
||||
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
||||
|
||||
RUN apk add --no-cache ca-certificates sqlite
|
||||
RUN apk add --no-cache ca-certificates
|
||||
|
||||
# Copy binary from builder
|
||||
COPY --from=builder /vaultik /usr/local/bin/vaultik
|
||||
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
# Lint image.
|
||||
#
|
||||
# Every lint run in this repo happens inside this image, invoked through
|
||||
# script/lint, and linting is a BUILD STEP rather than a container
|
||||
# command: a successful build of this file IS a clean lint. That shape
|
||||
# also works where the docker daemon is remote and bind mounts are
|
||||
# impossible, which `docker run` against a mounted worktree does not.
|
||||
#
|
||||
# This FROM line is the single source of truth for the linter version in
|
||||
# this repo. Nothing else pins golangci-lint: the product Dockerfile has
|
||||
# no lint stage, deliberately, so there is no second digest to bump and
|
||||
# no pair of pins that can drift apart. Bump the tag AND the digest here
|
||||
# and nowhere else.
|
||||
#
|
||||
# Note for readers coming from REPO_POLICIES.md: that document still
|
||||
# describes the older pattern, a lint stage inside the product
|
||||
# Dockerfile wired up with `COPY --from=lint /src/go.sum /dev/null`.
|
||||
# That pattern is superseded here by the owner's ruling recorded in
|
||||
# https://git.eeqj.de/sneak/vaultik/issues/113 -- lint runs in its own
|
||||
# image, per run, with its own cache and its own lock, which is what
|
||||
# makes concurrent runs on one host safe. The policy text is org-wide
|
||||
# and is being amended separately; this file is what this repo does.
|
||||
#
|
||||
# golangci/golangci-lint:v2.12.2, 2026-08-10
|
||||
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
# Copy the dependency manifests first so the module download layer stays
|
||||
# cached until they change. Everything above the ARG below is cacheable
|
||||
# on purpose; a cold module download on every lint would make the inner
|
||||
# loop unusable and buys nothing, because it is not what the gate is
|
||||
# asserting.
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
|
||||
# Force the check layers to execute on every invocation.
|
||||
#
|
||||
# CHECK_EPOCH must stay immediately above the RUNs below. Those layers
|
||||
# are keyed on its value, so they are cache-eligible only for a value
|
||||
# already built against this same tree; script/lint and script/cibuild
|
||||
# each pass a fresh value on every invocation, which is what makes their
|
||||
# green mean the linter really ran. Without it, `docker build -f
|
||||
# Dockerfile.lint .` on an unchanged tree exits 0 in well under a second
|
||||
# having linted nothing.
|
||||
#
|
||||
# The value is expanded into each check command itself rather than left
|
||||
# to a bare declaration, so the cache miss does not depend on BuildKit's
|
||||
# unreferenced-ARG handling staying as it is. It also puts the epoch in
|
||||
# the build log, where a reader can see the layer was keyed fresh.
|
||||
#
|
||||
# The guard is what makes a build that omits --build-arg fail instead of
|
||||
# lie. An unset ARG is an empty string, and an empty string is a
|
||||
# perfectly stable cache key: without the guard the first such build
|
||||
# lints and every one after it on an unchanged tree replays this layer,
|
||||
# executes nothing, and still exits 0. Failed steps are never cached, so
|
||||
# the guard fails on EVERY invocation rather than once. Do not give
|
||||
# CHECK_EPOCH a default value; a default would satisfy the guard with a
|
||||
# constant and restore the hole.
|
||||
ARG CHECK_EPOCH
|
||||
RUN [ -n "$CHECK_EPOCH" ] || exit 1
|
||||
|
||||
# Validate .golangci.yml before linting with it.
|
||||
#
|
||||
# This is not belt-and-braces; it closes a hole that `golangci-lint run`
|
||||
# leaves wide open. `run` rejects YAML it cannot PARSE, but it silently
|
||||
# IGNORES an unknown top-level KEY. Renaming `linters:` to `linterz:` --
|
||||
# one character -- discards `default: all`, the whole disable list and
|
||||
# every threshold, leaves only golangci-lint's small default linter set
|
||||
# running, and exits 0 reporting `0 issues.` on a tree the real config
|
||||
# fails. Demonstrated on this repo at this pin, recorded on
|
||||
# https://git.eeqj.de/sneak/vaultik/pulls/114: with a planted
|
||||
# over-length line, `script/lint` exits 1 naming the `revive` finding
|
||||
# with `linters:` and exits 0 with `linterz:`. A set-but-ineffective
|
||||
# config quietly falling back to defaults is precisely the false-green
|
||||
# class this gate exists to eliminate, so it must not sit in the gate's
|
||||
# own configuration.
|
||||
#
|
||||
# `config verify` catches it, and it does so OFFLINE at this pinned
|
||||
# version -- verified, not assumed. Under `docker run --network none`
|
||||
# against the pinned digest it exits 0 on this repo's config and exits 3
|
||||
# on the `linterz:` variant with `additional properties 'linterz' not
|
||||
# allowed`. An earlier revision of this file asserted the opposite, that
|
||||
# the schema is fetched over live HTTPS from an unpinned URL, and used
|
||||
# that to justify omitting this line. That claim was false at v2.12.2;
|
||||
# the schema is embedded. If a future bump reintroduces a network fetch
|
||||
# the failure is loud and this comment is where to record it.
|
||||
#
|
||||
# It is keyed on CHECK_EPOCH, like the lint run below, so it executes on
|
||||
# every invocation. Content-addressing alone would arguably be enough --
|
||||
# .golangci.yml arrives through `COPY . .`, so a cache hit here implies
|
||||
# a byte-identical config was validated when the layer really ran. That
|
||||
# argument is exactly the one that would also excuse caching the lint
|
||||
# layer, and this repo has ruled it insufficient: a cached check layer
|
||||
# checks nothing, and the cost of being wrong is silent. Forcing it costs
|
||||
# milliseconds and puts the epoch in the log, where a reader can see that
|
||||
# this validation ran rather than being replayed.
|
||||
RUN echo "check epoch: ${CHECK_EPOCH}" && \
|
||||
golangci-lint config verify --config .golangci.yml
|
||||
|
||||
RUN echo "check epoch: ${CHECK_EPOCH}" && \
|
||||
golangci-lint run --config .golangci.yml ./...
|
||||
@@ -1,51 +1,106 @@
|
||||
.PHONY: test fmt lint fmt-check check build clean all docker hooks
|
||||
.PHONY: all bootstrap setup check test lint lint-fix fmt fmt-check build clean deps test-coverage local install release release-snapshot docker hooks
|
||||
|
||||
# Version number
|
||||
VERSION := 0.0.1
|
||||
# Version number, derived from git by script/version -- the tag when
|
||||
# HEAD is on one, otherwise dev-<sha>. This used to be a hardcoded
|
||||
# constant, which meant every local build claimed to be a release that
|
||||
# had never been tagged.
|
||||
VERSION := $(shell script/version)
|
||||
|
||||
# $(shell) discards exit status, so a script/version that is missing,
|
||||
# non-executable or broken would otherwise leave VERSION empty and every
|
||||
# binary built here would print "vaultik " with no version at all. A
|
||||
# build that cannot determine what it is must not produce an artifact.
|
||||
ifeq ($(strip $(VERSION)),)
|
||||
$(error script/version produced no version string; a build that cannot \
|
||||
determine its version will not be made. Check that script/version exists \
|
||||
and is executable)
|
||||
endif
|
||||
|
||||
# Build variables
|
||||
GIT_REVISION := $(shell git rev-parse HEAD 2>/dev/null || echo "unknown")
|
||||
GIT_COMMIT_DATE := $(shell git show -s --format=%cs HEAD 2>/dev/null || echo "unknown")
|
||||
|
||||
# Linker flags
|
||||
LDFLAGS := -X 'git.eeqj.de/sneak/vaultik/internal/globals.Version=$(VERSION)' \
|
||||
-X 'git.eeqj.de/sneak/vaultik/internal/globals.Commit=$(GIT_REVISION)'
|
||||
LDFLAGS := -X 'sneak.berlin/go/vaultik/internal/globals.Version=$(VERSION)' \
|
||||
-X 'sneak.berlin/go/vaultik/internal/globals.Commit=$(GIT_REVISION)' \
|
||||
-X 'sneak.berlin/go/vaultik/internal/globals.CommitDate=$(GIT_COMMIT_DATE)'
|
||||
|
||||
# Default target
|
||||
all: vaultik
|
||||
|
||||
# Run tests
|
||||
# Install all development dependencies.
|
||||
bootstrap:
|
||||
@script/bootstrap
|
||||
|
||||
# Prepare a fresh clone: bootstrap plus pre-commit hook.
|
||||
setup:
|
||||
@script/setup
|
||||
|
||||
# Combined pre-commit/CI gate: tests, lint, format check.
|
||||
check:
|
||||
@script/check
|
||||
|
||||
# Run tests only. This runs the ENTIRE suite -- there is no separate
|
||||
# integration target and no build-tagged subset held back. In
|
||||
# particular internal/vaultik/integration_test.go, which does full
|
||||
# chunk -> pack -> encrypt -> upload -> restore round-trips, runs here.
|
||||
# A `test-integration` target used to exist and was removed: no file in
|
||||
# the repo carried a build tag, so `-tags=integration` selected nothing
|
||||
# extra and the target was an exact duplicate of this one.
|
||||
test:
|
||||
go test -race -timeout 30s ./...
|
||||
@script/test
|
||||
|
||||
# Check if code is formatted (read-only)
|
||||
# Check if code is formatted (read-only).
|
||||
fmt-check:
|
||||
@test -z "$$(gofmt -l .)" || (echo "Files not formatted:" && gofmt -l . && exit 1)
|
||||
@script/fmt-check
|
||||
|
||||
# Format code
|
||||
# Format code.
|
||||
fmt:
|
||||
go fmt ./...
|
||||
@script/fmt
|
||||
|
||||
# Run linter
|
||||
# Run linter only.
|
||||
lint:
|
||||
golangci-lint run ./...
|
||||
@script/lint
|
||||
|
||||
# Apply the linter's autofixes (rewrites files).
|
||||
lint-fix:
|
||||
@script/lint-fix
|
||||
|
||||
# Build binary. `build` is the name the org convention reaches for and
|
||||
# the one a caller checks the exit code of; `vaultik` is the file rule
|
||||
# that does the work, so an unchanged tree still short-circuits.
|
||||
#
|
||||
# This alias is not decorative. `build` was listed in .PHONY with no
|
||||
# rule, and a phony target with no prerequisites and no recipe is
|
||||
# already satisfied: `make build` printed "Nothing to be done" and
|
||||
# exited 0 without producing a binary (issue #110). Every name in
|
||||
# .PHONY needs a rule for that reason; TestPhonyTargetsAllHaveRules in
|
||||
# cmd/vaultik keeps it that way.
|
||||
build: vaultik
|
||||
|
||||
# Build binary
|
||||
vaultik: internal/*/*.go cmd/vaultik/*.go
|
||||
go build -ldflags "$(LDFLAGS)" -o $@ ./cmd/vaultik
|
||||
|
||||
# Clean build artifacts
|
||||
# Clean build artifacts.
|
||||
clean:
|
||||
rm -f vaultik
|
||||
go clean
|
||||
|
||||
# Run tests with coverage
|
||||
test-coverage:
|
||||
go test -v -coverprofile=coverage.out ./...
|
||||
go tool cover -html=coverage.out -o coverage.html
|
||||
# Install dependencies. The linter is deliberately not installed here:
|
||||
# script/lint lints by building Dockerfile.lint, whose FROM line is the
|
||||
# single source of truth for the linter version. A second, separately
|
||||
# pinned copy on PATH could drift from it and make a local `make lint`
|
||||
# disagree with CI.
|
||||
deps:
|
||||
go mod download
|
||||
|
||||
# Run integration tests
|
||||
test-integration:
|
||||
go test -v -tags=integration ./...
|
||||
# Run tests with coverage. -count=1 for the same reason script/test
|
||||
# uses it: without it an unchanged package is served from Go's test
|
||||
# result cache, and a coverage profile assembled from cached results
|
||||
# describes a run that did not happen.
|
||||
test-coverage:
|
||||
go test -v -count=1 -coverprofile=coverage.out ./...
|
||||
go tool cover -html=coverage.out -o coverage.html
|
||||
|
||||
local:
|
||||
VAULTIK_CONFIG=$(HOME)/etc/vaultik/config.yml ./vaultik snapshot --debug list 2>&1
|
||||
@@ -54,16 +109,18 @@ local:
|
||||
install: vaultik
|
||||
cp ./vaultik $(HOME)/bin/
|
||||
|
||||
# Run all checks (formatting, linting, tests) without modifying files
|
||||
check: fmt-check lint test
|
||||
# Build and publish release artifacts (linux/darwin × amd64/arm64) via goreleaser.
|
||||
release:
|
||||
@script/release
|
||||
|
||||
# Build Docker image
|
||||
# Dry-run a release build without publishing or tagging.
|
||||
release-snapshot:
|
||||
@script/release-snapshot
|
||||
|
||||
# Build Docker image.
|
||||
docker:
|
||||
docker build -t vaultik .
|
||||
@script/docker
|
||||
|
||||
# Install pre-commit hook
|
||||
# Install pre-commit hook.
|
||||
hooks:
|
||||
@printf '#!/bin/sh\nset -e\n' > .git/hooks/pre-commit
|
||||
@printf 'go mod tidy\ngo fmt ./...\ngit diff --exit-code -- go.mod go.sum || { echo "go mod tidy changed files; please stage and retry"; exit 1; }\n' >> .git/hooks/pre-commit
|
||||
@printf 'make check\n' >> .git/hooks/pre-commit
|
||||
@chmod +x .git/hooks/pre-commit
|
||||
@script/install-precommit
|
||||
|
||||
-556
@@ -1,556 +0,0 @@
|
||||
# Vaultik Snapshot Creation Process
|
||||
|
||||
This document describes the lifecycle of objects during snapshot creation, with a focus on database transactions and foreign key constraints.
|
||||
|
||||
## Database Schema Overview
|
||||
|
||||
### Tables and Foreign Key Dependencies
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────┐
|
||||
│ FOREIGN KEY GRAPH │
|
||||
│ │
|
||||
│ snapshots ◄────── snapshot_files ────────► files │
|
||||
│ │ │ │
|
||||
│ └───────── snapshot_blobs ────────► blobs │ │
|
||||
│ │ │ │
|
||||
│ │ ├──► file_chunks ◄── chunks│
|
||||
│ │ │ ▲ │
|
||||
│ │ └──► chunk_files ────┘ │
|
||||
│ │ │
|
||||
│ └──► blob_chunks ─────────────┘│
|
||||
│ │
|
||||
│ uploads ───────► blobs.blob_hash │
|
||||
│ └──────────► snapshots.id │
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Critical Constraint: `chunks` Must Exist First
|
||||
|
||||
These tables reference `chunks.chunk_hash` **without CASCADE**:
|
||||
- `file_chunks.chunk_hash` → `chunks.chunk_hash`
|
||||
- `chunk_files.chunk_hash` → `chunks.chunk_hash`
|
||||
- `blob_chunks.chunk_hash` → `chunks.chunk_hash`
|
||||
|
||||
**Implication**: A chunk record MUST be committed to the database BEFORE any of these referencing records can be created.
|
||||
|
||||
### Order of Operations Required by Schema
|
||||
|
||||
```
|
||||
1. snapshots (created first, before scan)
|
||||
2. blobs (created when packer starts new blob)
|
||||
3. chunks (created during file processing)
|
||||
4. blob_chunks (created immediately after chunk added to packer)
|
||||
5. files (created after file fully chunked)
|
||||
6. file_chunks (created with file record)
|
||||
7. chunk_files (created with file record)
|
||||
8. snapshot_files (created with file record)
|
||||
9. snapshot_blobs (created after blob uploaded)
|
||||
10. uploads (created after blob uploaded)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Snapshot Creation Phases
|
||||
|
||||
### Phase 0: Initialization
|
||||
|
||||
**Actions:**
|
||||
1. Snapshot record created in database (Transaction T0)
|
||||
2. Known files loaded into memory from `files` table
|
||||
3. Known chunks loaded into memory from `chunks` table
|
||||
|
||||
**Transactions:**
|
||||
```
|
||||
T0: INSERT INTO snapshots (id, hostname, ...) VALUES (...)
|
||||
COMMIT
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Phase 1: Scan Directory
|
||||
|
||||
**Actions:**
|
||||
1. Walk filesystem directory tree
|
||||
2. For each file, compare against in-memory `knownFiles` map
|
||||
3. Classify files as: unchanged, new, or modified
|
||||
4. Collect unchanged file IDs for later association
|
||||
5. Collect new/modified files for processing
|
||||
|
||||
**Transactions:**
|
||||
```
|
||||
(None during scan - all in-memory)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Phase 1b: Associate Unchanged Files
|
||||
|
||||
**Actions:**
|
||||
1. For unchanged files, add entries to `snapshot_files` table
|
||||
2. Done in batches of 1000
|
||||
|
||||
**Transactions:**
|
||||
```
|
||||
For each batch of 1000 file IDs:
|
||||
T: BEGIN
|
||||
INSERT INTO snapshot_files (snapshot_id, file_id) VALUES (?, ?)
|
||||
... (up to 1000 inserts)
|
||||
COMMIT
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Phase 2: Process Files
|
||||
|
||||
For each file that needs processing:
|
||||
|
||||
#### Step 2a: Open and Chunk File
|
||||
|
||||
**Location:** `processFileStreaming()`
|
||||
|
||||
For each chunk produced by content-defined chunking:
|
||||
|
||||
##### Step 2a-1: Check Chunk Existence
|
||||
```go
|
||||
chunkExists := s.chunkExists(chunk.Hash) // In-memory lookup
|
||||
```
|
||||
|
||||
##### Step 2a-2: Create Chunk Record (if new)
|
||||
```go
|
||||
// TRANSACTION: Create chunk in database
|
||||
err := s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||
dbChunk := &database.Chunk{ChunkHash: chunk.Hash, Size: chunk.Size}
|
||||
return s.repos.Chunks.Create(txCtx, tx, dbChunk)
|
||||
})
|
||||
// COMMIT immediately after WithTx returns
|
||||
|
||||
// Update in-memory cache
|
||||
s.addKnownChunk(chunk.Hash)
|
||||
```
|
||||
|
||||
**Transaction:**
|
||||
```
|
||||
T_chunk: BEGIN
|
||||
INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)
|
||||
COMMIT
|
||||
```
|
||||
|
||||
##### Step 2a-3: Add Chunk to Packer
|
||||
|
||||
```go
|
||||
s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data})
|
||||
```
|
||||
|
||||
**Inside packer.AddChunk → addChunkToCurrentBlob():**
|
||||
|
||||
```go
|
||||
// TRANSACTION: Create blob_chunks record IMMEDIATELY
|
||||
if p.repos != nil {
|
||||
blobChunk := &database.BlobChunk{
|
||||
BlobID: p.currentBlob.id,
|
||||
ChunkHash: chunk.Hash,
|
||||
Offset: offset,
|
||||
Length: chunkSize,
|
||||
}
|
||||
err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return p.repos.BlobChunks.Create(ctx, tx, blobChunk)
|
||||
})
|
||||
// COMMIT immediately
|
||||
}
|
||||
```
|
||||
|
||||
**Transaction:**
|
||||
```
|
||||
T_blob_chunk: BEGIN
|
||||
INSERT INTO blob_chunks (blob_id, chunk_hash, offset, length) VALUES (?, ?, ?, ?)
|
||||
COMMIT
|
||||
```
|
||||
|
||||
**⚠️ CRITICAL DEPENDENCY**: This transaction requires `chunks.chunk_hash` to exist (FK constraint).
|
||||
The chunk MUST be committed in Step 2a-2 BEFORE this can succeed.
|
||||
|
||||
---
|
||||
|
||||
#### Step 2b: Blob Size Limit Handling
|
||||
|
||||
If adding a chunk would exceed blob size limit:
|
||||
|
||||
```go
|
||||
if err == blob.ErrBlobSizeLimitExceeded {
|
||||
if err := s.packer.FinalizeBlob(); err != nil { ... }
|
||||
// Retry adding the chunk
|
||||
if err := s.packer.AddChunk(...); err != nil { ... }
|
||||
}
|
||||
```
|
||||
|
||||
**FinalizeBlob() transactions:**
|
||||
```
|
||||
T_blob_finish: BEGIN
|
||||
UPDATE blobs SET blob_hash=?, uncompressed_size=?, compressed_size=?, finished_ts=? WHERE id=?
|
||||
COMMIT
|
||||
```
|
||||
|
||||
Then blob handler is called (handleBlobReady):
|
||||
```
|
||||
(Upload to S3 - no transaction)
|
||||
|
||||
T_blob_uploaded: BEGIN
|
||||
UPDATE blobs SET uploaded_ts=? WHERE id=?
|
||||
INSERT INTO snapshot_blobs (snapshot_id, blob_id, blob_hash) VALUES (?, ?, ?)
|
||||
INSERT INTO uploads (blob_hash, snapshot_id, uploaded_at, size, duration_ms) VALUES (?, ?, ?, ?, ?)
|
||||
COMMIT
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### Step 2c: Queue File for Batch Insertion
|
||||
|
||||
After all chunks for a file are processed:
|
||||
|
||||
```go
|
||||
// Build file data (in-memory, no DB)
|
||||
fileChunks := make([]database.FileChunk, len(chunks))
|
||||
chunkFiles := make([]database.ChunkFile, len(chunks))
|
||||
|
||||
// Queue for batch insertion
|
||||
return s.addPendingFile(ctx, pendingFileData{
|
||||
file: fileToProcess.File,
|
||||
fileChunks: fileChunks,
|
||||
chunkFiles: chunkFiles,
|
||||
})
|
||||
```
|
||||
|
||||
**No transaction yet** - just adds to `pendingFiles` slice.
|
||||
|
||||
If `len(pendingFiles) >= fileBatchSize (100)`, triggers `flushPendingFiles()`.
|
||||
|
||||
---
|
||||
|
||||
### Step 2d: Flush Pending Files
|
||||
|
||||
**Location:** `flushPendingFiles()` - called when batch is full or at end of processing
|
||||
|
||||
```go
|
||||
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||
for _, data := range files {
|
||||
// 1. Create file record
|
||||
s.repos.Files.Create(txCtx, tx, data.file) // INSERT OR REPLACE
|
||||
|
||||
// 2. Delete old associations
|
||||
s.repos.FileChunks.DeleteByFileID(txCtx, tx, data.file.ID)
|
||||
s.repos.ChunkFiles.DeleteByFileID(txCtx, tx, data.file.ID)
|
||||
|
||||
// 3. Create file_chunks records
|
||||
for _, fc := range data.fileChunks {
|
||||
s.repos.FileChunks.Create(txCtx, tx, &fc) // FK: chunks.chunk_hash
|
||||
}
|
||||
|
||||
// 4. Create chunk_files records
|
||||
for _, cf := range data.chunkFiles {
|
||||
s.repos.ChunkFiles.Create(txCtx, tx, &cf) // FK: chunks.chunk_hash
|
||||
}
|
||||
|
||||
// 5. Add file to snapshot
|
||||
s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, data.file.ID)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// COMMIT (all or nothing for the batch)
|
||||
```
|
||||
|
||||
**Transaction:**
|
||||
```
|
||||
T_files_batch: BEGIN
|
||||
-- For each file in batch:
|
||||
INSERT OR REPLACE INTO files (...) VALUES (...)
|
||||
DELETE FROM file_chunks WHERE file_id = ?
|
||||
DELETE FROM chunk_files WHERE file_id = ?
|
||||
INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES (?, ?, ?) -- FK: chunks
|
||||
INSERT INTO chunk_files (chunk_hash, file_id, ...) VALUES (?, ?, ...) -- FK: chunks
|
||||
INSERT INTO snapshot_files (snapshot_id, file_id) VALUES (?, ?)
|
||||
-- Repeat for each file
|
||||
COMMIT
|
||||
```
|
||||
|
||||
**⚠️ CRITICAL DEPENDENCY**: `file_chunks` and `chunk_files` require `chunks.chunk_hash` to exist.
|
||||
|
||||
---
|
||||
|
||||
### Phase 2 End: Final Flush
|
||||
|
||||
```go
|
||||
// Flush any remaining pending files
|
||||
if err := s.flushAllPending(ctx); err != nil { ... }
|
||||
|
||||
// Final packer flush
|
||||
s.packer.Flush()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## The Current Bug
|
||||
|
||||
### Problem
|
||||
|
||||
The current code attempts to batch file insertions, but `file_chunks` and `chunk_files` have foreign keys to `chunks.chunk_hash`. The batched file flush tries to insert these records, but if the chunks haven't been committed yet, the FK constraint fails.
|
||||
|
||||
### Why It's Happening
|
||||
|
||||
Looking at the sequence:
|
||||
|
||||
1. Process file A, chunk X
|
||||
2. Create chunk X in DB (Transaction commits)
|
||||
3. Add chunk X to packer
|
||||
4. Packer creates blob_chunks for chunk X (needs chunk X - OK, committed in step 2)
|
||||
5. Queue file A with chunk references
|
||||
6. Process file B, chunk Y
|
||||
7. Create chunk Y in DB (Transaction commits)
|
||||
8. ... etc ...
|
||||
9. At end: flushPendingFiles()
|
||||
10. Insert file_chunks for file A referencing chunk X (chunk X committed - should work)
|
||||
|
||||
The chunks ARE being created individually. But something is going wrong.
|
||||
|
||||
### Actual Issue
|
||||
|
||||
Wait - let me re-read the code. The issue is:
|
||||
|
||||
In `processFileStreaming`, when we queue file data:
|
||||
```go
|
||||
fileChunks[i] = database.FileChunk{
|
||||
FileID: fileToProcess.File.ID,
|
||||
Idx: ci.fileChunk.Idx,
|
||||
ChunkHash: ci.fileChunk.ChunkHash,
|
||||
}
|
||||
```
|
||||
|
||||
The `FileID` is set, but `fileToProcess.File.ID` might be empty at this point because the file record hasn't been created yet!
|
||||
|
||||
Looking at `checkFileInMemory`:
|
||||
```go
|
||||
// For new files:
|
||||
if !exists {
|
||||
return file, true // file.ID is empty string!
|
||||
}
|
||||
|
||||
// For existing files:
|
||||
file.ID = existingFile.ID // Reuse existing ID
|
||||
```
|
||||
|
||||
**For NEW files, `file.ID` is empty!**
|
||||
|
||||
Then in `flushPendingFiles`:
|
||||
```go
|
||||
s.repos.Files.Create(txCtx, tx, data.file) // This generates/uses the ID
|
||||
```
|
||||
|
||||
But `data.fileChunks` was built with the EMPTY ID!
|
||||
|
||||
### The Real Problem
|
||||
|
||||
For new files:
|
||||
1. `checkFileInMemory` creates file record with empty ID
|
||||
2. `processFileStreaming` queues file_chunks with empty `FileID`
|
||||
3. `flushPendingFiles` creates file (generates ID), but file_chunks still have empty `FileID`
|
||||
|
||||
Wait, but `Files.Create` should be INSERT OR REPLACE by path, and the file struct should get updated... Let me check.
|
||||
|
||||
Actually, looking more carefully at the code path - the file IS created first in the flush, but the `fileChunks` slice was already built with the old (possibly empty) ID. The ID isn't updated after the file is created.
|
||||
|
||||
Hmm, but looking at the current code:
|
||||
```go
|
||||
fileChunks[i] = database.FileChunk{
|
||||
FileID: fileToProcess.File.ID, // This uses the ID from the File struct
|
||||
```
|
||||
|
||||
And in `checkFileInMemory` for new files, we create a file struct but don't set the ID. However, looking at the database repository, `Files.Create` should be doing `INSERT OR REPLACE` and the ID should be pre-generated...
|
||||
|
||||
Let me check if IDs are being generated. Looking at the File struct usage, it seems like UUIDs should be generated somewhere...
|
||||
|
||||
Actually, looking at the test failures again:
|
||||
```
|
||||
creating file chunk: inserting file_chunk: constraint failed: FOREIGN KEY constraint failed (787)
|
||||
```
|
||||
|
||||
Error 787 is SQLite's foreign key constraint error. The failing FK is on `file_chunks.chunk_hash → chunks.chunk_hash`.
|
||||
|
||||
So the chunks ARE NOT in the database when we try to insert file_chunks. Let me trace through more carefully...
|
||||
|
||||
---
|
||||
|
||||
## Transaction Timing Issue
|
||||
|
||||
The problem is transaction visibility in SQLite.
|
||||
|
||||
Each `WithTx` creates a new transaction that commits at the end. But with batched file insertion:
|
||||
|
||||
1. Chunk transactions commit one at a time
|
||||
2. File batch transaction runs later
|
||||
|
||||
If chunks are being inserted but something goes wrong with transaction isolation, the file batch might not see them.
|
||||
|
||||
But actually SQLite in WAL mode should have SERIALIZABLE isolation by default, so committed transactions should be visible.
|
||||
|
||||
Let me check if the in-memory cache is masking a database problem...
|
||||
|
||||
Actually, wait. Let me re-check the current broken code more carefully. The issue might be simpler.
|
||||
|
||||
---
|
||||
|
||||
## Current Code Flow Analysis
|
||||
|
||||
Looking at `processFileStreaming` in the current broken state:
|
||||
|
||||
```go
|
||||
// For each chunk:
|
||||
if !chunkExists {
|
||||
err := s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||
dbChunk := &database.Chunk{ChunkHash: chunk.Hash, Size: chunk.Size}
|
||||
return s.repos.Chunks.Create(txCtx, tx, dbChunk)
|
||||
})
|
||||
// ... check error ...
|
||||
s.addKnownChunk(chunk.Hash)
|
||||
}
|
||||
|
||||
// ... add to packer (creates blob_chunks) ...
|
||||
|
||||
// Collect chunk info for file
|
||||
chunks = append(chunks, chunkInfo{...})
|
||||
```
|
||||
|
||||
Then at end of function:
|
||||
```go
|
||||
// Queue file for batch insertion
|
||||
return s.addPendingFile(ctx, pendingFileData{
|
||||
file: fileToProcess.File,
|
||||
fileChunks: fileChunks,
|
||||
chunkFiles: chunkFiles,
|
||||
})
|
||||
```
|
||||
|
||||
At end of `processPhase`:
|
||||
```go
|
||||
if err := s.flushAllPending(ctx); err != nil { ... }
|
||||
```
|
||||
|
||||
The chunks are being created one-by-one with individual transactions. By the time `flushPendingFiles` runs, all chunk transactions should have committed.
|
||||
|
||||
Unless... there's a bug in how the chunks are being referenced. Let me check if the chunk_hash values are correct.
|
||||
|
||||
Or... maybe the test database is being recreated between operations somehow?
|
||||
|
||||
Actually, let me check the test setup. Maybe the issue is specific to the test environment.
|
||||
|
||||
---
|
||||
|
||||
## Summary of Object Lifecycle
|
||||
|
||||
| Object | When Created | Transaction | Dependencies |
|
||||
|--------|--------------|-------------|--------------|
|
||||
| snapshot | Before scan | Individual tx | None |
|
||||
| blob | When packer needs new blob | Individual tx | None |
|
||||
| chunk | During file chunking (each chunk) | Individual tx | None |
|
||||
| blob_chunks | Immediately after adding chunk to packer | Individual tx | chunks, blobs |
|
||||
| files | Batched at end of processing | Batch tx | None |
|
||||
| file_chunks | With file (batched) | Batch tx | files, chunks |
|
||||
| chunk_files | With file (batched) | Batch tx | files, chunks |
|
||||
| snapshot_files | With file (batched) | Batch tx | snapshots, files |
|
||||
| snapshot_blobs | After blob upload | Individual tx | snapshots, blobs |
|
||||
| uploads | After blob upload | Same tx as snapshot_blobs | blobs, snapshots |
|
||||
|
||||
---
|
||||
|
||||
## Root Cause Analysis
|
||||
|
||||
After detailed analysis, I believe the issue is one of the following:
|
||||
|
||||
### Hypothesis 1: File ID Not Set
|
||||
|
||||
Looking at `checkFileInMemory()` for NEW files:
|
||||
```go
|
||||
if !exists {
|
||||
return file, true // file.ID is empty string!
|
||||
}
|
||||
```
|
||||
|
||||
For new files, `file.ID` is empty. Then in `processFileStreaming`:
|
||||
```go
|
||||
fileChunks[i] = database.FileChunk{
|
||||
FileID: fileToProcess.File.ID, // Empty for new files!
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
The `FileID` in the built `fileChunks` slice is empty.
|
||||
|
||||
Then in `flushPendingFiles`:
|
||||
```go
|
||||
s.repos.Files.Create(txCtx, tx, data.file) // This generates the ID
|
||||
// But data.fileChunks still has empty FileID!
|
||||
for i := range data.fileChunks {
|
||||
s.repos.FileChunks.Create(...) // Uses empty FileID
|
||||
}
|
||||
```
|
||||
|
||||
**Solution**: Generate file IDs upfront in `checkFileInMemory()`:
|
||||
```go
|
||||
file := &database.File{
|
||||
ID: uuid.New().String(), // Generate ID immediately
|
||||
Path: path,
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
### Hypothesis 2: Transaction Isolation
|
||||
|
||||
SQLite with a single connection pool (`MaxOpenConns(1)`) should serialize all transactions. Committed data should be visible to subsequent transactions.
|
||||
|
||||
However, there might be a subtle issue with how `context.Background()` is used in the packer vs the scanner's context.
|
||||
|
||||
## Recommended Fix
|
||||
|
||||
**Step 1: Generate file IDs upfront**
|
||||
|
||||
In `checkFileInMemory()`, generate the UUID for new files immediately:
|
||||
```go
|
||||
file := &database.File{
|
||||
ID: uuid.New().String(), // Always generate ID
|
||||
Path: path,
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
This ensures `file.ID` is set when building `fileChunks` and `chunkFiles` slices.
|
||||
|
||||
**Step 2: Verify by reverting to per-file transactions**
|
||||
|
||||
If Step 1 doesn't fix it, revert to non-batched file insertion to isolate the issue:
|
||||
|
||||
```go
|
||||
// Instead of queuing:
|
||||
// return s.addPendingFile(ctx, pendingFileData{...})
|
||||
|
||||
// Do immediate insertion:
|
||||
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||
// Create file
|
||||
s.repos.Files.Create(txCtx, tx, fileToProcess.File)
|
||||
// Delete old associations
|
||||
s.repos.FileChunks.DeleteByFileID(...)
|
||||
s.repos.ChunkFiles.DeleteByFileID(...)
|
||||
// Create new associations
|
||||
for _, fc := range fileChunks {
|
||||
s.repos.FileChunks.Create(...)
|
||||
}
|
||||
for _, cf := range chunkFiles {
|
||||
s.repos.ChunkFiles.Create(...)
|
||||
}
|
||||
// Add to snapshot
|
||||
s.repos.Snapshots.AddFileByID(...)
|
||||
return nil
|
||||
})
|
||||
```
|
||||
|
||||
**Step 3: If batching is still desired**
|
||||
|
||||
After confirming per-file transactions work, re-implement batching with the ID fix in place, and add debug logging to trace exactly which chunk_hash is failing and why.
|
||||
@@ -0,0 +1,408 @@
|
||||
---
|
||||
title: Repository Policies
|
||||
last_modified: 2026-07-06
|
||||
---
|
||||
|
||||
This document covers repository structure, tooling, and workflow standards. Code
|
||||
style conventions are in separate documents:
|
||||
|
||||
- [Code Styleguide](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE.md)
|
||||
(general, bash, Docker)
|
||||
- [Go](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_GO.md)
|
||||
- [JavaScript](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_JS.md)
|
||||
- [Python](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_PYTHON.md)
|
||||
- [Go HTTP Server Conventions](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/GO_HTTP_SERVER_CONVENTIONS.md)
|
||||
|
||||
---
|
||||
|
||||
- Cross-project documentation (such as this file) must include
|
||||
`last_modified: YYYY-MM-DD` in the YAML front matter so it can be kept in sync
|
||||
with the authoritative source as policies evolve.
|
||||
|
||||
- **ALL external references must be pinned by cryptographic hash.** This
|
||||
includes Docker base images, Go modules, npm packages, GitHub Actions, and
|
||||
anything else fetched from a remote source. Version tags (`@v4`, `@latest`,
|
||||
`:3.21`, etc.) are server-mutable and therefore remote code execution
|
||||
vulnerabilities. The ONLY acceptable way to reference an external dependency
|
||||
is by its content hash (Docker `@sha256:...`, Go module hash in `go.sum`, npm
|
||||
integrity hash in lockfile, GitHub Actions `@<commit-sha>`). No exceptions.
|
||||
This also means never `curl | bash` to install tools like pyenv, nvm, rustup,
|
||||
etc. Instead, download a specific release archive from GitHub, verify its hash
|
||||
(hardcoded in the Dockerfile or script), and only then install. Unverified
|
||||
install scripts are arbitrary remote code execution. This is the single most
|
||||
important rule in this document. Double-check every external reference in
|
||||
every file before committing. There are zero exceptions to this rule.
|
||||
|
||||
- Every repo with software must have a root `Makefile` with these targets:
|
||||
`make bootstrap`, `make setup`, `make test`, `make lint`, `make fmt` (writes),
|
||||
`make fmt-check` (read-only), `make check` (runs `test`, `lint`, `fmt-check`),
|
||||
`make docker`, and `make hooks` (installs pre-commit hook). A model Makefile
|
||||
is at `https://git.eeqj.de/sneak/prompts/raw/branch/main/Makefile`.
|
||||
|
||||
- Repos follow the
|
||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||
pattern: the implementation of each Makefile target lives in an executable
|
||||
script in `script/` (`script/bootstrap`, `script/setup`, `script/test`,
|
||||
`script/lint`, `script/fmt`, `script/fmt-check`, `script/check`,
|
||||
`script/docker`), and the Makefile targets are thin shims that call them. The
|
||||
scripts must be POSIX sh (`#!/bin/sh`, `set -eu`, no bashisms) so they run in
|
||||
minimal containers (e.g. alpine images have no bash); locate the repo root
|
||||
with `$(cd "$(dirname "$0")/.." && pwd -P)` and `cd` there before acting. From
|
||||
the standard's canonical set we use `bootstrap`, `setup` (make the repo ready
|
||||
for development after a fresh clone: runs `bootstrap`, then
|
||||
`install-precommit`, plus any repo-specific initialization), `test`, and
|
||||
`cibuild`. `script/bootstrap` installs all dependencies idempotently and
|
||||
assumes nothing is present: base tools come from nix, apt, brew, or apk
|
||||
(detected in that order; apt runs noninteractive). For node it uses the
|
||||
installed node if present; otherwise it installs a PINNED node version via
|
||||
nvm, first installing nvm itself if missing — from a hash-verified GitHub
|
||||
release archive (never `curl | sh`), with bash installed as an explicit
|
||||
prerequisite since nvm requires bash. yarn is then pinned via
|
||||
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
|
||||
always exact versions. `script/cibuild` runs the CI build: it changes to the
|
||||
repo root and runs `docker build .`; the Gitea workflow calls it. Four further
|
||||
scripts are our own extensions to the standard: `script/check` runs
|
||||
`script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
|
||||
what the git pre-commit hook runs, and it calls `script/check`;
|
||||
`script/install-precommit` installs the git pre-commit hook (the `make hooks`
|
||||
target shims to it); and `script/projectname` (literally that filename) simply
|
||||
outputs the project's name. Scripts that need the name call
|
||||
`script/projectname` — e.g. `script/docker` assembles its image tag from it —
|
||||
so those scripts stay byte-identical across all repos. Repo-type-specific
|
||||
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
|
||||
`script/precommit`, not in the hook itself. Model scripts are at
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
||||
must document the provided scripts in an **Entrypoints** section (see the
|
||||
README requirements below).
|
||||
|
||||
- Always use Makefile targets (`make fmt`, `make test`, `make lint`, etc.)
|
||||
instead of invoking the underlying tools directly. The Makefile is the single
|
||||
source of truth for how these operations are run.
|
||||
|
||||
- The Makefile is authoritative documentation for how the repo is used. Beyond
|
||||
the required targets above, it should have targets for every common operation:
|
||||
running a local development server (`make run`, `make dev`), re-initializing
|
||||
or migrating the database (`make db-reset`, `make migrate`), building
|
||||
artifacts (`make build`), generating code, seeding data, or anything else a
|
||||
developer would do regularly. If someone checks out the repo and types
|
||||
`make<tab>`, they should see every meaningful operation available. A new
|
||||
contributor should be able to understand the entire development workflow by
|
||||
reading the Makefile.
|
||||
|
||||
- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
|
||||
as a build step so the build fails if the branch is not green. For non-server
|
||||
repos, the Dockerfile should bring up a development environment and run
|
||||
`make check`. For server repos, `make check` should run as an early build
|
||||
stage before the final image is assembled. Dockerfiles install development
|
||||
prerequisites by running `script/bootstrap` rather than duplicating installs
|
||||
inline; COPY `script/` and the dependency manifests (`package.json` +
|
||||
`yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
|
||||
layer stays cached until dependencies change.
|
||||
|
||||
- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
|
||||
repos use a multistage build where linting runs in an independent stage based
|
||||
on the `golangci/golangci-lint` image (pinned by hash). This stage runs
|
||||
`make fmt-check` and `make lint` before the full build begins. The build stage
|
||||
then declares an explicit dependency on the lint stage via
|
||||
`COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
|
||||
linting before proceeding to compilation and tests. This ensures lint failures
|
||||
surface in seconds rather than minutes, without blocking on dependency
|
||||
download or compilation in the build stage.
|
||||
|
||||
The standard pattern for a Go repo Dockerfile is:
|
||||
|
||||
```dockerfile
|
||||
# Lint stage — fast feedback on formatting and lint issues
|
||||
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
||||
FROM golangci/golangci-lint@sha256:... AS lint
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN make fmt-check
|
||||
RUN make lint
|
||||
|
||||
# Build stage
|
||||
# golang:1.x-alpine, YYYY-MM-DD
|
||||
FROM golang@sha256:... AS builder
|
||||
WORKDIR /src
|
||||
|
||||
# Force BuildKit to run the lint stage before proceeding
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN make test
|
||||
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
||||
-o /app ./cmd/app/
|
||||
|
||||
# Runtime stage
|
||||
FROM alpine@sha256:...
|
||||
COPY --from=builder /app /usr/local/bin/app
|
||||
ENTRYPOINT ["app"]
|
||||
```
|
||||
|
||||
Key points:
|
||||
- The lint stage uses the `golangci/golangci-lint` image directly (it
|
||||
includes both Go and the linter), so there is no need to install the
|
||||
linter separately.
|
||||
- `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
|
||||
a stage dependency. BuildKit runs stages in parallel by default; without
|
||||
this line, the build stage would not wait for lint to finish and a lint
|
||||
failure might not fail the overall build.
|
||||
- If the project uses `//go:embed` directives that reference build artifacts
|
||||
(e.g. a web frontend compiled in a separate stage), the lint stage must
|
||||
create placeholder files so the embed directives resolve. Example:
|
||||
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
|
||||
The lint stage should not depend on the actual build output — it exists to
|
||||
fail fast.
|
||||
- If the project requires CGO or system libraries for linting (e.g.
|
||||
`vips-dev`), install them in the lint stage with `apk add`.
|
||||
- The build stage runs `make test` after compilation setup. Tests run in the
|
||||
build stage, not the lint stage, because they may require compiled
|
||||
artifacts or heavier dependencies.
|
||||
|
||||
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
|
||||
runs `script/cibuild` (which runs `docker build .`) on push. Since the
|
||||
Dockerfile already runs `make check`, a successful build implies all checks
|
||||
pass.
|
||||
|
||||
- Use platform-standard formatters: `black` for Python, `prettier` for
|
||||
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
||||
two exceptions: four-space indents (except Go), and `proseWrap: always` for
|
||||
Markdown (hard-wrap at 80 columns). Documentation and writing repos (Markdown,
|
||||
HTML, CSS) should also have `.prettierrc` and `.prettierignore`.
|
||||
|
||||
- Pre-commit hook: runs `script/precommit`, which calls `script/check`. If local
|
||||
testing is not possible in the repo, `script/precommit` may skip `script/test`
|
||||
and run only `script/lint` and `script/fmt-check`. The hook is installed by
|
||||
`script/install-precommit`; the Makefile must provide a `make hooks` target
|
||||
that shims to it.
|
||||
|
||||
- All repos with software must have tests that run via the platform-standard
|
||||
test framework (`go test`, `pytest`, `jest`/`vitest`, etc.). If no meaningful
|
||||
tests exist yet, add the most minimal test possible — e.g. importing the
|
||||
module under test to verify it compiles/parses. There is no excuse for
|
||||
`make test` to be a no-op.
|
||||
|
||||
- `make test` must complete in under 20 seconds. Add a 30-second timeout in the
|
||||
Makefile.
|
||||
|
||||
- **`make test` should use the conditional verbose rerun pattern.** Run tests
|
||||
without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
|
||||
show full output. This keeps CI logs and `docker build` output clean on
|
||||
success (just package/suite summaries) while providing full diagnostic detail
|
||||
on failure (every test case, every assertion). The general shell pattern:
|
||||
|
||||
```makefile
|
||||
test:
|
||||
@<test-command> || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
<test-command-with-v>; exit 1; }
|
||||
```
|
||||
|
||||
Go example:
|
||||
|
||||
```makefile
|
||||
test:
|
||||
@go test -timeout 30s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -timeout 30s -race -v ./...; exit 1; }
|
||||
```
|
||||
|
||||
Python example:
|
||||
|
||||
```makefile
|
||||
test:
|
||||
@python -m pytest || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
python -m pytest -v; exit 1; }
|
||||
```
|
||||
|
||||
The `exit 1` ensures the target always fails after a rerun — the first run
|
||||
already proved the tests are broken, so the build must not pass even if a
|
||||
flaky test happens to succeed on the second attempt. The rerun exists solely
|
||||
for diagnostic output.
|
||||
|
||||
- Docker builds must complete in under 5 minutes.
|
||||
|
||||
- `make check` must not modify any files in the repo. Tests may use temporary
|
||||
directories.
|
||||
|
||||
- `main` must always pass `make check`, no exceptions.
|
||||
|
||||
- Never commit secrets. `.env` files, credentials, API keys, and private keys
|
||||
must be in `.gitignore`. No exceptions.
|
||||
|
||||
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
|
||||
editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
|
||||
Fetch the standard `.gitignore` from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
|
||||
a new repo.
|
||||
|
||||
- **No build artifacts in version control.** Code-derived data (compiled
|
||||
bundles, minified output, generated assets) must never be committed to the
|
||||
repository if it can be avoided. The build process (e.g. Dockerfile, Makefile)
|
||||
should generate these at build time. Notable exception: Go protobuf generated
|
||||
files (`.pb.go`) ARE committed because repos need to work with `go get`, which
|
||||
downloads code but does not execute code generation.
|
||||
|
||||
- Never use `git add -A` or `git add .`. Always stage files explicitly by name.
|
||||
|
||||
- Never force-push to `main`.
|
||||
|
||||
- Make all changes on a feature branch. You can do whatever you want on a
|
||||
feature branch.
|
||||
|
||||
- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
|
||||
manually by the user. Fetch from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
|
||||
|
||||
- When pinning images or packages by hash, add a comment above the reference
|
||||
with the version and date (YYYY-MM-DD).
|
||||
|
||||
- Use `yarn`, not `npm`.
|
||||
|
||||
- Write all dates as YYYY-MM-DD (ISO 8601).
|
||||
|
||||
- Simple projects should be configured with environment variables.
|
||||
|
||||
- Dockerized web services listen on port 8080 by default, overridable with
|
||||
`PORT`.
|
||||
|
||||
- **HTTP/web services must be hardened for production internet exposure before
|
||||
tagging 1.0.** This means full compliance with security best practices
|
||||
including, without limitation, all of the following:
|
||||
- **Security headers** on every response:
|
||||
- `Strict-Transport-Security` (HSTS) with `max-age` of at least one year
|
||||
and `includeSubDomains`.
|
||||
- `Content-Security-Policy` (CSP) with a restrictive default policy
|
||||
(`default-src 'self'` as a baseline, tightened per-resource as
|
||||
needed). Never use `unsafe-inline` or `unsafe-eval` unless
|
||||
unavoidable, and document the reason.
|
||||
- `X-Frame-Options: DENY` (or `SAMEORIGIN` if framing is required).
|
||||
Prefer the `frame-ancestors` CSP directive as the primary control.
|
||||
- `X-Content-Type-Options: nosniff`.
|
||||
- `Referrer-Policy: strict-origin-when-cross-origin` (or stricter).
|
||||
- `Permissions-Policy` restricting access to browser features the
|
||||
application does not use (camera, microphone, geolocation, etc.).
|
||||
- **Request and response limits:**
|
||||
- Maximum request body size enforced on all endpoints (e.g. Go
|
||||
`http.MaxBytesReader`). Choose a sane default per-route; never accept
|
||||
unbounded input.
|
||||
- Maximum response body size where applicable (e.g. paginated APIs).
|
||||
- `ReadTimeout` and `ReadHeaderTimeout` on the `http.Server` to defend
|
||||
against slowloris attacks.
|
||||
- `WriteTimeout` on the `http.Server`.
|
||||
- `IdleTimeout` on the `http.Server`.
|
||||
- Per-handler execution time limits via `context.WithTimeout` or
|
||||
chi/stdlib `middleware.Timeout`.
|
||||
- **Authentication and session security:**
|
||||
- Rate limiting on password-based authentication endpoints. API keys are
|
||||
high-entropy and not susceptible to brute force, so they are exempt.
|
||||
- CSRF tokens on all state-mutating HTML forms. API endpoints
|
||||
authenticated via `Authorization` header (Bearer token, API key) are
|
||||
exempt because the browser does not attach these automatically.
|
||||
- Passwords stored using bcrypt, scrypt, or argon2 — never plain-text,
|
||||
MD5, or SHA.
|
||||
- Session cookies set with `HttpOnly`, `Secure`, and `SameSite=Lax` (or
|
||||
`Strict`) attributes.
|
||||
- **Reverse proxy awareness:**
|
||||
- True client IP detection when behind a reverse proxy
|
||||
(`X-Forwarded-For`, `X-Real-IP`). The application must accept
|
||||
forwarded headers only from a configured set of trusted proxy
|
||||
addresses — never trust `X-Forwarded-For` unconditionally.
|
||||
- **CORS:**
|
||||
- Authenticated endpoints must restrict `Access-Control-Allow-Origin` to
|
||||
an explicit allowlist of known origins. Wildcard (`*`) is acceptable
|
||||
only for public, unauthenticated read-only APIs.
|
||||
- **Error handling:**
|
||||
- Internal errors must never leak stack traces, SQL queries, file paths,
|
||||
or other implementation details to the client. Return generic error
|
||||
messages in production; detailed errors only when `DEBUG` is enabled.
|
||||
- **TLS:**
|
||||
- Services never terminate TLS directly. They are always deployed behind
|
||||
a TLS-terminating reverse proxy. The service itself listens on plain
|
||||
HTTP. However, HSTS headers and `Secure` cookie flags must still be
|
||||
set by the application so that the browser enforces HTTPS end-to-end.
|
||||
|
||||
This list is non-exhaustive. Apply defense-in-depth: if a standard security
|
||||
hardening measure exists for HTTP services and is not listed here, it is
|
||||
still expected. When in doubt, harden.
|
||||
|
||||
- `README.md` is the primary documentation. Required sections:
|
||||
- **Description**: First line must include the project name, purpose,
|
||||
category (web server, SPA, CLI tool, etc.), license, and author. Example:
|
||||
"µPaaS is an MIT-licensed Go web application by @sneak that receives
|
||||
git-frontend webhooks and deploys applications via Docker in realtime."
|
||||
- **Getting Started**: Copy-pasteable install/usage code block.
|
||||
- **Entrypoints**: Opens by stating that the repo adheres to the
|
||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||
standard (with that link), then documents each provided `script/`
|
||||
entrypoint and its purpose.
|
||||
- **Rationale**: Why does this exist?
|
||||
- **Design**: How is the program structured?
|
||||
- **TODO**: Update meticulously, even between commits. When planning, put
|
||||
the todo list in the README so a new agent can pick up where the last one
|
||||
left off.
|
||||
- **License**: MIT, GPL, or WTFPL. Ask the user for new projects. Include a
|
||||
`LICENSE` file in the repo root and a License section in the README.
|
||||
- **Author**: [@sneak](https://sneak.berlin).
|
||||
|
||||
- First commit of a new repo should contain only `README.md`.
|
||||
|
||||
- Go module root: `sneak.berlin/go/<name>`. Always run `go mod tidy` before
|
||||
committing.
|
||||
|
||||
- Use SemVer.
|
||||
|
||||
- Database migrations live in `internal/db/migrations/` and must be embedded in
|
||||
the binary.
|
||||
- `000_migration.sql` — contains ONLY the creation of the migrations
|
||||
tracking table itself. Nothing else.
|
||||
- `001_schema.sql` — the full application schema.
|
||||
- **Pre-1.0.0:** never add additional migration files (002, 003, etc.).
|
||||
There is no installed base to migrate. Edit `001_schema.sql` directly.
|
||||
- **Post-1.0.0:** add new numbered migration files for each schema change.
|
||||
Never edit existing migrations after release.
|
||||
|
||||
- All repos should have an `.editorconfig` enforcing the project's indentation
|
||||
settings.
|
||||
|
||||
- Avoid putting files in the repo root unless necessary. Root should contain
|
||||
only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
|
||||
`LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
|
||||
language-specific config). Everything else goes in a subdirectory. Canonical
|
||||
subdirectory names:
|
||||
- `bin/` — executable scripts and tools
|
||||
- `cmd/` — Go command entrypoints
|
||||
- `configs/` — configuration templates and examples
|
||||
- `deploy/` — deployment manifests (k8s, compose, terraform)
|
||||
- `docs/` — documentation and markdown (README.md stays in root)
|
||||
- `internal/` — Go internal packages
|
||||
- `internal/db/migrations/` — database migrations
|
||||
- `pkg/` — Go library packages
|
||||
- `share/` — systemd units, data files
|
||||
- `static/` — static assets (images, fonts, etc.)
|
||||
- `web/` — web frontend source
|
||||
|
||||
- When setting up a new repo, files from the `prompts` repo may be used as
|
||||
templates. Fetch them from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/<path>`.
|
||||
|
||||
- New repos must contain at minimum:
|
||||
- `README.md`, `.git`, `.gitignore`, `.editorconfig`
|
||||
- `LICENSE`, `REPO_POLICIES.md` (copy from the `prompts` repo)
|
||||
- `Makefile`
|
||||
- `script/` entrypoints (`bootstrap`, `setup`, `projectname`, `test`,
|
||||
`lint`, `fmt`, `fmt-check`, `check`, `docker`, `cibuild`, `precommit`,
|
||||
`install-precommit`)
|
||||
- `Dockerfile`, `.dockerignore`
|
||||
- `.gitea/workflows/check.yml`
|
||||
- Go: `go.mod`, `go.sum`, `.golangci.yml`
|
||||
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
|
||||
- Python: `pyproject.toml`
|
||||
@@ -1,128 +1,632 @@
|
||||
# Vaultik 1.0 TODO
|
||||
# Workflow
|
||||
|
||||
Linear list of tasks to complete before 1.0 release.
|
||||
* branch (from `main`)
|
||||
* do the work in Next Step
|
||||
* move Next Step to the top of Completed Steps
|
||||
* move the top item of Future Steps into Next Step
|
||||
* commit (`TODO.md` changes in the same commit as the work)
|
||||
* merge to `main` if the branch is not protected, otherwise open a PR
|
||||
* push
|
||||
|
||||
## Rclone Storage Backend (Complete)
|
||||
# Status
|
||||
|
||||
Add rclone as a storage backend via Go library import, allowing vaultik to use any of rclone's 70+ supported cloud storage providers.
|
||||
pre-1.0
|
||||
|
||||
**Configuration:**
|
||||
```yaml
|
||||
storage_url: "rclone://myremote/path/to/backups"
|
||||
```
|
||||
User must have rclone configured separately (via `rclone config`).
|
||||
# Next Step
|
||||
|
||||
**Implementation Steps:**
|
||||
1. [x] Add rclone dependency to go.mod
|
||||
2. [x] Create `internal/storage/rclone.go` implementing `Storer` interface
|
||||
- `NewRcloneStorer(remote, path)` - init with `configfile.Install()` and `fs.NewFs()`
|
||||
- `Put` / `PutWithProgress` - use `operations.Rcat()`
|
||||
- `Get` - use `fs.NewObject()` then `obj.Open()`
|
||||
- `Stat` - use `fs.NewObject()` for size/metadata
|
||||
- `Delete` - use `obj.Remove()`
|
||||
- `List` / `ListStream` - use `operations.ListFn()`
|
||||
- `Info` - return remote name
|
||||
3. [x] Update `internal/storage/url.go` - parse `rclone://remote/path` URLs
|
||||
4. [x] Update `internal/storage/module.go` - add rclone case to `storerFromURL()`
|
||||
5. [x] Test with real rclone remote
|
||||
Define the remaining scope for the first tagged release under the 1.0.0
|
||||
milestone, then cut that tag. The mechanism to cut it now exists and is
|
||||
exercised; what is left is the scope decision, which is the owner's.
|
||||
This step deliberately names one version number: it previously said
|
||||
"cut v0.1.0" while the `Makefile` baked in `1.0.0-rc.1` and the issue
|
||||
milestone said 1.0.0, and three different answers to "what is the next
|
||||
release" is exactly the contradiction
|
||||
[issue #65](https://git.eeqj.de/sneak/vaultik/issues/65) was filed over.
|
||||
|
||||
**Error Mapping:**
|
||||
- `fs.ErrorObjectNotFound` → `ErrNotFound`
|
||||
- `fs.ErrorDirNotFound` → `ErrNotFound`
|
||||
- `fs.ErrorNotFoundInConfigFile` → `ErrRemoteNotFound` (new)
|
||||
# Completed Steps
|
||||
|
||||
---
|
||||
- 2026-09-21: Stopped `prune` from reporting a failed row count as 0
|
||||
([issue #96](https://git.eeqj.de/sneak/vaultik/issues/96)). The seven
|
||||
`getTableCount` reads in `PruneDatabase` discarded their error, so a
|
||||
query that could not run became a plausible `0` and the before/after
|
||||
delta computed from it looked like real work. Each read now logs at
|
||||
warn on failure and renders as `unknown`, never `0`, so an empty table
|
||||
is distinguishable from one that could not be queried. The counts have
|
||||
no `--json` representation — under `--json` the summary is suppressed
|
||||
entirely — so nothing there can show a false `0`.
|
||||
|
||||
## CLI Polish (Priority)
|
||||
- 2026-09-21: Stopped `--json` from silencing stderr diagnostics
|
||||
([issue #112](https://git.eeqj.de/sneak/vaultik/issues/112)). `--json`
|
||||
used to be folded into `Quiet`, which pinned the log level to `WARN`,
|
||||
so `prune --json` gave a machine consumer no record of the local index
|
||||
rows it deleted even under `--verbose`. `--json` now quiets only the
|
||||
stdout UI (the JSON document must stay clean, per
|
||||
[issue #108](https://git.eeqj.de/sneak/vaultik/issues/108)); the stderr
|
||||
log level follows `--verbose`/`--debug` again. The coupling was
|
||||
removed the same way for `snapshot verify`, `snapshot remove`, and
|
||||
`remote info`, which carried it for the same outdated reason.
|
||||
|
||||
1. Improve error messages throughout
|
||||
- Ensure all errors include actionable context
|
||||
- Add suggestions for common issues (e.g., "did you set VAULTIK_AGE_SECRET_KEY?")
|
||||
- 2026-09-21: Made the s3 storage backend report a missing object as
|
||||
`storage.ErrNotFound`, like the `file` and `rclone` backends and as the
|
||||
`Storer` interface documents. `S3Storer.Get` and `Stat` returned the raw
|
||||
AWS SDK error, so `errors.Is(err, storage.ErrNotFound)` was false on s3
|
||||
and callers branched differently per backend. Added a small `s3.IsNotFound`
|
||||
helper (reused by `HeadObject`) and a test that a missing key maps to
|
||||
`ErrNotFound`
|
||||
([issue #129](https://git.eeqj.de/sneak/vaultik/issues/129)).
|
||||
- 2026-09-21: Fixed `verify --deep` reporting healthy snapshots as
|
||||
corrupt. Its final blob-integrity check hashed the encrypted
|
||||
downloaded bytes with a single SHA256 and compared that to the blob
|
||||
ID, which is the double SHA256 of the plaintext, so the two could
|
||||
never match. It now hashes the decompressed plaintext and compares the
|
||||
double SHA256. Added a test that backs up a real snapshot, deep-verifies
|
||||
it, then flips a byte in one stored blob and confirms deep verification
|
||||
then fails
|
||||
([issue #131](https://git.eeqj.de/sneak/vaultik/issues/131)).
|
||||
|
||||
## Security (Priority)
|
||||
- 2026-09-21: Made `snapshot create` VACUUM the per-snapshot metadata
|
||||
database through the `modernc.org/sqlite` driver instead of shelling
|
||||
out to the external `sqlite` command-line binary (issue #120). A
|
||||
backup no longer needs that binary on `PATH`, so `make check` passes
|
||||
on a stock `go install` host; `script/bootstrap` and the `Dockerfile`
|
||||
(both the test-build and the shipped runtime stage) no longer install
|
||||
it, and a new test asserts the uploaded database keeps no pages from
|
||||
deleted rows. Dropped the now-false note on the 2026-08-07 entry below
|
||||
that said bootstrap installs it.
|
||||
- 2026-09-21: Made `.gitea/workflows/check.yml` run on pushes to `main`
|
||||
and `next` and on pull requests against either, so unit PRs (whose
|
||||
base is `next`) and `next` itself get a CI run instead of relying on a
|
||||
local `make check`
|
||||
([issue #122](https://git.eeqj.de/sneak/vaultik/issues/122)).
|
||||
|
||||
1. Audit encryption implementation
|
||||
- Verify age encryption is used correctly
|
||||
- Ensure no plaintext leaks in logs or errors
|
||||
- Verify blob hashes are computed correctly
|
||||
- 2026-09-21: Hash-verified the Go toolchain in the release workflow
|
||||
([issue #105](https://git.eeqj.de/sneak/vaultik/issues/105)). New
|
||||
`script/install-go` downloads the exact `go.dev` archive for `go.mod`'s
|
||||
`go` directive and refuses it unless its sha256 matches a value
|
||||
committed in the script; `.gitea/workflows/release.yml` calls it
|
||||
instead of `actions/setup-go`, which verified the downloaded toolchain
|
||||
against nothing in the repo. `GOTOOLCHAIN: local` on the release step
|
||||
keeps that exact compiler from auto-switching. Bumping Go now touches
|
||||
`go.mod`, the checksum, and the `Dockerfile` `golang` digest together.
|
||||
|
||||
1. Secure memory handling for secrets
|
||||
- Clear S3 credentials from memory after client init
|
||||
- Document that age_secret_key is env-var only (already implemented)
|
||||
- 2026-09-21: Collapsed the two duration parsers into one and fixed the
|
||||
`--older-than` months example
|
||||
([issue #123](https://git.eeqj.de/sneak/vaultik/issues/123)). Two
|
||||
functions named `parseDuration` existed with different grammars;
|
||||
`snapshot purge --older-than` and `--keep-newer-than` both already went
|
||||
through the one in `internal/vaultik`, while the richer copy in
|
||||
`internal/cli/duration.go` was reachable only from its own test. Kept
|
||||
the live-path parser and deleted the unused one, so no flag's accepted
|
||||
grammar changes. The trap the issue was filed over: `README.md`
|
||||
documented `6m` as the months example for `--older-than`, but `m` is
|
||||
minutes, so the documented command deleted every snapshot older than
|
||||
six minutes on a destructive flag. Corrected the doc to `6mo` and put
|
||||
both flags' help text on one example list that states `m` is minutes
|
||||
and `mo` is months. The surviving parser now rejects negatives, which
|
||||
it previously accepted (`-5h`) or silently made positive (`-5d`).
|
||||
Table-driven tests cover every unit, `6m` as six minutes, `6mo` as 180
|
||||
days, and rejection of a bare number, an unknown unit, and a negative.
|
||||
|
||||
## Testing
|
||||
- 2026-08-10: Moved every lint run into its own container, as a build
|
||||
step ([issue #113](https://git.eeqj.de/sneak/vaultik/issues/113)).
|
||||
New root `Dockerfile.lint`, built by `script/lint`, runs
|
||||
`golangci-lint run --config .golangci.yml ./...` as a `RUN`
|
||||
instruction in the digest-pinned `golangci/golangci-lint` image: a
|
||||
successful build of that file *is* a clean lint, and it works even
|
||||
where the daemon is remote and bind mounts are impossible. That
|
||||
`FROM` line is now the only pin of the linter version in the repo.
|
||||
|
||||
1. Write integration tests for restore command
|
||||
This supersedes the per-worktree cache isolation landed for
|
||||
[issue #99](https://git.eeqj.de/sneak/vaultik/issues/99). Isolation
|
||||
fixed cross-worktree contamination but not lock contention — two
|
||||
concurrent runs with entirely separate cache directories still
|
||||
collided. A container per run has its own cache and its own lock, so
|
||||
the whole class is gone, and with it the per-worktree cache
|
||||
machinery, the lock-retry loop, and `script/lint-audit`, which
|
||||
existed to catch replayed findings from a cache that no longer
|
||||
exists. The host lint path went too: no escape hatch, no
|
||||
`VAULTIK_LINT_IN_CONTAINER`, no version detection. Nothing lints on
|
||||
the host at any version.
|
||||
|
||||
1. Write end-to-end integration test
|
||||
- Create backup
|
||||
- Verify backup
|
||||
- Restore backup
|
||||
- Compare restored files to originals
|
||||
A cached build lints nothing, so the same `CHECK_EPOCH` mechanism the
|
||||
product `Dockerfile` already used is what makes the green mean
|
||||
something: `ARG CHECK_EPOCH` with no default below the module layers,
|
||||
a `RUN [ -n "$CHECK_EPOCH" ] || exit 1` guard, the value expanded
|
||||
into each check command, and a fresh `$(date +%s%N)$$` per invocation
|
||||
computed as a bare assignment. `cmd/vaultik/lintdocker_test.go`
|
||||
parses both Dockerfiles and both scripts and fails if any part of
|
||||
that is dropped, because every way of losing it is silent. No test
|
||||
asserts that no script runs the host linter: `script/lint` is the one
|
||||
lint entry point and runs `golangci-lint` only inside the container,
|
||||
and keeping it that way is a review matter, not something a test
|
||||
proves.
|
||||
|
||||
1. Add tests for edge cases
|
||||
- Empty directories
|
||||
- Symlinks
|
||||
- Special characters in filenames
|
||||
- Very large files (multi-GB)
|
||||
- Many small files (100k+)
|
||||
The product `Dockerfile` lost its lint stage rather than gaining a
|
||||
second linter pin: `make lint` is now `docker build`, so the stage
|
||||
would have been docker-in-docker with no daemon, and calling
|
||||
`golangci-lint` directly there would have restored the two-pins drift
|
||||
of [issue #78](https://git.eeqj.de/sneak/vaultik/issues/78).
|
||||
`make fmt-check` moved beside `make test` in the builder stage, and
|
||||
`script/cibuild` now builds `Dockerfile.lint` and then `Dockerfile`,
|
||||
each with its own fresh epoch. Consequence, stated rather than left
|
||||
to be found: `script/docker` builds the product image only and no
|
||||
longer lints; the gates are `script/check` and `script/cibuild`.
|
||||
|
||||
1. Add tests for error conditions
|
||||
- Network failures during upload
|
||||
- Disk full during restore
|
||||
- Corrupted blobs
|
||||
- Missing blobs
|
||||
`golangci-lint config verify` runs as its own epoch-keyed layer,
|
||||
above the lint. `golangci-lint run` rejects a config it cannot parse
|
||||
but silently ignores an unknown top-level *key*: renaming `linters:`
|
||||
to `linterz:` discarded `default: all` and every threshold and still
|
||||
exited 0 on a tree the real config fails. `config verify` catches
|
||||
that, and it does so with the network off at this pin — checked under
|
||||
`docker run --network none`, not assumed. An earlier revision omitted
|
||||
it on the claim that it fetches its schema over live HTTPS; that
|
||||
claim was false at v2.12.2.
|
||||
|
||||
## Performance
|
||||
`script/lint-fix` is kept, reimplemented as a
|
||||
bind-mounted `docker run` against the image parsed out of
|
||||
`Dockerfile.lint` — it cannot be a build step, because fixes have to
|
||||
land in the worktree — and marked in its header as a developer
|
||||
convenience that no gate reads.
|
||||
|
||||
1. Profile and optimize restore performance
|
||||
- Parallel blob downloads
|
||||
- Streaming decompression/decryption
|
||||
- Efficient chunk reassembly
|
||||
- 2026-08-09: Finished the `--json` stdout contract and gave `make build`
|
||||
a rule ([issue #108](https://git.eeqj.de/sneak/vaultik/issues/108),
|
||||
[issue #110](https://git.eeqj.de/sneak/vaultik/issues/110)). Two
|
||||
unrelated defects of the same shape — a command reporting something it
|
||||
did not do — landed together because both are small.
|
||||
|
||||
1. Add bandwidth limiting option
|
||||
- `--bwlimit` flag for upload/download speed limiting
|
||||
`CleanupLocalSnapshots` wrote three prose lines to stdout with no
|
||||
`--json` awareness, covering every branch of the function, so no input
|
||||
avoided them and `vaultik prune --json | jq` failed even after
|
||||
[issue #106](https://git.eeqj.de/sneak/vaultik/issues/106) removed the
|
||||
banner. `-q` never helped either: `printlnStdout` and `stdoutf` write
|
||||
straight to `Vaultik.Stdout` and never consult `Vaultik.UI`, which is
|
||||
what `SetQuiet` affects. The issue offered three fixes and asked for a
|
||||
decision. Taken: thread `*PruneOptions` into the function and gate each
|
||||
write on `!opts.JSON`, matching `PruneBlobs` (its sibling phase, which
|
||||
already takes the same struct), `RemoveSnapshot` and `remote info`, so
|
||||
the package has one pattern rather than two. Rejected: moving the lines
|
||||
to `log.Info`, because the logger's default level is `slog.LevelWarn`,
|
||||
so that would not relocate them to stderr — it would delete them from a
|
||||
plain `vaultik prune`, and the removal of rows from the local index is
|
||||
not something to narrate only under `--verbose`. Also rejected: putting
|
||||
the stale-record count into `PruneBlobsResult`, whose every field is
|
||||
blob-scoped and which is produced by the later phase; a prune document
|
||||
covering both phases is a reasonable thing to want, but it is a schema
|
||||
design question and not a stream-hygiene fix. The narration is
|
||||
duplicated as `log.Info` records, which `PruneBlobs` already does
|
||||
alongside its own prints, so the events survive on stderr for anyone
|
||||
running `--verbose`.
|
||||
|
||||
## Documentation
|
||||
`make build` printed "Nothing to be done for 'build'" and exited 0
|
||||
without producing a binary: `build` was listed in `.PHONY` with no
|
||||
`build:` rule anywhere, and declaring a name phony is exactly what
|
||||
converts make's "No rule to make target" error into a silent success.
|
||||
Fixed with `build: vaultik`, keeping `vaultik:` as the file rule. The
|
||||
audit the issue asked for covers all 19 `.PHONY` names; `build` was the
|
||||
only one without a rule, and `vaultik` is correctly absent from
|
||||
`.PHONY`, being a real file target.
|
||||
|
||||
1. Add man page or --help improvements
|
||||
- Detailed help for each command
|
||||
- Examples in help output
|
||||
Tests, each verified to fail with the fix reverted rather than assumed
|
||||
to: `CleanupLocalSnapshots` leaves stdout untouched under `--json` in
|
||||
all three branches (stale records, none, empty index) and still emits
|
||||
every line without it, so the guard cannot be satisfied by deleting the
|
||||
output; `prune --json` run end to end through `Entry`, cobra and fx
|
||||
over the process's real stdout descriptor against a `file://` store,
|
||||
asserting exactly one JSON document, in both the stale and non-stale
|
||||
branches; and a parse of the `Makefile` asserting every `.PHONY` name
|
||||
has a rule and that `build` reaches the rule that produces the binary,
|
||||
which keeps the audit true for names added later. That last one is a
|
||||
parse rather than an invocation of `make`, since `make test` is what
|
||||
runs it and shelling back into `make build` would nest a build inside
|
||||
the test run. The property a parse cannot establish — that the recipe
|
||||
still fails when the build fails — was verified by hand against a
|
||||
deliberately broken tree: `make build` exits 2 and produces nothing.
|
||||
`cmd/vaultik` gains its first test file, so `make test` now reports 16
|
||||
packages `ok` where it reported 15.
|
||||
|
||||
## Final Polish
|
||||
- 2026-08-09: Stopped the startup banner from contaminating `--json`
|
||||
documents ([issue #106](https://git.eeqj.de/sneak/vaultik/issues/106)).
|
||||
`Entry` writes the banner to stdout before cobra parses anything, and
|
||||
the flag scan that suppresses it knew `--quiet`, `-q` and `--cron` but
|
||||
not `--json`, so every `--json` document arrived behind two lines of
|
||||
prose and a blank line, and `vaultik snapshot list --json | jq` failed.
|
||||
With the logger already on stderr from
|
||||
[issue #82](https://git.eeqj.de/sneak/vaultik/issues/82), this was the
|
||||
last writer that could put something on stdout that the caller did not
|
||||
ask for. The design question the issue raised — extend the raw-argv
|
||||
scan, or move the banner after parsing — is answered in favour of the
|
||||
scan: the banner is printed first deliberately, so that it still
|
||||
appears when cobra rejects the arguments and on `--help`, and after
|
||||
parsing there is no single place that covers those paths. The stated
|
||||
cost of the scan, that `--json` is a subcommand flag matched anywhere
|
||||
in the vector, is a cost `--cron` already carries — it exists only on
|
||||
`snapshot create` — so this adds an instance of an accepted
|
||||
imprecision rather than a new kind, and the two error directions are
|
||||
not symmetric: a false positive loses a decorative banner, a false
|
||||
negative corrupts a document. Regression tests at the CLI layer, where
|
||||
`internal/vaultik`'s existing guard cannot reach: one runs `Entry`
|
||||
itself over the process's real stdout descriptor, through cobra and fx
|
||||
to the document, made hermetic by `file://` storage; a second covers
|
||||
the argument vectors of all five `--json` commands; a third asserts the
|
||||
banner is still printed without a suppressing flag, so the first
|
||||
cannot be satisfied by deleting the banner. Also corrected `AGENTS.md`
|
||||
policy 9, which still keyed the structured-log format on stdout's
|
||||
TTY-ness after #82 moved that decision to stderr — a rules file that
|
||||
misdescribes the code misleads exactly the readers who trust it most.
|
||||
Two smaller findings from the same review: `bytesAttrKey`'s
|
||||
human-readable byte formatting silently stopped applying under an open
|
||||
group, because the key reaching the comparison is group-qualified
|
||||
(`transfer.bytes`), now matched on its final segment and tested both
|
||||
ways; and `listEnv.stderr` in `snapshot_list_test.go`, assigned but
|
||||
never read since those tests began capturing the process's stderr, is
|
||||
removed. `Vaultik.Stderr` is kept — nothing writes to it today, which
|
||||
its comment now says outright.
|
||||
|
||||
1. Ensure version is set correctly in releases
|
||||
- 2026-08-09: Moved the logger to stderr and fixed `TTYHandler`'s
|
||||
discarded attributes
|
||||
([issue #82](https://git.eeqj.de/sneak/vaultik/issues/82),
|
||||
[issue #97](https://git.eeqj.de/sneak/vaultik/issues/97)). Two defects
|
||||
in `internal/log`, fixed together because both live in the handler
|
||||
construction path. The first: both handlers were built over
|
||||
`os.Stdout`, and `WARN`/`ERROR` are never suppressed, so a config file
|
||||
with group- or world-readable permissions was enough to put a log
|
||||
record inside a `--json` document and break `jq`. Diagnostics now go
|
||||
to stderr, and the TTY/JSON format choice follows stderr rather than
|
||||
stdout — testing the wrong stream would colorize records on a
|
||||
redirected stderr whenever stdout happened to be a terminal. This is
|
||||
user-visible: `--verbose` and `--debug` output moves to stderr too,
|
||||
which is documented in `README.md` under "stdout and stderr". It also
|
||||
let the local workaround in `internal/vaultik/snapshot_list.go` go:
|
||||
`warnWhileListing` had been hand-rolling structured-log formatting to
|
||||
reach a non-stdout writer, and the `jsonOutput` parameter threaded
|
||||
through the remote-listing helpers existed only to choose between the
|
||||
two writers. The collect-then-emit machinery around `listingWarning`
|
||||
stays, but on its remaining merit — warnings emitted in key order
|
||||
after `group.Wait()` are deterministic run to run, where emitting from
|
||||
the fetch workers would order them by network timing. The second
|
||||
defect: `TTYHandler.WithAttrs` and `WithGroup` discarded their
|
||||
arguments and returned the receiver while their doc comments claimed
|
||||
otherwise, so `log.With` attributes vanished on a terminal and
|
||||
appeared correctly in CI — failing precisely when someone is debugging
|
||||
interactively. Both now return a new handler (the receiver is never
|
||||
written to, since `slog` permits concurrent derivation), attributes
|
||||
persist across records, and grouping is implemented as dotted key
|
||||
prefixes, which is the only honest rendering for a format with nowhere
|
||||
to nest. New tests cover both, including one that feeds the same
|
||||
derivation chain to the TTY and JSON handlers and compares the
|
||||
attribute sets, so the two paths cannot drift apart again. Found and
|
||||
filed while verifying: the startup banner is written to stdout and
|
||||
`--json` does not suppress it
|
||||
([issue #106](https://git.eeqj.de/sneak/vaultik/issues/106)), which is
|
||||
a separate writer on a separate path and the remaining source of
|
||||
stdout contamination.
|
||||
|
||||
1. Create release process
|
||||
- Binary releases for supported platforms
|
||||
- Checksums for binaries
|
||||
- Release notes template
|
||||
- 2026-08-09: Made the tagged-release path actually work on Gitea
|
||||
([issue #65](https://git.eeqj.de/sneak/vaultik/issues/65)). Three
|
||||
independent blockers, one of which was the whole
|
||||
release: `.goreleaser.yaml` had no `gitea_urls:` block, so goreleaser
|
||||
defaulted to the GitHub API and a `goreleaser release` from this repo
|
||||
would have failed or published where nobody is looking. It now points
|
||||
at `https://git.eeqj.de/api/v1`. The version is the second: it was a
|
||||
hardcoded `VERSION := 1.0.0-rc.1` in the `Makefile`, so every local
|
||||
build claimed to be a release candidate that had never been tagged and
|
||||
did not exist, while `git tag -l` was empty and `internal/globals`
|
||||
defaulted to `dev`. Version now comes from git via the new
|
||||
`script/version` — the exact tag with a leading `v` stripped (so a
|
||||
`make` build and a goreleaser build of one commit report the same
|
||||
string, and it matches the archive names), otherwise `dev-<12-char
|
||||
sha>`, with `-dirty` appended in either case when tracked files are
|
||||
modified. Untracked files are deliberately not counted, matching
|
||||
`git describe --dirty`. The same honesty was owed by the snapshot
|
||||
path: `snapshot.version_template` was `{{ incpatch .Version }}-next`,
|
||||
which manufactures a release number from the last tag and, with no
|
||||
tags at all, from goreleaser's fabricated `v0.0.0`; it now emits the
|
||||
same `dev-<sha>`. The one non-obvious consequence is that
|
||||
`internal/cli/version.go` gated its "this is a development build"
|
||||
notice on the version being exactly `dev`, so the moment untagged
|
||||
builds began carrying a commit sha that notice would have gone silent
|
||||
and an unreleased binary would have read as a release — the gate is
|
||||
now `globals.IsDevVersion`, which is a predicate over a string rather
|
||||
than a comparison against a global precisely so it can be tested, and
|
||||
it is tested at the boundary (`1.0.0-dev` is a release, `dev-<sha>`
|
||||
is not). Release automation is the third blocker: a tag-triggered
|
||||
`.gitea/workflows/release.yml` runs the build in CI rather than from
|
||||
a laptop, with `fetch-depth: 0` because a shallow checkout has no
|
||||
tags and would silently mislabel the release, and with the
|
||||
`RELEASE_TOKEN` repository secret passed as `GITEA_TOKEN` (documented
|
||||
in `README.md`; the runner's automatic token is not used because it
|
||||
is not guaranteed to carry release write scope). `script/release`
|
||||
unsets any `GITHUB_TOKEN`/`GITLAB_TOKEN` it finds, since goreleaser
|
||||
chooses its forge from whichever token variable is set and refuses to
|
||||
run when it sees more than one — a runner-provided token must not get
|
||||
to decide where these artifacts are published. `make release` and
|
||||
`make release-snapshot`, the last two Makefile targets that were not
|
||||
shims, now call `script/release` and `script/release-snapshot`, which
|
||||
resolve goreleaser exactly the way `script/lint` resolves the linter:
|
||||
a `PATH` binary is used only at the pinned version, never as a silent
|
||||
fallback. `script/bootstrap` installs it, from a sha256-verified
|
||||
GitHub release archive per `REPO_POLICIES.md`, via a separate
|
||||
`script/install-goreleaser` — separate because `script/bootstrap`
|
||||
hard-fails without a usable Docker daemon by design, and the release
|
||||
runner needs goreleaser without needing Docker. Verified by running
|
||||
the thing rather than reading it: `make release-snapshot` produced
|
||||
four archives and `checksums.txt`, and the linux/amd64 binary from
|
||||
`dist/` reports `dev-<sha>` with the development-build notice. Tag
|
||||
handling was exercised in a throwaway repository rather than by
|
||||
tagging this one; no tag was created here, since that is the owner's
|
||||
call. Signing, SBOM, reproducible builds, completions and a man page
|
||||
are out of scope by the issue.
|
||||
|
||||
1. Final code review
|
||||
- Remove debug statements
|
||||
- Ensure consistent code style
|
||||
- 2026-08-09: Isolated the lint cache per worktree and context-gated the
|
||||
native lint path (issues #99, #80). One defect seen twice:
|
||||
`script/lint` decided whether it could skip the pinned image by asking
|
||||
what version was on `PATH` rather than where it was running, and cache
|
||||
isolation is part of that same question. The cache was one directory
|
||||
per repo, shared by every worktree on the host, so two checkouts with
|
||||
identical Go file contents collided and golangci-lint replayed the
|
||||
stored analysis — paths and all. The loud direction of that failure
|
||||
(a clean tree failed by a dirty sibling) is the harmless one; the
|
||||
silent direction, a dirty tree **passed** by a clean sibling, is a
|
||||
sixth way for a gate here to report a green it did not earn. The cache
|
||||
is now keyed on a digest of the worktree path, and every run is
|
||||
audited by the new `script/lint-audit`, which rejects output citing any
|
||||
file that is not in the tree being linted — a backstop that runs on
|
||||
clean output too, because that is the case nobody investigates. Caches
|
||||
record the worktree they belong to and are collected when it
|
||||
disappears, so throwaway worktrees do not accumulate them; the whole
|
||||
tree lives under `XDG_CACHE_HOME` and is disposable. The
|
||||
`parallel golangci-lint is running` refusal is now a bounded retry
|
||||
rather than a verdict: it is not a lint result, and exiting non-zero
|
||||
on it is indistinguishable to a caller from real findings (#88 showed
|
||||
a private cache does not remove that contention). The native path now
|
||||
requires `VAULTIK_LINT_IN_CONTAINER=1`, set only by the `Dockerfile`
|
||||
lint stage, in addition to matching the pin, so a developer's locally
|
||||
installed 2.12.2 no longer bypasses the digest pin; `/.dockerenv` was
|
||||
rejected as the signal because `dockerd` creates it for `docker run`
|
||||
and it is not reliably present during a BuildKit `docker build`, which
|
||||
is the case the exception exists for. Version detection uses
|
||||
`golangci-lint version --short` with the old banner scrape kept only
|
||||
as a fallback. `script/bootstrap` no longer prints `bootstrap
|
||||
complete` on a machine that cannot run the gate: a missing docker, or
|
||||
one whose daemon is unreachable, is a hard failure naming exactly what
|
||||
breaks. Verification was by reproduction rather than inspection — two
|
||||
concurrent lints from two worktrees of differing cleanliness, a real
|
||||
run made to report an outside path, a matching linter shimmed onto
|
||||
`PATH`, and a `PATH` with docker removed — and is recorded on the pull
|
||||
request.
|
||||
- 2026-08-09: Closed the fifth false-green mechanism (issues #93, #69).
|
||||
`script/test` omitted `-count=1`, so Go's test result cache could
|
||||
satisfy the gate outright: a second back-to-back `make test` printed
|
||||
the full set of 14 `ok` lines, every one marked `(cached)`, having
|
||||
executed no test at all. Since `ok <pkg> (cached)` is an `ok` line,
|
||||
the "14 `ok` lines means the suite ran" signal this repo leans on was
|
||||
forgeable, one level below the Docker layer cache that #85 addressed.
|
||||
Fixed with `-count=1` unconditionally rather than only in the
|
||||
container, because the pre-commit hook runs the same script and a
|
||||
gate honest only in CI is dishonest where people rely on it most;
|
||||
`test-coverage` got the same flag, and `script/check` inherits it by
|
||||
calling `script/test`. In the same area, `make test-integration` was
|
||||
deleted rather than made real: no file in the repo carried a build
|
||||
tag, so `-tags=integration` selected nothing and the target was an
|
||||
exact duplicate of `make test`. Tagging a subset was rejected because
|
||||
the entire suite runs in well under a minute, and a scheme whose
|
||||
failure mode is "some tests silently stopped running" is a poor trade
|
||||
for those seconds in a repo with this particular history. The
|
||||
`-timeout` was raised from 30s after measuring rather than after
|
||||
assuming: the standing claim that cold-cache compilation is charged
|
||||
against `-timeout` is **false**, disproved by a containerised run
|
||||
that spent 46s compiling and still reported per-package durations
|
||||
within noise of a warm host run. `-timeout` reaches the test binary
|
||||
as `-test.timeout` and its clock starts inside `testing.M.Run`, after
|
||||
the build. The real exposure was margin, not compilation. The 120s
|
||||
landed on is a **deliberate, documented divergence** from
|
||||
`REPO_POLICIES.md:192`, which mandates 30s, and from that file's
|
||||
canonical recipe at `:212-214`; the divergence is recorded in
|
||||
`script/test`'s comment because `REPO_POLICIES.md` is org-canonical
|
||||
and not editable here, and issue #101 proposes amending the policy
|
||||
text upstream. Numbers and the full verification are recorded once,
|
||||
on the pull request, and are deliberately not restated here.
|
||||
|
||||
1. Tag and release v1.0.0
|
||||
- 2026-08-09: Triaged all fifteen stale remote branches (issue #71) and
|
||||
deleted fourteen of them; the full per-branch disposition with
|
||||
evidence is recorded on that issue. Method mattered more than the
|
||||
outcome here: a three-dot `git diff main...branch` diffs from the
|
||||
merge base, so it replays everything that landed on `main` after the
|
||||
branch diverged and makes any old branch look like it holds unlanded
|
||||
work. That artifact is what made `golangci-v2.12.2` appear to carry
|
||||
126 files of unpushed changes when its tree was byte-identical to
|
||||
`main`'s. Every containment claim here therefore rests on two-dot tip
|
||||
diffs, tree-hash equality, `git cherry`, and `git branch -r --merged`.
|
||||
Nine branches were plain ancestors of `main` with zero `git cherry`
|
||||
`+` commits. `golangci-v2.12.2` had landed squashed as `cc58583`,
|
||||
whose tree hash equals the branch tip's exactly; note the hash
|
||||
recorded in the issue had gone stale because `main` advanced, so the
|
||||
check had to be redone rather than repeated.
|
||||
`fix/sync-snapshot-cleanup` was redundant, its one line already on
|
||||
`main` in `syncWithRemote`. `feature/restore-progress-bar` was
|
||||
superseded by `printRestoreProgress` and the disk-backed blob cache,
|
||||
and had become actively regressive — it would have deleted
|
||||
`internal/blobgen/compress_test.go`, the #28 regression test that
|
||||
landed separately. The two branches this issue was filed for both
|
||||
turned out to be closed questions that `main` had already moved past
|
||||
by a recorded decision, so neither was landed and no regression test
|
||||
was owed: `ctime` no longer exists anywhere in the codebase after
|
||||
`1c72a37` removed the column, the `File.CTime` field and every use
|
||||
(#54/#55), and change detection compares size, mtime, mode, uid and
|
||||
gid only, exactly as `ARCHITECTURE.md` documents — so the
|
||||
silently-skipped-file data-loss risk that made this a 1.0 item does
|
||||
not exist. The SQL allow-list branch would have reverted `bfd7334`,
|
||||
which replaced that very allow-list with regex sanitisation on review
|
||||
feedback, and would have broken `getTableCount("snapshots")` because
|
||||
its allow-list omits that table. `feature/daemon-mode` is untouched
|
||||
and deferred to #94 pending an owner decision, so it is the one
|
||||
branch besides `main` still on the remote. The stale `TODO.md` entry
|
||||
named in the issue needed no fix: `e496aa3` had already removed it.
|
||||
No product code changed.
|
||||
- 2026-08-09: Adopted the remaining upstream `CHECK_EPOCH` hardening
|
||||
(issue #91), closing the gap #85 knowingly left open. Four changes,
|
||||
all four decided as adopt upstream in `sneak/prompts` #26. (1) Each
|
||||
check stage now asserts `[ -n "$CHECK_EPOCH" ] || exit 1` before
|
||||
running anything, so a build that supplies no `--build-arg` fails
|
||||
instead of lying. This is the item that mattered: an unset `ARG` is
|
||||
an empty string and an empty string is a stable cache key, so the
|
||||
second and every later bare `docker build .` on an unchanged tree
|
||||
replayed all three check layers and still exited 0 — and `docker
|
||||
build .` is the command `REPO_POLICIES.md` names verbatim as a thing
|
||||
that must be green, so the documented command was precisely the one
|
||||
that lied. Failed steps are never cached, which is what makes the
|
||||
guard fire on every invocation rather than once. (2) The epoch is now
|
||||
expanded into each check command rather than left as a bare
|
||||
declaration, so the cache miss no longer depends on BuildKit's
|
||||
unreferenced-`ARG` handling staying as it is, and the value appears
|
||||
in the build log. (3) `script/cibuild` uses
|
||||
`epoch="$(date +%s%N)$$"`, unique per invocation rather than per
|
||||
second; `%N` alone is insufficient because busybox drops it silently
|
||||
and exits 0, and `$$` is what makes the guarantee hold regardless.
|
||||
The bare-assignment form is kept deliberately — inlined in an
|
||||
argument, a failing substitution does not abort under `set -eu` and
|
||||
would yield an empty constant epoch, restoring the exact false green
|
||||
being fixed. (4) `script/docker` passes the same fresh arg, so the
|
||||
two entrypoints cannot disagree about whether the tree is green;
|
||||
local builds are almost always warm, which made it the likelier
|
||||
fooling in practice. The `ARG` placement from #85 is unchanged, below
|
||||
`apk add`, `COPY go.mod go.sum` and `go mod download`, so dependency
|
||||
layers still cache and the build is not cold. Verified by negative
|
||||
control rather than inspection — a bare `docker build .` run twice
|
||||
back to back, plus back-to-back pairs of both scripts and a host-side
|
||||
`make check`; the measurements are recorded once, in the PR
|
||||
verification comment, rather than restated here. `.golangci.yml`, the
|
||||
lint-stage `FROM` line and its digest, `script/lint`,
|
||||
`REPO_POLICIES.md` and `.gitea/workflows/check.yml` are all
|
||||
untouched.
|
||||
- 2026-08-09: Stopped `script/cibuild` from reporting a green it did
|
||||
not earn (issue #85). A bare `docker build .` let Docker serve the
|
||||
check layers from the layer cache whenever the tree had not changed:
|
||||
the checks never executed and the build still exited 0. The fix is an
|
||||
`ARG CHECK_EPOCH` declared immediately above the check `RUN`s in both
|
||||
the lint stage and the builder stage (`ARG` scope is per-stage, so
|
||||
each declares its own), with `script/cibuild` assigning
|
||||
`epoch="$(date +%s)"` and passing `--build-arg CHECK_EPOCH="$epoch"`.
|
||||
The assignment is separate on purpose: under `set -eu` a command
|
||||
substitution that fails inside an argument does not abort the script,
|
||||
which would leave an empty constant `CHECK_EPOCH` and restore the
|
||||
very false green being fixed. Placement is the rest of the point —
|
||||
the `ARG` sits below the `apk add`, `COPY go.mod go.sum`, and `go mod
|
||||
download` layers, so only the checks are invalidated and the
|
||||
dependency layers still cache. The guarantee is conditional on a
|
||||
fresh value rather than absolute: a bare `docker build .` gets an
|
||||
empty `CHECK_EPOCH` and can still serve the check layers from cache,
|
||||
which `README.md` and the `Dockerfile` now say plainly, with issue
|
||||
#91 tracking the upstream hardening (expanded `ARG` form, unset
|
||||
guard, per-invocation epoch, `script/docker`) that would close it.
|
||||
Verified by re-running the reproduction plus the withheld-`--build-arg`
|
||||
counterfactual; the measurements are recorded once, in the PR #89
|
||||
verification comment, rather than restated here. `.golangci.yml`, the
|
||||
lint-stage `FROM` line and its digest, `script/lint`, and
|
||||
`.gitea/workflows/check.yml` are all untouched.
|
||||
- 2026-08-09: Corrected the `Vaultik.UI` doc comment (issue #84). It
|
||||
claimed the cli layer replaces the writer with a discarding one in
|
||||
`--cron` mode; the actual mechanism is `UI.SetQuiet(true)` in
|
||||
`setupGlobals`, which drops Begin/Complete/Info/Notice/Detail/
|
||||
Progress/Banner but still emits Warning and Error. The `--cron` line
|
||||
in `README.md` said "Silent unless error", which understated what
|
||||
survives, and now names warnings too. The other `--cron` comments
|
||||
(`internal/log/log.go`, `internal/cli/snapshot.go`,
|
||||
`internal/vaultik/snapshot.go`) were audited and already accurate.
|
||||
Comments and docs only, no behavior change.
|
||||
- 2026-08-09: Made `snapshot list` list the destination store without
|
||||
the private key (issue #64). The listing is now the union of the
|
||||
local index and a single streamed listing of the `metadata/` prefix,
|
||||
with no `age_secret_key` gate — the manifest is unencrypted, so a
|
||||
host holding only the public key can enumerate its own backups and a
|
||||
host that lost its local index can still see them. A remote-only
|
||||
snapshot's hostname and name are deliberately not recovered (they are
|
||||
not recoverable without the private key, and making them so would
|
||||
undo the privacy property tracked in issue #81); such rows are
|
||||
labelled by an abbreviation of their remote key and carry the real
|
||||
timestamp and compressed size from the manifest, with `<remote only>`
|
||||
in the two columns that require the local index. Local-only snapshots
|
||||
are reported as drift, and the hint now names `vaultik prune`, which
|
||||
exists, instead of `vaultik snapshot cleanup`, which does not.
|
||||
`reportRemoteDrift` collapsed into the merged view. Every remote
|
||||
manifest read in the codebase now goes through
|
||||
`downloadManifestByKey`, so issue #81 has one call site to change.
|
||||
Review rework: snapshot timestamps now normalize to UTC in
|
||||
`scanSnapshotRows`, the one place they enter the domain, so the merged
|
||||
TIMESTAMP column cannot show local time for a locally tracked row and
|
||||
UTC for a remote-only row on a non-UTC host; `GetIncompleteByHostname`
|
||||
was folded onto that same scanner. `--json` now reports the
|
||||
unreadable-manifest count and the 1000-row truncation on stderr
|
||||
instead of returning a silently short document (the document's shape
|
||||
is unchanged). The two per-snapshot `log.Warn` calls on the listing
|
||||
path now route through the same JSON-aware writer as the existing
|
||||
workaround, so one corrupt manifest can no longer put a log line on
|
||||
stdout ahead of the document and break `| jq` — still a local
|
||||
workaround pending issue #82. Verified with `script/cibuild` and with
|
||||
an uncached `make check` (`0 issues.`, no cached test packages), plus
|
||||
end to end against a `file://` destination with no secret key present.
|
||||
- 2026-08-09: Closed the gap between `make lint` and CI (issue #78).
|
||||
`script/lint` now runs the digest-pinned `golangci-lint` image taken
|
||||
from the `Dockerfile` lint stage, which is the single source of truth
|
||||
for the linter version; the duplicate pin in the `Makefile` `deps`
|
||||
target and the unpinned `golangci-lint` install in `script/bootstrap`
|
||||
are gone. A `golangci-lint` on `PATH` is used only when its version is
|
||||
exactly the pinned one (which is how the lint stage runs it inside the
|
||||
container); anything else goes through Docker, and a missing or
|
||||
unreachable Docker daemon is a hard error rather than a silent
|
||||
fallback. Only the **lint** leg of `make check` became equivalent to
|
||||
`script/cibuild`; its tests and `gofmt` still run on the host against
|
||||
the host toolchain, as `README.md` states. An earlier version of this
|
||||
entry claimed `make check` was "as trustworthy as `script/cibuild`"
|
||||
outright, which overstated it; corrected under issue #80.
|
||||
- 2026-08-09: Finished the lint remediation under the canonical
|
||||
`.golangci.yml` (issue #61, which also unblocks issue #59). The
|
||||
remaining findings were fixed behavior-preservingly: `wsl_v5`
|
||||
whitespace, `sqlclosecheck`, and `prealloc`. The `sqlclosecheck` sites
|
||||
now close `sql.Rows` in a deferred closure instead of via the
|
||||
`CloseRows` helper, which the linter could not see through. Only the
|
||||
`revive` package-name findings remain suppressed, with per-site
|
||||
`//nolint` directives; the package-rename question behind them is
|
||||
tracked in issue #76. Verified with `script/cibuild`, which exits 0 —
|
||||
that is the only trustworthy gate, because `script/lint` runs whatever
|
||||
`golangci-lint` happens to be on `PATH` rather than the pinned
|
||||
v2.12.2 that CI and the `Dockerfile` use, so `make check` can report
|
||||
green on findings CI still fails. That tooling gap is tracked in issue
|
||||
#78.
|
||||
- 2026-08-09: The earlier next step "reconcile the uncommitted
|
||||
`ARCHITECTURE.md` edits on `main`" needed no work: the working tree is
|
||||
clean and `ARCHITECTURE.md` is committed on `main`.
|
||||
- 2026-08-07: Updated golangci-lint to v2.12.2 everywhere it is pinned
|
||||
(`Dockerfile` lint stage, `Makefile` deps target), replaced
|
||||
`.golangci.yml` with the canonical config (v2 schema, `default: all`),
|
||||
and remediated the bulk of the lint findings it surfaced (issue #61):
|
||||
behavior-preserving fixes across every package, 2,990 findings down to
|
||||
80. `make test` and `make fmt-check` were green at that point but
|
||||
`make lint` was still red; the commit message claiming `make check`
|
||||
was green was wrong.
|
||||
- 2026-08-07: Added the standard `.golangci.yml` and `.editorconfig`
|
||||
(issue #59); lint findings under the new config are tracked in issue
|
||||
#61.
|
||||
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
|
||||
Makefile shims, README Entrypoints section
|
||||
- 2026-07-02: Consolidated CLI verbs, retired overlapping commands; bound
|
||||
the local index to its backup destination URL.
|
||||
- 2026-06-28: snapshot rm now removes metadata only and prints the prune
|
||||
command; restore skips chown when running as non-root.
|
||||
- 2026-06-26: Snapshot IDs hashed at the storage boundary; snapshot list
|
||||
made resilient to bad remote entries.
|
||||
- 2026-06-24: Collapsed snapshot prune into vaultik prune; restore streams
|
||||
blobs to disk and restores files in blob-locality order; cron output
|
||||
fixes.
|
||||
- 2026-06-17: Restore overhaul: ReadAt chunk reads from cached blobs,
|
||||
reference-counted blob sweeper, integration tests; new internal/ui
|
||||
output layer, banner, and progress lines.
|
||||
- 2025-12-18: Added ARCHITECTURE.md and godoc coverage for exported API.
|
||||
- 2025-07-26: End-to-end integration tests; manifest format refactor;
|
||||
renamed backup to snapshot; afero filesystem abstraction.
|
||||
- 2025-07-20: Initial design and implementation: cobra + fx CLI skeleton,
|
||||
SQLite index database, UUID blob storage with streaming chunking.
|
||||
|
||||
---
|
||||
# Future Steps
|
||||
|
||||
## Post-1.0 (Daemon Mode)
|
||||
|
||||
1. Implement inotify file watcher for Linux
|
||||
- Watch source directories for changes
|
||||
- Track dirty paths in memory
|
||||
|
||||
1. Implement FSEvents watcher for macOS
|
||||
- Watch source directories for changes
|
||||
- Track dirty paths in memory
|
||||
|
||||
1. Implement backup scheduler in daemon mode
|
||||
- Respect backup_interval config
|
||||
- Trigger backup when dirty paths exist and interval elapsed
|
||||
- Implement full_scan_interval for periodic full scans
|
||||
|
||||
1. Add proper signal handling for daemon
|
||||
- Graceful shutdown on SIGTERM/SIGINT
|
||||
- Complete in-progress backup before exit
|
||||
|
||||
1. Write tests for daemon mode
|
||||
None queued; the release-scoping item is now the Next Step.
|
||||
|
||||
@@ -0,0 +1,363 @@
|
||||
package main_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// This file guards the shape of the lint gate. Every property asserted
|
||||
// here is one whose loss is SILENT: the build still exits 0, the gate
|
||||
// still looks green, and nothing was linted or tested.
|
||||
//
|
||||
// The gate is a build step. script/lint builds Dockerfile.lint, which
|
||||
// runs golangci-lint as a RUN instruction, so a successful build is a
|
||||
// clean lint. BuildKit will happily replay that RUN from cache on an
|
||||
// unchanged tree in well under a second, which is why the check layers
|
||||
// are keyed on a CHECK_EPOCH build arg that the calling script
|
||||
// regenerates per invocation, and why an empty value is a hard error
|
||||
// rather than a stable cache key.
|
||||
//
|
||||
// These are parses rather than invocations. Shelling out to docker from
|
||||
// the test suite would nest a build inside `make test`, which itself
|
||||
// runs inside a build in CI. The one property a parse cannot establish
|
||||
// -- that a real finding actually fails the build -- is verified by
|
||||
// hand against a deliberately broken tree, recorded on the pull
|
||||
// request.
|
||||
//
|
||||
// One property is deliberately NOT tested here: that no script runs the
|
||||
// linter on the host. script/lint is the only lint entry point, and it
|
||||
// runs golangci-lint only inside the container; keeping it that way is a
|
||||
// review matter, not something a test in this file establishes.
|
||||
|
||||
// The files under guard, relative to the repository root.
|
||||
const (
|
||||
lintDockerfile = "Dockerfile.lint"
|
||||
productDockerfile = "Dockerfile"
|
||||
lintScript = "script/lint"
|
||||
cibuildScript = "script/cibuild"
|
||||
)
|
||||
|
||||
// linterBinary is the linter's command name, used to locate the
|
||||
// config-verify and lint steps in Dockerfile.lint.
|
||||
const linterBinary = "golangci-lint"
|
||||
|
||||
// checkEpochARG is the declaration, with no default value. A default
|
||||
// would satisfy the non-empty guard with a constant, and a constant is
|
||||
// a stable cache key: the checks would be replayed from cache forever
|
||||
// after the first build.
|
||||
const checkEpochARG = "ARG CHECK_EPOCH"
|
||||
|
||||
// checkEpochGuard is what turns a build that omits --build-arg into a
|
||||
// loud failure instead of a quiet green. Failed steps are never cached,
|
||||
// so it fires on every such invocation rather than once.
|
||||
const checkEpochGuard = `RUN [ -n "$CHECK_EPOCH" ] || exit 1`
|
||||
|
||||
// freshEpoch is the epoch computation the calling scripts must use, as
|
||||
// a bare assignment on its own line. Inline in an argument, a failing
|
||||
// `date` would not abort under `set -eu`; CHECK_EPOCH would become the
|
||||
// empty string, and the guard above would be the only thing standing
|
||||
// between that and a permanently cached green. `$$` is required because
|
||||
// `date +%s` is second-granular and busybox silently drops `%N`, so
|
||||
// without the pid two concurrent runs in one second can collide.
|
||||
const freshEpoch = `epoch="$(date +%s%N)$$"`
|
||||
|
||||
// TestLintDockerfilePinsTheLinterByDigest fails if the lint image stops
|
||||
// being pinned. An unpinned tag makes the gate's verdict depend on
|
||||
// whatever the registry currently serves under that name.
|
||||
func TestLintDockerfilePinsTheLinterByDigest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
from := ""
|
||||
|
||||
for _, instruction := range instructions(t, lintDockerfile) {
|
||||
if strings.HasPrefix(instruction, "FROM ") {
|
||||
from = instruction
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
require.NotEmpty(t, from, "%s declares no FROM", lintDockerfile)
|
||||
assert.Contains(t, from, "golangci/golangci-lint",
|
||||
"the lint image must be the golangci-lint image")
|
||||
assert.Contains(t, from, "@sha256:",
|
||||
"the lint image must be pinned by digest, not by tag alone")
|
||||
}
|
||||
|
||||
// TestLintDockerfileCannotBeCachedGreen pins the whole cache-busting
|
||||
// mechanism in the file that lints: the declaration with no default,
|
||||
// the non-empty guard, and the value expanded into the lint command
|
||||
// itself rather than merely declared.
|
||||
func TestLintDockerfileCannotBeCachedGreen(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
found := instructions(t, lintDockerfile)
|
||||
|
||||
argAt := indexOf(found, checkEpochARG)
|
||||
require.GreaterOrEqual(t, argAt, 0,
|
||||
"%s must declare `%s` with no default value",
|
||||
lintDockerfile, checkEpochARG)
|
||||
|
||||
assert.GreaterOrEqual(t, indexOf(found, checkEpochGuard), argAt,
|
||||
"%s must guard against an empty CHECK_EPOCH with `%s`",
|
||||
lintDockerfile, checkEpochGuard)
|
||||
|
||||
assertEpochExpandedInto(t, found[argAt:], "golangci-lint run")
|
||||
|
||||
// Dependency layers must stay above the ARG, or every lint run
|
||||
// re-downloads the module cache and the inner loop becomes
|
||||
// unusable.
|
||||
download := indexOf(found, "RUN go mod download")
|
||||
require.GreaterOrEqual(t, download, 0,
|
||||
"%s must download modules in their own layer", lintDockerfile)
|
||||
assert.Less(t, download, argAt,
|
||||
"`%s` must come after `go mod download` so dependency layers"+
|
||||
" still cache", checkEpochARG)
|
||||
}
|
||||
|
||||
// TestLintDockerfileVerifiesTheLinterConfig guards the validation of
|
||||
// .golangci.yml itself. `golangci-lint run` rejects a config it cannot
|
||||
// parse but silently IGNORES an unknown top-level key, so renaming
|
||||
// `linters:` to `linterz:` discards `default: all` and every threshold
|
||||
// and still exits 0 reporting no issues. `config verify` is what turns
|
||||
// that into a failure, and it has to run BEFORE the lint, or the lint
|
||||
// spends a minute reporting a verdict from a config already known to be
|
||||
// wrong.
|
||||
func TestLintDockerfileVerifiesTheLinterConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
found := instructions(t, lintDockerfile)
|
||||
verify := linterBinary + " config verify"
|
||||
|
||||
verifyAt := indexContaining(found, verify)
|
||||
require.GreaterOrEqual(t, verifyAt, 0,
|
||||
"%s must run `%s --config .golangci.yml`: without it a typo'd"+
|
||||
" top-level key in .golangci.yml is silently ignored and the"+
|
||||
" gate passes with only the default linter set", lintDockerfile,
|
||||
verify)
|
||||
|
||||
runAt := indexContaining(found, linterBinary+" run")
|
||||
require.GreaterOrEqual(t, runAt, 0, "%s must lint", lintDockerfile)
|
||||
assert.Less(t, verifyAt, runAt,
|
||||
"%s must verify the config before linting with it", lintDockerfile)
|
||||
|
||||
// Keyed on the epoch like every other check layer, so it executes
|
||||
// per invocation rather than being replayed. A cached validation
|
||||
// validates nothing.
|
||||
assertEpochExpandedInto(t, found, verify)
|
||||
}
|
||||
|
||||
// TestProductDockerfileCannotBeCachedGreen holds the same line for the
|
||||
// checks that remain in the product image build.
|
||||
func TestProductDockerfileCannotBeCachedGreen(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
found := instructions(t, productDockerfile)
|
||||
|
||||
argAt := indexOf(found, checkEpochARG)
|
||||
require.GreaterOrEqual(t, argAt, 0,
|
||||
"%s must declare `%s` with no default value",
|
||||
productDockerfile, checkEpochARG)
|
||||
|
||||
assert.GreaterOrEqual(t, indexOf(found, checkEpochGuard), argAt,
|
||||
"%s must guard against an empty CHECK_EPOCH", productDockerfile)
|
||||
|
||||
assertEpochExpandedInto(t, found[argAt:], "make fmt-check")
|
||||
assertEpochExpandedInto(t, found[argAt:], "make test")
|
||||
}
|
||||
|
||||
// TestProductDockerfileDoesNotLint records the split deliberately: the
|
||||
// linter lives in Dockerfile.lint and nowhere else, so there is exactly
|
||||
// one digest pinning it. A lint stage reintroduced here would either be
|
||||
// docker-in-docker (`make lint` is now `docker build`) or a second,
|
||||
// independently bumpable pin.
|
||||
func TestProductDockerfileDoesNotLint(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
contents := readRepoFile(t, productDockerfile)
|
||||
|
||||
for _, forbidden := range []string{"golangci", "make lint"} {
|
||||
assert.NotContains(t, instructionText(contents), forbidden,
|
||||
"%s must not lint: the linter is pinned once, in %s",
|
||||
productDockerfile, lintDockerfile)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLintScriptBuildsTheLintDockerfileWithAFreshEpoch is the other
|
||||
// half of the mechanism. The Dockerfile's guard only rejects an EMPTY
|
||||
// epoch; a constant non-empty one would satisfy it and still be served
|
||||
// from cache forever.
|
||||
func TestLintScriptBuildsTheLintDockerfileWithAFreshEpoch(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
script := readRepoFile(t, lintScript)
|
||||
|
||||
assertBareEpochAssignment(t, script, lintScript)
|
||||
assert.Contains(t, script, `--build-arg CHECK_EPOCH="$epoch"`,
|
||||
"%s must pass the fresh epoch to the build", lintScript)
|
||||
assert.Contains(t, script, lintDockerfile,
|
||||
"%s must build %s", lintScript, lintDockerfile)
|
||||
}
|
||||
|
||||
// TestCibuildBuildsBothDockerfilesWithFreshEpochs guards the CI gate:
|
||||
// dropping either build silently removes a whole class of check from
|
||||
// CI while leaving it green.
|
||||
func TestCibuildBuildsBothDockerfilesWithFreshEpochs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
script := readRepoFile(t, cibuildScript)
|
||||
|
||||
assertBareEpochAssignment(t, script, cibuildScript)
|
||||
assert.Equal(t, 2, strings.Count(script, freshEpoch),
|
||||
"%s must compute a fresh epoch for each of its two builds",
|
||||
cibuildScript)
|
||||
assert.Equal(t, 2,
|
||||
strings.Count(script, `--build-arg CHECK_EPOCH="$epoch"`),
|
||||
"%s must pass a fresh epoch to both builds", cibuildScript)
|
||||
assert.Contains(t, script, "-f Dockerfile.lint",
|
||||
"%s must build %s", cibuildScript, lintDockerfile)
|
||||
}
|
||||
|
||||
// assertEpochExpandedInto fails unless some instruction runs the named
|
||||
// command with the epoch expanded into it. Expansion, not mere
|
||||
// declaration: an ARG that no instruction references is not guaranteed
|
||||
// to key the layer, and the expansion also puts the value in the build
|
||||
// log where a reader can see the layer was keyed fresh.
|
||||
func assertEpochExpandedInto(t *testing.T, found []string, command string) {
|
||||
t.Helper()
|
||||
|
||||
for _, instruction := range found {
|
||||
if !strings.HasPrefix(instruction, "RUN ") {
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.Contains(instruction, command) &&
|
||||
strings.Contains(instruction, "${CHECK_EPOCH}") {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
assert.Fail(t, "no epoch-keyed layer runs the command",
|
||||
"`%s` must run in a layer that expands ${CHECK_EPOCH}, or it"+
|
||||
" will be replayed from cache without executing", command)
|
||||
}
|
||||
|
||||
// assertBareEpochAssignment fails unless the script computes the epoch
|
||||
// as a bare assignment on its own line.
|
||||
func assertBareEpochAssignment(t *testing.T, script, name string) {
|
||||
t.Helper()
|
||||
|
||||
for line := range strings.SplitSeq(script, "\n") {
|
||||
if strings.TrimSpace(line) == freshEpoch {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
assert.Fail(t, "no bare epoch assignment",
|
||||
"%s must compute `%s` as a bare assignment on its own line, so"+
|
||||
" `set -e` catches a failing date instead of quietly"+
|
||||
" building with an empty epoch", name, freshEpoch)
|
||||
}
|
||||
|
||||
// instructions returns the Dockerfile's instructions, one per element,
|
||||
// with comments and blank lines dropped and continuation lines joined,
|
||||
// so a multi-line RUN is one string.
|
||||
func instructions(t *testing.T, name string) []string {
|
||||
t.Helper()
|
||||
|
||||
return strings.Split(instructionText(readRepoFile(t, name)), "\n")
|
||||
}
|
||||
|
||||
// instructionText is instructions' parse, before splitting: it is also
|
||||
// what a "must not contain" assertion should look at, so that a word
|
||||
// appearing only in a comment is not mistaken for behaviour.
|
||||
func instructionText(contents string) string {
|
||||
var (
|
||||
out []string
|
||||
continued string
|
||||
isContinued bool
|
||||
)
|
||||
|
||||
for line := range strings.SplitSeq(contents, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if !isContinued && (trimmed == "" || strings.HasPrefix(trimmed, "#")) {
|
||||
continue
|
||||
}
|
||||
|
||||
isContinued = strings.HasSuffix(trimmed, `\`)
|
||||
continued += strings.TrimSuffix(trimmed, `\`)
|
||||
|
||||
if isContinued {
|
||||
continue
|
||||
}
|
||||
|
||||
out = append(out, strings.Join(strings.Fields(continued), " "))
|
||||
continued = ""
|
||||
}
|
||||
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
// indexOf returns the position of the first instruction equal to, or
|
||||
// beginning with, want; -1 if there is none.
|
||||
func indexOf(found []string, want string) int {
|
||||
for i, instruction := range found {
|
||||
if instruction == want || strings.HasPrefix(instruction, want+" ") {
|
||||
return i
|
||||
}
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
// indexContaining returns the position of the first instruction
|
||||
// containing want; -1 if there is none.
|
||||
func indexContaining(found []string, want string) int {
|
||||
for i, instruction := range found {
|
||||
if strings.Contains(instruction, want) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
// readRepoFile reads a file by its path relative to the repository
|
||||
// root.
|
||||
func readRepoFile(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
|
||||
//nolint:gosec // G304: the path is a constant relative to this repo
|
||||
contents, err := os.ReadFile(filepath.Join(repoRoot(t), name))
|
||||
require.NoError(t, err)
|
||||
|
||||
return string(contents)
|
||||
}
|
||||
|
||||
// repoRoot returns the repository root. The test binary runs with its
|
||||
// package directory as the working directory, so the root is found by
|
||||
// walking up until the module file appears.
|
||||
func repoRoot(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
dir, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
|
||||
for {
|
||||
_, err = os.Stat(filepath.Join(dir, "go.mod"))
|
||||
if err == nil {
|
||||
return dir
|
||||
}
|
||||
|
||||
parent := filepath.Dir(dir)
|
||||
require.NotEqual(t, dir, parent,
|
||||
"walked to the filesystem root without finding a go.mod")
|
||||
|
||||
dir = parent
|
||||
}
|
||||
}
|
||||
+15
-6
@@ -1,3 +1,4 @@
|
||||
// Package main is the vaultik command-line entry point.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -5,37 +6,45 @@ import (
|
||||
"runtime"
|
||||
"runtime/pprof"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/cli"
|
||||
"sneak.berlin/go/vaultik/internal/cli"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// CPU profiling: set VAULTIK_CPUPROFILE=/path/to/cpu.prof
|
||||
if cpuProfile := os.Getenv("VAULTIK_CPUPROFILE"); cpuProfile != "" {
|
||||
f, err := os.Create(cpuProfile)
|
||||
f, err := os.Create(cpuProfile) //nolint:gosec // G304: operator-set path
|
||||
if err != nil {
|
||||
panic("could not create CPU profile: " + err.Error())
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
if err := pprof.StartCPUProfile(f); err != nil {
|
||||
|
||||
err = pprof.StartCPUProfile(f)
|
||||
if err != nil {
|
||||
panic("could not start CPU profile: " + err.Error())
|
||||
}
|
||||
|
||||
defer pprof.StopCPUProfile()
|
||||
}
|
||||
|
||||
// Memory profiling: set VAULTIK_MEMPROFILE=/path/to/mem.prof
|
||||
if memProfile := os.Getenv("VAULTIK_MEMPROFILE"); memProfile != "" {
|
||||
defer func() {
|
||||
f, err := os.Create(memProfile)
|
||||
f, err := os.Create(memProfile) //nolint:gosec // G304: operator-set path
|
||||
if err != nil {
|
||||
panic("could not create memory profile: " + err.Error())
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
runtime.GC() // get up-to-date statistics
|
||||
if err := pprof.WriteHeapProfile(f); err != nil {
|
||||
|
||||
err = pprof.WriteHeapProfile(f)
|
||||
if err != nil {
|
||||
panic("could not write memory profile: " + err.Error())
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
cli.CLIEntry()
|
||||
cli.Entry()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
package main_test
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// This file guards the Makefile that builds this program, which is why
|
||||
// it lives beside it rather than in a package of its own.
|
||||
//
|
||||
// Issue #110: `build` was listed in .PHONY with no `build:` rule
|
||||
// anywhere in the file. That combination is silently successful — make
|
||||
// considers a phony target with no prerequisites and no recipe already
|
||||
// satisfied, so `rm -f vaultik && make build` printed "Nothing to be
|
||||
// done for 'build'" and exited 0 with no binary produced. Declaring the
|
||||
// name phony is precisely what converts the "No rule to make target"
|
||||
// error into a green.
|
||||
//
|
||||
// The guard is a parse of the Makefile rather than an invocation of
|
||||
// make. `make test` is what runs these tests, so shelling back into
|
||||
// `make build` here would nest a build inside the test run and drop a
|
||||
// binary into the tree as a side effect of testing. The one property a
|
||||
// parse cannot establish — that the recipe still fails when the build
|
||||
// fails — is not testable from inside the build either; it is verified
|
||||
// by hand against a deliberately broken tree.
|
||||
|
||||
// phonyDirective introduces the list of phony target names.
|
||||
const phonyDirective = ".PHONY:"
|
||||
|
||||
// ruleLine matches a rule's target list: a target starts in column
|
||||
// zero, so recipe lines (tab-indented) and the continuation lines of a
|
||||
// variable assignment (space-indented) are excluded by construction.
|
||||
//
|
||||
// The trailing (?:[^=]|$) rejects `:=` assignments such as
|
||||
// `VERSION := $(shell script/version)`, which are not rules. Directives
|
||||
// and function calls (`.PHONY:`, `ifeq`, `$(error ...)`) do not match
|
||||
// because a target here must begin with a letter, digit or underscore.
|
||||
var ruleLine = regexp.MustCompile(`^([A-Za-z0-9_][A-Za-z0-9_./ -]*):(?:[^=]|$)`)
|
||||
|
||||
// TestPhonyTargetsAllHaveRules fails on any name in .PHONY that has no
|
||||
// rule in the Makefile. Such a name is not a build target at all: it is
|
||||
// a command that reports success without doing anything, which is worse
|
||||
// than one that does not exist, because a caller checking the exit code
|
||||
// cannot tell the difference.
|
||||
func TestPhonyTargetsAllHaveRules(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
makefile := readMakefile(t)
|
||||
|
||||
phony := phonyTargets(makefile)
|
||||
require.NotEmpty(t, phony, "no .PHONY names found; the parser is broken")
|
||||
|
||||
rules := declaredRules(makefile)
|
||||
|
||||
// Sanity check on the rule parser before trusting its verdict: a
|
||||
// parser that found nothing would pass this test by accident.
|
||||
require.Contains(t, rules, "vaultik",
|
||||
"the file rule that builds the binary must be recognized")
|
||||
|
||||
for _, target := range phony {
|
||||
assert.Contains(t, rules, target,
|
||||
"`.PHONY` lists %q but the Makefile declares no %q rule, so "+
|
||||
"`make %s` exits 0 without doing anything", target, target, target)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildTargetBuildsTheBinary pins the specific shape of issue #110:
|
||||
// `make build` has to reach the rule that produces the binary. The test
|
||||
// above would also pass if `build:` were given an empty recipe of its
|
||||
// own, which would be the same silent success under a different
|
||||
// spelling.
|
||||
func TestBuildTargetBuildsTheBinary(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
prerequisites := rulePrerequisites(readMakefile(t), "build")
|
||||
require.NotNil(t, prerequisites, "the Makefile declares no `build` rule")
|
||||
|
||||
assert.Contains(t, prerequisites, "vaultik",
|
||||
"`make build` must depend on the rule that builds the binary")
|
||||
}
|
||||
|
||||
// readMakefile returns the contents of the repository's Makefile. The
|
||||
// root is located by the shared walk in lintdocker_test.go.
|
||||
func readMakefile(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
return readRepoFile(t, "Makefile")
|
||||
}
|
||||
|
||||
// phonyTargets returns every name declared phony, across all .PHONY
|
||||
// lines.
|
||||
func phonyTargets(makefile string) []string {
|
||||
var targets []string
|
||||
|
||||
for line := range strings.SplitSeq(makefile, "\n") {
|
||||
if !strings.HasPrefix(line, phonyDirective) {
|
||||
continue
|
||||
}
|
||||
|
||||
targets = append(targets,
|
||||
strings.Fields(strings.TrimPrefix(line, phonyDirective))...)
|
||||
}
|
||||
|
||||
return targets
|
||||
}
|
||||
|
||||
// declaredRules returns the set of target names that have a rule.
|
||||
func declaredRules(makefile string) map[string]bool {
|
||||
rules := make(map[string]bool)
|
||||
|
||||
for line := range strings.SplitSeq(makefile, "\n") {
|
||||
match := ruleLine.FindStringSubmatch(line)
|
||||
if match == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// One rule may name several targets: `a b: prereq`.
|
||||
for target := range strings.FieldsSeq(match[1]) {
|
||||
rules[target] = true
|
||||
}
|
||||
}
|
||||
|
||||
return rules
|
||||
}
|
||||
|
||||
// rulePrerequisites returns the prerequisites of the named rule, or nil
|
||||
// if no such rule exists. A rule with none returns an empty slice, so
|
||||
// "declared with nothing to do" is distinguishable from "not declared".
|
||||
func rulePrerequisites(makefile, target string) []string {
|
||||
for line := range strings.SplitSeq(makefile, "\n") {
|
||||
match := ruleLine.FindStringSubmatch(line)
|
||||
if match == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if !slices.Contains(strings.Fields(match[1]), target) {
|
||||
continue
|
||||
}
|
||||
|
||||
_, after, _ := strings.Cut(line, ":")
|
||||
|
||||
return append([]string{}, strings.Fields(after)...)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -291,21 +291,6 @@ storage_url: "rclone://las1stor1//srv/pool.2024.04/backups/heraklion"
|
||||
# # Default: 5MB
|
||||
# #part_size: 5MB
|
||||
|
||||
# How often to run backups in daemon mode
|
||||
# Format: 1h, 30m, 24h, etc
|
||||
# Default: 1h
|
||||
#backup_interval: 1h
|
||||
|
||||
# How often to do a full filesystem scan in daemon mode
|
||||
# Between full scans, inotify is used to detect changes
|
||||
# Default: 24h
|
||||
#full_scan_interval: 24h
|
||||
|
||||
# Minimum time between backup runs in daemon mode
|
||||
# Prevents backups from running too frequently
|
||||
# Default: 15m
|
||||
#min_time_between_run: 15m
|
||||
|
||||
# Path to local SQLite index database
|
||||
# This database tracks file state for incremental backups
|
||||
# Default: /var/lib/vaultik/index.sqlite
|
||||
|
||||
+13
-5
@@ -5,8 +5,14 @@
|
||||
Vaultik uses a local SQLite database to track file metadata, chunk mappings, and blob associations during the backup process. This database serves as an index for incremental backups and enables efficient deduplication.
|
||||
|
||||
**Important Notes:**
|
||||
- **No Migration Support**: Vaultik does not support database schema migrations. If the schema changes, the local database must be deleted and recreated by performing a full backup.
|
||||
- **Version Compatibility**: In rare cases, you may need to use the same version of Vaultik to restore a backup as was used to create it. This ensures compatibility with the metadata format stored in S3.
|
||||
- **No Migration Support (pre-1.0)**: Vaultik does not support database schema
|
||||
migrations. The local index is treated as disposable — if the schema changes,
|
||||
delete the local SQLite database (`vaultik database delete`) and run a full
|
||||
backup. The remote storage is unaffected; the new index will re-deduplicate
|
||||
against existing remote blobs.
|
||||
- **Version Compatibility**: In rare cases, you may need to use the same version
|
||||
of Vaultik to restore a backup as was used to create it. This ensures
|
||||
compatibility with the metadata format stored in S3.
|
||||
|
||||
## Database Tables
|
||||
|
||||
@@ -186,10 +192,12 @@ Tracks blob upload metrics.
|
||||
After a snapshot is completed:
|
||||
1. Copy database to temporary file
|
||||
2. Clean temporary database to contain only current snapshot data
|
||||
3. Export to SQL dump using sqlite3
|
||||
3. VACUUM the trimmed database so deleted rows leave no pages behind
|
||||
4. Compress with zstd and encrypt with age
|
||||
5. Upload to S3 as `metadata/{snapshot-id}/db.zst.age`
|
||||
6. Generate blob manifest and upload as `metadata/{snapshot-id}/manifest.json.zst`
|
||||
5. Upload to S3 as `metadata/{remote-key}/db.zst.age`
|
||||
6. Generate blob manifest and upload as `metadata/{remote-key}/manifest.json.zst`
|
||||
|
||||
The `{remote-key}` directory name is a one-way hash of the human snapshot ID, so the ID is never written to the store in plaintext; see [REPOSTRUCTURE.md](REPOSTRUCTURE.md#remote-key-derivation).
|
||||
|
||||
### 4. Restore Process
|
||||
|
||||
|
||||
+46
-25
@@ -17,11 +17,13 @@ Vaultik stores all backup data in an S3-compatible object store. The repository
|
||||
│ └── <hash[2:4]>/
|
||||
│ └── <full-hash>
|
||||
└── metadata/
|
||||
└── <snapshot-id>/
|
||||
└── <remote-key>/
|
||||
├── db.zst.age
|
||||
└── manifest.json.zst
|
||||
```
|
||||
|
||||
The metadata subdirectory is named with the **remote key**, a one-way hash of the snapshot ID, not with the human-readable snapshot ID itself. See [Remote Key Derivation](#remote-key-derivation).
|
||||
|
||||
## Blobs Directory (`blobs/`)
|
||||
|
||||
### Structure
|
||||
@@ -40,21 +42,37 @@ Blobs contain the actual file data from backups and must be encrypted for securi
|
||||
|
||||
## Metadata Directory (`metadata/`)
|
||||
|
||||
Each snapshot has its own subdirectory named with the snapshot ID.
|
||||
Each snapshot has its own subdirectory. The directory is **not** named with the human-readable snapshot ID; it is named with the remote key — a one-way hash of that ID. The human ID is never written to the destination store as a directory name (see [Remote Key Derivation](#remote-key-derivation)).
|
||||
|
||||
### Snapshot ID Format
|
||||
- **Format**: `<hostname>-<YYYYMMDD>-<HHMMSSZ>`
|
||||
- **Example**: `laptop-20240115-143052Z`
|
||||
|
||||
The human-readable snapshot ID is used in CLI arguments, log lines, and the local database. It is not written to the destination store.
|
||||
- **Format**: `<hostname>_<snapshot-name>_<RFC3339>` (or `<hostname>_<RFC3339>` if no
|
||||
name was specified)
|
||||
- **Example**: `laptop_home_2024-01-15T14:30:52Z`
|
||||
- **Components**:
|
||||
- Hostname (may contain hyphens)
|
||||
- Date in YYYYMMDD format
|
||||
- Time in HHMMSSZ format (Z indicates UTC)
|
||||
- Short hostname (everything before the first dot is stripped from the FQDN)
|
||||
- Snapshot name from the configured `snapshots:` map (optional)
|
||||
- RFC3339 UTC timestamp
|
||||
|
||||
This ID reveals the hostname, the configured snapshot name, and the backup time, so it is never used as the on-disk directory name — the remote key is used instead.
|
||||
|
||||
### Remote Key Derivation
|
||||
|
||||
The remote key is `hex(SHA256(SHA256("vaultik|" + snapshot-id)))`: a double SHA-256 over the snapshot ID, with a `vaultik|` domain-separation prefix. The result is a 64-character hex string with no structure a remote observer can reverse. Implemented in `internal/snapshot/remotekey.go`.
|
||||
|
||||
Worked example:
|
||||
- Snapshot ID: `server1_home_2025-06-01T12:00:00Z`
|
||||
- Remote key: `17f97bcde958748af076b926af59823943db59e80ce7170b40f124dfa28f64aa`
|
||||
- Directory: `metadata/17f97bcde958748af076b926af59823943db59e80ce7170b40f124dfa28f64aa/`
|
||||
|
||||
Because the hash is one-way, a listing of the destination store reveals neither the hostname nor the snapshot name of any backup. The same remote key is stored in the manifest's `snapshot_id` field.
|
||||
|
||||
### Files in Each Snapshot Directory
|
||||
|
||||
#### `db.zst.age` - Encrypted Database Dump
|
||||
- **What it contains**: Complete SQLite database dump for this snapshot
|
||||
- **Format**: SQL dump → Zstandard compressed → Age encrypted
|
||||
#### `db.zst.age` - Encrypted Database
|
||||
- **What it contains**: Pruned binary SQLite database for this snapshot
|
||||
- **Format**: Binary SQLite → Zstandard compressed → Age encrypted
|
||||
- **Encryption**: Encrypted with Age
|
||||
- **Purpose**: Contains full file metadata, chunk mappings, and all relationships
|
||||
- **Why encrypted**: Contains sensitive metadata like file paths, permissions, and ownership
|
||||
@@ -67,16 +85,17 @@ Each snapshot has its own subdirectory named with the snapshot ID.
|
||||
- **Structure**:
|
||||
```json
|
||||
{
|
||||
"snapshot_id": "laptop-20240115-143052Z",
|
||||
"timestamp": "2024-01-15T14:30:52Z",
|
||||
"snapshot_id": "17f97bcde958748af076b926af59823943db59e80ce7170b40f124dfa28f64aa",
|
||||
"timestamp": "2025-06-01T12:00:00Z",
|
||||
"blob_count": 42,
|
||||
"total_compressed_size": 1048576,
|
||||
"blobs": [
|
||||
"cafebabe1234567890abcdef1234567890abcdef1234567890abcdef12345678",
|
||||
"deadbeef1234567890abcdef1234567890abcdef1234567890abcdef12345678",
|
||||
...
|
||||
{ "hash": "cafebabe1234567890abcdef1234567890abcdef1234567890abcdef12345678", "compressed_size": 24576 },
|
||||
{ "hash": "deadbeef1234567890abcdef1234567890abcdef1234567890abcdef12345678", "compressed_size": 32768 }
|
||||
]
|
||||
}
|
||||
```
|
||||
`snapshot_id` is the remote key (a hash), not the human ID; `timestamp` is written in the clear.
|
||||
|
||||
### Why Manifest is Unencrypted
|
||||
The manifest must be readable without the private key to enable:
|
||||
@@ -85,7 +104,7 @@ The manifest must be readable without the private key to enable:
|
||||
3. **Verification** - Checking blob existence without decryption
|
||||
4. **Cross-snapshot deduplication analysis** - Finding shared blobs between snapshots
|
||||
|
||||
The manifest only contains blob hashes, not file names or any other sensitive information.
|
||||
The manifest contains the remote key, the backup timestamp, the blob count and total compressed size, and each blob's hash and compressed size. It contains no file names, paths, or other decrypted metadata.
|
||||
|
||||
## Security Considerations
|
||||
|
||||
@@ -95,19 +114,21 @@ The manifest only contains blob hashes, not file names or any other sensitive in
|
||||
- **File-to-chunk mappings** (in db.zst.age)
|
||||
|
||||
### What's Not Encrypted
|
||||
- **Blob hashes** (in manifest.json.zst)
|
||||
- **Snapshot IDs** (directory names)
|
||||
- **Blob count per snapshot** (in manifest.json.zst)
|
||||
- **The remote key** — directory names and the manifest `snapshot_id`, a one-way hash of the snapshot ID (see [Remote Key Derivation](#remote-key-derivation))
|
||||
- **The backup timestamp** (in manifest.json.zst)
|
||||
- **Blob hashes and their compressed sizes** (in manifest.json.zst)
|
||||
- **Blob count and total compressed size per snapshot** (in manifest.json.zst)
|
||||
|
||||
### Privacy Implications
|
||||
From the unencrypted data, an observer can determine:
|
||||
- When backups were taken (from snapshot IDs)
|
||||
- Which hostname created backups (from snapshot IDs)
|
||||
- How many blobs each snapshot references
|
||||
- Which blobs are shared between snapshots (deduplication patterns)
|
||||
- The size of each encrypted blob
|
||||
From the unencrypted data, an observer of the destination store can determine:
|
||||
- **When each backup was taken** — not from the directory name, which is a one-way hash, but from the plaintext `timestamp` field in manifest.json.zst, which is published in the clear
|
||||
- How many blobs each snapshot references, and the total compressed size
|
||||
- The compressed size of each blob, and which blobs are shared between snapshots (deduplication patterns)
|
||||
|
||||
Together these give an observer a timing-and-size profile of every snapshot. This is an accepted, documented property of the format, not a defect: the manifest is unencrypted so that pruning can run without the private key, and the timing channel could not be closed by encrypting it anyway — object creation times and per-object sizes stay visible at the storage layer on both `s3://` and `file://` destinations regardless.
|
||||
|
||||
An observer cannot determine:
|
||||
- The hostname or snapshot name of any backup (the directory name and the manifest `snapshot_id` are one-way hashes of the human ID)
|
||||
- File names or paths
|
||||
- File contents
|
||||
- File permissions or ownership
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
module git.eeqj.de/sneak/vaultik
|
||||
module sneak.berlin/go/vaultik
|
||||
|
||||
go 1.26.1
|
||||
|
||||
@@ -17,9 +17,7 @@ require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/johannesboyne/gofakes3 v0.0.0-20250603205740-ed9094be7668
|
||||
github.com/klauspost/compress v1.18.1
|
||||
github.com/mattn/go-sqlite3 v1.14.29
|
||||
github.com/rclone/rclone v1.72.1
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/spf13/afero v1.15.0
|
||||
github.com/spf13/cobra v1.10.1
|
||||
github.com/stretchr/testify v1.11.1
|
||||
@@ -187,7 +185,6 @@ require (
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
@@ -218,7 +215,6 @@ require (
|
||||
github.com/relvacode/iso8601 v1.7.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rfjakob/eme v1.1.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
|
||||
|
||||
@@ -202,8 +202,6 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc=
|
||||
github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts=
|
||||
github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
|
||||
github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
|
||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 h1:z0uK8UQqjMVYzvk4tiiu3obv2B44+XBsvgEJREQfnO8=
|
||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9/go.mod h1:Jl2neWsQaDanWORdqZ4emBl50J4/aRBBS4FyyG9/PFo=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
@@ -593,16 +591,12 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
|
||||
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
||||
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mattn/go-sqlite3 v1.14.29 h1:1O6nRLJKvsi1H2Sj0Hzdfojwt8GiGKm+LOfLaBFaouQ=
|
||||
github.com/mattn/go-sqlite3 v1.14.29/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
|
||||
github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
|
||||
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
|
||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||
github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
|
||||
@@ -707,8 +701,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
|
||||
github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
@@ -723,8 +715,6 @@ github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDj
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
|
||||
github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw=
|
||||
github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
|
||||
github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc=
|
||||
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
|
||||
+14
-1
@@ -2,5 +2,18 @@ package blob
|
||||
|
||||
import "errors"
|
||||
|
||||
// ErrBlobSizeLimitExceeded is returned when adding a chunk would exceed the blob size limit
|
||||
// ErrBlobSizeLimitExceeded is returned when adding a chunk would exceed
|
||||
// the blob size limit.
|
||||
var ErrBlobSizeLimitExceeded = errors.New("adding chunk would exceed blob size limit")
|
||||
|
||||
// ErrNoRecipients is returned when a Packer is created without any age
|
||||
// recipients; blobs must always be encrypted.
|
||||
var ErrNoRecipients = errors.New("recipients are required - blobs must be encrypted")
|
||||
|
||||
// ErrInvalidMaxBlobSize is returned when the configured maximum blob size
|
||||
// is zero or negative.
|
||||
var ErrInvalidMaxBlobSize = errors.New("max blob size must be positive")
|
||||
|
||||
// ErrNoFilesystem is returned when a Packer is created without a filesystem
|
||||
// for temporary files.
|
||||
var ErrNoFilesystem = errors.New("filesystem is required")
|
||||
|
||||
+184
-92
@@ -18,34 +18,42 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/blobgen"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"github.com/google/uuid"
|
||||
"github.com/spf13/afero"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// BlobHandler is a callback function invoked when a blob is finalized and ready for upload.
|
||||
// The handler receives a BlobWithReader containing the blob metadata and a reader for
|
||||
// the compressed and encrypted blob content. The handler is responsible for uploading
|
||||
// the blob to storage and cleaning up any temporary files.
|
||||
type BlobHandler func(blob *BlobWithReader) error
|
||||
// Handler is a callback function invoked when a blob is finalized and
|
||||
// ready for upload. The handler receives a WithReader containing the
|
||||
// blob metadata and a reader for the compressed and encrypted blob content.
|
||||
// The handler is responsible for uploading the blob to storage and cleaning
|
||||
// up any temporary files.
|
||||
type Handler func(blob *WithReader) error
|
||||
|
||||
// PackerConfig holds configuration for creating a Packer.
|
||||
// All fields except BlobHandler are required.
|
||||
type PackerConfig struct {
|
||||
MaxBlobSize int64 // Maximum size of a blob before forcing finalization
|
||||
CompressionLevel int // Zstd compression level (1-19, higher = better compression)
|
||||
Recipients []string // Age recipients for encryption
|
||||
Repositories *database.Repositories // Database repositories for tracking blob metadata
|
||||
BlobHandler BlobHandler // Optional callback when blob is ready for upload
|
||||
Fs afero.Fs // Filesystem for temporary files
|
||||
// MaxBlobSize is the maximum size of a blob before forcing finalization.
|
||||
MaxBlobSize int64
|
||||
// CompressionLevel is the zstd level (1-19, higher = better compression).
|
||||
CompressionLevel int
|
||||
// Recipients holds the age recipients for encryption.
|
||||
Recipients []string
|
||||
// Repositories provides database access for tracking blob metadata.
|
||||
Repositories *database.Repositories
|
||||
// BlobHandler is an optional callback when a blob is ready for upload.
|
||||
BlobHandler Handler
|
||||
// Fs is the filesystem used for temporary files.
|
||||
Fs afero.Fs
|
||||
}
|
||||
|
||||
// PendingChunk represents a chunk waiting to be inserted into the database.
|
||||
@@ -61,7 +69,7 @@ type Packer struct {
|
||||
maxBlobSize int64
|
||||
compressionLevel int
|
||||
recipients []string // Age recipients for encryption
|
||||
blobHandler BlobHandler // Called when blob is ready
|
||||
blobHandler Handler // Called when blob is ready
|
||||
repos *database.Repositories // For creating blob records
|
||||
fs afero.Fs // Filesystem for temporary files
|
||||
|
||||
@@ -108,22 +116,23 @@ type FinishedBlob struct {
|
||||
ID string
|
||||
Hash string
|
||||
Data []byte // Compressed data
|
||||
Chunks []*BlobChunkRef
|
||||
Chunks []*ChunkPosition
|
||||
CreatedTS time.Time
|
||||
Uncompressed int64
|
||||
Compressed int64
|
||||
}
|
||||
|
||||
// BlobChunkRef represents a chunk's position within a blob
|
||||
type BlobChunkRef struct {
|
||||
// ChunkPosition represents a chunk's position within a blob
|
||||
type ChunkPosition struct {
|
||||
ChunkHash string
|
||||
Offset int64
|
||||
Length int64
|
||||
}
|
||||
|
||||
// BlobWithReader wraps a FinishedBlob with its data reader
|
||||
type BlobWithReader struct {
|
||||
// WithReader wraps a FinishedBlob with its data reader
|
||||
type WithReader struct {
|
||||
*FinishedBlob
|
||||
|
||||
Reader io.ReadSeeker
|
||||
TempFile afero.File // Optional, only set for disk-based blobs
|
||||
InsertedChunkHashes []string // Chunk hashes that were inserted to DB with this blob
|
||||
@@ -134,14 +143,17 @@ type BlobWithReader struct {
|
||||
// Returns an error if required configuration fields are missing or invalid.
|
||||
func NewPacker(cfg PackerConfig) (*Packer, error) {
|
||||
if len(cfg.Recipients) == 0 {
|
||||
return nil, fmt.Errorf("recipients are required - blobs must be encrypted")
|
||||
return nil, ErrNoRecipients
|
||||
}
|
||||
|
||||
if cfg.MaxBlobSize <= 0 {
|
||||
return nil, fmt.Errorf("max blob size must be positive")
|
||||
return nil, ErrInvalidMaxBlobSize
|
||||
}
|
||||
|
||||
if cfg.Fs == nil {
|
||||
return nil, fmt.Errorf("filesystem is required")
|
||||
return nil, ErrNoFilesystem
|
||||
}
|
||||
|
||||
return &Packer{
|
||||
maxBlobSize: cfg.MaxBlobSize,
|
||||
compressionLevel: cfg.CompressionLevel,
|
||||
@@ -157,9 +169,10 @@ func NewPacker(cfg PackerConfig) (*Packer, error) {
|
||||
// The handler is responsible for uploading the blob to storage.
|
||||
// If no handler is set, finalized blobs are stored in memory and can be
|
||||
// retrieved with GetFinishedBlobs().
|
||||
func (p *Packer) SetBlobHandler(handler BlobHandler) {
|
||||
func (p *Packer) SetBlobHandler(handler Handler) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.blobHandler = handler
|
||||
}
|
||||
|
||||
@@ -169,6 +182,7 @@ func (p *Packer) SetBlobHandler(handler BlobHandler) {
|
||||
func (p *Packer) AddPendingChunk(hash string, size int64) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.pendingChunks = append(p.pendingChunks, PendingChunk{Hash: hash, Size: size})
|
||||
}
|
||||
|
||||
@@ -177,13 +191,14 @@ func (p *Packer) AddPendingChunk(hash string, size int64) {
|
||||
// In this case, the caller should finalize the current blob and retry.
|
||||
// The chunk data is written immediately and can be garbage collected after this call.
|
||||
// Thread-safe.
|
||||
func (p *Packer) AddChunk(chunk *ChunkRef) error {
|
||||
func (p *Packer) AddChunk(ctx context.Context, chunk *ChunkRef) error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
// Initialize new blob if needed
|
||||
if p.currentBlob == nil {
|
||||
if err := p.startNewBlob(); err != nil {
|
||||
err := p.startNewBlob(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("starting new blob: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -202,7 +217,8 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error {
|
||||
}
|
||||
|
||||
// Add chunk to current blob
|
||||
if err := p.addChunkToCurrentBlob(chunk); err != nil {
|
||||
err := p.addChunkToCurrentBlob(chunk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -213,12 +229,13 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error {
|
||||
// This should be called after all chunks have been added to ensure no data is lost.
|
||||
// If a BlobHandler is set, it will be called with the finalized blob.
|
||||
// Thread-safe.
|
||||
func (p *Packer) Flush() error {
|
||||
func (p *Packer) Flush(ctx context.Context) error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
if p.currentBlob != nil && len(p.currentBlob.chunks) > 0 {
|
||||
if err := p.finalizeCurrentBlob(); err != nil {
|
||||
err := p.finalizeCurrentBlob(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("finalizing blob: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -232,7 +249,7 @@ func (p *Packer) Flush() error {
|
||||
// BlobHandler (if set) or stored internally.
|
||||
// Caller must handle retrying any chunk that triggered size limit exceeded.
|
||||
// Not thread-safe - caller must hold the lock.
|
||||
func (p *Packer) FinalizeBlob() error {
|
||||
func (p *Packer) FinalizeBlob(ctx context.Context) error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
@@ -240,7 +257,7 @@ func (p *Packer) FinalizeBlob() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
return p.finalizeCurrentBlob()
|
||||
return p.finalizeCurrentBlob(ctx)
|
||||
}
|
||||
|
||||
// GetFinishedBlobs returns all completed blobs and clears the internal list.
|
||||
@@ -253,11 +270,37 @@ func (p *Packer) GetFinishedBlobs() []*FinishedBlob {
|
||||
|
||||
blobs := p.finishedBlobs
|
||||
p.finishedBlobs = make([]*FinishedBlob, 0)
|
||||
|
||||
return blobs
|
||||
}
|
||||
|
||||
// PackChunks is a convenience method to pack multiple chunks at once.
|
||||
func (p *Packer) PackChunks(ctx context.Context, chunks []*ChunkRef) error {
|
||||
for _, chunk := range chunks {
|
||||
err := p.AddChunk(ctx, chunk)
|
||||
if errors.Is(err, ErrBlobSizeLimitExceeded) {
|
||||
// Finalize current blob and retry
|
||||
err = p.FinalizeBlob(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("finalizing blob before retry: %w", err)
|
||||
}
|
||||
|
||||
// Retry the chunk
|
||||
err = p.AddChunk(ctx, chunk)
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"adding chunk %s after finalize: %w", chunk.Hash, err)
|
||||
}
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("adding chunk %s: %w", chunk.Hash, err)
|
||||
}
|
||||
}
|
||||
|
||||
return p.Flush(ctx)
|
||||
}
|
||||
|
||||
// startNewBlob initializes a new blob (must be called with lock held)
|
||||
func (p *Packer) startNewBlob() error {
|
||||
func (p *Packer) startNewBlob(ctx context.Context) error {
|
||||
// Generate UUID for the blob
|
||||
blobID := uuid.New().String()
|
||||
|
||||
@@ -267,18 +310,24 @@ func (p *Packer) startNewBlob() error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing blob ID: %w", err)
|
||||
}
|
||||
|
||||
blob := &database.Blob{
|
||||
ID: blobIDTyped,
|
||||
Hash: types.BlobHash("temp-placeholder-" + blobID), // Temporary placeholder until finalized
|
||||
// Temporary placeholder hash until finalized.
|
||||
Hash: types.BlobHash("temp-placeholder-" + blobID),
|
||||
CreatedTS: time.Now().UTC(),
|
||||
FinishedTS: nil,
|
||||
UncompressedSize: 0,
|
||||
CompressedSize: 0,
|
||||
UploadedTS: nil,
|
||||
}
|
||||
if err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return p.repos.Blobs.Create(ctx, tx, blob)
|
||||
}); err != nil {
|
||||
|
||||
err = p.repos.WithTx(
|
||||
ctx,
|
||||
func(txCtx context.Context, tx *sql.Tx) error {
|
||||
return p.repos.Blobs.Create(txCtx, tx, blob)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating blob record: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -294,6 +343,7 @@ func (p *Packer) startNewBlob() error {
|
||||
if err != nil {
|
||||
_ = tempFile.Close()
|
||||
_ = p.fs.Remove(tempFile.Name())
|
||||
|
||||
return fmt.Errorf("creating blobgen writer: %w", err)
|
||||
}
|
||||
|
||||
@@ -307,15 +357,20 @@ func (p *Packer) startNewBlob() error {
|
||||
size: 0,
|
||||
}
|
||||
|
||||
log.Debug("Created new blob container", "blob_id", blobID, "temp_file", tempFile.Name())
|
||||
log.Debug("Created new blob container",
|
||||
"blob_id", blobID, "temp_file", tempFile.Name())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// addChunkToCurrentBlob adds a chunk to the current blob (must be called with lock held)
|
||||
// addChunkToCurrentBlob adds a chunk to the current blob (must be called
|
||||
// with lock held).
|
||||
func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error {
|
||||
// Skip if chunk already in current blob
|
||||
if p.currentBlob.chunkSet[chunk.Hash] {
|
||||
log.Debug("Skipping duplicate chunk already in current blob", "chunk_hash", chunk.Hash)
|
||||
log.Debug("Skipping duplicate chunk already in current blob",
|
||||
"chunk_hash", chunk.Hash)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -323,7 +378,8 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error {
|
||||
offset := p.currentBlob.size
|
||||
|
||||
// Write to the blobgen writer (compression -> encryption -> disk)
|
||||
if _, err := p.currentBlob.writer.Write(chunk.Data); err != nil {
|
||||
_, err := p.currentBlob.writer.Write(chunk.Data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing to blob stream: %w", err)
|
||||
}
|
||||
|
||||
@@ -356,7 +412,7 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error {
|
||||
}
|
||||
|
||||
// finalizeCurrentBlob completes the current blob (must be called with lock held)
|
||||
func (p *Packer) finalizeCurrentBlob() error {
|
||||
func (p *Packer) finalizeCurrentBlob(ctx context.Context) error {
|
||||
if p.currentBlob == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -371,7 +427,8 @@ func (p *Packer) finalizeCurrentBlob() error {
|
||||
chunksToInsert := p.pendingChunks
|
||||
p.pendingChunks = nil
|
||||
|
||||
if err := p.commitBlobToDatabase(blobHash, finalSize, chunksToInsert); err != nil {
|
||||
err = p.commitBlobToDatabase(ctx, blobHash, finalSize, chunksToInsert)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -399,44 +456,59 @@ func (p *Packer) finalizeCurrentBlob() error {
|
||||
return p.deliverFinishedBlob(finished, insertedChunkHashes)
|
||||
}
|
||||
|
||||
// closeBlobWriter closes the writer, syncs to disk, and returns the blob hash and final size
|
||||
// closeBlobWriter closes the writer, syncs to disk, and returns the blob
|
||||
// hash and final size.
|
||||
func (p *Packer) closeBlobWriter() (string, int64, error) {
|
||||
if err := p.currentBlob.writer.Close(); err != nil {
|
||||
err := p.currentBlob.writer.Close()
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return "", 0, fmt.Errorf("closing blobgen writer: %w", err)
|
||||
}
|
||||
if err := p.currentBlob.tempFile.Sync(); err != nil {
|
||||
|
||||
err = p.currentBlob.tempFile.Sync()
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return "", 0, fmt.Errorf("syncing temp file: %w", err)
|
||||
}
|
||||
|
||||
finalSize, err := p.currentBlob.tempFile.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return "", 0, fmt.Errorf("getting file size: %w", err)
|
||||
}
|
||||
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
|
||||
|
||||
_, err = p.currentBlob.tempFile.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return "", 0, fmt.Errorf("seeking to start: %w", err)
|
||||
}
|
||||
|
||||
finalHash := p.currentBlob.writer.Sum256()
|
||||
|
||||
return hex.EncodeToString(finalHash), finalSize, nil
|
||||
}
|
||||
|
||||
// buildChunkRefs creates BlobChunkRef entries from the current blob's chunks
|
||||
func (p *Packer) buildChunkRefs() []*BlobChunkRef {
|
||||
refs := make([]*BlobChunkRef, 0, len(p.currentBlob.chunks))
|
||||
// buildChunkRefs creates ChunkPosition entries from the current blob's chunks
|
||||
func (p *Packer) buildChunkRefs() []*ChunkPosition {
|
||||
refs := make([]*ChunkPosition, 0, len(p.currentBlob.chunks))
|
||||
for _, chunk := range p.currentBlob.chunks {
|
||||
refs = append(refs, &BlobChunkRef{
|
||||
refs = append(refs, &ChunkPosition{
|
||||
ChunkHash: chunk.Hash, Offset: chunk.Offset, Length: chunk.Size,
|
||||
})
|
||||
}
|
||||
|
||||
return refs
|
||||
}
|
||||
|
||||
// commitBlobToDatabase inserts pending chunks, blob_chunks, and updates the blob record
|
||||
func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksToInsert []PendingChunk) error {
|
||||
func (p *Packer) commitBlobToDatabase(
|
||||
ctx context.Context,
|
||||
blobHash string, finalSize int64, chunksToInsert []PendingChunk,
|
||||
) error {
|
||||
if p.repos == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -444,13 +516,46 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
|
||||
blobIDTyped, parseErr := types.ParseBlobID(p.currentBlob.id)
|
||||
if parseErr != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("parsing blob ID: %w", parseErr)
|
||||
}
|
||||
|
||||
err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
err := p.repos.WithTx(
|
||||
ctx,
|
||||
func(txCtx context.Context, tx *sql.Tx) error {
|
||||
return p.insertBlobRecords(txCtx, tx, blobIDTyped, blobHash,
|
||||
finalSize, chunksToInsert)
|
||||
})
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("finalizing blob transaction: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Committed blob transaction",
|
||||
"chunks_inserted", len(chunksToInsert),
|
||||
"blob_chunks_inserted", len(p.currentBlob.chunks))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// insertBlobRecords inserts pending chunks and blob_chunk rows, then marks
|
||||
// the blob finished, all within the supplied transaction.
|
||||
func (p *Packer) insertBlobRecords(
|
||||
ctx context.Context,
|
||||
tx *sql.Tx,
|
||||
blobIDTyped types.BlobID,
|
||||
blobHash string,
|
||||
finalSize int64,
|
||||
chunksToInsert []PendingChunk,
|
||||
) error {
|
||||
for _, chunk := range chunksToInsert {
|
||||
dbChunk := &database.Chunk{ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size}
|
||||
if err := p.repos.Chunks.Create(ctx, tx, dbChunk); err != nil {
|
||||
dbChunk := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size,
|
||||
}
|
||||
|
||||
err := p.repos.Chunks.Create(ctx, tx, dbChunk)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating chunk: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -460,62 +565,70 @@ func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksTo
|
||||
BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash),
|
||||
Offset: chunk.Offset, Length: chunk.Size,
|
||||
}
|
||||
if err := p.repos.BlobChunks.Create(ctx, tx, blobChunk); err != nil {
|
||||
|
||||
err := p.repos.BlobChunks.Create(ctx, tx, blobChunk)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating blob_chunk: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return p.repos.Blobs.UpdateFinished(ctx, tx, p.currentBlob.id, blobHash, p.currentBlob.size, finalSize)
|
||||
})
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
return fmt.Errorf("finalizing blob transaction: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Committed blob transaction",
|
||||
"chunks_inserted", len(chunksToInsert), "blob_chunks_inserted", len(p.currentBlob.chunks))
|
||||
return nil
|
||||
return p.repos.Blobs.UpdateFinished(ctx, tx, p.currentBlob.id, blobHash,
|
||||
p.currentBlob.size, finalSize)
|
||||
}
|
||||
|
||||
// deliverFinishedBlob passes the blob to the handler or stores it internally
|
||||
func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes []string) error {
|
||||
func (p *Packer) deliverFinishedBlob(
|
||||
finished *FinishedBlob, insertedChunkHashes []string,
|
||||
) error {
|
||||
if p.blobHandler != nil {
|
||||
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
|
||||
_, err := p.currentBlob.tempFile.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("seeking for handler: %w", err)
|
||||
}
|
||||
|
||||
blobWithReader := &BlobWithReader{
|
||||
blobWithReader := &WithReader{
|
||||
FinishedBlob: finished,
|
||||
Reader: p.currentBlob.tempFile,
|
||||
TempFile: p.currentBlob.tempFile,
|
||||
InsertedChunkHashes: insertedChunkHashes,
|
||||
}
|
||||
|
||||
if err := p.blobHandler(blobWithReader); err != nil {
|
||||
err = p.blobHandler(blobWithReader)
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("blob handler failed: %w", err)
|
||||
}
|
||||
|
||||
p.currentBlob = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// No handler - read data for legacy behavior
|
||||
log.Debug("No blob handler callback configured", "blob_hash", finished.Hash[:8]+"...")
|
||||
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
|
||||
|
||||
_, err := p.currentBlob.tempFile.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("seeking to read data: %w", err)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(p.currentBlob.tempFile)
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("reading blob data: %w", err)
|
||||
}
|
||||
|
||||
finished.Data = data
|
||||
p.finishedBlobs = append(p.finishedBlobs, finished)
|
||||
p.cleanupTempFile()
|
||||
p.currentBlob = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -527,24 +640,3 @@ func (p *Packer) cleanupTempFile() {
|
||||
_ = p.fs.Remove(name)
|
||||
}
|
||||
}
|
||||
|
||||
// PackChunks is a convenience method to pack multiple chunks at once
|
||||
func (p *Packer) PackChunks(chunks []*ChunkRef) error {
|
||||
for _, chunk := range chunks {
|
||||
err := p.AddChunk(chunk)
|
||||
if err == ErrBlobSizeLimitExceeded {
|
||||
// Finalize current blob and retry
|
||||
if err := p.FinalizeBlob(); err != nil {
|
||||
return fmt.Errorf("finalizing blob before retry: %w", err)
|
||||
}
|
||||
// Retry the chunk
|
||||
if err := p.AddChunk(chunk); err != nil {
|
||||
return fmt.Errorf("adding chunk %s after finalize: %w", chunk.Hash, err)
|
||||
}
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("adding chunk %s: %w", chunk.Hash, err)
|
||||
}
|
||||
}
|
||||
|
||||
return p.Flush()
|
||||
}
|
||||
|
||||
+167
-234
@@ -1,4 +1,4 @@
|
||||
package blob
|
||||
package blob_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -6,107 +6,113 @@ import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/spf13/afero"
|
||||
"sneak.berlin/go/vaultik/internal/blob"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
const (
|
||||
// Test key from test/insecure-integration-test.key
|
||||
testPrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5"
|
||||
testPrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7A" +
|
||||
"PHXA2QS2NJA5"
|
||||
testPublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"
|
||||
|
||||
defaultMaxBlobSize = 10 * 1024 * 1024 // 10MB
|
||||
testChunkSize = 1000
|
||||
testChunkCount = 10
|
||||
)
|
||||
|
||||
func TestPacker(t *testing.T) {
|
||||
// Initialize logger for tests
|
||||
log.Initialize(log.Config{})
|
||||
// parseTestIdentity parses the fixed test age identity.
|
||||
func parseTestIdentity(t *testing.T) *age.X25519Identity {
|
||||
t.Helper()
|
||||
|
||||
// Parse test identity
|
||||
identity, err := age.ParseX25519Identity(testPrivateKey)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse test identity: %v", err)
|
||||
}
|
||||
|
||||
t.Run("single chunk creates single blob", func(t *testing.T) {
|
||||
// Create test database
|
||||
return identity
|
||||
}
|
||||
|
||||
// newTestPacker creates a test database and a Packer backed by it.
|
||||
func newTestPacker(
|
||||
t *testing.T, maxBlobSize int64,
|
||||
) (*database.Repositories, *blob.Packer) {
|
||||
t.Helper()
|
||||
|
||||
db, err := database.NewTestDB()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test db: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
cfg := PackerConfig{
|
||||
MaxBlobSize: 10 * 1024 * 1024, // 10MB
|
||||
packer, err := blob.NewPacker(blob.PackerConfig{
|
||||
MaxBlobSize: maxBlobSize,
|
||||
CompressionLevel: 3,
|
||||
Recipients: []string{testPublicKey},
|
||||
Repositories: repos,
|
||||
Fs: afero.NewMemMapFs(),
|
||||
}
|
||||
packer, err := NewPacker(cfg)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create packer: %v", err)
|
||||
}
|
||||
|
||||
// Create a chunk
|
||||
data := []byte("Hello, World!")
|
||||
return repos, packer
|
||||
}
|
||||
|
||||
// makeChunk creates a ChunkRef for data and registers the chunk in the
|
||||
// database.
|
||||
func makeChunk(
|
||||
t *testing.T, repos *database.Repositories, data []byte,
|
||||
) *blob.ChunkRef {
|
||||
t.Helper()
|
||||
|
||||
hash := sha256.Sum256(data)
|
||||
hashStr := hex.EncodeToString(hash[:])
|
||||
|
||||
// Create chunk in database first
|
||||
dbChunk := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash(hashStr),
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
|
||||
err := repos.WithTx(
|
||||
context.Background(),
|
||||
func(ctx context.Context, tx *sql.Tx) error {
|
||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk in db: %v", err)
|
||||
}
|
||||
|
||||
chunk := &ChunkRef{
|
||||
return &blob.ChunkRef{
|
||||
Hash: hashStr,
|
||||
Data: data,
|
||||
}
|
||||
}
|
||||
|
||||
// Add chunk
|
||||
if err := packer.AddChunk(chunk); err != nil {
|
||||
t.Fatalf("failed to add chunk: %v", err)
|
||||
}
|
||||
// decryptAndDecompress reverses the blob pipeline: age decrypt, then zstd
|
||||
// decompress.
|
||||
func decryptAndDecompress(
|
||||
t *testing.T, blobData []byte, identity *age.X25519Identity,
|
||||
) []byte {
|
||||
t.Helper()
|
||||
|
||||
// Flush
|
||||
if err := packer.Flush(); err != nil {
|
||||
t.Fatalf("failed to flush: %v", err)
|
||||
}
|
||||
|
||||
// Get finished blobs
|
||||
blobs := packer.GetFinishedBlobs()
|
||||
if len(blobs) != 1 {
|
||||
t.Fatalf("expected 1 blob, got %d", len(blobs))
|
||||
}
|
||||
|
||||
blob := blobs[0]
|
||||
if len(blob.Chunks) != 1 {
|
||||
t.Errorf("expected 1 chunk in blob, got %d", len(blob.Chunks))
|
||||
}
|
||||
|
||||
// Note: Very small data may not compress well
|
||||
t.Logf("Compression: %d -> %d bytes", blob.Uncompressed, blob.Compressed)
|
||||
|
||||
// Decrypt the blob data
|
||||
decrypted, err := age.Decrypt(bytes.NewReader(blob.Data), identity)
|
||||
decrypted, err := age.Decrypt(bytes.NewReader(blobData), identity)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decrypt blob: %v", err)
|
||||
}
|
||||
|
||||
// Decompress the decrypted data
|
||||
reader, err := zstd.NewReader(decrypted)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create decompressor: %v", err)
|
||||
@@ -114,157 +120,144 @@ func TestPacker(t *testing.T) {
|
||||
defer reader.Close()
|
||||
|
||||
var decompressed bytes.Buffer
|
||||
if _, err := io.Copy(&decompressed, reader); err != nil {
|
||||
|
||||
_, err = io.Copy(&decompressed, reader)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decompress: %v", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(decompressed.Bytes(), data) {
|
||||
t.Error("decompressed data doesn't match original")
|
||||
}
|
||||
})
|
||||
return decompressed.Bytes()
|
||||
}
|
||||
|
||||
t.Run("multiple chunks packed together", func(t *testing.T) {
|
||||
// Create test database
|
||||
db, err := database.NewTestDB()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test db: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
repos := database.NewRepositories(db)
|
||||
func TestPackerSingleChunk(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
cfg := PackerConfig{
|
||||
MaxBlobSize: 10 * 1024 * 1024, // 10MB
|
||||
CompressionLevel: 3,
|
||||
Recipients: []string{testPublicKey},
|
||||
Repositories: repos,
|
||||
Fs: afero.NewMemMapFs(),
|
||||
}
|
||||
packer, err := NewPacker(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create packer: %v", err)
|
||||
}
|
||||
identity := parseTestIdentity(t)
|
||||
repos, packer := newTestPacker(t, defaultMaxBlobSize)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create multiple small chunks
|
||||
chunks := make([]*ChunkRef, 10)
|
||||
for i := 0; i < 10; i++ {
|
||||
data := bytes.Repeat([]byte{byte(i)}, 1000)
|
||||
hash := sha256.Sum256(data)
|
||||
hashStr := hex.EncodeToString(hash[:])
|
||||
data := []byte("Hello, World!")
|
||||
chunk := makeChunk(t, repos, data)
|
||||
|
||||
// Create chunk in database first
|
||||
dbChunk := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash(hashStr),
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk in db: %v", err)
|
||||
}
|
||||
|
||||
chunks[i] = &ChunkRef{
|
||||
Hash: hashStr,
|
||||
Data: data,
|
||||
}
|
||||
}
|
||||
|
||||
// Add all chunks
|
||||
for _, chunk := range chunks {
|
||||
err := packer.AddChunk(chunk)
|
||||
err := packer.AddChunk(ctx, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to add chunk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Flush
|
||||
if err := packer.Flush(); err != nil {
|
||||
err = packer.Flush(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to flush: %v", err)
|
||||
}
|
||||
|
||||
// Should have one blob with all chunks
|
||||
blobs := packer.GetFinishedBlobs()
|
||||
if len(blobs) != 1 {
|
||||
t.Fatalf("expected 1 blob, got %d", len(blobs))
|
||||
}
|
||||
|
||||
if len(blobs[0].Chunks) != 10 {
|
||||
t.Errorf("expected 10 chunks in blob, got %d", len(blobs[0].Chunks))
|
||||
finished := blobs[0]
|
||||
if len(finished.Chunks) != 1 {
|
||||
t.Errorf("expected 1 chunk in blob, got %d", len(finished.Chunks))
|
||||
}
|
||||
|
||||
// Note: Very small data may not compress well
|
||||
t.Logf("Compression: %d -> %d bytes",
|
||||
finished.Uncompressed, finished.Compressed)
|
||||
|
||||
decompressed := decryptAndDecompress(t, finished.Data, identity)
|
||||
if !bytes.Equal(decompressed, data) {
|
||||
t.Error("decompressed data doesn't match original")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPackerMultipleChunks(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
repos, packer := newTestPacker(t, defaultMaxBlobSize)
|
||||
ctx := context.Background()
|
||||
|
||||
chunks := make([]*blob.ChunkRef, testChunkCount)
|
||||
for i := range testChunkCount {
|
||||
data := bytes.Repeat([]byte{byte(i)}, testChunkSize)
|
||||
chunks[i] = makeChunk(t, repos, data)
|
||||
}
|
||||
|
||||
for _, chunk := range chunks {
|
||||
err := packer.AddChunk(ctx, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to add chunk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
err := packer.Flush(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to flush: %v", err)
|
||||
}
|
||||
|
||||
blobs := packer.GetFinishedBlobs()
|
||||
if len(blobs) != 1 {
|
||||
t.Fatalf("expected 1 blob, got %d", len(blobs))
|
||||
}
|
||||
|
||||
if len(blobs[0].Chunks) != testChunkCount {
|
||||
t.Errorf("expected %d chunks in blob, got %d",
|
||||
testChunkCount, len(blobs[0].Chunks))
|
||||
}
|
||||
|
||||
// Verify offsets are correct
|
||||
expectedOffset := int64(0)
|
||||
|
||||
for i, chunkRef := range blobs[0].Chunks {
|
||||
if chunkRef.Offset != expectedOffset {
|
||||
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunkRef.Offset)
|
||||
t.Errorf("chunk %d: expected offset %d, got %d",
|
||||
i, expectedOffset, chunkRef.Offset)
|
||||
}
|
||||
if chunkRef.Length != 1000 {
|
||||
t.Errorf("chunk %d: expected length 1000, got %d", i, chunkRef.Length)
|
||||
|
||||
if chunkRef.Length != testChunkSize {
|
||||
t.Errorf("chunk %d: expected length %d, got %d",
|
||||
i, testChunkSize, chunkRef.Length)
|
||||
}
|
||||
|
||||
expectedOffset += chunkRef.Length
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("blob size limit enforced", func(t *testing.T) {
|
||||
// Create test database
|
||||
db, err := database.NewTestDB()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test db: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
repos := database.NewRepositories(db)
|
||||
func TestPackerSizeLimit(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
// Small blob size limit to force multiple blobs
|
||||
cfg := PackerConfig{
|
||||
MaxBlobSize: 5000, // 5KB max
|
||||
CompressionLevel: 3,
|
||||
Recipients: []string{testPublicKey},
|
||||
Repositories: repos,
|
||||
Fs: afero.NewMemMapFs(),
|
||||
}
|
||||
packer, err := NewPacker(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create packer: %v", err)
|
||||
}
|
||||
const (
|
||||
maxBlobSize = 5000 // 5KB max, forces multiple blobs
|
||||
maxBlobawoOverhead = 6000 // allow some overhead over the limit
|
||||
)
|
||||
|
||||
// Create chunks that will exceed the limit
|
||||
chunks := make([]*ChunkRef, 10)
|
||||
for i := 0; i < 10; i++ {
|
||||
data := bytes.Repeat([]byte{byte(i)}, 1000) // 1KB each
|
||||
hash := sha256.Sum256(data)
|
||||
hashStr := hex.EncodeToString(hash[:])
|
||||
repos, packer := newTestPacker(t, maxBlobSize)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create chunk in database first
|
||||
dbChunk := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash(hashStr),
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk in db: %v", err)
|
||||
}
|
||||
|
||||
chunks[i] = &ChunkRef{
|
||||
Hash: hashStr,
|
||||
Data: data,
|
||||
}
|
||||
chunks := make([]*blob.ChunkRef, testChunkCount)
|
||||
for i := range testChunkCount {
|
||||
data := bytes.Repeat([]byte{byte(i)}, testChunkSize) // 1KB each
|
||||
chunks[i] = makeChunk(t, repos, data)
|
||||
}
|
||||
|
||||
blobCount := 0
|
||||
|
||||
// Add chunks and handle size limit errors
|
||||
for _, chunk := range chunks {
|
||||
err := packer.AddChunk(chunk)
|
||||
if err == ErrBlobSizeLimitExceeded {
|
||||
err := packer.AddChunk(ctx, chunk)
|
||||
if errors.Is(err, blob.ErrBlobSizeLimitExceeded) {
|
||||
// Finalize current blob
|
||||
if err := packer.FinalizeBlob(); err != nil {
|
||||
err = packer.FinalizeBlob(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to finalize blob: %v", err)
|
||||
}
|
||||
|
||||
blobCount++
|
||||
|
||||
// Retry adding the chunk
|
||||
if err := packer.AddChunk(chunk); err != nil {
|
||||
err = packer.AddChunk(ctx, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to add chunk after finalize: %v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
@@ -272,114 +265,54 @@ func TestPacker(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining
|
||||
if err := packer.Flush(); err != nil {
|
||||
err := packer.Flush(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to flush: %v", err)
|
||||
}
|
||||
|
||||
// Get all blobs
|
||||
blobs := packer.GetFinishedBlobs()
|
||||
totalBlobs := blobCount + len(blobs)
|
||||
|
||||
// Should have multiple blobs due to size limit
|
||||
totalBlobs := blobCount + len(blobs)
|
||||
if totalBlobs < 2 {
|
||||
t.Errorf("expected multiple blobs due to size limit, got %d", totalBlobs)
|
||||
}
|
||||
|
||||
// Verify each blob respects size limit (approximately)
|
||||
for _, blob := range blobs {
|
||||
if blob.Compressed > 6000 { // Allow some overhead
|
||||
t.Errorf("blob size %d exceeds limit", blob.Compressed)
|
||||
for _, finished := range blobs {
|
||||
if finished.Compressed > maxBlobawoOverhead {
|
||||
t.Errorf("blob size %d exceeds limit", finished.Compressed)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("with encryption", func(t *testing.T) {
|
||||
// Create test database
|
||||
db, err := database.NewTestDB()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test db: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
repos := database.NewRepositories(db)
|
||||
func TestPackerEncryption(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
// Generate test identity (using the one from parent test)
|
||||
cfg := PackerConfig{
|
||||
MaxBlobSize: 10 * 1024 * 1024, // 10MB
|
||||
CompressionLevel: 3,
|
||||
Recipients: []string{testPublicKey},
|
||||
Repositories: repos,
|
||||
Fs: afero.NewMemMapFs(),
|
||||
}
|
||||
packer, err := NewPacker(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create packer: %v", err)
|
||||
}
|
||||
identity := parseTestIdentity(t)
|
||||
repos, packer := newTestPacker(t, defaultMaxBlobSize)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create test data
|
||||
data := bytes.Repeat([]byte("Test data for encryption!"), 100)
|
||||
hash := sha256.Sum256(data)
|
||||
hashStr := hex.EncodeToString(hash[:])
|
||||
chunk := makeChunk(t, repos, data)
|
||||
|
||||
// Create chunk in database first
|
||||
dbChunk := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash(hashStr),
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||
})
|
||||
err := packer.AddChunk(ctx, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk in db: %v", err)
|
||||
}
|
||||
|
||||
chunk := &ChunkRef{
|
||||
Hash: hashStr,
|
||||
Data: data,
|
||||
}
|
||||
|
||||
// Add chunk and flush
|
||||
if err := packer.AddChunk(chunk); err != nil {
|
||||
t.Fatalf("failed to add chunk: %v", err)
|
||||
}
|
||||
if err := packer.Flush(); err != nil {
|
||||
|
||||
err = packer.Flush(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to flush: %v", err)
|
||||
}
|
||||
|
||||
// Get blob
|
||||
blobs := packer.GetFinishedBlobs()
|
||||
if len(blobs) != 1 {
|
||||
t.Fatalf("expected 1 blob, got %d", len(blobs))
|
||||
}
|
||||
|
||||
blob := blobs[0]
|
||||
|
||||
// Decrypt the blob
|
||||
decrypted, err := age.Decrypt(bytes.NewReader(blob.Data), identity)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decrypt blob: %v", err)
|
||||
}
|
||||
|
||||
var decryptedData bytes.Buffer
|
||||
if _, err := decryptedData.ReadFrom(decrypted); err != nil {
|
||||
t.Fatalf("failed to read decrypted data: %v", err)
|
||||
}
|
||||
|
||||
// Decompress
|
||||
reader, err := zstd.NewReader(&decryptedData)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create decompressor: %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
var decompressed bytes.Buffer
|
||||
if _, err := decompressed.ReadFrom(reader); err != nil {
|
||||
t.Fatalf("failed to decompress: %v", err)
|
||||
}
|
||||
|
||||
// Verify data
|
||||
if !bytes.Equal(decompressed.Bytes(), data) {
|
||||
decompressed := decryptAndDecompress(t, blobs[0].Data, identity)
|
||||
if !bytes.Equal(decompressed, data) {
|
||||
t.Error("decrypted and decompressed data doesn't match original")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
// Package blobgen implements the blob data pipeline: streaming zstd
|
||||
// compression, age encryption, and SHA256 content hashing for blob
|
||||
// creation, plus the matching decrypt/decompress/verify reader.
|
||||
package blobgen
|
||||
|
||||
import (
|
||||
@@ -16,7 +19,9 @@ type CompressResult struct {
|
||||
}
|
||||
|
||||
// CompressData compresses and encrypts data, returning the result with hash
|
||||
func CompressData(data []byte, compressionLevel int, recipients []string) (*CompressResult, error) {
|
||||
func CompressData(
|
||||
data []byte, compressionLevel int, recipients []string,
|
||||
) (*CompressResult, error) {
|
||||
var buf bytes.Buffer
|
||||
|
||||
// Create writer
|
||||
@@ -26,13 +31,16 @@ func CompressData(data []byte, compressionLevel int, recipients []string) (*Comp
|
||||
}
|
||||
|
||||
// Write data
|
||||
if _, err := w.Write(data); err != nil {
|
||||
_, err = w.Write(data)
|
||||
if err != nil {
|
||||
_ = w.Close()
|
||||
|
||||
return nil, fmt.Errorf("writing data: %w", err)
|
||||
}
|
||||
|
||||
// Close to flush
|
||||
if err := w.Close(); err != nil {
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("closing writer: %w", err)
|
||||
}
|
||||
|
||||
@@ -44,8 +52,11 @@ func CompressData(data []byte, compressionLevel int, recipients []string) (*Comp
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CompressStream compresses and encrypts from reader to writer, returning hash
|
||||
func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipients []string) (written int64, hash string, err error) {
|
||||
// CompressStream compresses and encrypts from reader to writer, returning
|
||||
// the number of uncompressed bytes written and the content hash.
|
||||
func CompressStream(
|
||||
dst io.Writer, src io.Reader, compressionLevel int, recipients []string,
|
||||
) (int64, string, error) {
|
||||
// Create writer
|
||||
w, err := NewWriter(dst, compressionLevel, recipients)
|
||||
if err != nil {
|
||||
@@ -53,6 +64,7 @@ func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipien
|
||||
}
|
||||
|
||||
closed := false
|
||||
|
||||
defer func() {
|
||||
if !closed {
|
||||
_ = w.Close()
|
||||
@@ -60,14 +72,17 @@ func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipien
|
||||
}()
|
||||
|
||||
// Copy data
|
||||
if _, err := io.Copy(w, src); err != nil {
|
||||
_, err = io.Copy(w, src)
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("copying data: %w", err)
|
||||
}
|
||||
|
||||
// Close to flush
|
||||
if err := w.Close(); err != nil {
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("closing writer: %w", err)
|
||||
}
|
||||
|
||||
closed = true
|
||||
|
||||
return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package blobgen
|
||||
package blobgen_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// testRecipient is a static age recipient for tests.
|
||||
@@ -19,35 +20,47 @@ const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s8
|
||||
// the explicit Close() on the happy path combined with defer Close() would
|
||||
// cause a double close.
|
||||
func TestCompressStreamNoDoubleClose(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := []byte("regression test data for issue #28 double-close fix")
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
written, hash, err := CompressStream(&buf, bytes.NewReader(input), 3, []string{testRecipient})
|
||||
written, hash, err := blobgen.CompressStream(
|
||||
&buf, bytes.NewReader(input), 3, []string{testRecipient})
|
||||
require.NoError(t, err, "CompressStream should not return an error")
|
||||
assert.True(t, written > 0, "expected bytes written > 0")
|
||||
assert.Positive(t, written, "expected bytes written > 0")
|
||||
assert.NotEmpty(t, hash, "expected non-empty hash")
|
||||
assert.True(t, buf.Len() > 0, "expected non-empty output")
|
||||
assert.Positive(t, buf.Len(), "expected non-empty output")
|
||||
}
|
||||
|
||||
// TestCompressStreamLargeInput exercises CompressStream with a larger payload
|
||||
// to ensure no double-close issues surface under heavier I/O.
|
||||
func TestCompressStreamLargeInput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
data := make([]byte, 512*1024) // 512 KB
|
||||
_, err := rand.Read(data)
|
||||
require.NoError(t, err)
|
||||
|
||||
var buf bytes.Buffer
|
||||
written, hash, err := CompressStream(&buf, bytes.NewReader(data), 3, []string{testRecipient})
|
||||
|
||||
written, hash, err := blobgen.CompressStream(
|
||||
&buf, bytes.NewReader(data), 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, written > 0)
|
||||
assert.Positive(t, written)
|
||||
assert.NotEmpty(t, hash)
|
||||
}
|
||||
|
||||
// TestCompressStreamEmptyInput verifies CompressStream handles empty input
|
||||
// without double-close issues.
|
||||
func TestCompressStreamEmptyInput(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var buf bytes.Buffer
|
||||
_, hash, err := CompressStream(&buf, strings.NewReader(""), 3, []string{testRecipient})
|
||||
|
||||
_, hash, err := blobgen.CompressStream(
|
||||
&buf, strings.NewReader(""), 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, hash)
|
||||
}
|
||||
@@ -55,10 +68,13 @@ func TestCompressStreamEmptyInput(t *testing.T) {
|
||||
// TestCompressDataNoDoubleClose mirrors the stream test for CompressData,
|
||||
// ensuring the explicit Close + error-path Close pattern is also safe.
|
||||
func TestCompressDataNoDoubleClose(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
input := []byte("CompressData regression test for double-close")
|
||||
result, err := CompressData(input, 3, []string{testRecipient})
|
||||
|
||||
result, err := blobgen.CompressData(input, 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.CompressedSize > 0)
|
||||
assert.True(t, result.UncompressedSize == int64(len(input)))
|
||||
assert.Positive(t, result.CompressedSize)
|
||||
assert.Equal(t, result.UncompressedSize, int64(len(input)))
|
||||
assert.NotEmpty(t, result.SHA256)
|
||||
}
|
||||
|
||||
@@ -50,15 +50,17 @@ func NewReader(r io.Reader, identity age.Identity) (*Reader, error) {
|
||||
}
|
||||
|
||||
// Read implements io.Reader
|
||||
func (r *Reader) Read(p []byte) (n int, err error) {
|
||||
n, err = r.teeReader.Read(p)
|
||||
func (r *Reader) Read(p []byte) (int, error) {
|
||||
n, err := r.teeReader.Read(p)
|
||||
r.bytesRead += int64(n)
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close closes the decompressor
|
||||
func (r *Reader) Close() error {
|
||||
r.decompressor.Close()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+41
-15
@@ -2,6 +2,7 @@ package blobgen
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
@@ -11,6 +12,21 @@ import (
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// Zstd compression level bounds accepted by NewWriter.
|
||||
const (
|
||||
minCompressionLevel = 1
|
||||
maxCompressionLevel = 19
|
||||
)
|
||||
|
||||
// reservedCompressionCPUs is how many CPUs are left free of zstd
|
||||
// compression work for I/O and hashing.
|
||||
const reservedCompressionCPUs = 2
|
||||
|
||||
// ErrInvalidCompressionLevel is returned when the zstd compression level
|
||||
// is outside the accepted 1-19 range.
|
||||
var ErrInvalidCompressionLevel = errors.New(
|
||||
"invalid compression level: must be between 1 and 19")
|
||||
|
||||
// Writer wraps compression and encryption with SHA256 hashing.
|
||||
// Data flows: input -> tee(hasher, compressor -> encryptor -> destination)
|
||||
// The hash is computed on the uncompressed input for deterministic content-addressing.
|
||||
@@ -23,11 +39,15 @@ type Writer struct {
|
||||
bytesWritten int64
|
||||
}
|
||||
|
||||
// NewWriter creates a new Writer that compresses, encrypts, and hashes data.
|
||||
// The hash is computed on the uncompressed input for deterministic content-addressing.
|
||||
func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, error) {
|
||||
// NewWriter creates a new Writer that compresses, encrypts, and hashes
|
||||
// data. The hash is computed on the uncompressed input for deterministic
|
||||
// content-addressing.
|
||||
func NewWriter(
|
||||
w io.Writer, compressionLevel int, recipients []string,
|
||||
) (*Writer, error) {
|
||||
// Validate compression level
|
||||
if err := validateCompressionLevel(compressionLevel); err != nil {
|
||||
err := validateCompressionLevel(compressionLevel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -36,11 +56,13 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
||||
|
||||
// Parse recipients
|
||||
var ageRecipients []age.Recipient
|
||||
|
||||
for _, recipient := range recipients {
|
||||
r, err := age.ParseX25519Recipient(recipient)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing recipient %s: %w", recipient, err)
|
||||
}
|
||||
|
||||
ageRecipients = append(ageRecipients, r)
|
||||
}
|
||||
|
||||
@@ -51,10 +73,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
||||
}
|
||||
|
||||
// Calculate compression concurrency: CPUs - 2, minimum 1
|
||||
concurrency := runtime.NumCPU() - 2
|
||||
if concurrency < 1 {
|
||||
concurrency = 1
|
||||
}
|
||||
concurrency := max(runtime.NumCPU()-reservedCompressionCPUs, 1)
|
||||
|
||||
// Create compression writer with encryption as destination
|
||||
compressor, err := zstd.NewWriter(encWriter,
|
||||
@@ -63,6 +82,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
||||
)
|
||||
if err != nil {
|
||||
_ = encWriter.Close()
|
||||
|
||||
return nil, fmt.Errorf("creating compression writer: %w", err)
|
||||
}
|
||||
|
||||
@@ -79,21 +99,24 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
||||
}
|
||||
|
||||
// Write implements io.Writer
|
||||
func (w *Writer) Write(p []byte) (n int, err error) {
|
||||
n, err = w.teeWriter.Write(p)
|
||||
func (w *Writer) Write(p []byte) (int, error) {
|
||||
n, err := w.teeWriter.Write(p)
|
||||
w.bytesWritten += int64(n)
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close closes all layers and returns any errors
|
||||
func (w *Writer) Close() error {
|
||||
// Close compressor first
|
||||
if err := w.compressor.Close(); err != nil {
|
||||
err := w.compressor.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("closing compressor: %w", err)
|
||||
}
|
||||
|
||||
// Then close encryptor
|
||||
if err := w.encryptor.Close(); err != nil {
|
||||
err = w.encryptor.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("closing encryptor: %w", err)
|
||||
}
|
||||
|
||||
@@ -109,6 +132,7 @@ func (w *Writer) Sum256() []byte {
|
||||
firstHash := w.hasher.Sum(nil)
|
||||
// Second hash: SHA256(firstHash) - this is the blob ID
|
||||
secondHash := sha256.Sum256(firstHash)
|
||||
|
||||
return secondHash[:]
|
||||
}
|
||||
|
||||
@@ -119,9 +143,11 @@ func (w *Writer) BytesWritten() int64 {
|
||||
|
||||
func validateCompressionLevel(level int) error {
|
||||
// Zstd compression levels: 1-19 (default is 3)
|
||||
// SpeedFastest = 1, SpeedDefault = 3, SpeedBetterCompression = 7, SpeedBestCompression = 11
|
||||
if level < 1 || level > 19 {
|
||||
return fmt.Errorf("invalid compression level %d: must be between 1 and 19", level)
|
||||
// SpeedFastest = 1, SpeedDefault = 3, SpeedBetterCompression = 7,
|
||||
// SpeedBestCompression = 11
|
||||
if level < minCompressionLevel || level > maxCompressionLevel {
|
||||
return fmt.Errorf("%w: got %d", ErrInvalidCompressionLevel, level)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package blobgen
|
||||
package blobgen_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -9,12 +9,15 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// TestWriterHashIsDoubleHash verifies that Writer.Sum256() returns
|
||||
// the double hash SHA256(SHA256(plaintext)) for security.
|
||||
// Double hashing prevents attackers from confirming existence of known content.
|
||||
func TestWriterHashIsDoubleHash(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Test data - random data that doesn't compress well
|
||||
testData := make([]byte, 1024*1024) // 1MB
|
||||
_, err := rand.Read(testData)
|
||||
@@ -27,7 +30,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) {
|
||||
var encryptedBuf bytes.Buffer
|
||||
|
||||
// Create blobgen writer
|
||||
writer, err := NewWriter(&encryptedBuf, 3, []string{testRecipient})
|
||||
writer, err := blobgen.NewWriter(&encryptedBuf, 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Write test data
|
||||
@@ -67,6 +70,8 @@ func TestWriterHashIsDoubleHash(t *testing.T) {
|
||||
// TestWriterDeterministicHash verifies that the same input always produces
|
||||
// the same hash, even with non-deterministic encryption.
|
||||
func TestWriterDeterministicHash(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Test data
|
||||
testData := []byte("Hello, World! This is test data for deterministic hashing.")
|
||||
|
||||
@@ -76,13 +81,13 @@ func TestWriterDeterministicHash(t *testing.T) {
|
||||
// Create two writers and verify they produce the same hash
|
||||
var buf1, buf2 bytes.Buffer
|
||||
|
||||
writer1, err := NewWriter(&buf1, 3, []string{testRecipient})
|
||||
writer1, err := blobgen.NewWriter(&buf1, 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
_, err = writer1.Write(testData)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer1.Close())
|
||||
|
||||
writer2, err := NewWriter(&buf2, 3, []string{testRecipient})
|
||||
writer2, err := blobgen.NewWriter(&buf2, 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
_, err = writer2.Write(testData)
|
||||
require.NoError(t, err)
|
||||
|
||||
+50
-27
@@ -1,16 +1,21 @@
|
||||
// Package chunker splits input data into content-defined chunks using the
|
||||
// FastCDC algorithm so that identical data sequences produce identical
|
||||
// chunks regardless of their position in the file.
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Chunk represents a single chunk of data produced by the content-defined chunking algorithm.
|
||||
// Each chunk is identified by its SHA256 hash and contains the raw data along with
|
||||
// its position and size information from the original file.
|
||||
// Chunk represents a single chunk of data produced by the content-defined
|
||||
// chunking algorithm. Each chunk is identified by its SHA256 hash and
|
||||
// contains the raw data along with its position and size information from
|
||||
// the original file.
|
||||
type Chunk struct {
|
||||
Hash string // Content hash of the chunk
|
||||
Data []byte // Chunk data
|
||||
@@ -28,6 +33,10 @@ type Chunker struct {
|
||||
maxChunkSize int
|
||||
}
|
||||
|
||||
// chunkSizeSpread is the FastCDC-recommended factor between the average
|
||||
// chunk size and the minimum (avg/spread) and maximum (avg*spread) sizes.
|
||||
const chunkSizeSpread = 4
|
||||
|
||||
// NewChunker creates a new chunker with the specified average chunk size.
|
||||
// The actual chunk sizes will vary between avgChunkSize/4 and avgChunkSize*4
|
||||
// as recommended by the FastCDC algorithm. Typical values for avgChunkSize
|
||||
@@ -36,27 +45,31 @@ func NewChunker(avgChunkSize int64) *Chunker {
|
||||
// FastCDC recommends min = avg/4 and max = avg*4
|
||||
return &Chunker{
|
||||
avgChunkSize: int(avgChunkSize),
|
||||
minChunkSize: int(avgChunkSize / 4),
|
||||
maxChunkSize: int(avgChunkSize * 4),
|
||||
minChunkSize: int(avgChunkSize / chunkSizeSpread),
|
||||
maxChunkSize: int(avgChunkSize * chunkSizeSpread),
|
||||
}
|
||||
}
|
||||
|
||||
// ChunkReader splits the reader into content-defined chunks and returns all chunks at once.
|
||||
// This method loads all chunk data into memory, so it should only be used for
|
||||
// reasonably sized inputs. For large files or streams, use ChunkReaderStreaming instead.
|
||||
// ChunkReader splits the reader into content-defined chunks and returns all
|
||||
// chunks at once. This method loads all chunk data into memory, so it should
|
||||
// only be used for reasonably sized inputs. For large files or streams, use
|
||||
// ChunkReaderStreaming instead.
|
||||
// Returns an error if chunking fails or if reading from the input fails.
|
||||
func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) {
|
||||
chunker := AcquireReusableChunker(r, c.minChunkSize, c.avgChunkSize, c.maxChunkSize)
|
||||
chunker := AcquireReusableChunker(
|
||||
r, c.minChunkSize, c.avgChunkSize, c.maxChunkSize)
|
||||
defer chunker.Release()
|
||||
|
||||
var chunks []Chunk
|
||||
|
||||
offset := int64(0)
|
||||
|
||||
for {
|
||||
chunk, err := chunker.Next()
|
||||
if err == io.EOF {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading chunk: %w", err)
|
||||
}
|
||||
@@ -83,30 +96,36 @@ func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) {
|
||||
|
||||
// ChunkCallback is a function called for each chunk as it's processed.
|
||||
// The callback receives a Chunk containing the hash, data, offset, and size.
|
||||
// If the callback returns an error, chunk processing stops and the error is propagated.
|
||||
// If the callback returns an error, chunk processing stops and the error is
|
||||
// propagated.
|
||||
type ChunkCallback func(chunk Chunk) error
|
||||
|
||||
// ChunkReaderStreaming splits the reader into chunks and calls the callback for each chunk.
|
||||
// This is the preferred method for processing large files or streams as it doesn't
|
||||
// accumulate all chunks in memory. The callback is invoked for each chunk as it's
|
||||
// produced, allowing for streaming processing and immediate storage or transmission.
|
||||
// Returns the SHA256 hash of the entire file content and an error if chunking fails,
|
||||
// reading fails, or if the callback returns an error.
|
||||
func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (string, error) {
|
||||
// ChunkReaderStreaming splits the reader into chunks and calls the callback
|
||||
// for each chunk. This is the preferred method for processing large files or
|
||||
// streams as it doesn't accumulate all chunks in memory. The callback is
|
||||
// invoked for each chunk as it's produced, allowing for streaming processing
|
||||
// and immediate storage or transmission.
|
||||
// Returns the SHA256 hash of the entire file content and an error if
|
||||
// chunking fails, reading fails, or if the callback returns an error.
|
||||
func (c *Chunker) ChunkReaderStreaming(
|
||||
r io.Reader, callback ChunkCallback,
|
||||
) (string, error) {
|
||||
// Create a tee reader to calculate full file hash while chunking
|
||||
fileHasher := sha256.New()
|
||||
teeReader := io.TeeReader(r, fileHasher)
|
||||
|
||||
chunker := AcquireReusableChunker(teeReader, c.minChunkSize, c.avgChunkSize, c.maxChunkSize)
|
||||
chunker := AcquireReusableChunker(
|
||||
teeReader, c.minChunkSize, c.avgChunkSize, c.maxChunkSize)
|
||||
defer chunker.Release()
|
||||
|
||||
offset := int64(0)
|
||||
|
||||
for {
|
||||
chunk, err := chunker.Next()
|
||||
if err == io.EOF {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("reading chunk: %w", err)
|
||||
}
|
||||
@@ -114,15 +133,17 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str
|
||||
// Calculate chunk hash
|
||||
hash := sha256.Sum256(chunk.Data)
|
||||
|
||||
// Pass the data directly - caller must process it before we call Next() again
|
||||
// (chunker reuses its internal buffer, but since we process synchronously
|
||||
// and completely before continuing, no copy is needed)
|
||||
if err := callback(Chunk{
|
||||
// Pass the data directly - caller must process it before we call
|
||||
// Next() again (chunker reuses its internal buffer, but since we
|
||||
// process synchronously and completely before continuing, no copy
|
||||
// is needed)
|
||||
err = callback(Chunk{
|
||||
Hash: hex.EncodeToString(hash[:]),
|
||||
Data: chunk.Data,
|
||||
Offset: offset,
|
||||
Size: int64(len(chunk.Data)),
|
||||
}); err != nil {
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("callback error: %w", err)
|
||||
}
|
||||
|
||||
@@ -138,12 +159,14 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str
|
||||
// For large files, consider using ChunkReaderStreaming with a file handle instead.
|
||||
// Returns an error if the file cannot be opened or if chunking fails.
|
||||
func (c *Chunker) ChunkFile(path string) ([]Chunk, error) {
|
||||
file, err := os.Open(path)
|
||||
file, err := os.Open(path) //nolint:gosec // G304: path is caller-supplied by design
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening file: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := file.Close(); err != nil && err.Error() != "invalid argument" {
|
||||
err := file.Close()
|
||||
if err != nil && err.Error() != "invalid argument" {
|
||||
// Log error or handle as needed
|
||||
_ = err
|
||||
}
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
package chunker
|
||||
package chunker_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/chunker"
|
||||
)
|
||||
|
||||
func TestChunkerExpectedChunkCount(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
fileSize int
|
||||
@@ -38,16 +42,19 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
chunker := NewChunker(tt.avgChunkSize)
|
||||
t.Parallel()
|
||||
|
||||
c := chunker.NewChunker(tt.avgChunkSize)
|
||||
|
||||
// Create data with some variation to trigger chunk boundaries
|
||||
data := make([]byte, tt.fileSize)
|
||||
for i := 0; i < len(data); i++ {
|
||||
for i := range data {
|
||||
// Use a pattern that should create boundaries
|
||||
//nolint:gosec // G115: intentional byte truncation
|
||||
data[i] = byte((i * 17) ^ (i >> 5))
|
||||
}
|
||||
|
||||
chunks, err := chunker.ChunkReader(bytes.NewReader(data))
|
||||
chunks, err := c.ChunkReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("chunking failed: %v", err)
|
||||
}
|
||||
@@ -59,6 +66,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
|
||||
t.Errorf("too few chunks: got %d, expected at least %d",
|
||||
len(chunks), tt.minExpected)
|
||||
}
|
||||
|
||||
if len(chunks) > tt.maxExpected {
|
||||
t.Errorf("too many chunks: got %d, expected at most %d",
|
||||
len(chunks), tt.maxExpected)
|
||||
@@ -69,6 +77,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
|
||||
for _, chunk := range chunks {
|
||||
reconstructed = append(reconstructed, chunk.Data...)
|
||||
}
|
||||
|
||||
if !bytes.Equal(data, reconstructed) {
|
||||
t.Error("reconstructed data doesn't match original")
|
||||
}
|
||||
|
||||
@@ -1,17 +1,20 @@
|
||||
package chunker
|
||||
package chunker_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/chunker"
|
||||
)
|
||||
|
||||
func TestChunker(t *testing.T) {
|
||||
t.Run("small file produces single chunk", func(t *testing.T) {
|
||||
chunker := NewChunker(1024 * 1024) // 1MB average
|
||||
func TestChunkerSmallFileSingleChunk(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c := chunker.NewChunker(1024 * 1024) // 1MB average
|
||||
data := bytes.Repeat([]byte("hello"), 100) // 500 bytes
|
||||
|
||||
chunks, err := chunker.ChunkReader(bytes.NewReader(data))
|
||||
chunks, err := c.ChunkReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("chunking failed: %v", err)
|
||||
}
|
||||
@@ -23,29 +26,34 @@ func TestChunker(t *testing.T) {
|
||||
if chunks[0].Size != int64(len(data)) {
|
||||
t.Errorf("expected chunk size %d, got %d", len(data), chunks[0].Size)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("large file produces multiple chunks", func(t *testing.T) {
|
||||
chunker := NewChunker(256 * 1024) // 256KB average chunk size
|
||||
func TestChunkerLargeFileMultipleChunks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c := chunker.NewChunker(256 * 1024) // 256KB average chunk size
|
||||
|
||||
// Generate 2MB of random data
|
||||
data := make([]byte, 2*1024*1024)
|
||||
if _, err := rand.Read(data); err != nil {
|
||||
|
||||
_, err := rand.Read(data)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate random data: %v", err)
|
||||
}
|
||||
|
||||
chunks, err := chunker.ChunkReader(bytes.NewReader(data))
|
||||
chunks, err := c.ChunkReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("chunking failed: %v", err)
|
||||
}
|
||||
|
||||
// Should produce multiple chunks - with FastCDC we expect around 8 chunks for 2MB with 256KB average
|
||||
// Should produce multiple chunks - with FastCDC we expect around 8
|
||||
// chunks for 2MB with 256KB average
|
||||
if len(chunks) < 4 || len(chunks) > 16 {
|
||||
t.Errorf("expected 4-16 chunks, got %d", len(chunks))
|
||||
}
|
||||
|
||||
// Verify chunks reconstruct original data
|
||||
var reconstructed []byte
|
||||
reconstructed := make([]byte, 0, len(data))
|
||||
for _, chunk := range chunks {
|
||||
reconstructed = append(reconstructed, chunk.Data...)
|
||||
}
|
||||
@@ -56,17 +64,22 @@ func TestChunker(t *testing.T) {
|
||||
|
||||
// Verify offsets
|
||||
var expectedOffset int64
|
||||
|
||||
for i, chunk := range chunks {
|
||||
if chunk.Offset != expectedOffset {
|
||||
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset)
|
||||
t.Errorf("chunk %d: expected offset %d, got %d",
|
||||
i, expectedOffset, chunk.Offset)
|
||||
}
|
||||
|
||||
expectedOffset += chunk.Size
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("deterministic chunking", func(t *testing.T) {
|
||||
chunker1 := NewChunker(256 * 1024)
|
||||
chunker2 := NewChunker(256 * 1024)
|
||||
func TestChunkerDeterministic(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
chunker1 := chunker.NewChunker(256 * 1024)
|
||||
chunker2 := chunker.NewChunker(256 * 1024)
|
||||
|
||||
// Use deterministic data
|
||||
data := bytes.Repeat([]byte("abcdefghijklmnopqrstuvwxyz"), 20000) // ~520KB
|
||||
@@ -83,22 +96,25 @@ func TestChunker(t *testing.T) {
|
||||
|
||||
// Should produce same chunks
|
||||
if len(chunks1) != len(chunks2) {
|
||||
t.Fatalf("different number of chunks: %d vs %d", len(chunks1), len(chunks2))
|
||||
t.Fatalf("different number of chunks: %d vs %d",
|
||||
len(chunks1), len(chunks2))
|
||||
}
|
||||
|
||||
for i := range chunks1 {
|
||||
if chunks1[i].Hash != chunks2[i].Hash {
|
||||
t.Errorf("chunk %d: different hashes", i)
|
||||
}
|
||||
|
||||
if chunks1[i].Size != chunks2[i].Size {
|
||||
t.Errorf("chunk %d: different sizes", i)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestChunkBoundaries(t *testing.T) {
|
||||
chunker := NewChunker(256 * 1024) // 256KB average
|
||||
t.Parallel()
|
||||
|
||||
c := chunker.NewChunker(256 * 1024) // 256KB average
|
||||
|
||||
// FastCDC uses avg/4 for min and avg*4 for max
|
||||
avgSize := int64(256 * 1024)
|
||||
@@ -107,11 +123,13 @@ func TestChunkBoundaries(t *testing.T) {
|
||||
|
||||
// Test that minimum chunk size is respected
|
||||
data := make([]byte, minSize+1024)
|
||||
if _, err := rand.Read(data); err != nil {
|
||||
|
||||
_, err := rand.Read(data)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate random data: %v", err)
|
||||
}
|
||||
|
||||
chunks, err := chunker.ChunkReader(bytes.NewReader(data))
|
||||
chunks, err := c.ChunkReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("chunking failed: %v", err)
|
||||
}
|
||||
@@ -119,10 +137,13 @@ func TestChunkBoundaries(t *testing.T) {
|
||||
for i, chunk := range chunks {
|
||||
// Last chunk can be smaller than minimum
|
||||
if i < len(chunks)-1 && chunk.Size < minSize {
|
||||
t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize)
|
||||
t.Errorf("chunk %d size %d is below minimum %d",
|
||||
i, chunk.Size, minSize)
|
||||
}
|
||||
|
||||
if chunk.Size > maxSize {
|
||||
t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize)
|
||||
t.Errorf("chunk %d size %d exceeds maximum %d",
|
||||
i, chunk.Size, maxSize)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+77
-46
@@ -1,6 +1,7 @@
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"sync"
|
||||
@@ -27,32 +28,52 @@ type ReusableChunker struct {
|
||||
}
|
||||
|
||||
// reusableChunkerPool pools ReusableChunker instances to avoid allocations.
|
||||
//
|
||||
//nolint:gochecknoglobals // process-wide object pool by design
|
||||
var reusableChunkerPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
New: func() any {
|
||||
return &ReusableChunker{}
|
||||
},
|
||||
}
|
||||
|
||||
// bufferPools contains pools for different buffer sizes.
|
||||
// Key is the buffer size.
|
||||
//
|
||||
//nolint:gochecknoglobals // process-wide buffer pools by design
|
||||
var bufferPools = sync.Map{}
|
||||
|
||||
func getBuffer(size int) []byte {
|
||||
poolI, _ := bufferPools.LoadOrStore(size, &sync.Pool{
|
||||
New: func() interface{} {
|
||||
New: func() any {
|
||||
buf := make([]byte, size)
|
||||
|
||||
return &buf
|
||||
},
|
||||
})
|
||||
pool := poolI.(*sync.Pool)
|
||||
return *pool.Get().(*[]byte)
|
||||
|
||||
pool, ok := poolI.(*sync.Pool)
|
||||
if !ok {
|
||||
panic("bufferPools holds a non-pool value")
|
||||
}
|
||||
|
||||
buf, ok := pool.Get().(*[]byte)
|
||||
if !ok {
|
||||
panic("buffer pool holds a non-buffer value")
|
||||
}
|
||||
|
||||
return *buf
|
||||
}
|
||||
|
||||
func putBuffer(buf []byte) {
|
||||
size := cap(buf)
|
||||
|
||||
poolI, ok := bufferPools.Load(size)
|
||||
if ok {
|
||||
pool := poolI.(*sync.Pool)
|
||||
pool, isPool := poolI.(*sync.Pool)
|
||||
if !isPool {
|
||||
panic("bufferPools holds a non-pool value")
|
||||
}
|
||||
|
||||
b := buf[:size]
|
||||
pool.Put(&b)
|
||||
}
|
||||
@@ -66,17 +87,28 @@ type FastCDCChunk struct {
|
||||
Fingerprint uint64
|
||||
}
|
||||
|
||||
// AcquireReusableChunker gets a chunker from the pool and initializes it for the given reader.
|
||||
func AcquireReusableChunker(rd io.Reader, minSize, avgSize, maxSize int) *ReusableChunker {
|
||||
c := reusableChunkerPool.Get().(*ReusableChunker)
|
||||
// bufSizeFactor sizes the internal read buffer relative to the maximum
|
||||
// chunk size so a full chunk plus read-ahead always fits.
|
||||
const bufSizeFactor = 2
|
||||
|
||||
bufSize := maxSize * 2
|
||||
// AcquireReusableChunker gets a chunker from the pool and initializes it
|
||||
// for the given reader.
|
||||
func AcquireReusableChunker(
|
||||
rd io.Reader, minSize, avgSize, maxSize int,
|
||||
) *ReusableChunker {
|
||||
c, ok := reusableChunkerPool.Get().(*ReusableChunker)
|
||||
if !ok {
|
||||
panic("reusableChunkerPool holds a non-chunker value")
|
||||
}
|
||||
|
||||
bufSize := maxSize * bufSizeFactor
|
||||
|
||||
// Reuse buffer if it's the right size, otherwise get a new one
|
||||
if c.buf == nil || cap(c.buf) != bufSize {
|
||||
if c.buf != nil {
|
||||
putBuffer(c.buf)
|
||||
}
|
||||
|
||||
c.buf = getBuffer(bufSize)
|
||||
} else {
|
||||
// Restore buffer to full capacity (may have been truncated by previous EOF)
|
||||
@@ -108,41 +140,14 @@ func (c *ReusableChunker) Release() {
|
||||
reusableChunkerPool.Put(c)
|
||||
}
|
||||
|
||||
func (c *ReusableChunker) fillBuffer() error {
|
||||
n := len(c.buf) - c.cursor
|
||||
if n >= c.maxSize {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Move all data after the cursor to the start of the buffer
|
||||
copy(c.buf[:n], c.buf[c.cursor:])
|
||||
c.cursor = 0
|
||||
|
||||
if c.eof {
|
||||
c.buf = c.buf[:n]
|
||||
return nil
|
||||
}
|
||||
|
||||
// Restore buffer to full capacity for reading
|
||||
c.buf = c.buf[:c.bufSize]
|
||||
|
||||
// Fill the rest of the buffer
|
||||
m, err := io.ReadFull(c.rd, c.buf[n:])
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
c.buf = c.buf[:n+m]
|
||||
c.eof = true
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Next returns the next chunk or io.EOF when done.
|
||||
// The returned Data slice is only valid until the next call to Next.
|
||||
func (c *ReusableChunker) Next() (FastCDCChunk, error) {
|
||||
if err := c.fillBuffer(); err != nil {
|
||||
err := c.fillBuffer()
|
||||
if err != nil {
|
||||
return FastCDCChunk{}, err
|
||||
}
|
||||
|
||||
if len(c.buf) == 0 {
|
||||
return FastCDCChunk{}, io.EOF
|
||||
}
|
||||
@@ -162,6 +167,37 @@ func (c *ReusableChunker) Next() (FastCDCChunk, error) {
|
||||
return chunk, nil
|
||||
}
|
||||
|
||||
func (c *ReusableChunker) fillBuffer() error {
|
||||
n := len(c.buf) - c.cursor
|
||||
if n >= c.maxSize {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Move all data after the cursor to the start of the buffer
|
||||
copy(c.buf[:n], c.buf[c.cursor:])
|
||||
c.cursor = 0
|
||||
|
||||
if c.eof {
|
||||
c.buf = c.buf[:n]
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Restore buffer to full capacity for reading
|
||||
c.buf = c.buf[:c.bufSize]
|
||||
|
||||
// Fill the rest of the buffer
|
||||
m, err := io.ReadFull(c.rd, c.buf[n:])
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
c.buf = c.buf[:n+m]
|
||||
c.eof = true
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) {
|
||||
fp := uint64(0)
|
||||
i := c.minSize
|
||||
@@ -189,14 +225,9 @@ func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) {
|
||||
return i, fp
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// 256 random uint64s for the rolling hash function (from FastCDC paper)
|
||||
//
|
||||
//nolint:gochecknoglobals // immutable FastCDC gear lookup table
|
||||
var table = [256]uint64{
|
||||
0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c,
|
||||
0xeac874b86f0f57b9, 0xa5ccedec95ec79c7, 0xe15a3320ad42ac0a, 0x5ed3583fa63cec15,
|
||||
|
||||
+174
-23
@@ -1,3 +1,6 @@
|
||||
// Package cli implements the vaultik command-line interface: cobra
|
||||
// commands, fx application wiring, and process-level concerns such as
|
||||
// signal handling and the PID lock.
|
||||
package cli
|
||||
|
||||
import (
|
||||
@@ -7,48 +10,84 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/pidlock"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/adrg/xdg"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/globals"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/pidlock"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// shutdownTimeout bounds how long a signal-triggered graceful shutdown
|
||||
// may take before we give up.
|
||||
const shutdownTimeout = 30 * time.Second
|
||||
|
||||
// AppOptions contains common options for creating the fx application.
|
||||
// It includes the configuration file path, logging options, and additional
|
||||
// fx modules and invocations that should be included in the application.
|
||||
type AppOptions struct {
|
||||
ConfigPath string
|
||||
LogOptions log.LogOptions
|
||||
LogOptions log.Options
|
||||
Modules []fx.Option
|
||||
Invokes []fx.Option
|
||||
}
|
||||
|
||||
// setupGlobals sets up the globals with application startup time
|
||||
func setupGlobals(lc fx.Lifecycle, g *globals.Globals) {
|
||||
// setupGlobals records the startup time and, when an output-suppression
|
||||
// flag is active, marks the UI writer quiet so that Begin/Complete/
|
||||
// Info/Notice/Detail/Progress are silenced. Warning and Error are NOT
|
||||
// silenced — per the documented convention that --quiet suppresses
|
||||
// non-error output only. The startup banner is printed by Entry
|
||||
// before cobra parses arguments, gated by the same arg-level check.
|
||||
//
|
||||
// --json quiets the UI here too, because stdout then carries a JSON
|
||||
// document and human narration would corrupt it. Unlike Quiet it does
|
||||
// not lower the stderr log level (issue #112), so --verbose/--debug
|
||||
// still surface diagnostics alongside the document.
|
||||
func setupGlobals(
|
||||
lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts log.Options,
|
||||
) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
OnStart: func(_ context.Context) error {
|
||||
g.StartTime = time.Now().UTC()
|
||||
|
||||
if opts.Cron || opts.Quiet || opts.JSON {
|
||||
v.UI.SetQuiet(true)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// writeStartupBanner prints the two-line application banner followed by a
|
||||
// blank line. Used both from the fx hook (for subcommand invocations) and
|
||||
// from the root cobra Run handler (for `vaultik` with no subcommand).
|
||||
func writeStartupBanner(w *ui.Writer, startTime time.Time, shortCommit string) {
|
||||
w.Bannerf("%s %s by %s (commit %s, built on %s) starting up at %s.",
|
||||
globals.Appname, globals.Version, globals.Author,
|
||||
shortCommit, globals.CommitDate,
|
||||
startTime.Format(time.RFC3339))
|
||||
w.Bannerf("%s", globals.Homepage)
|
||||
w.Bannerf("")
|
||||
}
|
||||
|
||||
// NewApp creates a new fx application with common modules.
|
||||
// It sets up the base modules (config, database, logging, globals) and
|
||||
// combines them with any additional modules specified in the options.
|
||||
// The returned fx.App is ready to be started with RunApp.
|
||||
func NewApp(opts AppOptions) *fx.App {
|
||||
baseModules := []fx.Option{
|
||||
fx.Supply(config.ConfigPath(opts.ConfigPath)),
|
||||
fx.Supply(config.Path(opts.ConfigPath)),
|
||||
fx.Supply(opts.LogOptions),
|
||||
fx.Provide(globals.New),
|
||||
fx.Provide(log.New),
|
||||
@@ -62,12 +101,46 @@ func NewApp(opts AppOptions) *fx.App {
|
||||
fx.NopLogger,
|
||||
}
|
||||
|
||||
allOptions := append(baseModules, opts.Modules...)
|
||||
capacity := len(baseModules) + len(opts.Modules) + len(opts.Invokes)
|
||||
allOptions := make([]fx.Option, 0, capacity)
|
||||
allOptions = append(allOptions, baseModules...)
|
||||
allOptions = append(allOptions, opts.Modules...)
|
||||
allOptions = append(allOptions, opts.Invokes...)
|
||||
|
||||
return fx.New(allOptions...)
|
||||
}
|
||||
|
||||
// startupError carries a startup failure message that has been cleaned
|
||||
// of fx dependency-injection noise. A distinct type (rather than
|
||||
// errors.New) keeps the dynamic message out of err113's sight while
|
||||
// preserving the exact user-facing text.
|
||||
type startupError struct {
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *startupError) Error() string {
|
||||
return e.msg
|
||||
}
|
||||
|
||||
// cleanStartupError strips fx's dependency-injection call-chain noise from
|
||||
// startup errors. fx wraps the underlying error with messages like
|
||||
//
|
||||
// could not build arguments for function "X" (file:line): failed to build T:
|
||||
// could not build arguments for function "Y" (file:line): failed to build U:
|
||||
// received non-nil error from function "Z" (file:line): <real error>
|
||||
//
|
||||
// Users care about the real error, not the DI plumbing. We strip everything
|
||||
// up through the last "): " (which is always the close-paren of an fx
|
||||
// function-location annotation followed by the wrapped error).
|
||||
func cleanStartupError(err error) error {
|
||||
msg := err.Error()
|
||||
if idx := strings.LastIndex(msg, "): "); idx >= 0 {
|
||||
msg = msg[idx+3:]
|
||||
}
|
||||
|
||||
return &startupError{msg: msg}
|
||||
}
|
||||
|
||||
// RunApp starts and stops the fx application within the given context.
|
||||
// It handles graceful shutdown on interrupt signals (SIGINT, SIGTERM) and
|
||||
// ensures the application stops cleanly. The function blocks until the
|
||||
@@ -82,36 +155,45 @@ func RunApp(ctx context.Context, app *fx.App) error {
|
||||
defer cancel()
|
||||
|
||||
// Start the app
|
||||
if err := app.Start(ctx); err != nil {
|
||||
return fmt.Errorf("failed to start app: %w", err)
|
||||
err := app.Start(ctx)
|
||||
if err != nil {
|
||||
return cleanStartupError(err)
|
||||
}
|
||||
|
||||
// Handle shutdown
|
||||
shutdownComplete := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
defer close(shutdownComplete)
|
||||
|
||||
<-sigChan
|
||||
log.Notice("Received interrupt signal, shutting down gracefully...")
|
||||
|
||||
// Create a timeout context for shutdown
|
||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
// Create a timeout context for shutdown. The parent ctx is being
|
||||
// cancelled, so detach from its cancellation but keep its values.
|
||||
shutdownCtx, shutdownCancel := context.WithTimeout(
|
||||
context.WithoutCancel(ctx), shutdownTimeout)
|
||||
defer shutdownCancel()
|
||||
|
||||
if err := app.Stop(shutdownCtx); err != nil {
|
||||
err := app.Stop(shutdownCtx)
|
||||
if err != nil {
|
||||
log.Error("Error during shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Wait for either the signal handler to complete shutdown or the app to request shutdown
|
||||
// Wait for the signal handler to complete shutdown or the app to
|
||||
// request shutdown.
|
||||
select {
|
||||
case <-shutdownComplete:
|
||||
// Shutdown completed via signal
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
// Context cancelled (shouldn't happen in normal operation)
|
||||
if err := app.Stop(context.Background()); err != nil {
|
||||
err := app.Stop(context.WithoutCancel(ctx))
|
||||
if err != nil {
|
||||
log.Error("Error stopping app", "error", err)
|
||||
}
|
||||
|
||||
return ctx.Err()
|
||||
case <-app.Done():
|
||||
// App finished running (e.g., backup completed)
|
||||
@@ -119,26 +201,95 @@ func RunApp(ctx context.Context, app *fx.App) error {
|
||||
}
|
||||
}
|
||||
|
||||
// runVaultikApp runs the standard single-operation command lifecycle
|
||||
// shared by the list/purge/verify/remove/remote-info subcommands:
|
||||
// resolve the config, start the fx app, run op against the Vaultik
|
||||
// instance in a goroutine, report a failure prefixed with failMsg
|
||||
// (suppressed while suppressErrors is true, e.g. under --json), then
|
||||
// trigger shutdown. The operation is cancelled when the app stops.
|
||||
// jsonOutput marks a command whose stdout is a JSON document: it quiets
|
||||
// the UI but, unlike Quiet, leaves the stderr log level alone.
|
||||
func runVaultikApp(
|
||||
cmd *cobra.Command, jsonOutput, suppressErrors bool,
|
||||
failMsg string, op func(v *vaultik.Vaultik) error,
|
||||
) error {
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
JSON: jsonOutput,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(_ context.Context) error {
|
||||
go func() {
|
||||
err := op(v)
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
if !suppressErrors {
|
||||
log.Error(failMsg, "error", err)
|
||||
ReportErrorf("%s: %v", failMsg, err)
|
||||
}
|
||||
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
err = v.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(_ context.Context) error {
|
||||
v.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// RunWithApp is a helper that creates and runs an fx app with the given options.
|
||||
// It combines NewApp and RunApp into a single convenient function. This is the
|
||||
// preferred way to run CLI commands that need the full application context.
|
||||
// It acquires a PID lock before starting to prevent concurrent instances.
|
||||
func RunWithApp(ctx context.Context, opts AppOptions) error {
|
||||
// Acquire PID lock to prevent concurrent instances
|
||||
lockDir := filepath.Join(xdg.DataHome, "berlin.sneak.app.vaultik")
|
||||
lockDir := filepath.Join(xdg.DataHome, "vaultik")
|
||||
|
||||
lock, err := pidlock.Acquire(lockDir)
|
||||
if err != nil {
|
||||
if errors.Is(err, pidlock.ErrAlreadyRunning) {
|
||||
return fmt.Errorf("cannot start: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Errorf("failed to acquire lock: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := lock.Release(); err != nil {
|
||||
err := lock.Release()
|
||||
if err != nil {
|
||||
log.Warn("Failed to release PID lock", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
app := NewApp(opts)
|
||||
|
||||
return RunApp(ctx, app)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package cli //nolint:testpackage // needs access to unexported cleanStartupError
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCleanStartupError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "real fx error chain",
|
||||
in: `could not build arguments for function ` +
|
||||
`"sneak.berlin/go/vaultik/internal/cli".newSnapshotCreateCommand.func1.1 ` +
|
||||
`(/Users/user/dev/vaultik/internal/cli/snapshot.go:71): ` +
|
||||
`failed to build *vaultik.Vaultik: ` +
|
||||
`could not build arguments for function ` +
|
||||
`"sneak.berlin/go/vaultik/internal/vaultik".New ` +
|
||||
`(/Users/user/dev/vaultik/internal/vaultik/vaultik.go:59): ` +
|
||||
`failed to build storage.Storer: ` +
|
||||
`received non-nil error from function ` +
|
||||
`"sneak.berlin/go/vaultik/internal/storage".NewStorer ` +
|
||||
`(/Users/user/dev/vaultik/internal/storage/module.go:23): ` +
|
||||
`creating base path: mkdir /Volumes/BACKUPS: permission denied`,
|
||||
want: `creating base path: mkdir /Volumes/BACKUPS: permission denied`,
|
||||
},
|
||||
{
|
||||
name: "no fx wrapping",
|
||||
in: "plain error",
|
||||
want: "plain error",
|
||||
},
|
||||
{
|
||||
name: "single fx wrapping",
|
||||
in: `received non-nil error from function "foo" (file.go:1): underlying problem`,
|
||||
want: "underlying problem",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
//nolint:err113 // test constructs errors from table input
|
||||
got := cleanStartupError(errors.New(tt.in)).Error()
|
||||
if got != tt.want {
|
||||
t.Errorf("got %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,642 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// configFileMode is the permission set for freshly written config files;
|
||||
// configs may hold S3 credentials, so keep them owner-only.
|
||||
const configFileMode = 0o600
|
||||
|
||||
// configSetArgs is the argument count of `config set <key> <value>`.
|
||||
const configSetArgs = 2
|
||||
|
||||
// configDirMode is the permission set for created config directories;
|
||||
// parent config dirs (e.g. ~/.config) are conventionally traversable.
|
||||
const configDirMode = 0o755
|
||||
|
||||
// configYAMLIndent matches the 2-space indentation of defaultConfigTemplate,
|
||||
// so `config set` writes the file back with the same indentation rather than
|
||||
// yaml.Marshal's 4-space default.
|
||||
const configYAMLIndent = 2
|
||||
|
||||
var (
|
||||
errConfigExists = errors.New("config file already exists")
|
||||
errEmptyConfig = errors.New("empty config file")
|
||||
errKeyNotFound = errors.New("key not found")
|
||||
errNeedNumericIndex = errors.New("key is a list; use a numeric index")
|
||||
errIndexOutOfRange = errors.New("index out of range")
|
||||
errNotMapOrList = errors.New("key is not a map or list")
|
||||
)
|
||||
|
||||
const defaultConfigTemplate = `# vaultik configuration
|
||||
# Documentation: https://sneak.berlin/go/vaultik
|
||||
|
||||
# ─── REQUIRED ────────────────────────────────────────────────────────────────
|
||||
|
||||
# Age recipient public keys for encryption.
|
||||
# Backups are encrypted to ALL listed recipients. Any one of the corresponding
|
||||
# private keys can decrypt. Generate a keypair with:
|
||||
# age-keygen -o vaultik_backup_private_key.txt
|
||||
# grep 'public key' vaultik_backup_private_key.txt
|
||||
age_recipients:
|
||||
- age1REPLACE_WITH_YOUR_PUBLIC_KEY
|
||||
|
||||
# Named snapshots. Each snapshot backs up one or more paths and can have its
|
||||
# own exclude patterns in addition to the global excludes below.
|
||||
#
|
||||
# Exclude pattern semantics:
|
||||
# - Patterns starting with / are anchored to the snapshot path root
|
||||
# (e.g. "/Library/Caches" matches only ~/Library/Caches in a ~ snapshot)
|
||||
# - Patterns without a leading / match anywhere in the tree
|
||||
# (e.g. ".cache" matches any directory named .cache at any depth)
|
||||
# - Globs are supported: *, **, ?
|
||||
snapshots:
|
||||
home:
|
||||
paths:
|
||||
- "~"
|
||||
exclude:
|
||||
# Trash, temp, and filesystem metadata
|
||||
- "/.Trash"
|
||||
- "/.Trashes"
|
||||
- "/.fseventsd"
|
||||
- "/.Spotlight-V100"
|
||||
- "/.TemporaryItems"
|
||||
- "/tmp"
|
||||
- "/.rnd"
|
||||
- ".DS_Store"
|
||||
# Caches and package manager state (rebuildable)
|
||||
- ".cache"
|
||||
- ".bundle"
|
||||
- "/.cpan/build"
|
||||
- "/.cpan/sources"
|
||||
- "/.gradle/caches"
|
||||
- "/.dropbox"
|
||||
- "/.minikube/cache"
|
||||
- "/.local/share/containers/podman/machine"
|
||||
- "/.persepolis"
|
||||
- "/Library/Caches"
|
||||
- "/Library/Logs"
|
||||
- "/Library/Cookies"
|
||||
- "/Library/Metadata"
|
||||
- "/Library/Suggestions"
|
||||
- "/Library/PubSub"
|
||||
- "/Library/Homebrew"
|
||||
- "/Library/Developer"
|
||||
- "/Library/Google/GoogleSoftwareUpdate"
|
||||
- "/Library/Preferences/Macromedia/Flash Player"
|
||||
- "/Library/Preferences/SDMHelpData"
|
||||
- "/Library/VoiceTrigger/SAT"
|
||||
# Language/toolchain package caches (rebuildable from registries)
|
||||
- "/.npm"
|
||||
- "/.cargo/registry"
|
||||
- "/.cargo/git"
|
||||
- "/.rustup/toolchains"
|
||||
- "/go/pkg/mod"
|
||||
- "/.m2/repository"
|
||||
- "/.vagrant.d/boxes"
|
||||
- "node_modules"
|
||||
- "__pycache__"
|
||||
- ".venv"
|
||||
# Virtual machine disk images (huge; remove these lines to back them up)
|
||||
- "/Parallels"
|
||||
- "/Virtual Machines.localized"
|
||||
- "/VirtualBox VMs"
|
||||
- "/.orbstack"
|
||||
- "/Library/Containers/com.utmapp.UTM"
|
||||
# Downloaded LLM models (huge, re-downloadable)
|
||||
- "/.ollama/models"
|
||||
- "/.lmstudio/models"
|
||||
# Cloud-synced storage. These are synced to a provider already, and on
|
||||
# modern macOS may contain dataless placeholder files that the backup
|
||||
# would force-download in full.
|
||||
- "/Library/CloudStorage"
|
||||
- "/Library/Mobile Documents"
|
||||
# Android SDK and emulator images (re-downloadable)
|
||||
- "/Library/Android/sdk"
|
||||
- "/.android/avd"
|
||||
# Cloud-synced or restorable-from-server data
|
||||
- "/Library/Mail"
|
||||
- "/Library/Mail Downloads"
|
||||
- "/Library/Safari"
|
||||
- "/Library/Application Support/Evernote"
|
||||
- "/Library/Application Support/MobileSync"
|
||||
- "/Library/Application Support/SyncServices"
|
||||
- "/Library/Application Support/protonmail/bridge/cache"
|
||||
- "/Library/Application Support/Syncthing/index-*"
|
||||
- "/Library/Syncthing/folders"
|
||||
- "/Documents/Dropbox/.dropbox.cache"
|
||||
# Large rebuildable app data (games, media caches, device backups)
|
||||
- "/Applications/Fortnite"
|
||||
- "/Documents/Steam Content"
|
||||
- "/Library/Application Support/Ableton"
|
||||
- "/Library/Application Support/CrossOver Games"
|
||||
- "/Library/Application Support/SecondLife/cache"
|
||||
- "/Library/Application Support/Steam/SteamApps"
|
||||
- "/Library/Containers/com.docker.docker"
|
||||
- "/Library/Group Containers/group.com.apple.secure-control-center-preferences"
|
||||
- "/Library/iTunes/iPad Software Updates"
|
||||
- "/Library/iTunes/iPhone Software Updates"
|
||||
- "/Movies/CacheClip"
|
||||
- "/Movies/ProxyMedia"
|
||||
- "/Music/iTunes/Album Artwork"
|
||||
- "/Pictures/iPod Photo Cache"
|
||||
|
||||
# Third-party applications. OS-provided apps live in /System/Applications
|
||||
# on modern macOS and are never in /Applications, but Apple-installed
|
||||
# App Store apps (Safari, GarageBand, iWork, iMovie) are excluded since
|
||||
# they are re-downloadable.
|
||||
apps:
|
||||
paths:
|
||||
- /Applications
|
||||
exclude:
|
||||
- ".DS_Store"
|
||||
- "/Safari.app"
|
||||
- "/GarageBand.app"
|
||||
- "/iMovie.app"
|
||||
- "/Keynote.app"
|
||||
- "/Numbers.app"
|
||||
- "/Pages.app"
|
||||
- "/Xcode.app"
|
||||
- "/Spotify.app"
|
||||
- "/Steam.app"
|
||||
- "/VirtualBox.app"
|
||||
- "/Utilities/Adobe Installers"
|
||||
|
||||
# Storage backend (pick ONE of the three forms below).
|
||||
#
|
||||
# S3-compatible:
|
||||
# storage_url: "s3://mybucket/backups?endpoint=s3.example.com®ion=us-east-1"
|
||||
# (also set s3.access_key_id and s3.secret_access_key below)
|
||||
#
|
||||
# Local filesystem:
|
||||
# storage_url: "file:///mnt/backups/vaultik"
|
||||
#
|
||||
# Rclone (requires rclone configured separately):
|
||||
# storage_url: "rclone://myremote/path/to/backups"
|
||||
storage_url: ""
|
||||
|
||||
# ─── S3 CREDENTIALS (required for s3:// storage_url) ────────────────────────
|
||||
|
||||
# s3:
|
||||
# access_key_id: YOUR_ACCESS_KEY
|
||||
# secret_access_key: YOUR_SECRET_KEY
|
||||
# # region: us-east-1 # Default: us-east-1
|
||||
# # use_ssl: true # Default: true
|
||||
# # part_size: 5MB # Multipart upload part size. Default: 5MB
|
||||
|
||||
# ─── OPTIONAL ────────────────────────────────────────────────────────────────
|
||||
|
||||
# Global exclude patterns applied to ALL snapshots.
|
||||
# Snapshot-specific excludes are additive.
|
||||
# exclude:
|
||||
# - "*.log"
|
||||
# - "*.tmp"
|
||||
# - ".git"
|
||||
# - "node_modules"
|
||||
|
||||
# Average chunk size for content-defined chunking (FastCDC).
|
||||
# Smaller = better deduplication but more metadata overhead.
|
||||
# Accepts: 1MB, 10M, 64KB, etc.
|
||||
# Default: 10MB
|
||||
# chunk_size: 10MB
|
||||
|
||||
# Maximum blob size before splitting into a new blob.
|
||||
# Accepts: 1GB, 10G, 500MB, etc.
|
||||
# Default: 10GB
|
||||
# blob_size_limit: 10GB
|
||||
|
||||
# Zstd compression level (1-19). Higher = better ratio but slower.
|
||||
# Default: 3
|
||||
# compression_level: 3
|
||||
|
||||
# Hostname used in snapshot IDs. Default: system hostname.
|
||||
# hostname: myserver
|
||||
|
||||
# Path to the local SQLite index database.
|
||||
# Default: the platform data directory, e.g.
|
||||
# macOS: ~/Library/Application Support/vaultik/index.sqlite
|
||||
# Linux: ~/.local/share/vaultik/index.sqlite
|
||||
# index_path: /path/to/index.sqlite
|
||||
`
|
||||
|
||||
// NewConfigCommand creates the config command group.
|
||||
func NewConfigCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "config",
|
||||
Short: "Manage the configuration file",
|
||||
Long: "Commands for creating, editing, and querying the vaultik config file.",
|
||||
}
|
||||
|
||||
cmd.AddCommand(newConfigInitCommand())
|
||||
cmd.AddCommand(newConfigEditCommand())
|
||||
cmd.AddCommand(newConfigGetCommand())
|
||||
cmd.AddCommand(newConfigSetCommand())
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// newConfigInitCommand creates the 'config init' subcommand.
|
||||
func newConfigInitCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "init",
|
||||
Short: "Write a default config file",
|
||||
Long: `Creates a default configuration file with commented explanations
|
||||
for every setting. If a config file already exists at the target path,
|
||||
the command refuses to overwrite it.
|
||||
|
||||
The config is written to the path from --config, $VAULTIK_CONFIG, or
|
||||
the platform default config directory (e.g. ~/Library/Application Support/
|
||||
on macOS, ~/.config/ on Linux, /etc/vaultik/ as root).`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(_ *cobra.Command, _ []string) error {
|
||||
path := configPathForInit()
|
||||
|
||||
_, err := os.Stat(path)
|
||||
if err == nil {
|
||||
return fmt.Errorf("%w: %s", errConfigExists, path)
|
||||
}
|
||||
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
err = os.MkdirAll(dir, configDirMode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating config directory %s: %w", dir, err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing config file: %w", err)
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintf(os.Stdout, "Config written to %s\n", path)
|
||||
_, _ = fmt.Fprintln(os.Stdout,
|
||||
"Edit it to set your age_recipients, snapshots, and storage_url.")
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// newConfigEditCommand creates the 'config edit' subcommand.
|
||||
func newConfigEditCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "edit",
|
||||
Short: "Open the config file in $EDITOR",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
path, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
editor := os.Getenv("EDITOR")
|
||||
if editor == "" {
|
||||
editor = "vi"
|
||||
}
|
||||
|
||||
//nolint:gosec // G204: launching the operator's own $EDITOR is the point
|
||||
ed := exec.CommandContext(cmd.Context(), editor, path)
|
||||
ed.Stdin = os.Stdin
|
||||
ed.Stdout = os.Stdout
|
||||
ed.Stderr = os.Stderr
|
||||
|
||||
return ed.Run()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// newConfigGetCommand creates the 'config get' subcommand.
|
||||
func newConfigGetCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "get <key>",
|
||||
Short: "Print a config value by dotted path (e.g. storage_url, compression_level)",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(_ *cobra.Command, args []string) error {
|
||||
path, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
root, err := loadYAMLFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
node, err := yamlPathGet(root, strings.Split(args[0], "."))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if node.Kind == yaml.ScalarNode {
|
||||
_, _ = fmt.Fprintln(os.Stdout, node.Value)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
out, err := yaml.Marshal(node)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshaling value: %w", err)
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprint(os.Stdout, string(out))
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// newConfigSetCommand creates the 'config set' subcommand.
|
||||
func newConfigSetCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "set <key> <value>",
|
||||
Short: "Set a config value by dotted path (e.g. compression_level 5)",
|
||||
Long: `Sets a scalar config value addressed by dotted YAML path and writes
|
||||
the file back, preserving comments and formatting. Intermediate maps
|
||||
are created as needed.
|
||||
|
||||
Examples:
|
||||
vaultik config set storage_url "file:///mnt/backups"
|
||||
vaultik config set storage_url "s3://bucket/prefix?endpoint=host®ion=us-east-1"
|
||||
vaultik config set compression_level 9
|
||||
vaultik config set s3.bucket mybucket # legacy S3 fields still supported`,
|
||||
Args: cobra.ExactArgs(configSetArgs),
|
||||
RunE: func(_ *cobra.Command, args []string) error {
|
||||
path, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
root, err := loadYAMLFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = yamlPathSet(root, strings.Split(args[0], "."), args[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
out, err := marshalConfigYAML(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshaling config: %w", err)
|
||||
}
|
||||
|
||||
mode := os.FileMode(configFileMode)
|
||||
|
||||
info, statErr := os.Stat(path)
|
||||
if statErr == nil {
|
||||
mode = info.Mode().Perm()
|
||||
}
|
||||
|
||||
err = os.WriteFile(path, out, mode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing config file: %w", err)
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintf(os.Stdout, "%s = %s\n", args[0], args[1])
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// marshalConfigYAML renders a config document tree with 2-space indentation,
|
||||
// matching defaultConfigTemplate. yaml.Marshal defaults to 4 spaces, which
|
||||
// would reindent the whole file on the first `config set` despite the promise
|
||||
// to preserve formatting.
|
||||
func marshalConfigYAML(root *yaml.Node) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
|
||||
enc := yaml.NewEncoder(&buf)
|
||||
enc.SetIndent(configYAMLIndent)
|
||||
|
||||
err := enc.Encode(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = enc.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// loadYAMLFile parses a YAML file into a yaml.Node document tree,
|
||||
// which preserves comments and ordering for round-tripping.
|
||||
func loadYAMLFile(path string) (*yaml.Node, error) {
|
||||
data, err := os.ReadFile(path) //nolint:gosec // G304: config path is operator-supplied
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading config file: %w", err)
|
||||
}
|
||||
|
||||
var root yaml.Node
|
||||
|
||||
err = yaml.Unmarshal(data, &root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing config file: %w", err)
|
||||
}
|
||||
|
||||
// An empty file yields a zero node; normalize to an empty mapping document.
|
||||
if root.Kind == 0 {
|
||||
root = yaml.Node{
|
||||
Kind: yaml.DocumentNode,
|
||||
Content: []*yaml.Node{{Kind: yaml.MappingNode}},
|
||||
}
|
||||
}
|
||||
|
||||
return &root, nil
|
||||
}
|
||||
|
||||
// yamlPathGet navigates a dotted key path through mapping and sequence
|
||||
// nodes and returns the value node. Numeric path components index into
|
||||
// sequences (e.g. "age_recipients.0").
|
||||
func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
|
||||
node := root
|
||||
if node.Kind == yaml.DocumentNode {
|
||||
if len(node.Content) == 0 {
|
||||
return nil, errEmptyConfig
|
||||
}
|
||||
|
||||
node = node.Content[0]
|
||||
}
|
||||
|
||||
for i, key := range keys {
|
||||
switch node.Kind {
|
||||
case yaml.MappingNode:
|
||||
found := false
|
||||
|
||||
for j := 0; j+1 < len(node.Content); j += 2 {
|
||||
if node.Content[j].Value == key {
|
||||
node = node.Content[j+1]
|
||||
found = true
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%w: %s",
|
||||
errKeyNotFound, strings.Join(keys[:i+1], "."))
|
||||
}
|
||||
case yaml.SequenceNode:
|
||||
idx, err := strconv.Atoi(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s",
|
||||
errNeedNumericIndex, strings.Join(keys[:i], "."))
|
||||
}
|
||||
|
||||
if idx < 0 || idx >= len(node.Content) {
|
||||
return nil, fmt.Errorf("%w: index %d for %s (len %d)",
|
||||
errIndexOutOfRange, idx, strings.Join(keys[:i], "."),
|
||||
len(node.Content))
|
||||
}
|
||||
|
||||
node = node.Content[idx]
|
||||
case yaml.DocumentNode, yaml.ScalarNode, yaml.AliasNode:
|
||||
return nil, fmt.Errorf("%w: %s",
|
||||
errNotMapOrList, strings.Join(keys[:i], "."))
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %s",
|
||||
errNotMapOrList, strings.Join(keys[:i], "."))
|
||||
}
|
||||
}
|
||||
|
||||
return node, nil
|
||||
}
|
||||
|
||||
// yamlPathSet navigates a dotted key path, creating intermediate maps as
|
||||
// needed, and sets the final key to the given scalar value. Numeric path
|
||||
// components index into sequences; an index equal to the sequence length
|
||||
// appends a new element (e.g. "age_recipients.1" on a 1-element list).
|
||||
func yamlPathSet(root *yaml.Node, keys []string, value string) error {
|
||||
node := root
|
||||
if node.Kind == yaml.DocumentNode {
|
||||
if len(node.Content) == 0 {
|
||||
node.Content = []*yaml.Node{{Kind: yaml.MappingNode}}
|
||||
}
|
||||
|
||||
node = node.Content[0]
|
||||
}
|
||||
|
||||
for i, key := range keys {
|
||||
last := i == len(keys)-1
|
||||
|
||||
switch node.Kind {
|
||||
case yaml.MappingNode:
|
||||
node = yamlSetInMapping(node, key, value, last)
|
||||
case yaml.SequenceNode:
|
||||
next, err := yamlSetInSequence(node, keys, i, value, last)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
node = next
|
||||
case yaml.DocumentNode, yaml.ScalarNode, yaml.AliasNode:
|
||||
return fmt.Errorf("%w: %s",
|
||||
errNotMapOrList, strings.Join(keys[:i], "."))
|
||||
default:
|
||||
return fmt.Errorf("%w: %s",
|
||||
errNotMapOrList, strings.Join(keys[:i], "."))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// yamlSetInMapping resolves (creating if needed) the value node for key
|
||||
// within a mapping node, setting it to value when it is the final path
|
||||
// element, and returns the node to descend into.
|
||||
func yamlSetInMapping(node *yaml.Node, key, value string, last bool) *yaml.Node {
|
||||
var valueNode *yaml.Node
|
||||
|
||||
for j := 0; j+1 < len(node.Content); j += 2 {
|
||||
if node.Content[j].Value == key {
|
||||
valueNode = node.Content[j+1]
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if valueNode == nil {
|
||||
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
|
||||
|
||||
valueNode = &yaml.Node{Kind: yaml.MappingNode}
|
||||
if last {
|
||||
valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
|
||||
}
|
||||
|
||||
node.Content = append(node.Content, keyNode, valueNode)
|
||||
} else if last {
|
||||
setScalar(valueNode, value)
|
||||
}
|
||||
|
||||
return valueNode
|
||||
}
|
||||
|
||||
// yamlSetInSequence indexes (or appends to) a sequence node using the
|
||||
// numeric path element keys[i], setting the element to value when it is
|
||||
// the final path element, and returns the node to descend into.
|
||||
func yamlSetInSequence(
|
||||
node *yaml.Node, keys []string, i int, value string, last bool,
|
||||
) (*yaml.Node, error) {
|
||||
idx, err := strconv.Atoi(keys[i])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s",
|
||||
errNeedNumericIndex, strings.Join(keys[:i], "."))
|
||||
}
|
||||
|
||||
if idx < 0 || idx > len(node.Content) {
|
||||
return nil, fmt.Errorf("%w: index %d for %s (len %d)",
|
||||
errIndexOutOfRange, idx, strings.Join(keys[:i], "."),
|
||||
len(node.Content))
|
||||
}
|
||||
|
||||
if idx == len(node.Content) {
|
||||
newNode := &yaml.Node{Kind: yaml.MappingNode}
|
||||
if last {
|
||||
newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
|
||||
}
|
||||
|
||||
node.Content = append(node.Content, newNode)
|
||||
} else if last {
|
||||
setScalar(node.Content[idx], value)
|
||||
}
|
||||
|
||||
return node.Content[idx], nil
|
||||
}
|
||||
|
||||
// setScalar overwrites a node in place with a plain scalar value.
|
||||
func setScalar(n *yaml.Node, value string) {
|
||||
n.Kind = yaml.ScalarNode
|
||||
n.Tag = ""
|
||||
n.Value = value
|
||||
n.Content = nil
|
||||
n.Style = 0
|
||||
}
|
||||
|
||||
// configPathForInit returns the config path to write, checking --config flag,
|
||||
// VAULTIK_CONFIG env, and the platform default.
|
||||
func configPathForInit() string {
|
||||
if rootFlags.ConfigPath != "" {
|
||||
return rootFlags.ConfigPath
|
||||
}
|
||||
|
||||
if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" {
|
||||
return envPath
|
||||
}
|
||||
|
||||
return DefaultConfigPath()
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
package cli //nolint:testpackage // exercises unexported yamlPathGet/yamlPathSet
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
)
|
||||
|
||||
// TestDefaultConfigTemplateParses ensures the init template is valid YAML
|
||||
// that unmarshals into the Config struct with the expected snapshots.
|
||||
func TestDefaultConfigTemplateParses(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var cfg config.Config
|
||||
|
||||
err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("default config template is not valid YAML: %v", err)
|
||||
}
|
||||
|
||||
if len(cfg.AgeRecipients) != 1 {
|
||||
t.Errorf("expected 1 placeholder age recipient, got %d", len(cfg.AgeRecipients))
|
||||
}
|
||||
|
||||
home, ok := cfg.Snapshots["home"]
|
||||
if !ok {
|
||||
t.Fatal("expected 'home' snapshot in default config")
|
||||
}
|
||||
|
||||
if len(home.Paths) == 0 {
|
||||
t.Error("home snapshot should have at least one path")
|
||||
}
|
||||
|
||||
if len(home.Exclude) == 0 {
|
||||
t.Error("home snapshot should have exclude patterns")
|
||||
}
|
||||
|
||||
apps, ok := cfg.Snapshots["apps"]
|
||||
if !ok {
|
||||
t.Fatal("expected 'apps' snapshot in default config")
|
||||
}
|
||||
|
||||
if len(apps.Paths) != 1 || apps.Paths[0] != "/Applications" {
|
||||
t.Errorf("apps snapshot should back up /Applications, got %v", apps.Paths)
|
||||
}
|
||||
|
||||
if len(apps.Exclude) == 0 {
|
||||
t.Error("apps snapshot should have exclude patterns")
|
||||
}
|
||||
}
|
||||
|
||||
const testYAML = `# top comment
|
||||
compression_level: 3
|
||||
age_recipients:
|
||||
- age1aaa
|
||||
s3:
|
||||
bucket: oldbucket # inline comment
|
||||
region: us-east-1
|
||||
snapshots:
|
||||
home:
|
||||
paths:
|
||||
- "~"
|
||||
`
|
||||
|
||||
func parseTestYAML(t *testing.T) *yaml.Node {
|
||||
t.Helper()
|
||||
|
||||
var root yaml.Node
|
||||
|
||||
err := yaml.Unmarshal([]byte(testYAML), &root)
|
||||
if err != nil {
|
||||
t.Fatalf("parsing test yaml: %v", err)
|
||||
}
|
||||
|
||||
return &root
|
||||
}
|
||||
|
||||
func TestYAMLPathGet(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := parseTestYAML(t)
|
||||
|
||||
tests := []struct {
|
||||
path string
|
||||
want string
|
||||
err bool
|
||||
}{
|
||||
{"compression_level", "3", false},
|
||||
{"s3.bucket", "oldbucket", false},
|
||||
{"s3.region", "us-east-1", false},
|
||||
{"age_recipients.0", "age1aaa", false},
|
||||
{"age_recipients.5", "", true},
|
||||
{"age_recipients.notanumber", "", true},
|
||||
{"s3.nonexistent", "", true},
|
||||
{"nonexistent", "", true},
|
||||
{"compression_level.sub", "", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.path, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
node, err := yamlPathGet(root, splitPath(tt.path))
|
||||
if tt.err {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for %q", tt.path)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if node.Value != tt.want {
|
||||
t.Errorf("get %q = %q, want %q", tt.path, node.Value, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestYAMLPathSet(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := parseTestYAML(t)
|
||||
|
||||
// Overwrite existing nested value
|
||||
err := yamlPathSet(root, splitPath("s3.bucket"), "newbucket")
|
||||
if err != nil {
|
||||
t.Fatalf("set s3.bucket: %v", err)
|
||||
}
|
||||
|
||||
// Create new nested key with intermediate map
|
||||
err = yamlPathSet(root, splitPath("s3.endpoint"), "s3.example.com")
|
||||
if err != nil {
|
||||
t.Fatalf("set s3.endpoint: %v", err)
|
||||
}
|
||||
|
||||
err = yamlPathSet(root, splitPath("newmap.newkey"), "val")
|
||||
if err != nil {
|
||||
t.Fatalf("set newmap.newkey: %v", err)
|
||||
}
|
||||
|
||||
// Overwrite a sequence element and append a new one
|
||||
err = yamlPathSet(root, splitPath("age_recipients.0"), "age1bbb")
|
||||
if err != nil {
|
||||
t.Fatalf("set age_recipients.0: %v", err)
|
||||
}
|
||||
|
||||
err = yamlPathSet(root, splitPath("age_recipients.1"), "age1ccc")
|
||||
if err != nil {
|
||||
t.Fatalf("append age_recipients.1: %v", err)
|
||||
}
|
||||
|
||||
err = yamlPathSet(root, splitPath("age_recipients.5"), "age1ddd")
|
||||
if err == nil {
|
||||
t.Error("expected out-of-range append to fail")
|
||||
}
|
||||
|
||||
// Round-trip and verify values + comment preservation
|
||||
out, err := yaml.Marshal(root)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
|
||||
text := string(out)
|
||||
|
||||
wants := []string{
|
||||
"newbucket", "s3.example.com", "newkey: val",
|
||||
"# top comment", "# inline comment", "age1bbb", "age1ccc",
|
||||
}
|
||||
for _, want := range wants {
|
||||
if !contains(text, want) {
|
||||
t.Errorf("round-tripped YAML missing %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := yamlPathGet(root, splitPath("s3.bucket"))
|
||||
if err != nil {
|
||||
t.Fatalf("get after set: %v", err)
|
||||
}
|
||||
|
||||
if got.Value != "newbucket" {
|
||||
t.Errorf("s3.bucket = %q after set, want newbucket", got.Value)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigSetPreservesFormatting asserts the `config set` write path
|
||||
// (marshalConfigYAML) round-trips a 2-space-indented file without reindenting
|
||||
// it to yaml.Marshal's 4-space default, and keeps comments.
|
||||
func TestConfigSetPreservesFormatting(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := parseTestYAML(t)
|
||||
|
||||
err := yamlPathSet(root, splitPath("s3.bucket"), "newbucket")
|
||||
if err != nil {
|
||||
t.Fatalf("set s3.bucket: %v", err)
|
||||
}
|
||||
|
||||
out, err := marshalConfigYAML(root)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
|
||||
text := string(out)
|
||||
|
||||
for _, want := range []string{"# top comment", "# inline comment"} {
|
||||
if !contains(text, want) {
|
||||
t.Errorf("round-tripped YAML dropped comment %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
|
||||
// Nested map keys stay at 2-space indent; the bug reindented them to 4.
|
||||
if !contains(text, "\n bucket: newbucket") {
|
||||
t.Errorf("expected 2-space indent for s3.bucket, got:\n%s", text)
|
||||
}
|
||||
|
||||
if contains(text, "\n bucket:") {
|
||||
t.Errorf("s3.bucket reindented to 4 spaces:\n%s", text)
|
||||
}
|
||||
|
||||
// Sequence items under a key also stay at 2 spaces.
|
||||
if !contains(text, "\n - age1aaa") {
|
||||
t.Errorf("expected 2-space indent for sequence item, got:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func splitPath(s string) []string {
|
||||
return strings.Split(s, ".")
|
||||
}
|
||||
|
||||
func contains(haystack, needle string) bool {
|
||||
return strings.Contains(haystack, needle)
|
||||
}
|
||||
+34
-19
@@ -4,9 +4,9 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"github.com/spf13/cobra"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
// NewDatabaseCommand creates the database command group
|
||||
@@ -18,32 +18,37 @@ func NewDatabaseCommand() *cobra.Command {
|
||||
}
|
||||
|
||||
cmd.AddCommand(
|
||||
newDatabasePurgeCommand(),
|
||||
newDatabaseDeleteCommand(),
|
||||
)
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// newDatabasePurgeCommand creates the database purge command
|
||||
func newDatabasePurgeCommand() *cobra.Command {
|
||||
// newDatabaseDeleteCommand creates the database delete command.
|
||||
// (Renamed from "purge"; the operation removes the SQLite file
|
||||
// entirely, which is a delete, not a purge of content.)
|
||||
func newDatabaseDeleteCommand() *cobra.Command {
|
||||
var force bool
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "purge",
|
||||
Short: "Delete the local state database",
|
||||
Use: "delete",
|
||||
Short: "Delete the local state database file",
|
||||
Long: `Completely removes the local SQLite state database.
|
||||
|
||||
This will erase all local tracking of:
|
||||
- File metadata and change detection state
|
||||
- Chunk and blob mappings
|
||||
- Local snapshot records
|
||||
- The storage-binding record
|
||||
|
||||
The remote storage is NOT affected. After purging, the next backup will
|
||||
perform a full scan and re-deduplicate against existing remote blobs.
|
||||
The remote storage is NOT affected. After deletion, the next backup
|
||||
will perform a full scan and re-deduplicate against existing remote
|
||||
blobs, and the local index will re-bind to the currently configured
|
||||
storage destination on that run.
|
||||
|
||||
Use --force to skip the confirmation prompt.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
RunE: func(_ *cobra.Command, _ []string) error {
|
||||
// Resolve config path
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
@@ -59,24 +64,33 @@ Use --force to skip the confirmation prompt.`,
|
||||
dbPath := cfg.IndexPath
|
||||
|
||||
// Check if database exists
|
||||
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
|
||||
fmt.Printf("Database does not exist: %s\n", dbPath)
|
||||
_, err = os.Stat(dbPath)
|
||||
if os.IsNotExist(err) {
|
||||
_, _ = fmt.Fprintf(os.Stdout, "Database does not exist: %s\n", dbPath)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Confirm unless --force
|
||||
if !force {
|
||||
fmt.Printf("This will delete the local state database at:\n %s\n\n", dbPath)
|
||||
fmt.Print("Are you sure? Type 'yes' to confirm: ")
|
||||
_, _ = fmt.Fprintf(os.Stdout,
|
||||
"This will delete the local state database at:\n %s\n\n", dbPath)
|
||||
_, _ = fmt.Fprint(os.Stdout, "Are you sure? Type 'yes' to confirm: ")
|
||||
|
||||
var confirm string
|
||||
if _, err := fmt.Scanln(&confirm); err != nil || confirm != "yes" {
|
||||
fmt.Println("Aborted.")
|
||||
|
||||
_, err = fmt.Scanln(&confirm)
|
||||
if err != nil || confirm != "yes" {
|
||||
_, _ = fmt.Fprintln(os.Stdout, "Aborted.")
|
||||
|
||||
//nolint:nilerr // a failed/aborted confirmation is a clean abort
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the database file
|
||||
if err := os.Remove(dbPath); err != nil {
|
||||
err = os.Remove(dbPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete database: %w", err)
|
||||
}
|
||||
|
||||
@@ -88,10 +102,11 @@ Use --force to skip the confirmation prompt.`,
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
if !rootFlags.Quiet {
|
||||
fmt.Printf("Database purged: %s\n", dbPath)
|
||||
_, _ = fmt.Fprintf(os.Stdout, "Database deleted: %s\n", dbPath)
|
||||
}
|
||||
|
||||
log.Info("Local state database purged", "path", dbPath)
|
||||
log.Info("Local state database deleted", "path", dbPath)
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// parseDuration parses duration strings. Supports standard Go duration format
|
||||
// (e.g., "3h30m", "1h45m30s") as well as extended units:
|
||||
// - d: days (e.g., "30d", "7d")
|
||||
// - w: weeks (e.g., "2w", "4w")
|
||||
// - mo: months (30 days) (e.g., "6mo", "1mo")
|
||||
// - y: years (365 days) (e.g., "1y", "2y")
|
||||
//
|
||||
// Can combine units: "1y6mo", "2w3d", "1d12h30m"
|
||||
func parseDuration(s string) (time.Duration, error) {
|
||||
// First try standard Go duration parsing
|
||||
if d, err := time.ParseDuration(s); err == nil {
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// Extended duration parsing
|
||||
// Check for negative values
|
||||
if strings.HasPrefix(strings.TrimSpace(s), "-") {
|
||||
return 0, fmt.Errorf("negative durations are not supported")
|
||||
}
|
||||
|
||||
// Pattern matches: number + unit, repeated
|
||||
re := regexp.MustCompile(`(\d+(?:\.\d+)?)\s*([a-zA-Z]+)`)
|
||||
matches := re.FindAllStringSubmatch(s, -1)
|
||||
|
||||
if len(matches) == 0 {
|
||||
return 0, fmt.Errorf("invalid duration format: %q", s)
|
||||
}
|
||||
|
||||
var total time.Duration
|
||||
|
||||
for _, match := range matches {
|
||||
valueStr := match[1]
|
||||
unit := strings.ToLower(match[2])
|
||||
|
||||
value, err := strconv.ParseFloat(valueStr, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid number %q: %w", valueStr, err)
|
||||
}
|
||||
|
||||
var d time.Duration
|
||||
switch unit {
|
||||
// Standard time units
|
||||
case "ns", "nanosecond", "nanoseconds":
|
||||
d = time.Duration(value)
|
||||
case "us", "µs", "microsecond", "microseconds":
|
||||
d = time.Duration(value * float64(time.Microsecond))
|
||||
case "ms", "millisecond", "milliseconds":
|
||||
d = time.Duration(value * float64(time.Millisecond))
|
||||
case "s", "sec", "second", "seconds":
|
||||
d = time.Duration(value * float64(time.Second))
|
||||
case "m", "min", "minute", "minutes":
|
||||
d = time.Duration(value * float64(time.Minute))
|
||||
case "h", "hr", "hour", "hours":
|
||||
d = time.Duration(value * float64(time.Hour))
|
||||
// Extended units
|
||||
case "d", "day", "days":
|
||||
d = time.Duration(value * float64(24*time.Hour))
|
||||
case "w", "week", "weeks":
|
||||
d = time.Duration(value * float64(7*24*time.Hour))
|
||||
case "mo", "month", "months":
|
||||
// Using 30 days as approximation
|
||||
d = time.Duration(value * float64(30*24*time.Hour))
|
||||
case "y", "year", "years":
|
||||
// Using 365 days as approximation
|
||||
d = time.Duration(value * float64(365*24*time.Hour))
|
||||
default:
|
||||
// Try parsing as standard Go duration unit
|
||||
testStr := fmt.Sprintf("1%s", unit)
|
||||
if _, err := time.ParseDuration(testStr); err == nil {
|
||||
// It's a valid Go duration unit, parse the full value
|
||||
fullStr := fmt.Sprintf("%g%s", value, unit)
|
||||
if d, err = time.ParseDuration(fullStr); err != nil {
|
||||
return 0, fmt.Errorf("invalid duration %q: %w", fullStr, err)
|
||||
}
|
||||
} else {
|
||||
return 0, fmt.Errorf("unknown time unit %q", unit)
|
||||
}
|
||||
}
|
||||
|
||||
total += d
|
||||
}
|
||||
|
||||
return total, nil
|
||||
}
|
||||
@@ -1,263 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestParseDuration(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected time.Duration
|
||||
wantErr bool
|
||||
}{
|
||||
// Standard Go durations
|
||||
{
|
||||
name: "standard seconds",
|
||||
input: "30s",
|
||||
expected: 30 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "standard minutes",
|
||||
input: "45m",
|
||||
expected: 45 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "standard hours",
|
||||
input: "2h",
|
||||
expected: 2 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "standard combined",
|
||||
input: "3h30m",
|
||||
expected: 3*time.Hour + 30*time.Minute,
|
||||
},
|
||||
{
|
||||
name: "standard complex",
|
||||
input: "1h45m30s",
|
||||
expected: 1*time.Hour + 45*time.Minute + 30*time.Second,
|
||||
},
|
||||
{
|
||||
name: "standard with milliseconds",
|
||||
input: "1s500ms",
|
||||
expected: 1*time.Second + 500*time.Millisecond,
|
||||
},
|
||||
// Extended units - days
|
||||
{
|
||||
name: "single day",
|
||||
input: "1d",
|
||||
expected: 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "multiple days",
|
||||
input: "7d",
|
||||
expected: 7 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "fractional days",
|
||||
input: "1.5d",
|
||||
expected: 36 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "days spelled out",
|
||||
input: "3days",
|
||||
expected: 3 * 24 * time.Hour,
|
||||
},
|
||||
// Extended units - weeks
|
||||
{
|
||||
name: "single week",
|
||||
input: "1w",
|
||||
expected: 7 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "multiple weeks",
|
||||
input: "4w",
|
||||
expected: 4 * 7 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "weeks spelled out",
|
||||
input: "2weeks",
|
||||
expected: 2 * 7 * 24 * time.Hour,
|
||||
},
|
||||
// Extended units - months
|
||||
{
|
||||
name: "single month",
|
||||
input: "1mo",
|
||||
expected: 30 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "multiple months",
|
||||
input: "6mo",
|
||||
expected: 6 * 30 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "months spelled out",
|
||||
input: "3months",
|
||||
expected: 3 * 30 * 24 * time.Hour,
|
||||
},
|
||||
// Extended units - years
|
||||
{
|
||||
name: "single year",
|
||||
input: "1y",
|
||||
expected: 365 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "multiple years",
|
||||
input: "2y",
|
||||
expected: 2 * 365 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "years spelled out",
|
||||
input: "1year",
|
||||
expected: 365 * 24 * time.Hour,
|
||||
},
|
||||
// Combined extended units
|
||||
{
|
||||
name: "weeks and days",
|
||||
input: "2w3d",
|
||||
expected: 2*7*24*time.Hour + 3*24*time.Hour,
|
||||
},
|
||||
{
|
||||
name: "years and months",
|
||||
input: "1y6mo",
|
||||
expected: 365*24*time.Hour + 6*30*24*time.Hour,
|
||||
},
|
||||
{
|
||||
name: "days and hours",
|
||||
input: "1d12h",
|
||||
expected: 24*time.Hour + 12*time.Hour,
|
||||
},
|
||||
{
|
||||
name: "complex combination",
|
||||
input: "1y2mo3w4d5h6m7s",
|
||||
expected: 365*24*time.Hour + 2*30*24*time.Hour + 3*7*24*time.Hour + 4*24*time.Hour + 5*time.Hour + 6*time.Minute + 7*time.Second,
|
||||
},
|
||||
{
|
||||
name: "with spaces",
|
||||
input: "1d 12h 30m",
|
||||
expected: 24*time.Hour + 12*time.Hour + 30*time.Minute,
|
||||
},
|
||||
// Edge cases
|
||||
{
|
||||
name: "zero duration",
|
||||
input: "0s",
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "large duration",
|
||||
input: "10y",
|
||||
expected: 10 * 365 * 24 * time.Hour,
|
||||
},
|
||||
// Error cases
|
||||
{
|
||||
name: "empty string",
|
||||
input: "",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid format",
|
||||
input: "abc",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unknown unit",
|
||||
input: "5x",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid number",
|
||||
input: "xyzd",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "negative not supported",
|
||||
input: "-5d",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := parseDuration(tt.input)
|
||||
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err, "expected error for input %q", tt.input)
|
||||
return
|
||||
}
|
||||
|
||||
assert.NoError(t, err, "unexpected error for input %q", tt.input)
|
||||
assert.Equal(t, tt.expected, got, "duration mismatch for input %q", tt.input)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDurationSpecialCases(t *testing.T) {
|
||||
// Test that standard Go durations work exactly as expected
|
||||
standardDurations := []string{
|
||||
"300ms",
|
||||
"1.5h",
|
||||
"2h45m",
|
||||
"72h",
|
||||
"1us",
|
||||
"1µs",
|
||||
"1ns",
|
||||
}
|
||||
|
||||
for _, d := range standardDurations {
|
||||
expected, err := time.ParseDuration(d)
|
||||
assert.NoError(t, err)
|
||||
|
||||
got, err := parseDuration(d)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, expected, got, "standard duration %q should parse identically", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDurationRealWorldExamples(t *testing.T) {
|
||||
// Test real-world snapshot purge scenarios
|
||||
tests := []struct {
|
||||
description string
|
||||
input string
|
||||
olderThan time.Duration
|
||||
}{
|
||||
{
|
||||
description: "keep snapshots from last 30 days",
|
||||
input: "30d",
|
||||
olderThan: 30 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
description: "keep snapshots from last 6 months",
|
||||
input: "6mo",
|
||||
olderThan: 6 * 30 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
description: "keep snapshots from last year",
|
||||
input: "1y",
|
||||
olderThan: 365 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
description: "keep snapshots from last week and a half",
|
||||
input: "1w3d",
|
||||
olderThan: 10 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
description: "keep snapshots from last 90 days",
|
||||
input: "90d",
|
||||
olderThan: 90 * 24 * time.Hour,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.description, func(t *testing.T) {
|
||||
got, err := parseDuration(tt.input)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.olderThan, got)
|
||||
|
||||
// Verify the duration makes sense for snapshot purging
|
||||
assert.Greater(t, got, time.Hour, "snapshot purge duration should be at least an hour")
|
||||
})
|
||||
}
|
||||
}
|
||||
+93
-5
@@ -1,15 +1,103 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/globals"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
)
|
||||
|
||||
// CLIEntry is the main entry point for the CLI application.
|
||||
// It creates the root command, executes it, and exits with status 1
|
||||
// if an error occurs. This function should be called from main().
|
||||
func CLIEntry() {
|
||||
// shortCommitLen is the number of git commit hash characters shown in
|
||||
// the startup banner.
|
||||
const shortCommitLen = 12
|
||||
|
||||
// Entry is the main entry point for the CLI application.
|
||||
// It prints the startup banner to stdout (unless a banner-suppressing
|
||||
// flag is present in os.Args — see bannerSuppressedInArgs), executes the
|
||||
// root cobra command, and routes any returned error through the
|
||||
// ui.Writer so the user sees a properly formatted "🛑 ERROR:" line.
|
||||
func Entry() {
|
||||
emitStartupBanner(os.Args[1:], os.Stdout)
|
||||
|
||||
rootCmd := NewRootCommand()
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
rootCmd.SilenceErrors = true
|
||||
|
||||
err := rootCmd.Execute()
|
||||
if err != nil {
|
||||
ReportErrorf("%s", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// emitStartupBanner writes the startup banner to w unless args (the
|
||||
// argument vector with the program name already stripped) contains a
|
||||
// flag that suppresses it. Split out of Entry so that the decision — the
|
||||
// only thing standing between a --json invocation and a parseable
|
||||
// stdout — is reachable from a test without running the whole CLI.
|
||||
func emitStartupBanner(args []string, w io.Writer) {
|
||||
if bannerSuppressedInArgs(args) {
|
||||
return
|
||||
}
|
||||
|
||||
short := globals.Commit
|
||||
if len(short) > shortCommitLen {
|
||||
short = short[:shortCommitLen]
|
||||
}
|
||||
|
||||
writeStartupBanner(ui.New(w), time.Now().UTC(), short)
|
||||
}
|
||||
|
||||
// ReportErrorf emits a user-facing error to stderr in the standard
|
||||
// 🛑 ERROR: format. Use it from goroutine error paths (where returning
|
||||
// an error to cobra isn't an option) and anywhere else a CLI command
|
||||
// must surface a failure outside the normal RunE return path.
|
||||
func ReportErrorf(format string, args ...any) {
|
||||
ui.New(os.Stderr).Errorf(format, args...)
|
||||
}
|
||||
|
||||
// bannerSuppressedInArgs reports whether any of args is a flag that
|
||||
// should suppress the startup banner (--quiet/-q/--cron/--json). Stops
|
||||
// at the "--" argument terminator. Recognizes both long forms and short
|
||||
// -q, including combined short flags like "-qv".
|
||||
//
|
||||
// This scans the raw argument vector because the banner is printed
|
||||
// before cobra parses anything — deliberately, so that it still appears
|
||||
// when cobra rejects the arguments and on --help. The consequence is
|
||||
// that a flag is matched wherever it occurs in the vector, including
|
||||
// positions where the command it belongs to would not accept it.
|
||||
// --json is a subcommand flag rather than a persistent one, but so is
|
||||
// --cron (it exists only on `snapshot create`), so this adds no new
|
||||
// class of imprecision. The only cost of a false positive is a missing
|
||||
// decorative banner; the cost of a false negative is a corrupt document
|
||||
// on stdout, so the scan errs deliberately in that direction.
|
||||
func bannerSuppressedInArgs(args []string) bool {
|
||||
for _, a := range args {
|
||||
if a == "--" {
|
||||
return false
|
||||
}
|
||||
|
||||
switch a {
|
||||
case "--quiet", "-q", "--cron", "--json":
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasPrefix(a, "--quiet=") ||
|
||||
strings.HasPrefix(a, "--cron=") ||
|
||||
strings.HasPrefix(a, "--json=") {
|
||||
return true
|
||||
}
|
||||
// Combined short flags like -qv or -vq.
|
||||
if len(a) > 1 && a[0] == '-' && a[1] != '-' {
|
||||
for _, c := range a[1:] {
|
||||
if c == 'q' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,300 @@
|
||||
package cli //nolint:testpackage // needs access to unexported emitStartupBanner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/adrg/xdg"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Command words and flags used to build argument vectors below. They are
|
||||
// constants rather than repeated literals so that a rename shows up as a
|
||||
// compile error in one place.
|
||||
const (
|
||||
cmdSnapshot = "snapshot"
|
||||
cmdList = "list"
|
||||
cmdCreate = "create"
|
||||
cmdVerify = "verify"
|
||||
cmdRemove = "remove"
|
||||
cmdPrune = "prune"
|
||||
cmdRemote = "remote"
|
||||
cmdInfo = "info"
|
||||
|
||||
flagJSON = "--json"
|
||||
flagQuiet = "--quiet"
|
||||
flagConfig = "--config"
|
||||
|
||||
// programName is argv[0] as the real process receives it. Entry
|
||||
// strips it before scanning, so it has to be present.
|
||||
programName = "vaultik"
|
||||
|
||||
// someSnapshotID is any snapshot identifier: these tests never run
|
||||
// the command, so it only has to occupy the positional argument.
|
||||
someSnapshotID = "host_2026-01-01T00:00:00Z"
|
||||
)
|
||||
|
||||
// placeholderJSONDocument stands in for whatever document a --json
|
||||
// command writes to stdout. `snapshot list --json` with no snapshots
|
||||
// prints exactly this; the other --json commands print an object rather
|
||||
// than an array, but this test is not about their shape. It is about
|
||||
// what is on stdout *before* them, which is the same for all of them
|
||||
// because Entry prints the banner before cobra has parsed anything and
|
||||
// therefore before it can know which command is running.
|
||||
const placeholderJSONDocument = "[]\n"
|
||||
|
||||
// jsonArgumentVectors are the argument vectors of every --json
|
||||
// invocation the CLI accepts, with the program name stripped exactly as
|
||||
// Entry strips it. Each one must leave stdout untouched by the banner.
|
||||
//
|
||||
//nolint:gochecknoglobals // read-only test fixture shared by two tests
|
||||
var jsonArgumentVectors = map[string][]string{
|
||||
"snapshot list": {cmdSnapshot, cmdList, flagJSON},
|
||||
"snapshot verify": {cmdSnapshot, cmdVerify, someSnapshotID, flagJSON},
|
||||
"snapshot remove": {cmdSnapshot, cmdRemove, someSnapshotID, flagJSON},
|
||||
"prune": {cmdPrune, flagJSON},
|
||||
"remote info": {cmdRemote, cmdInfo, flagJSON},
|
||||
|
||||
// --json before the subcommand, and with an explicit value: the
|
||||
// scan is positional, so both forms have to be recognized.
|
||||
"json first": {flagJSON, cmdSnapshot, cmdList},
|
||||
"json with value": {cmdSnapshot, cmdList, flagJSON + "=true"},
|
||||
|
||||
// A --json invocation that also carries a flag with a value, so the
|
||||
// scan cannot be fooled by an argument that consumes the next one.
|
||||
"json with config": {
|
||||
flagConfig, "/nonexistent/vaultik.yml", cmdSnapshot, cmdList, flagJSON,
|
||||
},
|
||||
}
|
||||
|
||||
// TestJSONInvocationStdoutIsExactlyOneDocument is the CLI-layer
|
||||
// regression guard for issue #106: `vaultik snapshot list --json | jq`
|
||||
// must work with no other flags.
|
||||
//
|
||||
// internal/vaultik's TestListSnapshots_JSONStdoutIsOnlyTheDocument
|
||||
// guards the same contract one layer down, but it calls the library
|
||||
// function directly and so cannot see Entry, which is where the
|
||||
// contamination was: the startup banner is written to stdout before
|
||||
// cobra parses anything, and the suppression scan did not know about
|
||||
// --json. The two banner lines and the blank line landed ahead of the
|
||||
// document and `jq` refused the result.
|
||||
//
|
||||
// The document is a constant here because this test is about the
|
||||
// argument vectors, one per --json command; the one that runs a real
|
||||
// command end to end is TestEntryJSONStdoutIsExactlyOneDocument below.
|
||||
func TestJSONInvocationStdoutIsExactlyOneDocument(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for name, argv := range jsonArgumentVectors {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var stdout bytes.Buffer
|
||||
|
||||
emitStartupBanner(argv, &stdout)
|
||||
|
||||
require.Empty(t, stdout.String(),
|
||||
"nothing may reach stdout ahead of a --json document")
|
||||
|
||||
_, err := stdout.WriteString(placeholderJSONDocument)
|
||||
require.NoError(t, err)
|
||||
|
||||
requireExactlyOneJSONDocument(t, stdout.String())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBannerStillPrintedWithoutSuppressingFlag pins the other half of
|
||||
// the contract. Without it, deleting the banner outright would satisfy
|
||||
// the test above, and the banner is wanted on interactive invocations.
|
||||
func TestBannerStillPrintedWithoutSuppressingFlag(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for name, argv := range map[string][]string{
|
||||
"no flags": {cmdSnapshot, cmdList},
|
||||
"verbose": {cmdSnapshot, cmdList, "--verbose"},
|
||||
"after the terminator": {
|
||||
cmdSnapshot, "restore", "--", flagJSON,
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var stdout bytes.Buffer
|
||||
|
||||
emitStartupBanner(argv, &stdout)
|
||||
|
||||
assert.Contains(t, stdout.String(), "starting up at",
|
||||
"the banner belongs on invocations that did not opt out")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBannerSuppressedInArgs covers the suppression scan directly,
|
||||
// including the flags that suppressed the banner before --json joined
|
||||
// them, so that adding --json cannot regress them.
|
||||
func TestBannerSuppressedInArgs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for name, testCase := range map[string]struct {
|
||||
args []string
|
||||
suppressed bool
|
||||
}{
|
||||
"quiet long": {[]string{cmdSnapshot, cmdCreate, flagQuiet}, true},
|
||||
"quiet short": {[]string{cmdSnapshot, cmdCreate, "-q"}, true},
|
||||
"quiet combined": {[]string{cmdSnapshot, cmdCreate, "-qv"}, true},
|
||||
"cron": {[]string{cmdSnapshot, cmdCreate, "--cron"}, true},
|
||||
"json": {[]string{cmdSnapshot, cmdList, flagJSON}, true},
|
||||
"nothing": {[]string{cmdSnapshot, cmdList}, false},
|
||||
"empty": {nil, false},
|
||||
"json after dashes": {
|
||||
[]string{cmdSnapshot, cmdList, "--", flagJSON}, false,
|
||||
},
|
||||
"quiet after dashes": {
|
||||
[]string{cmdSnapshot, cmdCreate, "--", "-q"}, false,
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.Equal(t, testCase.suppressed,
|
||||
bannerSuppressedInArgs(testCase.args))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// hermeticConfig is a complete, valid config that needs no network and
|
||||
// no credentials: file:// storage is exempt from the S3 credential
|
||||
// checks, and FileStorer over a directory that does not exist lists
|
||||
// zero objects without erroring. Chunk, blob and compression settings
|
||||
// are filled in by config.Load.
|
||||
const hermeticConfig = `age_recipients:
|
||||
- age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj
|
||||
snapshots:
|
||||
test:
|
||||
paths:
|
||||
- %s
|
||||
storage_url: file://%s
|
||||
index_path: %s
|
||||
hostname: test-host
|
||||
`
|
||||
|
||||
// TestEntryJSONStdoutIsExactlyOneDocument runs the real thing: Entry,
|
||||
// with a real argument vector, over the process's real stdout file
|
||||
// descriptor, all the way through cobra and the fx graph to the
|
||||
// document. It is the assertion the issue asks for — `vaultik snapshot
|
||||
// list --json | jq .` with no other flags — with the pipe replaced by a
|
||||
// decoder.
|
||||
//
|
||||
// `snapshot list` is the command chosen because it is the only --json
|
||||
// command that reaches its document without a populated destination
|
||||
// store: it reads the local index, streams `metadata/` (empty here),
|
||||
// and treats a barren destination as an empty list rather than a
|
||||
// failure.
|
||||
//
|
||||
// Not parallel: it replaces os.Args, os.Stdout and the xdg globals.
|
||||
func TestEntryJSONStdoutIsExactlyOneDocument(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
configPath := filepath.Join(dir, "config.yml")
|
||||
|
||||
contents := fmt.Sprintf(hermeticConfig,
|
||||
filepath.Join(dir, "source"),
|
||||
filepath.Join(dir, "store"),
|
||||
filepath.Join(dir, "index.sqlite"))
|
||||
|
||||
require.NoError(t,
|
||||
os.WriteFile(configPath, []byte(contents), configFileMode))
|
||||
|
||||
// The PID lock lives under xdg.DataHome, which xdg resolves at
|
||||
// package init; point it at the temp dir so the test neither
|
||||
// touches nor collides with the real one.
|
||||
t.Setenv("XDG_DATA_HOME", filepath.Join(dir, "data"))
|
||||
xdg.Reload()
|
||||
t.Cleanup(xdg.Reload)
|
||||
|
||||
previousArgs := os.Args
|
||||
|
||||
t.Cleanup(func() {
|
||||
os.Args = previousArgs
|
||||
rootFlags = RootFlags{}
|
||||
})
|
||||
|
||||
os.Args = []string{
|
||||
programName, flagConfig, configPath, cmdSnapshot, cmdList, flagJSON,
|
||||
}
|
||||
|
||||
stdout := captureProcessStdout(t, Entry)
|
||||
|
||||
requireExactlyOneJSONDocument(t, stdout)
|
||||
|
||||
var snapshots []any
|
||||
|
||||
require.NoError(t, json.Unmarshal([]byte(stdout), &snapshots))
|
||||
assert.Empty(t, snapshots,
|
||||
"a destination store with no snapshots lists none")
|
||||
}
|
||||
|
||||
// captureProcessStdout redirects the process's own stdout to a pipe for
|
||||
// the duration of fn and returns what was written to it. The redirection
|
||||
// has to be at the file-descriptor level rather than through an injected
|
||||
// writer, because the banner and the JSON encoder reach os.Stdout
|
||||
// independently and the point of the test is that both land in the same
|
||||
// place.
|
||||
//
|
||||
// Not parallel-safe: os.Stdout is process-global.
|
||||
func captureProcessStdout(t *testing.T, fn func()) string {
|
||||
t.Helper()
|
||||
|
||||
reader, writer, err := os.Pipe()
|
||||
require.NoError(t, err)
|
||||
|
||||
previous := os.Stdout
|
||||
os.Stdout = writer
|
||||
|
||||
captured := make(chan string, 1)
|
||||
|
||||
go func() {
|
||||
var buf bytes.Buffer
|
||||
|
||||
_, _ = io.Copy(&buf, reader)
|
||||
captured <- buf.String()
|
||||
}()
|
||||
|
||||
fn()
|
||||
|
||||
os.Stdout = previous
|
||||
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
out := <-captured
|
||||
|
||||
require.NoError(t, reader.Close())
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// requireExactlyOneJSONDocument fails unless stdout decodes as a single
|
||||
// JSON value with nothing before or after it — the property that makes
|
||||
// `| jq` work.
|
||||
func requireExactlyOneJSONDocument(t *testing.T, stdout string) {
|
||||
t.Helper()
|
||||
|
||||
decoder := json.NewDecoder(strings.NewReader(stdout))
|
||||
|
||||
var document any
|
||||
|
||||
err := decoder.Decode(&document)
|
||||
require.NoError(t, err,
|
||||
"stdout must parse as JSON, got:\n%s", stdout)
|
||||
|
||||
_, err = decoder.Token()
|
||||
require.ErrorIs(t, err, io.EOF,
|
||||
"stdout must hold exactly one JSON document, got:\n%s", stdout)
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package cli //nolint:testpackage // shares the prune fixtures and capture helpers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// staleRecordLogMessage is the local-cleanup audit line CleanupLocalSnapshots
|
||||
// logs for each stale record. It is exactly the signal issue #112 says a
|
||||
// machine consumer lost under --json: gated off stdout, and pinned below
|
||||
// the log level on stderr because --json used to force Quiet.
|
||||
const staleRecordLogMessage = "Removing stale local snapshot record"
|
||||
|
||||
// TestEntryPruneJSONStderrHonoursVerbosity is the end-to-end regression
|
||||
// guard for issue #112. Under --json the log level must still follow
|
||||
// --verbose/--debug rather than being pinned to WARN, so the
|
||||
// local-cleanup records reach stderr under --verbose while stdout stays
|
||||
// exactly one JSON document; without --verbose they stay below the
|
||||
// level, as they do without --json.
|
||||
//
|
||||
// Both halves are asserted together on the same run, because the fix has
|
||||
// to keep the document clean (issue #108) while freeing stderr.
|
||||
//
|
||||
// Not parallel: it replaces os.Args, os.Stdout, os.Stderr and the xdg
|
||||
// globals.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntryPruneJSONStderrHonoursVerbosity(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
verbose bool
|
||||
wantOnStderr bool
|
||||
}{
|
||||
{
|
||||
name: "verbose json surfaces the cleanup record on stderr",
|
||||
verbose: true,
|
||||
wantOnStderr: true,
|
||||
},
|
||||
{
|
||||
name: "json alone keeps the cleanup record below the level",
|
||||
verbose: false,
|
||||
wantOnStderr: false,
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
configPath := writeHermeticPruneConfig(t, true)
|
||||
|
||||
previousArgs := os.Args
|
||||
|
||||
t.Cleanup(func() {
|
||||
os.Args = previousArgs
|
||||
rootFlags = RootFlags{}
|
||||
})
|
||||
|
||||
args := []string{
|
||||
programName, flagConfig, configPath, cmdPrune, flagJSON,
|
||||
}
|
||||
if testCase.verbose {
|
||||
args = append(args, "--verbose")
|
||||
}
|
||||
|
||||
os.Args = args
|
||||
|
||||
stdout, stderr := captureProcessStdoutAndStderr(t, Entry)
|
||||
|
||||
// The document stays clean in both cases: freeing stderr must
|
||||
// not regress issue #108.
|
||||
requireExactlyOneJSONDocument(t, stdout)
|
||||
|
||||
if testCase.wantOnStderr {
|
||||
assert.Contains(t, stderr, staleRecordLogMessage,
|
||||
"--verbose --json must emit the cleanup record on stderr")
|
||||
assert.Contains(t, stderr, stalePruneSnapshotID,
|
||||
"the record must name the snapshot it removed")
|
||||
} else {
|
||||
assert.NotContains(t, stderr, staleRecordLogMessage,
|
||||
"without --verbose the record stays below the log level")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// captureProcessStdoutAndStderr redirects both of the process's own
|
||||
// standard streams to pipes for the duration of fn and returns what was
|
||||
// written to each. The redirection is at the file-descriptor level
|
||||
// because the logger binds os.Stderr when it initializes inside fn, and
|
||||
// the JSON document reaches os.Stdout independently; the point is to see
|
||||
// where each actually lands.
|
||||
//
|
||||
// Not parallel-safe: os.Stdout and os.Stderr are process-global.
|
||||
func captureProcessStdoutAndStderr(t *testing.T, fn func()) (string, string) {
|
||||
t.Helper()
|
||||
|
||||
outReader, outWriter, err := os.Pipe()
|
||||
require.NoError(t, err)
|
||||
|
||||
errReader, errWriter, err := os.Pipe()
|
||||
require.NoError(t, err)
|
||||
|
||||
previousOut, previousErr := os.Stdout, os.Stderr
|
||||
os.Stdout, os.Stderr = outWriter, errWriter
|
||||
|
||||
capturedOut := drain(outReader)
|
||||
capturedErr := drain(errReader)
|
||||
|
||||
fn()
|
||||
|
||||
os.Stdout, os.Stderr = previousOut, previousErr
|
||||
|
||||
require.NoError(t, outWriter.Close())
|
||||
require.NoError(t, errWriter.Close())
|
||||
|
||||
out, errOut := <-capturedOut, <-capturedErr
|
||||
|
||||
require.NoError(t, outReader.Close())
|
||||
require.NoError(t, errReader.Close())
|
||||
|
||||
return out, errOut
|
||||
}
|
||||
|
||||
// drain copies a reader to a string on a goroutine and delivers the
|
||||
// result once the writer end is closed.
|
||||
func drain(reader io.Reader) <-chan string {
|
||||
captured := make(chan string, 1)
|
||||
|
||||
go func() {
|
||||
var buf bytes.Buffer
|
||||
|
||||
_, _ = io.Copy(&buf, reader)
|
||||
captured <- buf.String()
|
||||
}()
|
||||
|
||||
return captured
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package cli //nolint:testpackage // shares hermeticConfig and the capture helpers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/adrg/xdg"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// pruneJSONDocument is the shape `prune --json` writes: the
|
||||
// PruneBlobsResult document, and nothing else.
|
||||
//
|
||||
//nolint:tagliatelle // snake_case is the established JSON output format
|
||||
type pruneJSONDocument struct {
|
||||
BlobsFound int `json:"blobs_found"`
|
||||
BlobsDeleted int `json:"blobs_deleted"`
|
||||
BytesFreed int64 `json:"bytes_freed"`
|
||||
}
|
||||
|
||||
// stalePruneSnapshotID is seeded into the local index with no manifest
|
||||
// on the destination store, which is exactly what makes it stale.
|
||||
const stalePruneSnapshotID = "test-host_test_2026-04-01T09:00:00Z"
|
||||
|
||||
// TestEntryPruneJSONStdoutIsExactlyOneDocument is the end-to-end
|
||||
// regression guard for issue #108: `vaultik prune --json | jq .` must
|
||||
// work with no other flags.
|
||||
//
|
||||
// It runs Entry over the process's real stdout descriptor, through
|
||||
// cobra and the fx graph, against a hermetic file:// destination store
|
||||
// — the same construction TestEntryJSONStdoutIsExactlyOneDocument uses
|
||||
// for `snapshot list`, with the pipe to jq replaced by a decoder.
|
||||
//
|
||||
// Both branches of the local-snapshot reconciliation are exercised
|
||||
// because the three stdout writes that broke this covered all of them:
|
||||
// one line per stale record and a summary when there were any, and a
|
||||
// "No stale local snapshots found." line when there were none. No input
|
||||
// avoided the contamination, so no single branch demonstrates the fix.
|
||||
//
|
||||
// Not parallel: it replaces os.Args, os.Stdout and the xdg globals.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout and the xdg globals
|
||||
func TestEntryPruneJSONStdoutIsExactlyOneDocument(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
seedStale bool
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "no stale local records",
|
||||
seedStale: false,
|
||||
description: "the empty-index branch used to print a 'No stale' line",
|
||||
},
|
||||
{
|
||||
name: "stale local records present",
|
||||
seedStale: true,
|
||||
description: "the removal branch used to print a line per record " +
|
||||
"plus a summary",
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
configPath := writeHermeticPruneConfig(t, testCase.seedStale)
|
||||
|
||||
previousArgs := os.Args
|
||||
|
||||
t.Cleanup(func() {
|
||||
os.Args = previousArgs
|
||||
rootFlags = RootFlags{}
|
||||
})
|
||||
|
||||
os.Args = []string{
|
||||
programName, flagConfig, configPath, cmdPrune, flagJSON,
|
||||
}
|
||||
|
||||
stdout := captureProcessStdout(t, Entry)
|
||||
|
||||
requireExactlyOneJSONDocument(t, stdout)
|
||||
|
||||
var document pruneJSONDocument
|
||||
|
||||
require.NoError(t, json.Unmarshal([]byte(stdout), &document),
|
||||
testCase.description)
|
||||
|
||||
// A destination store with no blobs has none to prune. The
|
||||
// assertion that matters is the one above; this one keeps the
|
||||
// test honest about which document it decoded.
|
||||
assert.Equal(t, 0, document.BlobsFound)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// writeHermeticPruneConfig builds a config over a temp directory and, if
|
||||
// seedStale is set, creates the index database up front with one
|
||||
// snapshot record that has no counterpart on the destination store.
|
||||
// Returns the config path.
|
||||
func writeHermeticPruneConfig(t *testing.T, seedStale bool) string {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
configPath := filepath.Join(dir, "config.yml")
|
||||
indexPath := filepath.Join(dir, "index.sqlite")
|
||||
|
||||
contents := fmt.Sprintf(hermeticConfig,
|
||||
filepath.Join(dir, "source"),
|
||||
filepath.Join(dir, "store"),
|
||||
indexPath)
|
||||
|
||||
require.NoError(t,
|
||||
os.WriteFile(configPath, []byte(contents), configFileMode))
|
||||
|
||||
// The PID lock lives under xdg.DataHome, which xdg resolves at
|
||||
// package init; point it at the temp dir so the test neither
|
||||
// touches nor collides with the real one.
|
||||
t.Setenv("XDG_DATA_HOME", filepath.Join(dir, "data"))
|
||||
xdg.Reload()
|
||||
t.Cleanup(xdg.Reload)
|
||||
|
||||
if seedStale {
|
||||
seedStaleSnapshotRecord(t, indexPath)
|
||||
}
|
||||
|
||||
return configPath
|
||||
}
|
||||
|
||||
// seedStaleSnapshotRecord creates the index database at path and
|
||||
// inserts one completed snapshot into it. Nothing is written to the
|
||||
// destination store, so `prune` finds the record stale and removes it —
|
||||
// the branch that printed a line per record.
|
||||
func seedStaleSnapshotRecord(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
db, err := database.New(ctx, path)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { require.NoError(t, db.Close()) }()
|
||||
|
||||
startedAt := time.Date(2026, 4, 1, 9, 0, 0, 0, time.UTC)
|
||||
completedAt := startedAt.Add(time.Minute)
|
||||
|
||||
snap := &database.Snapshot{
|
||||
ID: types.SnapshotID(stalePruneSnapshotID),
|
||||
Hostname: "test-host",
|
||||
VaultikVersion: "test",
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: &completedAt,
|
||||
}
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
return repos.Snapshots.Create(ctx, tx, snap)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -1,14 +1,18 @@
|
||||
package cli
|
||||
package cli_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/cli"
|
||||
)
|
||||
|
||||
// TestCLIEntry ensures the CLI can be imported and basic initialization works
|
||||
func TestCLIEntry(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// This test primarily serves as a compilation test
|
||||
// to ensure all imports resolve correctly
|
||||
cmd := NewRootCommand()
|
||||
cmd := cli.NewRootCommand()
|
||||
if cmd == nil {
|
||||
t.Fatal("NewRootCommand() returned nil")
|
||||
}
|
||||
@@ -18,15 +22,20 @@ func TestCLIEntry(t *testing.T) {
|
||||
}
|
||||
|
||||
// Verify all subcommands are registered
|
||||
expectedCommands := []string{"snapshot", "store", "restore", "prune", "verify", "info", "version"}
|
||||
expectedCommands := []string{
|
||||
"config", "snapshot", "prune", "info", "version", "remote", "database",
|
||||
}
|
||||
for _, expected := range expectedCommands {
|
||||
found := false
|
||||
|
||||
for _, cmd := range cmd.Commands() {
|
||||
if cmd.Use == expected || cmd.Name() == expected {
|
||||
found = true
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Errorf("Expected command '%s' not found", expected)
|
||||
}
|
||||
@@ -38,15 +47,20 @@ func TestCLIEntry(t *testing.T) {
|
||||
t.Errorf("Failed to find snapshot command: %v", err)
|
||||
} else {
|
||||
// Check snapshot subcommands
|
||||
expectedSubCommands := []string{"create", "list", "purge", "verify"}
|
||||
expectedSubCommands := []string{
|
||||
"create", "list", "purge", "verify", "remove", "restore",
|
||||
}
|
||||
for _, expected := range expectedSubCommands {
|
||||
found := false
|
||||
|
||||
for _, subcmd := range snapshotCmd.Commands() {
|
||||
if subcmd.Use == expected || subcmd.Name() == expected {
|
||||
found = true
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Errorf("Expected snapshot subcommand '%s' not found", expected)
|
||||
}
|
||||
|
||||
+17
-9
@@ -2,12 +2,13 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// NewInfoCommand creates the info command
|
||||
@@ -22,7 +23,7 @@ func NewInfoCommand() *cobra.Command {
|
||||
- Encryption configuration (recipients)
|
||||
- Local database statistics`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
@@ -31,9 +32,10 @@ func NewInfoCommand() *cobra.Command {
|
||||
|
||||
// Use the app framework
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
@@ -42,22 +44,28 @@ func NewInfoCommand() *cobra.Command {
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
OnStart: func(_ context.Context) error {
|
||||
go func() {
|
||||
if err := v.ShowInfo(); err != nil {
|
||||
if err != context.Canceled {
|
||||
err := v.ShowInfo()
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
log.Error("Failed to show info", "error", err)
|
||||
ReportErrorf("Failed to show info: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
|
||||
err = v.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
OnStop: func(_ context.Context) error {
|
||||
v.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
+30
-16
@@ -2,12 +2,13 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// NewPruneCommand creates the prune command
|
||||
@@ -16,16 +17,21 @@ func NewPruneCommand() *cobra.Command {
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "prune",
|
||||
Short: "Remove unreferenced blobs",
|
||||
Long: `Removes blobs that are not referenced by any snapshot.
|
||||
Short: "Tidy local database and remote storage",
|
||||
Long: `Removes orphaned data from both the local index database and
|
||||
unreferenced blobs from the backup destination store.
|
||||
|
||||
This command scans all snapshots and their manifests to build a list of
|
||||
referenced blobs, then removes any blobs in storage that are not in this list.
|
||||
Local cleanup drops incomplete snapshots and any files, chunks, or
|
||||
blobs no longer referenced by a completed snapshot. Remote cleanup
|
||||
scans every snapshot manifest in the destination store, builds the
|
||||
set of still-referenced blob hashes, and deletes any blob not in that
|
||||
set.
|
||||
|
||||
Use this command after deleting snapshots with 'vaultik purge' to reclaim
|
||||
storage space.`,
|
||||
Snapshot create --prune and snapshot remove run the same cleanup
|
||||
automatically; this command is the manual entry point for the same
|
||||
work (e.g. after a crashed backup or to reclaim storage).`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
@@ -34,40 +40,48 @@ storage space.`,
|
||||
|
||||
// Use the app framework like other commands
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet || opts.JSON,
|
||||
Quiet: rootFlags.Quiet,
|
||||
JSON: opts.JSON,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
OnStart: func(_ context.Context) error {
|
||||
// Start the prune operation in a goroutine
|
||||
go func() {
|
||||
// Run the prune operation
|
||||
if err := v.PruneBlobs(opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
err := v.Prune(opts)
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
if !opts.JSON {
|
||||
log.Error("Prune operation failed", "error", err)
|
||||
ReportErrorf("Prune failed: %v", err)
|
||||
}
|
||||
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when prune completes
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
err = v.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
OnStop: func(_ context.Context) error {
|
||||
log.Debug("Stopping prune operation")
|
||||
v.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// NewPurgeCommand creates the purge command
|
||||
func NewPurgeCommand() *cobra.Command {
|
||||
opts := &vaultik.SnapshotPurgeOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "purge",
|
||||
Short: "Purge old snapshots",
|
||||
Long: `Removes snapshots based on age or count criteria.
|
||||
|
||||
This command allows you to:
|
||||
- Keep only the latest snapshot per name (--keep-latest)
|
||||
- Remove snapshots older than a specific duration (--older-than)
|
||||
- Filter to a specific snapshot name (--name)
|
||||
|
||||
When --keep-latest is used, retention is applied per snapshot name. For example,
|
||||
if you have snapshots named "home" and "system", --keep-latest keeps the most
|
||||
recent of each.
|
||||
|
||||
Use --name to restrict the purge to a single snapshot name.
|
||||
|
||||
Config is located at /etc/vaultik/config.yml by default, but can be overridden by
|
||||
specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Validate flags
|
||||
if !opts.KeepLatest && opts.OlderThan == "" {
|
||||
return fmt.Errorf("must specify either --keep-latest or --older-than")
|
||||
}
|
||||
if opts.KeepLatest && opts.OlderThan != "" {
|
||||
return fmt.Errorf("cannot specify both --keep-latest and --older-than")
|
||||
}
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Use the app framework like other commands
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
// Start the purge operation in a goroutine
|
||||
go func() {
|
||||
// Run the purge operation
|
||||
if err := v.PurgeSnapshotsWithOptions(opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Purge operation failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when purge completes
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping purge operation")
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, "Keep only the latest snapshot per name")
|
||||
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "", "Remove snapshots older than duration (e.g. 30d, 6m, 1y)")
|
||||
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompts")
|
||||
cmd.Flags().StringVar(&opts.Name, "name", "", "Filter purge to a specific snapshot name")
|
||||
|
||||
return cmd
|
||||
}
|
||||
+56
-10
@@ -2,14 +2,19 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// errNukeNeedsForce guards the destructive 'remote nuke' subcommand.
|
||||
var errNukeNeedsForce = errors.New(
|
||||
"remote nuke requires --force (this deletes ALL remote snapshots and blobs)")
|
||||
|
||||
// NewRemoteCommand creates the remote command and subcommands
|
||||
func NewRemoteCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
@@ -20,6 +25,38 @@ func NewRemoteCommand() *cobra.Command {
|
||||
|
||||
// Add subcommands
|
||||
cmd.AddCommand(newRemoteInfoCommand())
|
||||
cmd.AddCommand(newRemoteNukeCommand())
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// newRemoteNukeCommand creates the 'remote nuke' subcommand.
|
||||
func newRemoteNukeCommand() *cobra.Command {
|
||||
var force bool
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "nuke",
|
||||
Short: "Delete ALL snapshot metadata and blobs from the backup destination store",
|
||||
Long: `Removes every snapshot's metadata and every blob from remote
|
||||
storage. After this command completes successfully the bucket prefix is
|
||||
empty and the next backup starts from scratch.
|
||||
|
||||
This is destructive and irreversible. Requires --force.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if !force {
|
||||
return errNukeNeedsForce
|
||||
}
|
||||
|
||||
return runVaultikApp(cmd, false, false, "Remote nuke failed",
|
||||
func(v *vaultik.Vaultik) error {
|
||||
return v.NukeRemote(true)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&force, "force", false,
|
||||
"Required: confirm destruction of ALL remote data")
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -37,7 +74,7 @@ func newRemoteInfoCommand() *cobra.Command {
|
||||
- Count and size of referenced blobs (from all manifests)
|
||||
- Count and size of orphaned blobs (not referenced by any manifest)`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
@@ -45,35 +82,44 @@ func newRemoteInfoCommand() *cobra.Command {
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet || jsonOutput,
|
||||
Quiet: rootFlags.Quiet,
|
||||
JSON: jsonOutput,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
OnStart: func(_ context.Context) error {
|
||||
go func() {
|
||||
if err := v.RemoteInfo(jsonOutput); err != nil {
|
||||
if err != context.Canceled {
|
||||
err := v.RemoteInfo(jsonOutput)
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
if !jsonOutput {
|
||||
log.Error("Failed to get remote info", "error", err)
|
||||
ReportErrorf("Failed to get remote info: %v", err)
|
||||
}
|
||||
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
|
||||
err = v.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
OnStop: func(_ context.Context) error {
|
||||
v.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
+82
-22
@@ -1,12 +1,19 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/adrg/xdg"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// errConfigNotFound is wrapped by all config-resolution failures.
|
||||
var errConfigNotFound = errors.New("config file not found")
|
||||
|
||||
// RootFlags holds global flags that apply to all commands.
|
||||
// These flags are defined on the root command and inherited by all subcommands.
|
||||
type RootFlags struct {
|
||||
@@ -14,8 +21,10 @@ type RootFlags struct {
|
||||
Verbose bool
|
||||
Debug bool
|
||||
Quiet bool
|
||||
SkipErrors bool
|
||||
}
|
||||
|
||||
//nolint:gochecknoglobals // cobra persistent flags bind to package state
|
||||
var rootFlags RootFlags
|
||||
|
||||
// NewRootCommand creates the root cobra command for the vaultik CLI.
|
||||
@@ -25,24 +34,36 @@ func NewRootCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "vaultik",
|
||||
Short: "Secure incremental backup tool with asymmetric encryption",
|
||||
Long: `vaultik is a secure incremental backup daemon that encrypts data using age
|
||||
Long: `vaultik is a secure incremental backup tool that encrypts data using age
|
||||
public keys and uploads to S3-compatible storage. No private keys are needed
|
||||
on the source system.`,
|
||||
SilenceUsage: true,
|
||||
// Bare 'vaultik' (no subcommand): print help. The banner is
|
||||
// printed once at process startup by Entry, before cobra
|
||||
// parses arguments, so it appears even when cobra rejects
|
||||
// args (e.g. "requires at least 2 arg(s)") and on --help.
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
_ = cmd.Help()
|
||||
},
|
||||
}
|
||||
|
||||
// Add global flags
|
||||
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "", "Path to config file (default: $VAULTIK_CONFIG or /etc/vaultik/config.yml)")
|
||||
cmd.PersistentFlags().BoolVarP(&rootFlags.Verbose, "verbose", "v", false, "Enable verbose output")
|
||||
cmd.PersistentFlags().BoolVar(&rootFlags.Debug, "debug", false, "Enable debug output")
|
||||
cmd.PersistentFlags().BoolVarP(&rootFlags.Quiet, "quiet", "q", false, "Suppress non-error output")
|
||||
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "",
|
||||
"Path to config file (default: $VAULTIK_CONFIG or platform config dir)")
|
||||
cmd.PersistentFlags().BoolVarP(&rootFlags.Verbose, "verbose", "v", false,
|
||||
"Enable verbose output")
|
||||
cmd.PersistentFlags().BoolVar(&rootFlags.Debug, "debug", false,
|
||||
"Enable debug output")
|
||||
cmd.PersistentFlags().BoolVarP(&rootFlags.Quiet, "quiet", "q", false,
|
||||
"Suppress non-error output")
|
||||
cmd.PersistentFlags().BoolVar(&rootFlags.SkipErrors, "skip-errors", false,
|
||||
"Continue past per-file errors instead of aborting "+
|
||||
"(applies to snapshot create and restore)")
|
||||
|
||||
// Add subcommands
|
||||
cmd.AddCommand(
|
||||
NewRestoreCommand(),
|
||||
NewConfigCommand(),
|
||||
NewPruneCommand(),
|
||||
NewVerifyCommand(),
|
||||
NewStoreCommand(),
|
||||
NewSnapshotCommand(),
|
||||
NewInfoCommand(),
|
||||
NewVersionCommand(),
|
||||
@@ -60,25 +81,64 @@ func GetRootFlags() RootFlags {
|
||||
}
|
||||
|
||||
// ResolveConfigPath resolves the config file path from flags, environment, or default.
|
||||
// It checks in order: 1) --config flag, 2) VAULTIK_CONFIG environment variable,
|
||||
// 3) default location /etc/vaultik/config.yml. Returns an error if no valid
|
||||
// config file can be found through any of these methods.
|
||||
// Search order: --config flag, VAULTIK_CONFIG env, XDG config dir,
|
||||
// /etc/vaultik/config.yml.
|
||||
// Explicit paths from --config and $VAULTIK_CONFIG are checked for existence
|
||||
// so the user gets a clear error instead of a downstream YAML parser failure.
|
||||
func ResolveConfigPath() (string, error) {
|
||||
// First check global flag
|
||||
if rootFlags.ConfigPath != "" {
|
||||
return rootFlags.ConfigPath, nil
|
||||
if path := rootFlags.ConfigPath; path != "" {
|
||||
_, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf(
|
||||
"%w: from --config: %s (run 'vaultik config init --config %s' to create it)",
|
||||
errConfigNotFound, path, path)
|
||||
}
|
||||
|
||||
// Then check environment variable
|
||||
if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" {
|
||||
return envPath, nil
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// Finally check default location
|
||||
defaultPath := "/etc/vaultik/config.yml"
|
||||
if _, err := os.Stat(defaultPath); err == nil {
|
||||
return defaultPath, nil
|
||||
if path := os.Getenv("VAULTIK_CONFIG"); path != "" {
|
||||
_, err := os.Stat(path) //nolint:gosec // G703: path is operator-supplied by design
|
||||
if err != nil {
|
||||
return "", fmt.Errorf(
|
||||
"%w: from $VAULTIK_CONFIG: %s (unset VAULTIK_CONFIG, point it at "+
|
||||
"an existing file, or run 'vaultik config init')",
|
||||
errConfigNotFound, path)
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("no config file specified, VAULTIK_CONFIG not set, and %s not found", defaultPath)
|
||||
return path, nil
|
||||
}
|
||||
|
||||
for _, path := range defaultConfigPaths() {
|
||||
_, err := os.Stat(path)
|
||||
if err == nil {
|
||||
return path, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf(
|
||||
"%w: searched %s (run 'vaultik config init' to create the default "+
|
||||
"config, or pass --config <path>)",
|
||||
errConfigNotFound, strings.Join(defaultConfigPaths(), " or "))
|
||||
}
|
||||
|
||||
// defaultConfigPaths returns the ordered list of config paths to search.
|
||||
// On macOS: ~/Library/Application Support/vaultik/config.yml
|
||||
// On Linux: ~/.config/vaultik/config.yml
|
||||
// Fallback: /etc/vaultik/config.yml
|
||||
func defaultConfigPaths() []string {
|
||||
return []string{
|
||||
filepath.Join(xdg.ConfigHome, "vaultik", "config.yml"),
|
||||
"/etc/vaultik/config.yml",
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultConfigPath returns the platform-appropriate default config path.
|
||||
// Used by the init command and in help text.
|
||||
func DefaultConfigPath() string {
|
||||
if os.Getuid() == 0 {
|
||||
return "/etc/vaultik/config.yml"
|
||||
}
|
||||
|
||||
return filepath.Join(xdg.ConfigHome, "vaultik", "config.yml")
|
||||
}
|
||||
|
||||
+121
-258
@@ -2,15 +2,42 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
var (
|
||||
errSnapshotIDRequired = errors.New("snapshot ID required")
|
||||
errWrongArgCount = errors.New("wrong argument count")
|
||||
errPurgeCriteriaNeeded = errors.New(
|
||||
"must specify either --keep-latest or --older-than")
|
||||
errPurgeCriteriaBoth = errors.New(
|
||||
"cannot specify both --keep-latest and --older-than")
|
||||
)
|
||||
|
||||
// requireSnapshotIDArg validates that exactly one positional argument
|
||||
// (the snapshot ID) was supplied, printing help otherwise.
|
||||
func requireSnapshotIDArg(cmd *cobra.Command, args []string) error {
|
||||
if len(args) != 1 {
|
||||
_ = cmd.Help()
|
||||
|
||||
if len(args) == 0 {
|
||||
return errSnapshotIDRequired
|
||||
}
|
||||
|
||||
return fmt.Errorf("%w: expected 1 argument, got %d",
|
||||
errWrongArgCount, len(args))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewSnapshotCommand creates the snapshot command and subcommands
|
||||
func NewSnapshotCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
@@ -25,7 +52,7 @@ func NewSnapshotCommand() *cobra.Command {
|
||||
cmd.AddCommand(newSnapshotPurgeCommand())
|
||||
cmd.AddCommand(newSnapshotVerifyCommand())
|
||||
cmd.AddCommand(newSnapshotRemoveCommand())
|
||||
cmd.AddCommand(newSnapshotPruneCommand())
|
||||
cmd.AddCommand(newSnapshotRestoreCommand())
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -48,6 +75,8 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Pass snapshot names from args
|
||||
opts.Snapshots = args
|
||||
// --skip-errors is a global flag on the root command.
|
||||
opts.SkipErrors = rootFlags.SkipErrors
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
@@ -56,9 +85,10 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
|
||||
// Use the backup functionality from cli package
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Cron: opts.Cron,
|
||||
@@ -68,27 +98,33 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
OnStart: func(_ context.Context) error {
|
||||
// Start the snapshot creation in a goroutine
|
||||
go func() {
|
||||
// Run the snapshot creation
|
||||
if err := v.CreateSnapshot(opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
// --cron suppression is wired through v.UI by setupGlobals.
|
||||
err := v.CreateSnapshot(opts)
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
log.Error("Snapshot creation failed", "error", err)
|
||||
ReportErrorf("Snapshot creation failed: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when snapshot completes
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
err = v.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
OnStop: func(_ context.Context) error {
|
||||
log.Debug("Stopping snapshot creation")
|
||||
// Cancel the Vaultik context
|
||||
v.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
@@ -98,10 +134,15 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.Daemon, "daemon", false, "Run in daemon mode with inotify monitoring")
|
||||
cmd.Flags().BoolVar(&opts.Cron, "cron", false, "Run in cron mode (silent unless error)")
|
||||
cmd.Flags().BoolVar(&opts.Prune, "prune", false, "Delete all previous snapshots and unreferenced blobs after backup")
|
||||
cmd.Flags().BoolVar(&opts.SkipErrors, "skip-errors", false, "Skip file read errors (log them loudly but continue)")
|
||||
cmd.Flags().BoolVar(&opts.Cron, "cron", false,
|
||||
"Run in cron mode (silent unless warning or error)")
|
||||
cmd.Flags().BoolVar(&opts.Prune, "prune", false,
|
||||
"After backup, drop older snapshots of the same name and remove "+
|
||||
"orphaned blobs")
|
||||
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "",
|
||||
"With --prune: keep snapshots newer than this duration "+
|
||||
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+
|
||||
"instead of only the latest")
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -116,46 +157,11 @@ func newSnapshotListCommand() *cobra.Command {
|
||||
Short: "List all snapshots",
|
||||
Long: "Lists all snapshots with their ID, timestamp, and compressed size",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if err := v.ListSnapshots(jsonOutput); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Failed to list snapshots", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
return runVaultikApp(cmd, false, false,
|
||||
"Failed to list snapshots",
|
||||
func(v *vaultik.Vaultik) error {
|
||||
return v.ListSnapshots(jsonOutput)
|
||||
})
|
||||
},
|
||||
}
|
||||
@@ -174,68 +180,36 @@ func newSnapshotPurgeCommand() *cobra.Command {
|
||||
Short: "Purge old snapshots",
|
||||
Long: `Removes snapshots based on age or count criteria.
|
||||
|
||||
When --keep-latest is used, retention is applied per snapshot name. For example,
|
||||
if you have snapshots named "home" and "system", --keep-latest keeps the most
|
||||
recent of each.
|
||||
|
||||
Use --name to restrict the purge to a single snapshot name.`,
|
||||
Retention is per-snapshot-name: --keep-latest keeps the latest of each
|
||||
configured snapshot name, not the latest globally. Use --snapshot to
|
||||
restrict the operation to specific snapshot names.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
// Validate flags
|
||||
if !opts.KeepLatest && opts.OlderThan == "" {
|
||||
return fmt.Errorf("must specify either --keep-latest or --older-than")
|
||||
return errPurgeCriteriaNeeded
|
||||
}
|
||||
|
||||
if opts.KeepLatest && opts.OlderThan != "" {
|
||||
return fmt.Errorf("cannot specify both --keep-latest and --older-than")
|
||||
return errPurgeCriteriaBoth
|
||||
}
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if err := v.PurgeSnapshotsWithOptions(opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Failed to purge snapshots", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
return runVaultikApp(cmd, false, false,
|
||||
"Failed to purge snapshots",
|
||||
func(v *vaultik.Vaultik) error {
|
||||
return v.PurgeSnapshotsWithOptions(opts)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, "Keep only the latest snapshot per name")
|
||||
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "", "Remove snapshots older than duration (e.g., 30d, 6m, 1y)")
|
||||
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false,
|
||||
"Keep only the latest snapshot of each name")
|
||||
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "",
|
||||
"Remove snapshots older than duration "+
|
||||
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months)")
|
||||
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt")
|
||||
cmd.Flags().StringVar(&opts.Name, "name", "", "Filter purge to a specific snapshot name")
|
||||
cmd.Flags().StringArrayVar(&opts.Names, "snapshot", nil,
|
||||
"Restrict to snapshots with these names (repeat for multiple)")
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -248,16 +222,7 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
||||
Use: "verify <snapshot-id>",
|
||||
Short: "Verify snapshot integrity",
|
||||
Long: "Verifies that all blobs referenced in a snapshot exist",
|
||||
Args: func(cmd *cobra.Command, args []string) error {
|
||||
if len(args) != 1 {
|
||||
_ = cmd.Help()
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("snapshot ID required")
|
||||
}
|
||||
return fmt.Errorf("expected 1 argument, got %d", len(args))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Args: requireSnapshotIDArg,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
snapshotID := args[0]
|
||||
|
||||
@@ -268,41 +233,44 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet || opts.JSON,
|
||||
Quiet: rootFlags.Quiet,
|
||||
JSON: opts.JSON,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
OnStart: func(_ context.Context) error {
|
||||
go func() {
|
||||
var err error
|
||||
if opts.Deep {
|
||||
err = v.RunDeepVerify(snapshotID, opts)
|
||||
} else {
|
||||
err = v.VerifySnapshotWithOptions(snapshotID, opts)
|
||||
}
|
||||
err := v.VerifySnapshotWithOptions(snapshotID, opts)
|
||||
if err != nil {
|
||||
if err != context.Canceled {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
if !opts.JSON {
|
||||
log.Error("Verification failed", "error", err)
|
||||
ReportErrorf("Verification failed: %v", err)
|
||||
}
|
||||
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
|
||||
err = v.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
OnStop: func(_ context.Context) error {
|
||||
v.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
@@ -323,150 +291,45 @@ func newSnapshotRemoveCommand() *cobra.Command {
|
||||
opts := &vaultik.RemoveOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "remove [snapshot-id]",
|
||||
Use: "remove <snapshot-id>",
|
||||
Aliases: []string{"rm"},
|
||||
Short: "Remove a snapshot from the local database",
|
||||
Long: `Removes a snapshot from the local database.
|
||||
Short: "Remove a snapshot from local index and remote metadata",
|
||||
Long: `Removes a snapshot.
|
||||
|
||||
By default, only removes from the local database. Use --remote to also remove
|
||||
the snapshot metadata from remote storage.
|
||||
By default, this removes the snapshot from the local index database and
|
||||
strips the snapshot's metadata from the backup destination store. Blobs
|
||||
are NOT touched: deleting them requires reading every remaining remote
|
||||
manifest (the destination store may hold snapshots this host doesn't
|
||||
know about), which is what 'vaultik prune' does. On success the command
|
||||
prints the exact 'vaultik prune' invocation to run as a follow-up.
|
||||
|
||||
Note: This does NOT remove blobs. Use 'vaultik prune' to remove orphaned blobs
|
||||
after removing snapshots.
|
||||
Use --local-only to skip the remote half (e.g. when you want to forget a
|
||||
snapshot locally without touching the destination store).
|
||||
|
||||
Use --all --force to remove all snapshots.`,
|
||||
Args: func(cmd *cobra.Command, args []string) error {
|
||||
all, _ := cmd.Flags().GetBool("all")
|
||||
if all {
|
||||
if len(args) > 0 {
|
||||
_ = cmd.Help()
|
||||
return fmt.Errorf("--all cannot be used with a snapshot ID")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if len(args) != 1 {
|
||||
_ = cmd.Help()
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("snapshot ID required (or use --all --force)")
|
||||
}
|
||||
return fmt.Errorf("expected 1 argument, got %d", len(args))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
If the remote is unreachable, the local-database removal still completes
|
||||
and a warning is emitted; rerun 'vaultik prune' once the destination store
|
||||
is reachable to finish remote cleanup.
|
||||
|
||||
To wipe the entire destination store and start over, use 'vaultik remote
|
||||
nuke --force' — it is the single supported entry point for that.`,
|
||||
Args: requireSnapshotIDArg,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return runVaultikApp(cmd, opts.JSON, opts.JSON,
|
||||
"Failed to remove snapshot",
|
||||
func(v *vaultik.Vaultik) error {
|
||||
_, err := v.RemoveSnapshot(args[0], opts)
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet || opts.JSON,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
var err error
|
||||
if opts.All {
|
||||
_, err = v.RemoveAllSnapshots(opts)
|
||||
} else {
|
||||
_, err = v.RemoveSnapshot(args[0], opts)
|
||||
}
|
||||
if err != nil {
|
||||
if err != context.Canceled {
|
||||
if !opts.JSON {
|
||||
log.Error("Failed to remove snapshot", "error", err)
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
return err
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVarP(&opts.Force, "force", "f", false, "Skip confirmation prompt")
|
||||
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Show what would be removed without removing")
|
||||
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false,
|
||||
"Show what would be removed without removing")
|
||||
cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output result as JSON")
|
||||
cmd.Flags().BoolVar(&opts.Remote, "remote", false, "Also remove snapshot metadata from remote storage")
|
||||
cmd.Flags().BoolVar(&opts.All, "all", false, "Remove all snapshots (requires --force)")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// newSnapshotPruneCommand creates the 'snapshot prune' subcommand
|
||||
func newSnapshotPruneCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "prune",
|
||||
Short: "Remove orphaned data from local database",
|
||||
Long: `Removes orphaned files, chunks, and blobs from the local database.
|
||||
|
||||
This cleans up data that is no longer referenced by any snapshot, which can
|
||||
accumulate from incomplete backups or deleted snapshots.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if _, err := v.PruneDatabase(); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Failed to prune database", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
cmd.Flags().BoolVar(&opts.LocalOnly, "local-only", false,
|
||||
"Skip remote cleanup; only touch the local index")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
@@ -2,16 +2,22 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/globals"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// restoreMinArgs is the minimum positional argument count of
|
||||
// `snapshot restore <snapshot-id> <target-dir> [paths...]`.
|
||||
const restoreMinArgs = 2
|
||||
|
||||
// RestoreOptions contains options for the restore command
|
||||
type RestoreOptions struct {
|
||||
TargetDir string
|
||||
@@ -28,40 +34,45 @@ type RestoreApp struct {
|
||||
Shutdowner fx.Shutdowner
|
||||
}
|
||||
|
||||
// NewRestoreCommand creates the restore command
|
||||
func NewRestoreCommand() *cobra.Command {
|
||||
// newSnapshotRestoreCommand creates the 'snapshot restore' subcommand
|
||||
func newSnapshotRestoreCommand() *cobra.Command {
|
||||
opts := &RestoreOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "restore <snapshot-id> <target-dir> [paths...]",
|
||||
Short: "Restore files from backup",
|
||||
Short: "Restore files from a snapshot",
|
||||
Long: `Download and decrypt files from a backup snapshot.
|
||||
|
||||
This command will restore files from the specified snapshot to the target directory.
|
||||
This command will restore files from the specified snapshot to the
|
||||
target directory.
|
||||
If no paths are specified, all files are restored.
|
||||
If paths are specified, only matching files/directories are restored.
|
||||
|
||||
Requires the VAULTIK_AGE_SECRET_KEY environment variable to be set with the age private key.
|
||||
Requires the VAULTIK_AGE_SECRET_KEY environment variable to be set with
|
||||
the age private key.
|
||||
|
||||
Examples:
|
||||
# Restore entire snapshot
|
||||
vaultik restore myhost_docs_2025-01-01T12:00:00Z /restore
|
||||
vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore
|
||||
|
||||
# Restore specific file
|
||||
vaultik restore myhost_docs_2025-01-01T12:00:00Z /restore /home/user/important.txt
|
||||
vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore \
|
||||
/home/user/important.txt
|
||||
|
||||
# Restore specific directory
|
||||
vaultik restore myhost_docs_2025-01-01T12:00:00Z /restore /home/user/documents/
|
||||
vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore \
|
||||
/home/user/documents/
|
||||
|
||||
# Restore and verify all files
|
||||
vaultik restore --verify myhost_docs_2025-01-01T12:00:00Z /restore`,
|
||||
Args: cobra.MinimumNArgs(2),
|
||||
vaultik snapshot restore --verify myhost_docs_2025-01-01T12:00:00Z /restore`,
|
||||
Args: cobra.MinimumNArgs(restoreMinArgs),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runRestore(cmd, args, opts)
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.Verify, "verify", false, "Verify restored files by checking chunk hashes")
|
||||
cmd.Flags().BoolVar(&opts.Verify, "verify", false,
|
||||
"Verify restored files by checking chunk hashes")
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -69,9 +80,10 @@ Examples:
|
||||
// runRestore parses arguments and runs the restore operation through the app framework
|
||||
func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
|
||||
snapshotID := args[0]
|
||||
|
||||
opts.TargetDir = args[1]
|
||||
if len(args) > 2 {
|
||||
opts.Paths = args[2:]
|
||||
if len(args) > restoreMinArgs {
|
||||
opts.Paths = args[restoreMinArgs:]
|
||||
}
|
||||
|
||||
// Use unified config resolution
|
||||
@@ -82,9 +94,10 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
|
||||
|
||||
// Use the app framework like other commands
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
@@ -117,7 +130,7 @@ func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option {
|
||||
return []fx.Option{
|
||||
fx.Invoke(func(app *RestoreApp, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
OnStart: func(_ context.Context) error {
|
||||
// Start the restore operation in a goroutine
|
||||
go func() {
|
||||
// Run the restore operation
|
||||
@@ -126,23 +139,31 @@ func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option {
|
||||
TargetDir: opts.TargetDir,
|
||||
Paths: opts.Paths,
|
||||
Verify: opts.Verify,
|
||||
SkipErrors: GetRootFlags().SkipErrors,
|
||||
}
|
||||
if err := app.Vaultik.Restore(restoreOpts); err != nil {
|
||||
if err != context.Canceled {
|
||||
|
||||
err := app.Vaultik.Restore(restoreOpts)
|
||||
if err != nil {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
log.Error("Restore operation failed", "error", err)
|
||||
ReportErrorf("Restore failed: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when restore completes
|
||||
if err := app.Shutdowner.Shutdown(); err != nil {
|
||||
err = app.Shutdowner.Shutdown()
|
||||
if err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
OnStop: func(_ context.Context) error {
|
||||
log.Debug("Stopping restore operation")
|
||||
app.Vaultik.Cancel()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
@@ -1,158 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// StoreApp contains dependencies for store commands
|
||||
type StoreApp struct {
|
||||
Storage storage.Storer
|
||||
Shutdowner fx.Shutdowner
|
||||
}
|
||||
|
||||
// NewStoreCommand creates the store command and subcommands
|
||||
func NewStoreCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "store",
|
||||
Short: "Storage information commands",
|
||||
Long: "Commands for viewing information about the storage backend",
|
||||
}
|
||||
|
||||
// Add subcommands
|
||||
cmd.AddCommand(newStoreInfoCommand())
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// newStoreInfoCommand creates the 'store info' subcommand
|
||||
func newStoreInfoCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "info",
|
||||
Short: "Display storage information",
|
||||
Long: "Shows storage configuration and statistics including snapshots and blobs",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runWithApp(cmd.Context(), func(app *StoreApp) error {
|
||||
return app.Info(cmd.Context())
|
||||
})
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Info displays storage information
|
||||
func (app *StoreApp) Info(ctx context.Context) error {
|
||||
// Get storage info
|
||||
storageInfo := app.Storage.Info()
|
||||
|
||||
fmt.Printf("Storage Information\n")
|
||||
fmt.Printf("==================\n\n")
|
||||
fmt.Printf("Storage Configuration:\n")
|
||||
fmt.Printf(" Type: %s\n", storageInfo.Type)
|
||||
fmt.Printf(" Location: %s\n\n", storageInfo.Location)
|
||||
|
||||
// Count snapshots by listing metadata/ prefix
|
||||
snapshotCount := 0
|
||||
snapshotCh := app.Storage.ListStream(ctx, "metadata/")
|
||||
snapshotDirs := make(map[string]bool)
|
||||
|
||||
for object := range snapshotCh {
|
||||
if object.Err != nil {
|
||||
return fmt.Errorf("listing snapshots: %w", object.Err)
|
||||
}
|
||||
// Extract snapshot ID from path like metadata/2024-01-15-143052-hostname/
|
||||
parts := strings.Split(object.Key, "/")
|
||||
if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" {
|
||||
snapshotDirs[parts[1]] = true
|
||||
}
|
||||
}
|
||||
snapshotCount = len(snapshotDirs)
|
||||
|
||||
// Count blobs and calculate total size by listing blobs/ prefix
|
||||
blobCount := 0
|
||||
var totalSize int64
|
||||
|
||||
blobCh := app.Storage.ListStream(ctx, "blobs/")
|
||||
for object := range blobCh {
|
||||
if object.Err != nil {
|
||||
return fmt.Errorf("listing blobs: %w", object.Err)
|
||||
}
|
||||
if !strings.HasSuffix(object.Key, "/") { // Skip directories
|
||||
blobCount++
|
||||
totalSize += object.Size
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("Storage Statistics:\n")
|
||||
fmt.Printf(" Snapshots: %d\n", snapshotCount)
|
||||
fmt.Printf(" Blobs: %d\n", blobCount)
|
||||
fmt.Printf(" Total Size: %s\n", formatBytes(totalSize))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// formatBytes formats bytes into human-readable format
|
||||
func formatBytes(bytes int64) string {
|
||||
const unit = 1024
|
||||
if bytes < unit {
|
||||
return fmt.Sprintf("%d B", bytes)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for n := bytes / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
// runWithApp creates the FX app and runs the given function
|
||||
func runWithApp(ctx context.Context, fn func(*StoreApp) error) error {
|
||||
var result error
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = RunWithApp(ctx, AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
},
|
||||
Modules: []fx.Option{
|
||||
fx.Provide(func(storer storage.Storer, shutdowner fx.Shutdowner) *StoreApp {
|
||||
return &StoreApp{
|
||||
Storage: storer,
|
||||
Shutdowner: shutdowner,
|
||||
}
|
||||
}),
|
||||
},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(app *StoreApp, shutdowner fx.Shutdowner) {
|
||||
result = fn(app)
|
||||
// Shutdown after command completes
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond) // Brief delay to ensure clean shutdown
|
||||
if err := shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
}),
|
||||
},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
package cli
|
||||
|
||||
import "time"
|
||||
|
||||
// SnapshotInfo represents snapshot information for listing
|
||||
type SnapshotInfo struct {
|
||||
ID string `json:"id"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
CompressedSize int64 `json:"compressed_size"`
|
||||
}
|
||||
@@ -1,98 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// NewVerifyCommand creates the verify command
|
||||
func NewVerifyCommand() *cobra.Command {
|
||||
opts := &vaultik.VerifyOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "verify <snapshot-id>",
|
||||
Short: "Verify snapshot integrity",
|
||||
Long: `Verifies that all blobs referenced in a snapshot exist and optionally verifies their contents.
|
||||
|
||||
Shallow verification (default):
|
||||
- Downloads and decompresses manifest
|
||||
- Checks existence of all blobs in S3
|
||||
- Reports missing blobs
|
||||
|
||||
Deep verification (--deep):
|
||||
- Downloads and decrypts database
|
||||
- Verifies blob lists match between manifest and database
|
||||
- Downloads, decrypts, and decompresses each blob
|
||||
- Verifies SHA256 hash of each chunk matches database
|
||||
- Ensures chunks are ordered correctly
|
||||
|
||||
The command will fail immediately on any verification error and exit with non-zero status.`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
snapshotID := args[0]
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Use the app framework for all verification
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet || opts.JSON, // Suppress log output in JSON mode
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
// Run the verify operation directly
|
||||
go func() {
|
||||
var err error
|
||||
if opts.Deep {
|
||||
err = v.RunDeepVerify(snapshotID, opts)
|
||||
} else {
|
||||
err = v.VerifySnapshotWithOptions(snapshotID, opts)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if err != context.Canceled {
|
||||
if !opts.JSON {
|
||||
log.Error("Verification failed", "error", err)
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping verify operation")
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.Deep, "deep", false, "Perform deep verification by downloading and verifying all blob contents")
|
||||
cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output verification results as JSON")
|
||||
|
||||
return cmd
|
||||
}
|
||||
+29
-6
@@ -2,10 +2,11 @@ package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"runtime"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"github.com/spf13/cobra"
|
||||
"sneak.berlin/go/vaultik/internal/globals"
|
||||
)
|
||||
|
||||
// NewVersionCommand creates the version command
|
||||
@@ -15,13 +16,35 @@ func NewVersionCommand() *cobra.Command {
|
||||
Short: "Print version information",
|
||||
Long: `Print version, git commit, and build information for vaultik.`,
|
||||
Args: cobra.NoArgs,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Printf("vaultik %s\n", globals.Version)
|
||||
fmt.Printf(" commit: %s\n", globals.Commit)
|
||||
fmt.Printf(" go: %s\n", runtime.Version())
|
||||
fmt.Printf(" os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
writeVersion(cmd.OutOrStdout())
|
||||
},
|
||||
}
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// writeVersion prints the version report. It takes a writer rather than
|
||||
// using os.Stdout directly so the output can be asserted on in tests.
|
||||
func writeVersion(w io.Writer) {
|
||||
_, _ = fmt.Fprintf(w, "vaultik %s\n", globals.Version)
|
||||
_, _ = fmt.Fprintf(w, " commit: %s\n", globals.Commit)
|
||||
_, _ = fmt.Fprintf(w, " build date: %s\n", globals.CommitDate)
|
||||
_, _ = fmt.Fprintf(w, " go: %s\n", runtime.Version())
|
||||
_, _ = fmt.Fprintf(w, " os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
|
||||
_, _ = fmt.Fprintf(w, " author: %s\n", globals.Author)
|
||||
_, _ = fmt.Fprintf(w, " homepage: %s\n", globals.Homepage)
|
||||
_, _ = fmt.Fprintf(w, " license: %s\n", globals.License)
|
||||
|
||||
if globals.IsDevVersion(globals.Version) {
|
||||
_, _ = fmt.Fprintln(w)
|
||||
_, _ = fmt.Fprintln(w,
|
||||
"This is a development build: it was not built from a tagged")
|
||||
_, _ = fmt.Fprintln(w,
|
||||
"commit, so it carries no release version. Released binaries")
|
||||
_, _ = fmt.Fprintf(w,
|
||||
"are published at %s\n", globals.ReleasesURL)
|
||||
_, _ = fmt.Fprintln(w,
|
||||
"and report their tag on the first line above.")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package cli_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/cli"
|
||||
"sneak.berlin/go/vaultik/internal/globals"
|
||||
)
|
||||
|
||||
// runVersionCommand executes `vaultik version` with its output
|
||||
// captured, and returns what it printed.
|
||||
func runVersionCommand(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
cmd := cli.NewVersionCommand()
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
cmd.SetOut(&out)
|
||||
cmd.SetErr(&out)
|
||||
cmd.SetArgs([]string{})
|
||||
|
||||
err := cmd.Execute()
|
||||
if err != nil {
|
||||
t.Fatalf("version command failed: %v", err)
|
||||
}
|
||||
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// TestVersionCommandReportsBuildVersion checks that the first line of
|
||||
// the report is the version the binary was actually built with. The
|
||||
// test binary carries no -ldflags, so that is the "dev" default -- the
|
||||
// same string an untagged `make vaultik` build stamps a prefix of.
|
||||
func TestVersionCommandReportsBuildVersion(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
out := runVersionCommand(t)
|
||||
|
||||
wantFirst := "vaultik " + globals.Version
|
||||
if first, _, _ := strings.Cut(out, "\n"); first != wantFirst {
|
||||
t.Errorf("first line = %q, want %q", first, wantFirst)
|
||||
}
|
||||
|
||||
if !strings.Contains(out, "commit:") {
|
||||
t.Error("output does not report the commit")
|
||||
}
|
||||
}
|
||||
|
||||
// TestVersionCommandFlagsDevelopmentBuild is the regression test for
|
||||
// the thing this command exists to prevent: a build that is not a
|
||||
// release must say so. The notice used to be gated on the version
|
||||
// being exactly "dev", so once untagged builds started carrying their
|
||||
// commit sha it would have gone silent and an unreleased binary would
|
||||
// have looked like a release.
|
||||
func TestVersionCommandFlagsDevelopmentBuild(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if !globals.IsDevVersion(globals.Version) {
|
||||
t.Skipf("test binary was stamped with release version %q",
|
||||
globals.Version)
|
||||
}
|
||||
|
||||
out := runVersionCommand(t)
|
||||
|
||||
if !strings.Contains(out, "development build") {
|
||||
t.Errorf("dev build did not print the development-build notice:\n%s",
|
||||
out)
|
||||
}
|
||||
|
||||
if !strings.Contains(out, globals.ReleasesURL) {
|
||||
t.Errorf("development-build notice does not point at %s:\n%s",
|
||||
globals.ReleasesURL, out)
|
||||
}
|
||||
}
|
||||
+119
-55
@@ -1,33 +1,72 @@
|
||||
// Package config loads, validates, and provides the vaultik YAML
|
||||
// configuration, including snapshot definitions, encryption recipients,
|
||||
// and storage settings.
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"filippo.io/age"
|
||||
"git.eeqj.de/sneak/smartconfig"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"github.com/adrg/xdg"
|
||||
"go.uber.org/fx"
|
||||
"gopkg.in/yaml.v3"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
const appName = "berlin.sneak.app.vaultik"
|
||||
const appName = "vaultik"
|
||||
|
||||
// Defaults and validation bounds for tunable settings.
|
||||
const (
|
||||
defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB
|
||||
defaultChunkSize = Size(10 * 1024 * 1024) // 10MB
|
||||
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MB
|
||||
defaultCompressionLevel = 3
|
||||
minChunkSize = 1024 * 1024 // 1MB
|
||||
minCompressionLevel = 1
|
||||
maxCompressionLevel = 19
|
||||
)
|
||||
|
||||
// Sentinel validation errors.
|
||||
var (
|
||||
errNoConfigPath = errors.New("config path not provided")
|
||||
errNoAgeRecipients = errors.New(
|
||||
"at least one age_recipient is required (generate with: age-keygen)")
|
||||
errNoSnapshots = errors.New(
|
||||
"at least one snapshot must be configured (see config.example.yml)")
|
||||
errSnapshotNoPaths = errors.New("snapshot must have at least one path")
|
||||
errChunkSizeTooSmall = errors.New("chunk_size must be at least 1MB")
|
||||
errBlobSizeTooSmall = errors.New("blob_size_limit must be at least chunk_size")
|
||||
errBadCompression = errors.New("compression_level must be between 1 and 19")
|
||||
errBadStorageScheme = errors.New(
|
||||
"storage_url must start with s3://, file://, or rclone://")
|
||||
errStorageNotConfigured = errors.New(
|
||||
"storage not configured; set storage_url or provide s3.endpoint + " +
|
||||
"s3.bucket + credentials")
|
||||
errS3BucketRequired = errors.New("s3.bucket is required (or set storage_url)")
|
||||
errS3KeyIDRequired = errors.New("s3.access_key_id is required")
|
||||
errS3SecretRequired = errors.New("s3.secret_access_key is required")
|
||||
)
|
||||
|
||||
// expandTilde expands ~ at the start of a path to the user's home directory.
|
||||
func expandTilde(path string) string {
|
||||
if path == "~" {
|
||||
home, _ := os.UserHomeDir()
|
||||
|
||||
return home
|
||||
}
|
||||
|
||||
if strings.HasPrefix(path, "~/") {
|
||||
home, _ := os.UserHomeDir()
|
||||
|
||||
return filepath.Join(home, path[2:])
|
||||
}
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
@@ -35,8 +74,10 @@ func expandTilde(path string) string {
|
||||
func expandTildeInURL(url string) string {
|
||||
if strings.HasPrefix(url, "file://~/") {
|
||||
home, _ := os.UserHomeDir()
|
||||
|
||||
return "file://" + filepath.Join(home, url[9:])
|
||||
}
|
||||
|
||||
return url
|
||||
}
|
||||
|
||||
@@ -64,6 +105,7 @@ func (c *Config) GetExcludes(snapshotName string) []string {
|
||||
combined := make([]string, 0, len(c.Exclude)+len(snap.Exclude))
|
||||
combined = append(combined, c.Exclude...)
|
||||
combined = append(combined, snap.Exclude...)
|
||||
|
||||
return combined
|
||||
}
|
||||
|
||||
@@ -75,6 +117,7 @@ func (c *Config) SnapshotNames() []string {
|
||||
}
|
||||
// Sort for deterministic order
|
||||
sort.Strings(names)
|
||||
|
||||
return names
|
||||
}
|
||||
|
||||
@@ -82,17 +125,17 @@ func (c *Config) SnapshotNames() []string {
|
||||
// It defines all settings for backup operations, including source directories,
|
||||
// encryption recipients, storage configuration, and performance tuning parameters.
|
||||
// Configuration is typically loaded from a YAML file.
|
||||
//
|
||||
//nolint:tagliatelle // snake_case is the established config-file format
|
||||
type Config struct {
|
||||
AgeRecipients []string `yaml:"age_recipients"`
|
||||
AgeSecretKey string `yaml:"age_secret_key"`
|
||||
BackupInterval time.Duration `yaml:"backup_interval"`
|
||||
BlobSizeLimit Size `yaml:"blob_size_limit"`
|
||||
ChunkSize Size `yaml:"chunk_size"`
|
||||
Exclude []string `yaml:"exclude"` // Global excludes applied to all snapshots
|
||||
FullScanInterval time.Duration `yaml:"full_scan_interval"`
|
||||
// Exclude holds global excludes applied to all snapshots.
|
||||
Exclude []string `yaml:"exclude"`
|
||||
Hostname string `yaml:"hostname"`
|
||||
IndexPath string `yaml:"index_path"`
|
||||
MinTimeBetweenRun time.Duration `yaml:"min_time_between_run"`
|
||||
S3 S3Config `yaml:"s3"`
|
||||
Snapshots map[string]SnapshotConfig `yaml:"snapshots"`
|
||||
CompressionLevel int `yaml:"compression_level"`
|
||||
@@ -102,13 +145,16 @@ type Config struct {
|
||||
// Supported formats:
|
||||
// - s3://bucket/prefix?endpoint=host®ion=us-east-1
|
||||
// - file:///path/to/backup
|
||||
// For S3 URLs, credentials are still read from s3.access_key_id and s3.secret_access_key.
|
||||
// For S3 URLs, credentials are still read from s3.access_key_id
|
||||
// and s3.secret_access_key.
|
||||
StorageURL string `yaml:"storage_url"`
|
||||
}
|
||||
|
||||
// S3Config represents S3 storage configuration for backup storage.
|
||||
// It supports both AWS S3 and S3-compatible storage services.
|
||||
// All fields except UseSSL and PartSize are required.
|
||||
//
|
||||
//nolint:tagliatelle // snake_case is the established config-file format
|
||||
type S3Config struct {
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Bucket string `yaml:"bucket"`
|
||||
@@ -120,17 +166,17 @@ type S3Config struct {
|
||||
PartSize Size `yaml:"part_size"`
|
||||
}
|
||||
|
||||
// ConfigPath wraps the config file path for fx dependency injection.
|
||||
// Path wraps the config file path for fx dependency injection.
|
||||
// This type allows the config file path to be injected as a distinct type
|
||||
// rather than a plain string, avoiding conflicts with other string dependencies.
|
||||
type ConfigPath string
|
||||
type Path string
|
||||
|
||||
// New creates a new Config instance by loading from the specified path.
|
||||
// This function is used by the fx dependency injection framework.
|
||||
// Returns an error if the path is empty or if loading fails.
|
||||
func New(path ConfigPath) (*Config, error) {
|
||||
func New(path Path) (*Config, error) {
|
||||
if path == "" {
|
||||
return nil, fmt.Errorf("config path not provided")
|
||||
return nil, errNoConfigPath
|
||||
}
|
||||
|
||||
cfg, err := Load(string(path))
|
||||
@@ -155,23 +201,22 @@ func Load(path string) (*Config, error) {
|
||||
|
||||
cfg := &Config{
|
||||
// Set defaults
|
||||
BlobSizeLimit: Size(10 * 1024 * 1024 * 1024), // 10GB
|
||||
ChunkSize: Size(10 * 1024 * 1024), // 10MB
|
||||
BackupInterval: 1 * time.Hour,
|
||||
FullScanInterval: 24 * time.Hour,
|
||||
MinTimeBetweenRun: 15 * time.Minute,
|
||||
BlobSizeLimit: defaultBlobSizeLimit,
|
||||
ChunkSize: defaultChunkSize,
|
||||
IndexPath: filepath.Join(xdg.DataHome, appName, "index.sqlite"),
|
||||
CompressionLevel: 3,
|
||||
CompressionLevel: defaultCompressionLevel,
|
||||
}
|
||||
|
||||
// Convert smartconfig data to YAML then unmarshal
|
||||
configData := sc.Data()
|
||||
|
||||
yamlBytes, err := yaml.Marshal(configData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal config data: %w", err)
|
||||
}
|
||||
|
||||
if err := yaml.Unmarshal(yamlBytes, cfg); err != nil {
|
||||
err = yaml.Unmarshal(yamlBytes, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
@@ -184,6 +229,7 @@ func Load(path string) (*Config, error) {
|
||||
for i, path := range snap.Paths {
|
||||
snap.Paths[i] = expandTilde(path)
|
||||
}
|
||||
|
||||
cfg.Snapshots[name] = snap
|
||||
}
|
||||
|
||||
@@ -203,6 +249,7 @@ func Load(path string) (*Config, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get hostname: %w", err)
|
||||
}
|
||||
|
||||
cfg.Hostname = hostname
|
||||
}
|
||||
|
||||
@@ -210,12 +257,15 @@ func Load(path string) (*Config, error) {
|
||||
if cfg.S3.Region == "" {
|
||||
cfg.S3.Region = "us-east-1"
|
||||
}
|
||||
|
||||
if cfg.S3.PartSize == 0 {
|
||||
cfg.S3.PartSize = Size(5 * 1024 * 1024) // 5MB
|
||||
cfg.S3.PartSize = defaultS3PartSize
|
||||
}
|
||||
|
||||
// Check config file permissions (warn if world or group readable)
|
||||
if info, err := os.Stat(path); err == nil {
|
||||
//nolint:gosec // G703: config path is operator-supplied by design
|
||||
info, statErr := os.Stat(path)
|
||||
if statErr == nil {
|
||||
mode := info.Mode().Perm()
|
||||
if mode&0044 != 0 { // group or world readable
|
||||
log.Warn("Config file has insecure permissions (contains S3 credentials)",
|
||||
@@ -225,7 +275,8 @@ func Load(path string) (*Config, error) {
|
||||
}
|
||||
}
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
err = cfg.Validate()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid config: %w", err)
|
||||
}
|
||||
|
||||
@@ -243,34 +294,36 @@ func Load(path string) (*Config, error) {
|
||||
// Returns an error describing the first validation failure encountered.
|
||||
func (c *Config) Validate() error {
|
||||
if len(c.AgeRecipients) == 0 {
|
||||
return fmt.Errorf("at least one age_recipient is required")
|
||||
return errNoAgeRecipients
|
||||
}
|
||||
|
||||
if len(c.Snapshots) == 0 {
|
||||
return fmt.Errorf("at least one snapshot must be configured")
|
||||
return errNoSnapshots
|
||||
}
|
||||
|
||||
for name, snap := range c.Snapshots {
|
||||
if len(snap.Paths) == 0 {
|
||||
return fmt.Errorf("snapshot %q must have at least one path", name)
|
||||
return fmt.Errorf("%w: %q", errSnapshotNoPaths, name)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate storage configuration
|
||||
if err := c.validateStorage(); err != nil {
|
||||
err := c.validateStorage()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if c.ChunkSize.Int64() < 1024*1024 { // 1MB minimum
|
||||
return fmt.Errorf("chunk_size must be at least 1MB")
|
||||
if c.ChunkSize.Int64() < minChunkSize {
|
||||
return errChunkSizeTooSmall
|
||||
}
|
||||
|
||||
if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() {
|
||||
return fmt.Errorf("blob_size_limit must be at least chunk_size")
|
||||
return errBlobSizeTooSmall
|
||||
}
|
||||
|
||||
if c.CompressionLevel < 1 || c.CompressionLevel > 19 {
|
||||
return fmt.Errorf("compression_level must be between 1 and 19")
|
||||
if c.CompressionLevel < minCompressionLevel ||
|
||||
c.CompressionLevel > maxCompressionLevel {
|
||||
return errBadCompression
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -282,48 +335,56 @@ func (c *Config) Validate() error {
|
||||
// If StorageURL is not set, legacy S3 configuration is required.
|
||||
func (c *Config) validateStorage() error {
|
||||
if c.StorageURL != "" {
|
||||
// URL-based configuration
|
||||
if strings.HasPrefix(c.StorageURL, "file://") {
|
||||
// File storage doesn't need S3 credentials
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(c.StorageURL, "s3://") {
|
||||
// S3 storage needs credentials
|
||||
if c.S3.AccessKeyID == "" {
|
||||
return fmt.Errorf("s3.access_key_id is required for s3:// URLs")
|
||||
}
|
||||
if c.S3.SecretAccessKey == "" {
|
||||
return fmt.Errorf("s3.secret_access_key is required for s3:// URLs")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(c.StorageURL, "rclone://") {
|
||||
// Rclone storage uses rclone's own config
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("storage_url must start with s3://, file://, or rclone://")
|
||||
return c.validateStorageURL()
|
||||
}
|
||||
|
||||
// Legacy S3 configuration
|
||||
if c.S3.Endpoint == "" {
|
||||
return fmt.Errorf("s3.endpoint is required (or set storage_url)")
|
||||
return errStorageNotConfigured
|
||||
}
|
||||
|
||||
if c.S3.Bucket == "" {
|
||||
return fmt.Errorf("s3.bucket is required (or set storage_url)")
|
||||
return errS3BucketRequired
|
||||
}
|
||||
|
||||
if c.S3.AccessKeyID == "" {
|
||||
return fmt.Errorf("s3.access_key_id is required")
|
||||
return errS3KeyIDRequired
|
||||
}
|
||||
|
||||
if c.S3.SecretAccessKey == "" {
|
||||
return fmt.Errorf("s3.secret_access_key is required")
|
||||
return errS3SecretRequired
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateStorageURL validates URL-based storage configuration. File and
|
||||
// rclone URLs need no credentials; S3 URLs require the legacy s3.*
|
||||
// credential fields.
|
||||
func (c *Config) validateStorageURL() error {
|
||||
switch {
|
||||
case strings.HasPrefix(c.StorageURL, "file://"):
|
||||
// File storage doesn't need S3 credentials
|
||||
return nil
|
||||
case strings.HasPrefix(c.StorageURL, "rclone://"):
|
||||
// Rclone storage uses rclone's own config
|
||||
return nil
|
||||
case strings.HasPrefix(c.StorageURL, "s3://"):
|
||||
// S3 storage needs credentials
|
||||
if c.S3.AccessKeyID == "" {
|
||||
return fmt.Errorf("%w for s3:// URLs", errS3KeyIDRequired)
|
||||
}
|
||||
|
||||
if c.S3.SecretAccessKey == "" {
|
||||
return fmt.Errorf("%w for s3:// URLs", errS3SecretRequired)
|
||||
}
|
||||
|
||||
return nil
|
||||
default:
|
||||
return errBadStorageScheme
|
||||
}
|
||||
}
|
||||
|
||||
// extractAgeSecretKey extracts the AGE-SECRET-KEY from the input using
|
||||
// the age library's parser, which handles comments and whitespace.
|
||||
func extractAgeSecretKey(input string) string {
|
||||
@@ -336,11 +397,14 @@ func extractAgeSecretKey(input string) string {
|
||||
if id, ok := identities[0].(*age.X25519Identity); ok {
|
||||
return id.String()
|
||||
}
|
||||
|
||||
return strings.TrimSpace(input)
|
||||
}
|
||||
|
||||
// Module exports the config module for fx dependency injection.
|
||||
// It provides the Config type to other modules in the application.
|
||||
//
|
||||
//nolint:gochecknoglobals // fx module definitions are package globals
|
||||
var Module = fx.Module("config",
|
||||
fx.Provide(New),
|
||||
)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package config
|
||||
package config //nolint:testpackage // exercises unexported extractAgeSecretKey
|
||||
|
||||
import (
|
||||
"os"
|
||||
@@ -7,15 +7,20 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
TEST_SNEAK_AGE_PUBLIC_KEY = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
|
||||
TEST_INTEGRATION_AGE_PUBLIC_KEY = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"
|
||||
TEST_INTEGRATION_AGE_PRIVATE_KEY = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5"
|
||||
testSneakAgePublicKey = "age1278m9q7dp3chsh2dcy82qk27v047zywyvt" +
|
||||
"xwnj4cvt0z65jw6a7q5dqhfj"
|
||||
testIntegrationAgePublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu" +
|
||||
"0x0ecq2f7tp8a05gl0sjq9q9wjg"
|
||||
testIntegrationAgePrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GX" +
|
||||
"VEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// Set up test environment
|
||||
testConfigPath := filepath.Join("..", "..", "test", "config.yaml")
|
||||
if absPath, err := filepath.Abs(testConfigPath); err == nil {
|
||||
|
||||
absPath, err := filepath.Abs(testConfigPath)
|
||||
if err == nil {
|
||||
_ = os.Setenv("VAULTIK_CONFIG", absPath)
|
||||
}
|
||||
|
||||
@@ -23,8 +28,11 @@ func TestMain(m *testing.M) {
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
// TestConfigLoad ensures the config package can be imported and basic functionality works
|
||||
// TestConfigLoad ensures the config package can be imported and basic
|
||||
// functionality works.
|
||||
func TestConfigLoad(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Use the test config file
|
||||
configPath := os.Getenv("VAULTIK_CONFIG")
|
||||
if configPath == "" {
|
||||
@@ -41,8 +49,10 @@ func TestConfigLoad(t *testing.T) {
|
||||
if len(cfg.AgeRecipients) != 2 {
|
||||
t.Errorf("Expected 2 age recipients, got %d", len(cfg.AgeRecipients))
|
||||
}
|
||||
if cfg.AgeRecipients[0] != TEST_SNEAK_AGE_PUBLIC_KEY {
|
||||
t.Errorf("Expected first age recipient to be %s, got '%s'", TEST_SNEAK_AGE_PUBLIC_KEY, cfg.AgeRecipients[0])
|
||||
|
||||
if cfg.AgeRecipients[0] != testSneakAgePublicKey {
|
||||
t.Errorf("Expected first age recipient to be %s, got '%s'",
|
||||
testSneakAgePublicKey, cfg.AgeRecipients[0])
|
||||
}
|
||||
|
||||
if len(cfg.Snapshots) != 1 {
|
||||
@@ -59,11 +69,13 @@ func TestConfigLoad(t *testing.T) {
|
||||
}
|
||||
|
||||
if testSnap.Paths[0] != "/tmp/vaultik-test-source" {
|
||||
t.Errorf("Expected first path to be '/tmp/vaultik-test-source', got '%s'", testSnap.Paths[0])
|
||||
t.Errorf("Expected first path to be '/tmp/vaultik-test-source', got '%s'",
|
||||
testSnap.Paths[0])
|
||||
}
|
||||
|
||||
if cfg.S3.Bucket != "vaultik-test-bucket" {
|
||||
t.Errorf("Expected S3 bucket to be 'vaultik-test-bucket', got '%s'", cfg.S3.Bucket)
|
||||
t.Errorf("Expected S3 bucket to be 'vaultik-test-bucket', got '%s'",
|
||||
cfg.S3.Bucket)
|
||||
}
|
||||
|
||||
if cfg.Hostname != "test-host" {
|
||||
@@ -73,19 +85,26 @@ func TestConfigLoad(t *testing.T) {
|
||||
|
||||
// TestConfigFromEnv tests loading config path from environment variable
|
||||
func TestConfigFromEnv(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
configPath := os.Getenv("VAULTIK_CONFIG")
|
||||
if configPath == "" {
|
||||
t.Skip("VAULTIK_CONFIG not set")
|
||||
}
|
||||
|
||||
// Verify the file exists
|
||||
if _, err := os.Stat(configPath); os.IsNotExist(err) {
|
||||
t.Errorf("Config file does not exist at path from VAULTIK_CONFIG: %s", configPath)
|
||||
//nolint:gosec // G703: test config path comes from the test environment
|
||||
_, err := os.Stat(configPath)
|
||||
if os.IsNotExist(err) {
|
||||
t.Errorf("Config file does not exist at path from VAULTIK_CONFIG: %s",
|
||||
configPath)
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractAgeSecretKey tests extraction of AGE-SECRET-KEY from various inputs
|
||||
func TestExtractAgeSecretKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
@@ -93,36 +112,32 @@ func TestExtractAgeSecretKey(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
name: "plain key",
|
||||
input: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
|
||||
expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
|
||||
input: testIntegrationAgePrivateKey,
|
||||
expected: testIntegrationAgePrivateKey,
|
||||
},
|
||||
{
|
||||
name: "key with trailing newline",
|
||||
input: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5\n",
|
||||
expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
|
||||
input: testIntegrationAgePrivateKey + "\n",
|
||||
expected: testIntegrationAgePrivateKey,
|
||||
},
|
||||
{
|
||||
name: "full age-keygen output",
|
||||
input: `# created: 2025-01-14T12:00:00Z
|
||||
# public key: age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg
|
||||
AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5
|
||||
`,
|
||||
expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
|
||||
input: "# created: 2025-01-14T12:00:00Z\n" +
|
||||
"# public key: " + testIntegrationAgePublicKey + "\n" +
|
||||
testIntegrationAgePrivateKey + "\n",
|
||||
expected: testIntegrationAgePrivateKey,
|
||||
},
|
||||
{
|
||||
name: "age-keygen output with extra blank lines",
|
||||
input: `# created: 2025-01-14T12:00:00Z
|
||||
# public key: age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg
|
||||
|
||||
AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5
|
||||
|
||||
`,
|
||||
expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
|
||||
input: "# created: 2025-01-14T12:00:00Z\n" +
|
||||
"# public key: " + testIntegrationAgePublicKey + "\n\n" +
|
||||
testIntegrationAgePrivateKey + "\n\n",
|
||||
expected: testIntegrationAgePrivateKey,
|
||||
},
|
||||
{
|
||||
name: "key with leading whitespace",
|
||||
input: " AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5 ",
|
||||
expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
|
||||
input: " " + testIntegrationAgePrivateKey + " ",
|
||||
expected: testIntegrationAgePrivateKey,
|
||||
},
|
||||
{
|
||||
name: "empty input",
|
||||
@@ -138,9 +153,12 @@ AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
result := extractAgeSecretKey(tt.input)
|
||||
if result != tt.expected {
|
||||
t.Errorf("extractAgeSecretKey(%q) = %q, want %q", tt.input, result, tt.expected)
|
||||
t.Errorf("extractAgeSecretKey(%q) = %q, want %q",
|
||||
tt.input, result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+29
-4
@@ -1,31 +1,45 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
)
|
||||
|
||||
var (
|
||||
errSizeType = errors.New("size must be a number or string")
|
||||
errSizeTooLarge = errors.New("size exceeds maximum supported value")
|
||||
)
|
||||
|
||||
// Size represents a byte size that can be specified in configuration files.
|
||||
// It can unmarshal from both numeric values (interpreted as bytes) and
|
||||
// human-readable strings like "10MB", "2.5GB", or "1TB".
|
||||
//
|
||||
//nolint:recvcheck // UnmarshalYAML requires a pointer; String/Int64 are value reads
|
||||
type Size int64
|
||||
|
||||
// UnmarshalYAML implements yaml.Unmarshaler for Size, allowing it to be
|
||||
// parsed from YAML configuration files. It accepts both numeric values
|
||||
// (interpreted as bytes) and string values with units (e.g., "10MB").
|
||||
func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
func (s *Size) UnmarshalYAML(unmarshal func(any) error) error {
|
||||
// Try to unmarshal as int64 first
|
||||
var intVal int64
|
||||
if err := unmarshal(&intVal); err == nil {
|
||||
|
||||
err := unmarshal(&intVal)
|
||||
if err == nil {
|
||||
*s = Size(intVal)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Try to unmarshal as string
|
||||
var strVal string
|
||||
if err := unmarshal(&strVal); err != nil {
|
||||
return fmt.Errorf("size must be a number or string")
|
||||
|
||||
err = unmarshal(&strVal)
|
||||
if err != nil {
|
||||
return errSizeType
|
||||
}
|
||||
|
||||
// Parse the string using go-humanize
|
||||
@@ -34,7 +48,12 @@ func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
return fmt.Errorf("invalid size format: %w", err)
|
||||
}
|
||||
|
||||
if bytes > math.MaxInt64 {
|
||||
return fmt.Errorf("%w: %s", errSizeTooLarge, strVal)
|
||||
}
|
||||
|
||||
*s = Size(bytes)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -49,6 +68,7 @@ func (s Size) Int64() int64 {
|
||||
// For example, 1048576 bytes would be formatted as "1.0 MB".
|
||||
// This implements the fmt.Stringer interface.
|
||||
func (s Size) String() string {
|
||||
//nolint:gosec // G115: sizes are non-negative by construction
|
||||
return humanize.Bytes(uint64(s))
|
||||
}
|
||||
|
||||
@@ -58,5 +78,10 @@ func ParseSize(s string) (Size, error) {
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid size format: %w", err)
|
||||
}
|
||||
|
||||
if bytes > math.MaxInt64 {
|
||||
return 0, fmt.Errorf("%w: %s", errSizeTooLarge, s)
|
||||
}
|
||||
|
||||
return Size(bytes), nil
|
||||
}
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
package crypto
|
||||
// Package crypto provides thread-safe age encryption and decryption
|
||||
// helpers used to protect blob and metadata content.
|
||||
package crypto //nolint:revive,nolintlint // stdlib crypto unused; see #76
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
@@ -10,6 +13,10 @@ import (
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// ErrNoRecipients is returned when an encryptor is created or updated
|
||||
// without any recipient public keys.
|
||||
var ErrNoRecipients = errors.New("at least one recipient is required")
|
||||
|
||||
// Encryptor provides thread-safe encryption using the age encryption library.
|
||||
// It supports encrypting data for multiple recipients simultaneously, allowing
|
||||
// any of the corresponding private keys to decrypt the data. This is useful
|
||||
@@ -25,7 +32,7 @@ type Encryptor struct {
|
||||
// public keys are invalid or if no recipients are specified.
|
||||
func NewEncryptor(publicKeys []string) (*Encryptor, error) {
|
||||
if len(publicKeys) == 0 {
|
||||
return nil, fmt.Errorf("at least one recipient is required")
|
||||
return nil, ErrNoRecipients
|
||||
}
|
||||
|
||||
recipients := make([]age.Recipient, 0, len(publicKeys))
|
||||
@@ -34,6 +41,7 @@ func NewEncryptor(publicKeys []string) (*Encryptor, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing age recipient %s: %w", key, err)
|
||||
}
|
||||
|
||||
recipients = append(recipients, recipient)
|
||||
}
|
||||
|
||||
@@ -60,12 +68,14 @@ func (e *Encryptor) Encrypt(data []byte) ([]byte, error) {
|
||||
}
|
||||
|
||||
// Write data
|
||||
if _, err := w.Write(data); err != nil {
|
||||
_, err = w.Write(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("writing encrypted data: %w", err)
|
||||
}
|
||||
|
||||
// Close to flush
|
||||
if err := w.Close(); err != nil {
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("closing encrypted writer: %w", err)
|
||||
}
|
||||
|
||||
@@ -88,12 +98,14 @@ func (e *Encryptor) EncryptStream(dst io.Writer, src io.Reader) error {
|
||||
}
|
||||
|
||||
// Copy data
|
||||
if _, err := io.Copy(w, src); err != nil {
|
||||
_, err = io.Copy(w, src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("copying encrypted data: %w", err)
|
||||
}
|
||||
|
||||
// Close to flush
|
||||
if err := w.Close(); err != nil {
|
||||
err = w.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("closing encrypted writer: %w", err)
|
||||
}
|
||||
|
||||
@@ -126,7 +138,7 @@ func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) {
|
||||
// of the public keys are invalid or if no recipients are specified.
|
||||
func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
|
||||
if len(publicKeys) == 0 {
|
||||
return fmt.Errorf("at least one recipient is required")
|
||||
return ErrNoRecipients
|
||||
}
|
||||
|
||||
recipients := make([]age.Recipient, 0, len(publicKeys))
|
||||
@@ -135,6 +147,7 @@ func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing age recipient %s: %w", key, err)
|
||||
}
|
||||
|
||||
recipients = append(recipients, recipient)
|
||||
}
|
||||
|
||||
@@ -206,4 +219,6 @@ func (d *Decryptor) DecryptStream(src io.Reader) (io.Reader, error) {
|
||||
}
|
||||
|
||||
// Module exports the crypto module for fx dependency injection.
|
||||
//
|
||||
//nolint:gochecknoglobals // fx module definitions are package globals
|
||||
var Module = fx.Module("crypto")
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
package crypto
|
||||
package crypto_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"sneak.berlin/go/vaultik/internal/crypto"
|
||||
)
|
||||
|
||||
func TestEncryptor(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Generate a test key pair
|
||||
identity, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
@@ -17,7 +20,7 @@ func TestEncryptor(t *testing.T) {
|
||||
publicKey := identity.Recipient().String()
|
||||
|
||||
// Create encryptor
|
||||
enc, err := NewEncryptor([]string{publicKey})
|
||||
enc, err := crypto.NewEncryptor([]string{publicKey})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create encryptor: %v", err)
|
||||
}
|
||||
@@ -43,7 +46,9 @@ func TestEncryptor(t *testing.T) {
|
||||
}
|
||||
|
||||
var decrypted bytes.Buffer
|
||||
if _, err := decrypted.ReadFrom(r); err != nil {
|
||||
|
||||
_, err = decrypted.ReadFrom(r)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read decrypted data: %v", err)
|
||||
}
|
||||
|
||||
@@ -53,15 +58,19 @@ func TestEncryptor(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEncryptorMultipleRecipients(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Generate three test key pairs
|
||||
identity1, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate identity1: %v", err)
|
||||
}
|
||||
|
||||
identity2, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate identity2: %v", err)
|
||||
}
|
||||
|
||||
identity3, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate identity3: %v", err)
|
||||
@@ -74,7 +83,7 @@ func TestEncryptorMultipleRecipients(t *testing.T) {
|
||||
}
|
||||
|
||||
// Create encryptor with multiple recipients
|
||||
enc, err := NewEncryptor(publicKeys)
|
||||
enc, err := crypto.NewEncryptor(publicKeys)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create encryptor: %v", err)
|
||||
}
|
||||
@@ -97,7 +106,9 @@ func TestEncryptorMultipleRecipients(t *testing.T) {
|
||||
}
|
||||
|
||||
var decrypted bytes.Buffer
|
||||
if _, err := decrypted.ReadFrom(r); err != nil {
|
||||
|
||||
_, err = decrypted.ReadFrom(r)
|
||||
if err != nil {
|
||||
t.Fatalf("recipient %d failed to read decrypted data: %v", i+1, err)
|
||||
}
|
||||
|
||||
@@ -108,6 +119,8 @@ func TestEncryptorMultipleRecipients(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEncryptorUpdateRecipients(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Generate two identities
|
||||
identity1, _ := age.GenerateX25519Identity()
|
||||
identity2, _ := age.GenerateX25519Identity()
|
||||
@@ -116,20 +129,22 @@ func TestEncryptorUpdateRecipients(t *testing.T) {
|
||||
publicKey2 := identity2.Recipient().String()
|
||||
|
||||
// Create encryptor with first key
|
||||
enc, err := NewEncryptor([]string{publicKey1})
|
||||
enc, err := crypto.NewEncryptor([]string{publicKey1})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create encryptor: %v", err)
|
||||
}
|
||||
|
||||
// Encrypt with first key
|
||||
plaintext := []byte("test data")
|
||||
|
||||
ciphertext1, err := enc.Encrypt(plaintext)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to encrypt: %v", err)
|
||||
}
|
||||
|
||||
// Update to second key
|
||||
if err := enc.UpdateRecipients([]string{publicKey2}); err != nil {
|
||||
err = enc.UpdateRecipients([]string{publicKey2})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to update recipients: %v", err)
|
||||
}
|
||||
|
||||
@@ -140,18 +155,24 @@ func TestEncryptorUpdateRecipients(t *testing.T) {
|
||||
}
|
||||
|
||||
// First ciphertext should only decrypt with first identity
|
||||
if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity1); err != nil {
|
||||
_, err = age.Decrypt(bytes.NewReader(ciphertext1), identity1)
|
||||
if err != nil {
|
||||
t.Error("failed to decrypt with identity1")
|
||||
}
|
||||
if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity2); err == nil {
|
||||
|
||||
_, err = age.Decrypt(bytes.NewReader(ciphertext1), identity2)
|
||||
if err == nil {
|
||||
t.Error("should not decrypt with identity2")
|
||||
}
|
||||
|
||||
// Second ciphertext should only decrypt with second identity
|
||||
if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity2); err != nil {
|
||||
_, err = age.Decrypt(bytes.NewReader(ciphertext2), identity2)
|
||||
if err != nil {
|
||||
t.Error("failed to decrypt with identity2")
|
||||
}
|
||||
if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity1); err == nil {
|
||||
|
||||
_, err = age.Decrypt(bytes.NewReader(ciphertext2), identity1)
|
||||
if err == nil {
|
||||
t.Error("should not decrypt with identity1")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,18 +3,25 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// BlobChunkRepository provides access to the blob_chunks table, which maps
|
||||
// blobs to the chunks they contain (with offset and length).
|
||||
type BlobChunkRepository struct {
|
||||
db *DB
|
||||
}
|
||||
|
||||
// NewBlobChunkRepository creates a BlobChunkRepository backed by db.
|
||||
func NewBlobChunkRepository(db *DB) *BlobChunkRepository {
|
||||
return &BlobChunkRepository{db: db}
|
||||
}
|
||||
|
||||
func (r *BlobChunkRepository) Create(ctx context.Context, tx *sql.Tx, bc *BlobChunk) error {
|
||||
// Create inserts a blob_chunks row, using tx when non-nil.
|
||||
func (r *BlobChunkRepository) Create(
|
||||
ctx context.Context, tx *sql.Tx, bc *BlobChunk,
|
||||
) error {
|
||||
query := `
|
||||
INSERT INTO blob_chunks (blob_id, chunk_hash, offset, length)
|
||||
VALUES (?, ?, ?, ?)
|
||||
@@ -34,7 +41,11 @@ func (r *BlobChunkRepository) Create(ctx context.Context, tx *sql.Tx, bc *BlobCh
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([]*BlobChunk, error) {
|
||||
// GetByBlobID returns all chunks contained in the given blob, ordered by
|
||||
// their offset within the blob.
|
||||
func (r *BlobChunkRepository) GetByBlobID(
|
||||
ctx context.Context, blobID string,
|
||||
) ([]*BlobChunk, error) {
|
||||
query := `
|
||||
SELECT blob_id, chunk_hash, offset, length
|
||||
FROM blob_chunks
|
||||
@@ -46,22 +57,35 @@ func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying blob chunks: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
var blobChunks []*BlobChunk
|
||||
|
||||
for rows.Next() {
|
||||
var bc BlobChunk
|
||||
|
||||
err := rows.Scan(&bc.BlobID, &bc.ChunkHash, &bc.Offset, &bc.Length)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning blob chunk: %w", err)
|
||||
}
|
||||
|
||||
blobChunks = append(blobChunks, &bc)
|
||||
}
|
||||
|
||||
return blobChunks, rows.Err()
|
||||
}
|
||||
|
||||
func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash string) (*BlobChunk, error) {
|
||||
// GetByChunkHash returns one blob_chunks row containing the given chunk,
|
||||
// or nil if the chunk is not packed in any blob.
|
||||
func (r *BlobChunkRepository) GetByChunkHash(
|
||||
ctx context.Context, chunkHash string,
|
||||
) (*BlobChunk, error) {
|
||||
query := `
|
||||
SELECT blob_id, chunk_hash, offset, length
|
||||
FROM blob_chunks
|
||||
@@ -70,7 +94,9 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
|
||||
`
|
||||
|
||||
LogSQL("GetByChunkHash", query, chunkHash)
|
||||
|
||||
var bc BlobChunk
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, chunkHash).Scan(
|
||||
&bc.BlobID,
|
||||
&bc.ChunkHash,
|
||||
@@ -78,21 +104,27 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
|
||||
&bc.Length,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
LogSQL("GetByChunkHash", "No rows found", chunkHash)
|
||||
return nil, nil
|
||||
|
||||
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
LogSQL("GetByChunkHash", "Error", chunkHash, err)
|
||||
|
||||
return nil, fmt.Errorf("querying blob chunk: %w", err)
|
||||
}
|
||||
|
||||
LogSQL("GetByChunkHash", "Found blob", chunkHash, "blob", bc.BlobID)
|
||||
|
||||
return &bc, nil
|
||||
}
|
||||
|
||||
// GetByChunkHashTx retrieves a blob chunk within a transaction
|
||||
func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx, chunkHash string) (*BlobChunk, error) {
|
||||
func (r *BlobChunkRepository) GetByChunkHashTx(
|
||||
ctx context.Context, tx *sql.Tx, chunkHash string,
|
||||
) (*BlobChunk, error) {
|
||||
query := `
|
||||
SELECT blob_id, chunk_hash, offset, length
|
||||
FROM blob_chunks
|
||||
@@ -101,7 +133,9 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
|
||||
`
|
||||
|
||||
LogSQL("GetByChunkHashTx", query, chunkHash)
|
||||
|
||||
var bc BlobChunk
|
||||
|
||||
err := tx.QueryRowContext(ctx, query, chunkHash).Scan(
|
||||
&bc.BlobID,
|
||||
&bc.ChunkHash,
|
||||
@@ -109,20 +143,25 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
|
||||
&bc.Length,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
LogSQL("GetByChunkHashTx", "No rows found", chunkHash)
|
||||
return nil, nil
|
||||
|
||||
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
LogSQL("GetByChunkHashTx", "Error", chunkHash, err)
|
||||
|
||||
return nil, fmt.Errorf("querying blob chunk: %w", err)
|
||||
}
|
||||
|
||||
LogSQL("GetByChunkHashTx", "Found blob", chunkHash, "blob", bc.BlobID)
|
||||
|
||||
return &bc, nil
|
||||
}
|
||||
|
||||
// DeleteOrphaned deletes blob_chunks entries where either the blob or chunk no longer exists
|
||||
// DeleteOrphaned deletes blob_chunks entries where either the blob or the
|
||||
// chunk no longer exists.
|
||||
func (r *BlobChunkRepository) DeleteOrphaned(ctx context.Context) error {
|
||||
// Delete blob_chunks where the blob doesn't exist
|
||||
query1 := `
|
||||
@@ -132,7 +171,9 @@ func (r *BlobChunkRepository) DeleteOrphaned(ctx context.Context) error {
|
||||
WHERE blobs.id = blob_chunks.blob_id
|
||||
)
|
||||
`
|
||||
if _, err := r.db.ExecWithLog(ctx, query1); err != nil {
|
||||
|
||||
_, err := r.db.ExecWithLog(ctx, query1)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting blob_chunks with missing blobs: %w", err)
|
||||
}
|
||||
|
||||
@@ -144,7 +185,9 @@ func (r *BlobChunkRepository) DeleteOrphaned(ctx context.Context) error {
|
||||
WHERE chunks.chunk_hash = blob_chunks.chunk_hash
|
||||
)
|
||||
`
|
||||
if _, err := r.db.ExecWithLog(ctx, query2); err != nil {
|
||||
|
||||
_, err = r.db.ExecWithLog(ctx, query2)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting blob_chunks with missing chunks: %w", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package database
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -6,71 +6,107 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func TestBlobChunkRepository(t *testing.T) {
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
// Chunk hashes used across the blob_chunks tests.
|
||||
const (
|
||||
chunk1Hash = "chunk1"
|
||||
chunk2Hash = "chunk2"
|
||||
chunk3Hash = "chunk3"
|
||||
)
|
||||
|
||||
// mustCreateChunks registers the given chunk hashes (1024 bytes each).
|
||||
func mustCreateChunks(
|
||||
t *testing.T,
|
||||
repos *database.Repositories,
|
||||
hashes ...types.ChunkHash,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
// Create blob first
|
||||
blob := &Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("blob1-hash"),
|
||||
CreatedTS: time.Now(),
|
||||
}
|
||||
err := repos.Blobs.Create(ctx, nil, blob)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob: %v", err)
|
||||
}
|
||||
|
||||
// Create chunks
|
||||
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
|
||||
for _, chunkHash := range chunks {
|
||||
chunk := &Chunk{
|
||||
for _, chunkHash := range hashes {
|
||||
chunk := &database.Chunk{
|
||||
ChunkHash: chunkHash,
|
||||
Size: 1024,
|
||||
}
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
|
||||
err := repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mustCreateBlob creates a blob row with the given hash.
|
||||
func mustCreateBlob(
|
||||
t *testing.T,
|
||||
repos *database.Repositories,
|
||||
hash types.BlobHash,
|
||||
) *database.Blob {
|
||||
t.Helper()
|
||||
|
||||
blob := &database.Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: hash,
|
||||
CreatedTS: time.Now(),
|
||||
}
|
||||
|
||||
err := repos.Blobs.Create(context.Background(), nil, blob)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob %s: %v", hash, err)
|
||||
}
|
||||
|
||||
return blob
|
||||
}
|
||||
|
||||
func TestBlobChunkRepository(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
blob := mustCreateBlob(t, repos, "blob1-hash")
|
||||
mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash)
|
||||
|
||||
// Test Create
|
||||
bc1 := &BlobChunk{
|
||||
bc1 := &database.BlobChunk{
|
||||
BlobID: blob.ID,
|
||||
ChunkHash: types.ChunkHash("chunk1"),
|
||||
ChunkHash: types.ChunkHash(chunk1Hash),
|
||||
Offset: 0,
|
||||
Length: 1024,
|
||||
}
|
||||
|
||||
err = repos.BlobChunks.Create(ctx, nil, bc1)
|
||||
err := repos.BlobChunks.Create(ctx, nil, bc1)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob chunk: %v", err)
|
||||
}
|
||||
|
||||
// Add more chunks to the same blob
|
||||
bc2 := &BlobChunk{
|
||||
bc2 := &database.BlobChunk{
|
||||
BlobID: blob.ID,
|
||||
ChunkHash: types.ChunkHash("chunk2"),
|
||||
ChunkHash: types.ChunkHash(chunk2Hash),
|
||||
Offset: 1024,
|
||||
Length: 2048,
|
||||
}
|
||||
|
||||
err = repos.BlobChunks.Create(ctx, nil, bc2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create second blob chunk: %v", err)
|
||||
}
|
||||
|
||||
bc3 := &BlobChunk{
|
||||
bc3 := &database.BlobChunk{
|
||||
BlobID: blob.ID,
|
||||
ChunkHash: types.ChunkHash("chunk3"),
|
||||
ChunkHash: types.ChunkHash(chunk3Hash),
|
||||
Offset: 3072,
|
||||
Length: 512,
|
||||
}
|
||||
|
||||
err = repos.BlobChunks.Create(ctx, nil, bc3)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create third blob chunk: %v", err)
|
||||
@@ -81,6 +117,7 @@ func TestBlobChunkRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(blobChunks) != 3 {
|
||||
t.Errorf("expected 3 chunks, got %d", len(blobChunks))
|
||||
}
|
||||
@@ -89,92 +126,97 @@ func TestBlobChunkRepository(t *testing.T) {
|
||||
expectedOffsets := []int64{0, 1024, 3072}
|
||||
for i, bc := range blobChunks {
|
||||
if bc.Offset != expectedOffsets[i] {
|
||||
t.Errorf("wrong chunk order: expected offset %d, got %d", expectedOffsets[i], bc.Offset)
|
||||
t.Errorf("wrong chunk order: expected offset %d, got %d",
|
||||
expectedOffsets[i], bc.Offset)
|
||||
}
|
||||
}
|
||||
|
||||
// Test GetByChunkHash
|
||||
bc, err := repos.BlobChunks.GetByChunkHash(ctx, "chunk2")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob chunk by chunk hash: %v", err)
|
||||
}
|
||||
if bc == nil {
|
||||
t.Fatal("expected blob chunk, got nil")
|
||||
}
|
||||
if bc.BlobID != blob.ID {
|
||||
t.Errorf("wrong blob ID: expected %s, got %s", blob.ID, bc.BlobID)
|
||||
}
|
||||
if bc.Offset != 1024 {
|
||||
t.Errorf("wrong offset: expected 1024, got %d", bc.Offset)
|
||||
}
|
||||
|
||||
// Test duplicate insert (should fail due to primary key constraint)
|
||||
err = repos.BlobChunks.Create(ctx, nil, bc1)
|
||||
if err == nil {
|
||||
t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "UNIQUE") && !strings.Contains(err.Error(), "constraint") {
|
||||
|
||||
if !strings.Contains(err.Error(), "UNIQUE") &&
|
||||
!strings.Contains(err.Error(), "constraint") {
|
||||
t.Fatalf("expected constraint error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobChunkRepositoryGetByChunkHash(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
blob := mustCreateBlob(t, repos, "blob-gbch-hash")
|
||||
mustCreateChunks(t, repos, chunk2Hash)
|
||||
|
||||
bc2 := &database.BlobChunk{
|
||||
BlobID: blob.ID,
|
||||
ChunkHash: types.ChunkHash(chunk2Hash),
|
||||
Offset: 1024,
|
||||
Length: 2048,
|
||||
}
|
||||
|
||||
err := repos.BlobChunks.Create(ctx, nil, bc2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob chunk: %v", err)
|
||||
}
|
||||
|
||||
// Test GetByChunkHash
|
||||
bc, err := repos.BlobChunks.GetByChunkHash(ctx, chunk2Hash)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob chunk by chunk hash: %v", err)
|
||||
}
|
||||
|
||||
if bc == nil {
|
||||
t.Fatal("expected blob chunk, got nil")
|
||||
}
|
||||
|
||||
if bc.BlobID != blob.ID {
|
||||
t.Errorf("wrong blob ID: expected %s, got %s", blob.ID, bc.BlobID)
|
||||
}
|
||||
|
||||
if bc.Offset != 1024 {
|
||||
t.Errorf("wrong offset: expected 1024, got %d", bc.Offset)
|
||||
}
|
||||
|
||||
// Test non-existent chunk
|
||||
bc, err = repos.BlobChunks.GetByChunkHash(ctx, "nonexistent")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if bc != nil {
|
||||
t.Error("expected nil for non-existent chunk")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Create blobs
|
||||
blob1 := &Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("blob1-hash"),
|
||||
CreatedTS: time.Now(),
|
||||
}
|
||||
blob2 := &Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("blob2-hash"),
|
||||
CreatedTS: time.Now(),
|
||||
}
|
||||
|
||||
err := repos.Blobs.Create(ctx, nil, blob1)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob1: %v", err)
|
||||
}
|
||||
err = repos.Blobs.Create(ctx, nil, blob2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob2: %v", err)
|
||||
}
|
||||
|
||||
// Create chunks
|
||||
chunkHashes := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
|
||||
for _, chunkHash := range chunkHashes {
|
||||
chunk := &Chunk{
|
||||
ChunkHash: chunkHash,
|
||||
Size: 1024,
|
||||
}
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||
}
|
||||
}
|
||||
blob1 := mustCreateBlob(t, repos, "blob1-hash")
|
||||
blob2 := mustCreateBlob(t, repos, "blob2-hash")
|
||||
mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash)
|
||||
|
||||
// Create chunks across multiple blobs
|
||||
// Some chunks are shared between blobs (deduplication scenario)
|
||||
blobChunks := []BlobChunk{
|
||||
{BlobID: blob1.ID, ChunkHash: types.ChunkHash("chunk1"), Offset: 0, Length: 1024},
|
||||
{BlobID: blob1.ID, ChunkHash: types.ChunkHash("chunk2"), Offset: 1024, Length: 1024},
|
||||
{BlobID: blob2.ID, ChunkHash: types.ChunkHash("chunk2"), Offset: 0, Length: 1024}, // chunk2 is shared
|
||||
{BlobID: blob2.ID, ChunkHash: types.ChunkHash("chunk3"), Offset: 1024, Length: 1024},
|
||||
blobChunks := []database.BlobChunk{
|
||||
{BlobID: blob1.ID, ChunkHash: chunk1Hash, Offset: 0, Length: 1024},
|
||||
{BlobID: blob1.ID, ChunkHash: chunk2Hash, Offset: 1024, Length: 1024},
|
||||
// chunk2 is shared between the blobs
|
||||
{BlobID: blob2.ID, ChunkHash: chunk2Hash, Offset: 0, Length: 1024},
|
||||
{BlobID: blob2.ID, ChunkHash: chunk3Hash, Offset: 1024, Length: 1024},
|
||||
}
|
||||
|
||||
for _, bc := range blobChunks {
|
||||
@@ -189,6 +231,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob1 chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(chunks) != 2 {
|
||||
t.Errorf("expected 2 chunks for blob1, got %d", len(chunks))
|
||||
}
|
||||
@@ -198,15 +241,17 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob2 chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(chunks) != 2 {
|
||||
t.Errorf("expected 2 chunks for blob2, got %d", len(chunks))
|
||||
}
|
||||
|
||||
// Verify shared chunk
|
||||
bc, err := repos.BlobChunks.GetByChunkHash(ctx, "chunk2")
|
||||
bc, err := repos.BlobChunks.GetByChunkHash(ctx, chunk2Hash)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get shared chunk: %v", err)
|
||||
}
|
||||
|
||||
if bc == nil {
|
||||
t.Fatal("expected shared chunk, got nil")
|
||||
}
|
||||
|
||||
+111
-53
@@ -3,31 +3,39 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
// BlobRepository provides access to the blobs table, which tracks the
|
||||
// packed, encrypted storage units uploaded to the destination.
|
||||
type BlobRepository struct {
|
||||
db *DB
|
||||
}
|
||||
|
||||
// NewBlobRepository creates a BlobRepository backed by db.
|
||||
func NewBlobRepository(db *DB) *BlobRepository {
|
||||
return &BlobRepository{db: db}
|
||||
}
|
||||
|
||||
// Create inserts a blob row, using tx when non-nil.
|
||||
func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) error {
|
||||
query := `
|
||||
INSERT INTO blobs (id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts)
|
||||
INSERT INTO blobs (id, blob_hash, created_ts, finished_ts,
|
||||
uncompressed_size, compressed_size, uploaded_ts)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
var finishedTS, uploadedTS *int64
|
||||
|
||||
if blob.FinishedTS != nil {
|
||||
ts := blob.FinishedTS.Unix()
|
||||
finishedTS = &ts
|
||||
}
|
||||
|
||||
if blob.UploadedTS != nil {
|
||||
ts := blob.UploadedTS.Unix()
|
||||
uploadedTS = &ts
|
||||
@@ -49,59 +57,49 @@ func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) err
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetByHash returns the blob with the given content hash, or nil if no
|
||||
// such blob exists.
|
||||
func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, error) {
|
||||
query := `
|
||||
SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts
|
||||
FROM blobs
|
||||
WHERE blob_hash = ?
|
||||
`
|
||||
|
||||
var blob Blob
|
||||
var createdTSUnix int64
|
||||
var finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, hash).Scan(
|
||||
&blob.ID,
|
||||
&blob.Hash,
|
||||
&createdTSUnix,
|
||||
&finishedTSUnix,
|
||||
&blob.UncompressedSize,
|
||||
&blob.CompressedSize,
|
||||
&uploadedTSUnix,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying blob: %w", err)
|
||||
}
|
||||
|
||||
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
|
||||
if finishedTSUnix.Valid {
|
||||
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
|
||||
blob.FinishedTS = &ts
|
||||
}
|
||||
if uploadedTSUnix.Valid {
|
||||
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
|
||||
blob.UploadedTS = &ts
|
||||
}
|
||||
return &blob, nil
|
||||
return r.getOne(ctx, "blob_hash", hash)
|
||||
}
|
||||
|
||||
// GetByID retrieves a blob by its ID
|
||||
func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error) {
|
||||
return r.getOne(ctx, "id", id)
|
||||
}
|
||||
|
||||
// GetAll returns every blob row keyed by blob ID. Useful at restore
|
||||
// start to translate the per-chunk blob_id references in chunkToBlobMap
|
||||
// into blob hashes without doing one GetByID query per chunk.
|
||||
func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
|
||||
query := `
|
||||
SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts
|
||||
SELECT id, blob_hash, created_ts, finished_ts,
|
||||
uncompressed_size, compressed_size, uploaded_ts
|
||||
FROM blobs
|
||||
WHERE id = ?
|
||||
`
|
||||
|
||||
var blob Blob
|
||||
var createdTSUnix int64
|
||||
var finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||
rows, err := r.db.conn.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying blobs: %w", err)
|
||||
}
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, id).Scan(
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
out := make(map[string]*Blob)
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
blob Blob
|
||||
createdTSUnix int64
|
||||
finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||
)
|
||||
|
||||
err := rows.Scan(
|
||||
&blob.ID,
|
||||
&blob.Hash,
|
||||
&createdTSUnix,
|
||||
@@ -110,12 +108,8 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
|
||||
&blob.CompressedSize,
|
||||
&uploadedTSUnix,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying blob: %w", err)
|
||||
return nil, fmt.Errorf("scanning blob: %w", err)
|
||||
}
|
||||
|
||||
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
|
||||
@@ -123,15 +117,26 @@ func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error)
|
||||
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
|
||||
blob.FinishedTS = &ts
|
||||
}
|
||||
|
||||
if uploadedTSUnix.Valid {
|
||||
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
|
||||
blob.UploadedTS = &ts
|
||||
}
|
||||
return &blob, nil
|
||||
|
||||
out[blob.ID.String()] = &blob
|
||||
}
|
||||
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// UpdateFinished updates a blob when it's finalized
|
||||
func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id string, hash string, uncompressedSize, compressedSize int64) error {
|
||||
func (r *BlobRepository) UpdateFinished(
|
||||
ctx context.Context,
|
||||
tx *sql.Tx,
|
||||
id string,
|
||||
hash string,
|
||||
uncompressedSize, compressedSize int64,
|
||||
) error {
|
||||
query := `
|
||||
UPDATE blobs
|
||||
SET blob_hash = ?, finished_ts = ?, uncompressed_size = ?, compressed_size = ?
|
||||
@@ -139,6 +144,7 @@ func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id stri
|
||||
`
|
||||
|
||||
now := time.Now().UTC().Unix()
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, hash, now, uncompressedSize, compressedSize, id)
|
||||
@@ -154,7 +160,9 @@ func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id stri
|
||||
}
|
||||
|
||||
// UpdateUploaded marks a blob as uploaded
|
||||
func (r *BlobRepository) UpdateUploaded(ctx context.Context, tx *sql.Tx, id string) error {
|
||||
func (r *BlobRepository) UpdateUploaded(
|
||||
ctx context.Context, tx *sql.Tx, id string,
|
||||
) error {
|
||||
query := `
|
||||
UPDATE blobs
|
||||
SET uploaded_ts = ?
|
||||
@@ -162,6 +170,7 @@ func (r *BlobRepository) UpdateUploaded(ctx context.Context, tx *sql.Tx, id stri
|
||||
`
|
||||
|
||||
now := time.Now().UTC().Unix()
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, now, id)
|
||||
@@ -198,3 +207,52 @@ func (r *BlobRepository) DeleteOrphaned(ctx context.Context) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getOne fetches a single blob row matched on the given column, or
|
||||
// (nil, nil) when no row matches.
|
||||
func (r *BlobRepository) getOne(
|
||||
ctx context.Context, column, value string,
|
||||
) (*Blob, error) {
|
||||
query := `
|
||||
SELECT id, blob_hash, created_ts, finished_ts,
|
||||
uncompressed_size, compressed_size, uploaded_ts
|
||||
FROM blobs
|
||||
WHERE ` + column + ` = ?`
|
||||
|
||||
var (
|
||||
blob Blob
|
||||
createdTSUnix int64
|
||||
finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||
)
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, value).Scan(
|
||||
&blob.ID,
|
||||
&blob.Hash,
|
||||
&createdTSUnix,
|
||||
&finishedTSUnix,
|
||||
&blob.UncompressedSize,
|
||||
&blob.CompressedSize,
|
||||
&uploadedTSUnix,
|
||||
)
|
||||
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying blob: %w", err)
|
||||
}
|
||||
|
||||
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
|
||||
if finishedTSUnix.Valid {
|
||||
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
|
||||
blob.FinishedTS = &ts
|
||||
}
|
||||
|
||||
if uploadedTSUnix.Valid {
|
||||
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
|
||||
blob.UploadedTS = &ts
|
||||
}
|
||||
|
||||
return &blob, nil
|
||||
}
|
||||
|
||||
@@ -1,22 +1,25 @@
|
||||
package database
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func TestBlobRepository(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewBlobRepository(db)
|
||||
repo := database.NewBlobRepository(db)
|
||||
|
||||
// Test Create
|
||||
blob := &Blob{
|
||||
blob := &database.Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("blobhash123"),
|
||||
CreatedTS: time.Now().Truncate(time.Second),
|
||||
@@ -32,14 +35,18 @@ func TestBlobRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob: %v", err)
|
||||
}
|
||||
|
||||
if retrieved == nil {
|
||||
t.Fatal("expected blob, got nil")
|
||||
}
|
||||
|
||||
if retrieved.Hash != blob.Hash {
|
||||
t.Errorf("blob hash mismatch: got %s, want %s", retrieved.Hash, blob.Hash)
|
||||
}
|
||||
|
||||
if !retrieved.CreatedTS.Equal(blob.CreatedTS) {
|
||||
t.Errorf("created timestamp mismatch: got %v, want %v", retrieved.CreatedTS, blob.CreatedTS)
|
||||
t.Errorf("created timestamp mismatch: got %v, want %v",
|
||||
retrieved.CreatedTS, blob.CreatedTS)
|
||||
}
|
||||
|
||||
// Test GetByID
|
||||
@@ -47,26 +54,51 @@ func TestBlobRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob by ID: %v", err)
|
||||
}
|
||||
|
||||
if retrievedByID == nil {
|
||||
t.Fatal("expected blob, got nil")
|
||||
}
|
||||
|
||||
if retrievedByID.ID != blob.ID {
|
||||
t.Errorf("blob ID mismatch: got %s, want %s", retrievedByID.ID, blob.ID)
|
||||
}
|
||||
|
||||
// Test with second blob
|
||||
blob2 := &Blob{
|
||||
blob2 := &database.Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("blobhash456"),
|
||||
CreatedTS: time.Now().Truncate(time.Second),
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, blob2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create second blob: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobRepositoryUpdates(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := database.NewBlobRepository(db)
|
||||
|
||||
blob := &database.Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("blobhash123"),
|
||||
CreatedTS: time.Now().Truncate(time.Second),
|
||||
}
|
||||
|
||||
err := repo.Create(ctx, nil, blob)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob: %v", err)
|
||||
}
|
||||
|
||||
// Test UpdateFinished
|
||||
now := time.Now()
|
||||
|
||||
err = repo.UpdateFinished(ctx, nil, blob.ID.String(), blob.Hash.String(), 1000, 500)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to update blob as finished: %v", err)
|
||||
@@ -77,12 +109,15 @@ func TestBlobRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get updated blob: %v", err)
|
||||
}
|
||||
|
||||
if updated.FinishedTS == nil {
|
||||
t.Fatal("expected finished timestamp to be set")
|
||||
}
|
||||
|
||||
if updated.UncompressedSize != 1000 {
|
||||
t.Errorf("expected uncompressed size 1000, got %d", updated.UncompressedSize)
|
||||
}
|
||||
|
||||
if updated.CompressedSize != 500 {
|
||||
t.Errorf("expected compressed size 500, got %d", updated.CompressedSize)
|
||||
}
|
||||
@@ -98,6 +133,7 @@ func TestBlobRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get uploaded blob: %v", err)
|
||||
}
|
||||
|
||||
if uploaded.UploadedTS == nil {
|
||||
t.Fatal("expected uploaded timestamp to be set")
|
||||
}
|
||||
@@ -108,13 +144,15 @@ func TestBlobRepository(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestBlobRepositoryDuplicate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewBlobRepository(db)
|
||||
repo := database.NewBlobRepository(db)
|
||||
|
||||
blob := &Blob{
|
||||
blob := &database.Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("duplicate_blob"),
|
||||
CreatedTS: time.Now().Truncate(time.Second),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
//nolint:testpackage // inspects the unexported database connection
|
||||
package database
|
||||
|
||||
import (
|
||||
@@ -6,26 +7,16 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// TestCascadeDeleteDebug tests cascade delete with debug output
|
||||
func TestCascadeDeleteDebug(t *testing.T) {
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
// createCascadeFixtures creates a file with three chunk mappings for the
|
||||
// cascade-delete test.
|
||||
func createCascadeFixtures(t *testing.T, repos *Repositories) *File {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
// Check if foreign keys are enabled
|
||||
var fkEnabled int
|
||||
err := db.conn.QueryRow("PRAGMA foreign_keys").Scan(&fkEnabled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("Foreign keys enabled: %d", fkEnabled)
|
||||
|
||||
// Create a file
|
||||
file := &File{
|
||||
Path: "/cascade-test.txt",
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
@@ -34,18 +25,21 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
err = repos.Files.Create(ctx, nil, file)
|
||||
|
||||
err := repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Created file with ID: %s", file.ID)
|
||||
|
||||
// Create chunks and file-chunk mappings
|
||||
for i := 0; i < 3; i++ {
|
||||
for i := range 3 {
|
||||
chunk := &Chunk{
|
||||
ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)),
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk: %v", err)
|
||||
@@ -56,33 +50,73 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
||||
Idx: i,
|
||||
ChunkHash: chunk.ChunkHash,
|
||||
}
|
||||
|
||||
err = repos.FileChunks.Create(ctx, nil, fc)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file chunk: %v", err)
|
||||
}
|
||||
t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash)
|
||||
|
||||
t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s",
|
||||
fc.FileID, fc.Idx, fc.ChunkHash)
|
||||
}
|
||||
|
||||
// Verify file chunks exist
|
||||
fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID)
|
||||
return file
|
||||
}
|
||||
|
||||
// logCascadeDebugInfo logs foreign-key state and the file_chunks table
|
||||
// definition for cascade-delete debugging.
|
||||
func logCascadeDebugInfo(ctx context.Context, t *testing.T, db *DB) {
|
||||
t.Helper()
|
||||
|
||||
// Check if foreign keys are enabled
|
||||
var fkEnabled int
|
||||
|
||||
err := db.conn.QueryRowContext(ctx, "PRAGMA foreign_keys").Scan(&fkEnabled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("File chunks before delete: %d", len(fileChunks))
|
||||
|
||||
t.Logf("Foreign keys enabled: %d", fkEnabled)
|
||||
|
||||
// Check the foreign key constraint
|
||||
var fkInfo string
|
||||
err = db.conn.QueryRow(`
|
||||
|
||||
err = db.conn.QueryRowContext(ctx, `
|
||||
SELECT sql FROM sqlite_master
|
||||
WHERE type='table' AND name='file_chunks'
|
||||
`).Scan(&fkInfo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("file_chunks table definition:\n%s", fkInfo)
|
||||
}
|
||||
|
||||
// TestCascadeDeleteDebug tests cascade delete with debug output
|
||||
func TestCascadeDeleteDebug(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
logCascadeDebugInfo(ctx, t, db)
|
||||
|
||||
file := createCascadeFixtures(t, repos)
|
||||
|
||||
// Verify file chunks exist
|
||||
fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("File chunks before delete: %d", len(fileChunks))
|
||||
|
||||
// Delete the file
|
||||
t.Log("Deleting file...")
|
||||
|
||||
err = repos.Files.DeleteByID(ctx, nil, file.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to delete file: %v", err)
|
||||
@@ -93,6 +127,7 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if deletedFile != nil {
|
||||
t.Error("file should have been deleted")
|
||||
} else {
|
||||
@@ -104,21 +139,27 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("File chunks after delete: %d", len(fileChunks))
|
||||
|
||||
// Manually check the database
|
||||
var count int
|
||||
err = db.conn.QueryRow("SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID).Scan(&count)
|
||||
|
||||
err = db.conn.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID,
|
||||
).Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("Manual count of file_chunks for deleted file: %d", count)
|
||||
|
||||
if len(fileChunks) != 0 {
|
||||
t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks))
|
||||
// List the remaining chunks
|
||||
for _, fc := range fileChunks {
|
||||
t.Logf("Remaining chunk: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash)
|
||||
t.Logf("Remaining chunk: file_id=%s, idx=%d, chunk=%s",
|
||||
fc.FileID, fc.Idx, fc.ChunkHash)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,19 +4,26 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// ChunkFileRepository provides access to the chunk_files table, the
|
||||
// reverse mapping from chunks to the files that contain them.
|
||||
type ChunkFileRepository struct {
|
||||
db *DB
|
||||
}
|
||||
|
||||
// NewChunkFileRepository creates a ChunkFileRepository backed by db.
|
||||
func NewChunkFileRepository(db *DB) *ChunkFileRepository {
|
||||
return &ChunkFileRepository{db: db}
|
||||
}
|
||||
|
||||
func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkFile) error {
|
||||
// Create inserts a chunk_files row (idempotently), using tx when non-nil.
|
||||
func (r *ChunkFileRepository) Create(
|
||||
ctx context.Context, tx *sql.Tx, cf *ChunkFile,
|
||||
) error {
|
||||
query := `
|
||||
INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length)
|
||||
VALUES (?, ?, ?, ?)
|
||||
@@ -25,9 +32,11 @@ func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkF
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
|
||||
_, err = tx.ExecContext(ctx, query,
|
||||
cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
|
||||
_, err = r.db.ExecWithLog(ctx, query,
|
||||
cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -37,7 +46,10 @@ func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkF
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ChunkFileRepository) GetByChunkHash(ctx context.Context, chunkHash types.ChunkHash) ([]*ChunkFile, error) {
|
||||
// GetByChunkHash returns all chunk_files rows for the given chunk hash.
|
||||
func (r *ChunkFileRepository) GetByChunkHash(
|
||||
ctx context.Context, chunkHash types.ChunkHash,
|
||||
) ([]*ChunkFile, error) {
|
||||
query := `
|
||||
SELECT chunk_hash, file_id, file_offset, length
|
||||
FROM chunk_files
|
||||
@@ -48,12 +60,21 @@ func (r *ChunkFileRepository) GetByChunkHash(ctx context.Context, chunkHash type
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying chunk files: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return r.scanChunkFiles(rows)
|
||||
}
|
||||
|
||||
func (r *ChunkFileRepository) GetByFilePath(ctx context.Context, filePath string) ([]*ChunkFile, error) {
|
||||
// GetByFilePath returns all chunk_files rows for the file at the given path.
|
||||
func (r *ChunkFileRepository) GetByFilePath(
|
||||
ctx context.Context, filePath string,
|
||||
) ([]*ChunkFile, error) {
|
||||
query := `
|
||||
SELECT cf.chunk_hash, cf.file_id, cf.file_offset, cf.length
|
||||
FROM chunk_files cf
|
||||
@@ -65,13 +86,21 @@ func (r *ChunkFileRepository) GetByFilePath(ctx context.Context, filePath string
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying chunk files: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return r.scanChunkFiles(rows)
|
||||
}
|
||||
|
||||
// GetByFileID retrieves chunk files by file ID
|
||||
func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.FileID) ([]*ChunkFile, error) {
|
||||
func (r *ChunkFileRepository) GetByFileID(
|
||||
ctx context.Context, fileID types.FileID,
|
||||
) ([]*ChunkFile, error) {
|
||||
query := `
|
||||
SELECT chunk_hash, file_id, file_offset, length
|
||||
FROM chunk_files
|
||||
@@ -82,34 +111,21 @@ func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.File
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying chunk files: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return r.scanChunkFiles(rows)
|
||||
}
|
||||
|
||||
// scanChunkFiles is a helper that scans chunk file rows
|
||||
func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) {
|
||||
var chunkFiles []*ChunkFile
|
||||
for rows.Next() {
|
||||
var cf ChunkFile
|
||||
var chunkHashStr, fileIDStr string
|
||||
err := rows.Scan(&chunkHashStr, &fileIDStr, &cf.FileOffset, &cf.Length)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning chunk file: %w", err)
|
||||
}
|
||||
cf.ChunkHash = types.ChunkHash(chunkHashStr)
|
||||
cf.FileID, err = types.ParseFileID(fileIDStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||
}
|
||||
chunkFiles = append(chunkFiles, &cf)
|
||||
}
|
||||
|
||||
return chunkFiles, rows.Err()
|
||||
}
|
||||
|
||||
// DeleteByFileID deletes all chunk_files entries for a given file ID
|
||||
func (r *ChunkFileRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fileID types.FileID) error {
|
||||
func (r *ChunkFileRepository) DeleteByFileID(
|
||||
ctx context.Context, tx *sql.Tx, fileID types.FileID,
|
||||
) error {
|
||||
query := `DELETE FROM chunk_files WHERE file_id = ?`
|
||||
|
||||
var err error
|
||||
@@ -127,7 +143,11 @@ func (r *ChunkFileRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fi
|
||||
}
|
||||
|
||||
// DeleteByFileIDs deletes all chunk_files for multiple files in a single statement.
|
||||
func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, fileIDs []types.FileID) error {
|
||||
//
|
||||
//nolint:dupl // symmetric implementation for a parallel association table
|
||||
func (r *ChunkFileRepository) DeleteByFileIDs(
|
||||
ctx context.Context, tx *sql.Tx, fileIDs []types.FileID,
|
||||
) error {
|
||||
if len(fileIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -136,14 +156,15 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
const batchSize = 500
|
||||
|
||||
for i := 0; i < len(fileIDs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(fileIDs) {
|
||||
end = len(fileIDs)
|
||||
}
|
||||
end := min(i+batchSize, len(fileIDs))
|
||||
|
||||
batch := fileIDs[i:end]
|
||||
|
||||
query := "DELETE FROM chunk_files WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
|
||||
args := make([]interface{}, len(batch))
|
||||
//nolint:gosec // G202: concatenates constant SQL and "?" placeholders only
|
||||
query := "DELETE FROM chunk_files WHERE file_id IN (?" +
|
||||
repeatPlaceholder(len(batch)-1) + ")"
|
||||
|
||||
args := make([]any, len(batch))
|
||||
for j, id := range batch {
|
||||
args[j] = id.String()
|
||||
}
|
||||
@@ -154,6 +175,7 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch deleting chunk_files: %w", err)
|
||||
}
|
||||
@@ -163,30 +185,43 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
}
|
||||
|
||||
// CreateBatch inserts multiple chunk_files in a single statement for efficiency.
|
||||
func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs []ChunkFile) error {
|
||||
func (r *ChunkFileRepository) CreateBatch(
|
||||
ctx context.Context, tx *sql.Tx, cfs []ChunkFile,
|
||||
) error {
|
||||
if len(cfs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Each ChunkFile has 4 values, so batch at 200 to be safe with SQLite's variable limit
|
||||
// Each chunk_files row binds this many SQL variables.
|
||||
const chunkFileCols = 4
|
||||
|
||||
// Batch at 200 rows to be safe with SQLite's variable limit.
|
||||
const batchSize = 200
|
||||
|
||||
for i := 0; i < len(cfs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(cfs) {
|
||||
end = len(cfs)
|
||||
}
|
||||
end := min(i+batchSize, len(cfs))
|
||||
|
||||
batch := cfs[i:end]
|
||||
|
||||
query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES "
|
||||
args := make([]interface{}, 0, len(batch)*4)
|
||||
|
||||
args := make([]any, 0, len(batch)*chunkFileCols)
|
||||
|
||||
var querySb183 strings.Builder
|
||||
|
||||
for j, cf := range batch {
|
||||
if j > 0 {
|
||||
query += ", "
|
||||
querySb183.WriteString(", ")
|
||||
}
|
||||
query += "(?, ?, ?, ?)"
|
||||
args = append(args, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
|
||||
|
||||
querySb183.WriteString("(?, ?, ?, ?)")
|
||||
|
||||
args = append(args,
|
||||
cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
|
||||
}
|
||||
|
||||
query += querySb183.String() //nolint:gosec // G202: appends "?" placeholders only
|
||||
|
||||
query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING"
|
||||
|
||||
var err error
|
||||
@@ -195,6 +230,7 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch inserting chunk_files: %w", err)
|
||||
}
|
||||
@@ -202,3 +238,31 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// scanChunkFiles is a helper that scans chunk file rows.
|
||||
func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) {
|
||||
var chunkFiles []*ChunkFile
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
cf ChunkFile
|
||||
chunkHashStr, fileIDStr string
|
||||
)
|
||||
|
||||
err := rows.Scan(&chunkHashStr, &fileIDStr, &cf.FileOffset, &cf.Length)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning chunk file: %w", err)
|
||||
}
|
||||
|
||||
cf.ChunkHash = types.ChunkHash(chunkHashStr)
|
||||
|
||||
cf.FileID, err = types.ParseFileID(fileIDStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||
}
|
||||
|
||||
chunkFiles = append(chunkFiles, &cf)
|
||||
}
|
||||
|
||||
return chunkFiles, rows.Err()
|
||||
}
|
||||
|
||||
@@ -1,120 +1,139 @@
|
||||
package database
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
const chunk4Hash = "chunk4"
|
||||
|
||||
// verifyChunkFilePair asserts that the chunk-file rows cover both test
|
||||
// files at their expected offsets.
|
||||
func verifyChunkFilePair(
|
||||
t *testing.T, chunkFiles []*database.ChunkFile,
|
||||
file1ID, file2ID types.FileID,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
foundFile1 := false
|
||||
foundFile2 := false
|
||||
|
||||
for _, cf := range chunkFiles {
|
||||
if cf.FileID == file1ID && cf.FileOffset == 0 {
|
||||
foundFile1 = true
|
||||
}
|
||||
|
||||
if cf.FileID == file2ID && cf.FileOffset == 2048 {
|
||||
foundFile2 = true
|
||||
}
|
||||
}
|
||||
|
||||
if !foundFile1 || !foundFile2 {
|
||||
t.Error("not all expected files found")
|
||||
}
|
||||
}
|
||||
|
||||
// createChunkFileTestFiles creates the two files used by the chunk-file
|
||||
// repository tests.
|
||||
func createChunkFileTestFiles(
|
||||
t *testing.T, fileRepo *database.FileRepository,
|
||||
) (*database.File, *database.File) {
|
||||
t.Helper()
|
||||
|
||||
testTime := time.Now().Truncate(time.Second)
|
||||
file1 := &database.File{
|
||||
Path: testFilePath1,
|
||||
MTime: testTime,
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
LinkTarget: "",
|
||||
}
|
||||
file2 := &database.File{
|
||||
Path: testFilePath2,
|
||||
MTime: testTime,
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
LinkTarget: "",
|
||||
}
|
||||
|
||||
mustCreateFile(t, fileRepo, file1)
|
||||
mustCreateFile(t, fileRepo, file2)
|
||||
|
||||
return file1, file2
|
||||
}
|
||||
|
||||
func TestChunkFileRepository(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewChunkFileRepository(db)
|
||||
fileRepo := NewFileRepository(db)
|
||||
chunksRepo := NewChunkRepository(db)
|
||||
repo := database.NewChunkFileRepository(db)
|
||||
fileRepo := database.NewFileRepository(db)
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Create test files first
|
||||
testTime := time.Now().Truncate(time.Second)
|
||||
file1 := &File{
|
||||
Path: "/file1.txt",
|
||||
MTime: testTime,
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
LinkTarget: "",
|
||||
}
|
||||
err := fileRepo.Create(ctx, nil, file1)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file1: %v", err)
|
||||
}
|
||||
|
||||
file2 := &File{
|
||||
Path: "/file2.txt",
|
||||
MTime: testTime,
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
LinkTarget: "",
|
||||
}
|
||||
err = fileRepo.Create(ctx, nil, file2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file2: %v", err)
|
||||
}
|
||||
|
||||
// Create chunk first
|
||||
chunk := &Chunk{
|
||||
ChunkHash: types.ChunkHash("chunk1"),
|
||||
Size: 1024,
|
||||
}
|
||||
err = chunksRepo.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk: %v", err)
|
||||
}
|
||||
file1, file2 := createChunkFileTestFiles(t, fileRepo)
|
||||
mustCreateChunks(t, repos, chunk1Hash)
|
||||
|
||||
// Test Create
|
||||
cf1 := &ChunkFile{
|
||||
ChunkHash: types.ChunkHash("chunk1"),
|
||||
cf1 := &database.ChunkFile{
|
||||
ChunkHash: types.ChunkHash(chunk1Hash),
|
||||
FileID: file1.ID,
|
||||
FileOffset: 0,
|
||||
Length: 1024,
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, cf1)
|
||||
err := repo.Create(ctx, nil, cf1)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk file: %v", err)
|
||||
}
|
||||
|
||||
// Add same chunk in different file (deduplication scenario)
|
||||
cf2 := &ChunkFile{
|
||||
ChunkHash: types.ChunkHash("chunk1"),
|
||||
cf2 := &database.ChunkFile{
|
||||
ChunkHash: types.ChunkHash(chunk1Hash),
|
||||
FileID: file2.ID,
|
||||
FileOffset: 2048,
|
||||
Length: 1024,
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, cf2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create second chunk file: %v", err)
|
||||
}
|
||||
|
||||
// Test GetByChunkHash
|
||||
chunkFiles, err := repo.GetByChunkHash(ctx, "chunk1")
|
||||
chunkFiles, err := repo.GetByChunkHash(ctx, chunk1Hash)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunk files: %v", err)
|
||||
}
|
||||
|
||||
if len(chunkFiles) != 2 {
|
||||
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
|
||||
}
|
||||
|
||||
// Verify both files are returned
|
||||
foundFile1 := false
|
||||
foundFile2 := false
|
||||
for _, cf := range chunkFiles {
|
||||
if cf.FileID == file1.ID && cf.FileOffset == 0 {
|
||||
foundFile1 = true
|
||||
}
|
||||
if cf.FileID == file2.ID && cf.FileOffset == 2048 {
|
||||
foundFile2 = true
|
||||
}
|
||||
}
|
||||
if !foundFile1 || !foundFile2 {
|
||||
t.Error("not all expected files found")
|
||||
}
|
||||
verifyChunkFilePair(t, chunkFiles, file1.ID, file2.ID)
|
||||
|
||||
// Test GetByFileID
|
||||
chunkFiles, err = repo.GetByFileID(ctx, file1.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks by file ID: %v", err)
|
||||
}
|
||||
|
||||
if len(chunkFiles) != 1 {
|
||||
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
|
||||
}
|
||||
if chunkFiles[0].ChunkHash != types.ChunkHash("chunk1") {
|
||||
|
||||
if chunkFiles[0].ChunkHash != types.ChunkHash(chunk1Hash) {
|
||||
t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash)
|
||||
}
|
||||
|
||||
@@ -126,60 +145,53 @@ func TestChunkFileRepository(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewChunkFileRepository(db)
|
||||
fileRepo := NewFileRepository(db)
|
||||
chunksRepo := NewChunkRepository(db)
|
||||
repo := database.NewChunkFileRepository(db)
|
||||
fileRepo := database.NewFileRepository(db)
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Create test files
|
||||
testTime := time.Now().Truncate(time.Second)
|
||||
file1 := &File{Path: "/file1.txt", MTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000}
|
||||
file2 := &File{Path: "/file2.txt", MTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000}
|
||||
file3 := &File{Path: "/file3.txt", MTime: testTime, Size: 2048, Mode: 0644, UID: 1000, GID: 1000}
|
||||
|
||||
if err := fileRepo.Create(ctx, nil, file1); err != nil {
|
||||
t.Fatalf("failed to create file1: %v", err)
|
||||
file1 := &database.File{
|
||||
Path: testFilePath1, MTime: testTime, Size: 3072,
|
||||
Mode: 0644, UID: 1000, GID: 1000,
|
||||
}
|
||||
if err := fileRepo.Create(ctx, nil, file2); err != nil {
|
||||
t.Fatalf("failed to create file2: %v", err)
|
||||
file2 := &database.File{
|
||||
Path: testFilePath2, MTime: testTime, Size: 3072,
|
||||
Mode: 0644, UID: 1000, GID: 1000,
|
||||
}
|
||||
if err := fileRepo.Create(ctx, nil, file3); err != nil {
|
||||
t.Fatalf("failed to create file3: %v", err)
|
||||
file3 := &database.File{
|
||||
Path: "/file3.txt", MTime: testTime, Size: 2048,
|
||||
Mode: 0644, UID: 1000, GID: 1000,
|
||||
}
|
||||
|
||||
// Create chunks first
|
||||
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3", "chunk4"}
|
||||
for _, chunkHash := range chunks {
|
||||
chunk := &Chunk{
|
||||
ChunkHash: chunkHash,
|
||||
Size: 1024,
|
||||
}
|
||||
err := chunksRepo.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||
}
|
||||
}
|
||||
mustCreateFile(t, fileRepo, file1)
|
||||
mustCreateFile(t, fileRepo, file2)
|
||||
mustCreateFile(t, fileRepo, file3)
|
||||
mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash, chunk4Hash)
|
||||
|
||||
// Simulate a scenario where multiple files share chunks
|
||||
// File1: chunk1, chunk2, chunk3
|
||||
// File2: chunk2, chunk3, chunk4
|
||||
// File3: chunk1, chunk4
|
||||
|
||||
chunkFiles := []ChunkFile{
|
||||
chunkFiles := []database.ChunkFile{
|
||||
// File1
|
||||
{ChunkHash: types.ChunkHash("chunk1"), FileID: file1.ID, FileOffset: 0, Length: 1024},
|
||||
{ChunkHash: types.ChunkHash("chunk2"), FileID: file1.ID, FileOffset: 1024, Length: 1024},
|
||||
{ChunkHash: types.ChunkHash("chunk3"), FileID: file1.ID, FileOffset: 2048, Length: 1024},
|
||||
{ChunkHash: chunk1Hash, FileID: file1.ID, FileOffset: 0, Length: 1024},
|
||||
{ChunkHash: chunk2Hash, FileID: file1.ID, FileOffset: 1024, Length: 1024},
|
||||
{ChunkHash: chunk3Hash, FileID: file1.ID, FileOffset: 2048, Length: 1024},
|
||||
// File2
|
||||
{ChunkHash: types.ChunkHash("chunk2"), FileID: file2.ID, FileOffset: 0, Length: 1024},
|
||||
{ChunkHash: types.ChunkHash("chunk3"), FileID: file2.ID, FileOffset: 1024, Length: 1024},
|
||||
{ChunkHash: types.ChunkHash("chunk4"), FileID: file2.ID, FileOffset: 2048, Length: 1024},
|
||||
{ChunkHash: chunk2Hash, FileID: file2.ID, FileOffset: 0, Length: 1024},
|
||||
{ChunkHash: chunk3Hash, FileID: file2.ID, FileOffset: 1024, Length: 1024},
|
||||
{ChunkHash: chunk4Hash, FileID: file2.ID, FileOffset: 2048, Length: 1024},
|
||||
// File3
|
||||
{ChunkHash: types.ChunkHash("chunk1"), FileID: file3.ID, FileOffset: 0, Length: 1024},
|
||||
{ChunkHash: types.ChunkHash("chunk4"), FileID: file3.ID, FileOffset: 1024, Length: 1024},
|
||||
{ChunkHash: chunk1Hash, FileID: file3.ID, FileOffset: 0, Length: 1024},
|
||||
{ChunkHash: chunk4Hash, FileID: file3.ID, FileOffset: 1024, Length: 1024},
|
||||
}
|
||||
|
||||
for _, cf := range chunkFiles {
|
||||
@@ -190,19 +202,21 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
||||
}
|
||||
|
||||
// Test chunk1 (used by file1 and file3)
|
||||
files, err := repo.GetByChunkHash(ctx, "chunk1")
|
||||
files, err := repo.GetByChunkHash(ctx, chunk1Hash)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get files for chunk1: %v", err)
|
||||
}
|
||||
|
||||
if len(files) != 2 {
|
||||
t.Errorf("expected 2 files for chunk1, got %d", len(files))
|
||||
}
|
||||
|
||||
// Test chunk2 (used by file1 and file2)
|
||||
files, err = repo.GetByChunkHash(ctx, "chunk2")
|
||||
files, err = repo.GetByChunkHash(ctx, chunk2Hash)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get files for chunk2: %v", err)
|
||||
}
|
||||
|
||||
if len(files) != 2 {
|
||||
t.Errorf("expected 2 files for chunk2, got %d", len(files))
|
||||
}
|
||||
@@ -212,6 +226,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks for file2: %v", err)
|
||||
}
|
||||
|
||||
if len(file2Chunks) != 3 {
|
||||
t.Errorf("expected 3 chunks for file2, got %d", len(file2Chunks))
|
||||
}
|
||||
|
||||
+49
-10
@@ -3,19 +3,25 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
// ChunkRepository provides access to the chunks table, which tracks
|
||||
// content-defined chunks by hash and size.
|
||||
type ChunkRepository struct {
|
||||
db *DB
|
||||
}
|
||||
|
||||
// NewChunkRepository creates a ChunkRepository backed by db.
|
||||
func NewChunkRepository(db *DB) *ChunkRepository {
|
||||
return &ChunkRepository{db: db}
|
||||
}
|
||||
|
||||
// Create inserts a chunk row (idempotently), using tx when non-nil.
|
||||
func (r *ChunkRepository) Create(ctx context.Context, tx *sql.Tx, chunk *Chunk) error {
|
||||
query := `
|
||||
INSERT INTO chunks (chunk_hash, size)
|
||||
@@ -37,6 +43,8 @@ func (r *ChunkRepository) Create(ctx context.Context, tx *sql.Tx, chunk *Chunk)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetByHash returns the chunk with the given hash, or nil if it is not
|
||||
// known to the index.
|
||||
func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, error) {
|
||||
query := `
|
||||
SELECT chunk_hash, size
|
||||
@@ -51,9 +59,10 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e
|
||||
&chunk.Size,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying chunk: %w", err)
|
||||
}
|
||||
@@ -61,7 +70,11 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e
|
||||
return &chunk, nil
|
||||
}
|
||||
|
||||
func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*Chunk, error) {
|
||||
// GetByHashes returns the chunks whose hashes appear in hashes, ordered by
|
||||
// chunk hash. Unknown hashes are silently omitted from the result.
|
||||
func (r *ChunkRepository) GetByHashes(
|
||||
ctx context.Context, hashes []string,
|
||||
) ([]*Chunk, error) {
|
||||
if len(hashes) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -71,23 +84,38 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
|
||||
FROM chunks
|
||||
WHERE chunk_hash IN (`
|
||||
|
||||
args := make([]interface{}, len(hashes))
|
||||
args := make([]any, len(hashes))
|
||||
|
||||
var querySb75 strings.Builder
|
||||
|
||||
for i, hash := range hashes {
|
||||
if i > 0 {
|
||||
query += ", "
|
||||
querySb75.WriteString(", ")
|
||||
}
|
||||
query += "?"
|
||||
|
||||
querySb75.WriteString("?")
|
||||
|
||||
args[i] = hash
|
||||
}
|
||||
|
||||
query += querySb75.String() //nolint:gosec // G202: appends "?" placeholders only
|
||||
|
||||
query += ") ORDER BY chunk_hash"
|
||||
|
||||
rows, err := r.db.conn.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying chunks: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
var chunks []*Chunk
|
||||
|
||||
for rows.Next() {
|
||||
var chunk Chunk
|
||||
|
||||
@@ -105,7 +133,11 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
|
||||
return chunks, rows.Err()
|
||||
}
|
||||
|
||||
func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk, error) {
|
||||
// ListUnpacked returns up to limit chunks that are not yet stored in any
|
||||
// blob, ordered by chunk hash.
|
||||
func (r *ChunkRepository) ListUnpacked(
|
||||
ctx context.Context, limit int,
|
||||
) ([]*Chunk, error) {
|
||||
query := `
|
||||
SELECT c.chunk_hash, c.size
|
||||
FROM chunks c
|
||||
@@ -119,9 +151,16 @@ func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying unpacked chunks: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
var chunks []*Chunk
|
||||
|
||||
for rows.Next() {
|
||||
var chunk Chunk
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// List returns every chunk in the index, ordered by chunk hash.
|
||||
func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) {
|
||||
query := `
|
||||
SELECT chunk_hash, size
|
||||
@@ -16,9 +17,16 @@ func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying chunks: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
var chunks []*Chunk
|
||||
|
||||
for rows.Next() {
|
||||
var chunk Chunk
|
||||
|
||||
|
||||
@@ -1,21 +1,24 @@
|
||||
package database
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func TestChunkRepository(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewChunkRepository(db)
|
||||
repo := database.NewChunkRepository(db)
|
||||
|
||||
// Test Create
|
||||
chunk := &Chunk{
|
||||
chunk := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash("chunkhash123"),
|
||||
Size: 4096,
|
||||
}
|
||||
@@ -30,12 +33,15 @@ func TestChunkRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunk: %v", err)
|
||||
}
|
||||
|
||||
if retrieved == nil {
|
||||
t.Fatal("expected chunk, got nil")
|
||||
}
|
||||
|
||||
if retrieved.ChunkHash != chunk.ChunkHash {
|
||||
t.Errorf("chunk hash mismatch: got %s, want %s", retrieved.ChunkHash, chunk.ChunkHash)
|
||||
}
|
||||
|
||||
if retrieved.Size != chunk.Size {
|
||||
t.Errorf("size mismatch: got %d, want %d", retrieved.Size, chunk.Size)
|
||||
}
|
||||
@@ -47,19 +53,23 @@ func TestChunkRepository(t *testing.T) {
|
||||
}
|
||||
|
||||
// Test GetByHashes
|
||||
chunk2 := &Chunk{
|
||||
chunk2 := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash("chunkhash456"),
|
||||
Size: 8192,
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, chunk2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create second chunk: %v", err)
|
||||
}
|
||||
|
||||
chunks, err := repo.GetByHashes(ctx, []string{chunk.ChunkHash.String(), chunk2.ChunkHash.String()})
|
||||
chunks, err := repo.GetByHashes(ctx, []string{
|
||||
chunk.ChunkHash.String(), chunk2.ChunkHash.String(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks by hashes: %v", err)
|
||||
}
|
||||
|
||||
if len(chunks) != 2 {
|
||||
t.Errorf("expected 2 chunks, got %d", len(chunks))
|
||||
}
|
||||
@@ -69,23 +79,27 @@ func TestChunkRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to list unpacked chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(unpacked) != 2 {
|
||||
t.Errorf("expected 2 unpacked chunks, got %d", len(unpacked))
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkRepositoryNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewChunkRepository(db)
|
||||
repo := database.NewChunkRepository(db)
|
||||
|
||||
// Test GetByHash with non-existent hash
|
||||
chunk, err := repo.GetByHash(ctx, "nonexistent")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if chunk != nil {
|
||||
t.Error("expected nil for non-existent chunk")
|
||||
}
|
||||
@@ -95,6 +109,7 @@ func TestChunkRepositoryNotFound(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if chunks != nil {
|
||||
t.Error("expected nil for empty hash list")
|
||||
}
|
||||
|
||||
+277
-69
@@ -6,24 +6,38 @@
|
||||
// multiple source files. Blobs are content-addressed, meaning their filename
|
||||
// is derived from their SHA256 hash after compression and encryption.
|
||||
//
|
||||
// The database does not support migrations. If the schema changes, delete
|
||||
// the local database and perform a full backup to recreate it.
|
||||
// Schema is managed via numbered SQL migrations embedded in the schema/
|
||||
// directory. Migration 000.sql bootstraps the schema_migrations tracking
|
||||
// table; subsequent migrations (001, 002, …) are applied in order.
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
_ "embed"
|
||||
"embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
// Register the pure-Go sqlite driver.
|
||||
_ "modernc.org/sqlite"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
//go:embed schema.sql
|
||||
var schemaSQL string
|
||||
// errInvalidMigrationFilename is returned when an embedded migration file
|
||||
// does not follow the "<version>[_<description>].sql" naming pattern.
|
||||
var errInvalidMigrationFilename = errors.New("invalid migration filename")
|
||||
|
||||
//go:embed schema/*.sql
|
||||
var schemaFS embed.FS
|
||||
|
||||
// bootstrapVersion is the migration that creates the schema_migrations
|
||||
// table itself. It is applied before the normal migration loop.
|
||||
const bootstrapVersion = 0
|
||||
|
||||
// DB represents the Vaultik local index database connection.
|
||||
// It uses SQLite to track file metadata, content-defined chunks, and blob associations.
|
||||
@@ -35,6 +49,48 @@ type DB struct {
|
||||
path string
|
||||
}
|
||||
|
||||
// ParseMigrationVersion extracts the numeric version prefix from a migration
|
||||
// filename. Filenames must follow the pattern "<version>.sql" or
|
||||
// "<version>_<description>.sql", where version is a zero-padded numeric
|
||||
// string (e.g. "001", "002"). Returns the version as an integer and an
|
||||
// error if the filename does not match the expected pattern.
|
||||
func ParseMigrationVersion(filename string) (int, error) {
|
||||
name := strings.TrimSuffix(filename, filepath.Ext(filename))
|
||||
if name == "" {
|
||||
return 0, fmt.Errorf("%w %q: empty name", errInvalidMigrationFilename, filename)
|
||||
}
|
||||
|
||||
// Split on underscore to separate version from description.
|
||||
// If there's no underscore, the entire stem is the version.
|
||||
versionStr := name
|
||||
if before, _, ok := strings.Cut(name, "_"); ok {
|
||||
versionStr = before
|
||||
}
|
||||
|
||||
if versionStr == "" {
|
||||
return 0, fmt.Errorf(
|
||||
"%w %q: empty version prefix", errInvalidMigrationFilename, filename,
|
||||
)
|
||||
}
|
||||
|
||||
// Validate the version is purely numeric.
|
||||
for _, ch := range versionStr {
|
||||
if ch < '0' || ch > '9' {
|
||||
return 0, fmt.Errorf(
|
||||
"%w %q: version %q contains non-numeric character %q",
|
||||
errInvalidMigrationFilename, filename, versionStr, string(ch),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
version, err := strconv.Atoi(versionStr)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid migration filename %q: %w", filename, err)
|
||||
}
|
||||
|
||||
return version, nil
|
||||
}
|
||||
|
||||
// New creates a new database connection at the specified path.
|
||||
// It creates the schema if needed and configures SQLite with WAL mode for
|
||||
// better concurrency. SQLite handles crash recovery automatically when
|
||||
@@ -50,61 +106,93 @@ func New(ctx context.Context, path string) (*DB, error) {
|
||||
|
||||
// First attempt with standard WAL mode
|
||||
log.Debug("Attempting to open database with WAL mode", "path", path)
|
||||
|
||||
conn, err := sql.Open(
|
||||
"sqlite",
|
||||
path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000&_locking_mode=NORMAL&_foreign_keys=ON",
|
||||
path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000"+
|
||||
"&_locking_mode=NORMAL&_foreign_keys=ON",
|
||||
)
|
||||
if err == nil {
|
||||
// Set connection pool settings
|
||||
// SQLite can handle multiple readers but only one writer at a time.
|
||||
// Setting MaxOpenConns to 1 ensures all writes are serialized through
|
||||
// a single connection, preventing SQLITE_BUSY errors.
|
||||
conn.SetMaxOpenConns(1)
|
||||
conn.SetMaxIdleConns(1)
|
||||
configureConnPool(conn)
|
||||
|
||||
if err := conn.PingContext(ctx); err == nil {
|
||||
err = conn.PingContext(ctx)
|
||||
if err == nil {
|
||||
// Success on first try
|
||||
log.Debug("Database opened successfully with WAL mode", "path", path)
|
||||
|
||||
// Enable foreign keys explicitly
|
||||
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys = ON"); err != nil {
|
||||
log.Warn("Failed to enable foreign keys", "error", err)
|
||||
return finishOpen(ctx, conn, path)
|
||||
}
|
||||
|
||||
db := &DB{conn: conn, path: path}
|
||||
if err := db.createSchema(ctx); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("creating schema: %w", err)
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
log.Debug("Failed to ping database, closing connection", "path", path, "error", err)
|
||||
log.Debug(
|
||||
"Failed to ping database, closing connection",
|
||||
"path", path, "error", err,
|
||||
)
|
||||
|
||||
_ = conn.Close()
|
||||
}
|
||||
|
||||
// If first attempt failed, try with TRUNCATE mode to clear any locks
|
||||
return openWithRecovery(ctx, path)
|
||||
}
|
||||
|
||||
// configureConnPool serializes all database access through one connection.
|
||||
// SQLite can handle multiple readers but only one writer at a time; setting
|
||||
// MaxOpenConns to 1 ensures all writes go through a single connection,
|
||||
// preventing SQLITE_BUSY errors.
|
||||
func configureConnPool(conn *sql.DB) {
|
||||
conn.SetMaxOpenConns(1)
|
||||
conn.SetMaxIdleConns(1)
|
||||
}
|
||||
|
||||
// finishOpen enables foreign keys, wraps the connection, and applies any
|
||||
// pending migrations. On migration failure the connection is closed.
|
||||
func finishOpen(ctx context.Context, conn *sql.DB, path string) (*DB, error) {
|
||||
// Enable foreign keys explicitly
|
||||
_, err := conn.ExecContext(ctx, "PRAGMA foreign_keys = ON")
|
||||
if err != nil {
|
||||
log.Warn("Failed to enable foreign keys", "path", path, "error", err)
|
||||
}
|
||||
|
||||
db := &DB{conn: conn, path: path}
|
||||
|
||||
err = applyMigrations(ctx, conn)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
|
||||
return nil, fmt.Errorf("applying migrations: %w", err)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// openWithRecovery retries opening the database in TRUNCATE journal mode to
|
||||
// clear stale locks, then switches back to WAL mode.
|
||||
func openWithRecovery(ctx context.Context, path string) (*DB, error) {
|
||||
log.Info(
|
||||
"Database appears locked, attempting recovery with TRUNCATE mode",
|
||||
"path", path,
|
||||
)
|
||||
conn, err = sql.Open(
|
||||
|
||||
conn, err := sql.Open(
|
||||
"sqlite",
|
||||
path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000&_foreign_keys=ON",
|
||||
path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000"+
|
||||
"&_foreign_keys=ON",
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening database in recovery mode: %w", err)
|
||||
}
|
||||
|
||||
// Set connection pool settings
|
||||
// SQLite can handle multiple readers but only one writer at a time.
|
||||
// Setting MaxOpenConns to 1 ensures all writes are serialized through
|
||||
// a single connection, preventing SQLITE_BUSY errors.
|
||||
conn.SetMaxOpenConns(1)
|
||||
conn.SetMaxIdleConns(1)
|
||||
configureConnPool(conn)
|
||||
|
||||
err = conn.PingContext(ctx)
|
||||
if err != nil {
|
||||
log.Debug(
|
||||
"Failed to ping database in recovery mode, closing",
|
||||
"path", path, "error", err,
|
||||
)
|
||||
|
||||
if err := conn.PingContext(ctx); err != nil {
|
||||
log.Debug("Failed to ping database in recovery mode, closing", "path", path, "error", err)
|
||||
_ = conn.Close()
|
||||
|
||||
return nil, fmt.Errorf(
|
||||
"database still locked after recovery attempt: %w",
|
||||
err,
|
||||
@@ -115,35 +203,44 @@ func New(ctx context.Context, path string) (*DB, error) {
|
||||
|
||||
// Switch back to WAL mode
|
||||
log.Debug("Switching database back to WAL mode", "path", path)
|
||||
if _, err := conn.ExecContext(ctx, "PRAGMA journal_mode=WAL"); err != nil {
|
||||
|
||||
_, err = conn.ExecContext(ctx, "PRAGMA journal_mode=WAL")
|
||||
if err != nil {
|
||||
log.Warn("Failed to switch back to WAL mode", "path", path, "error", err)
|
||||
}
|
||||
|
||||
// Ensure foreign keys are enabled
|
||||
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=ON"); err != nil {
|
||||
log.Warn("Failed to enable foreign keys", "path", path, "error", err)
|
||||
}
|
||||
|
||||
db := &DB{conn: conn, path: path}
|
||||
if err := db.createSchema(ctx); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("creating schema: %w", err)
|
||||
db, err := finishOpen(ctx, conn, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Debug("Database connection established successfully", "path", path)
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// NewTestDB creates an in-memory SQLite database for testing purposes.
|
||||
// The database is automatically initialized with the schema and is ready
|
||||
// for use. Each call creates a new independent database instance.
|
||||
func NewTestDB() (*DB, error) {
|
||||
return New(context.Background(), ":memory:")
|
||||
}
|
||||
|
||||
// Close closes the database connection.
|
||||
// It ensures all pending operations are completed before closing.
|
||||
// Returns an error if the database connection cannot be closed properly.
|
||||
func (db *DB) Close() error {
|
||||
log.Debug("Closing database connection", "path", db.path)
|
||||
if err := db.conn.Close(); err != nil {
|
||||
|
||||
err := db.conn.Close()
|
||||
if err != nil {
|
||||
log.Error("Failed to close database", "path", db.path, "error", err)
|
||||
|
||||
return fmt.Errorf("failed to close database: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Database connection closed successfully", "path", db.path)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -179,54 +276,165 @@ func (db *DB) BeginTx(
|
||||
func (db *DB) ExecWithLog(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
args ...interface{},
|
||||
args ...any,
|
||||
) (sql.Result, error) {
|
||||
LogSQL("Execute", query, args...)
|
||||
|
||||
return db.conn.ExecContext(ctx, query, args...)
|
||||
}
|
||||
|
||||
// QueryRowWithLog executes a query that returns at most one row with SQL logging.
|
||||
// This is useful for queries that modify data and return values (e.g., INSERT ... RETURNING).
|
||||
// SQLite handles its own locking internally.
|
||||
// The query and args parameters follow the same format as sql.DB.QueryRowContext.
|
||||
// QueryRowWithLog executes a query that returns at most one row with SQL
|
||||
// logging. This is useful for queries that modify data and return values
|
||||
// (e.g., INSERT ... RETURNING). SQLite handles its own locking internally.
|
||||
// The query and args parameters follow the same format as
|
||||
// sql.DB.QueryRowContext.
|
||||
func (db *DB) QueryRowWithLog(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
args ...interface{},
|
||||
args ...any,
|
||||
) *sql.Row {
|
||||
LogSQL("QueryRow", query, args...)
|
||||
|
||||
return db.conn.QueryRowContext(ctx, query, args...)
|
||||
}
|
||||
|
||||
func (db *DB) createSchema(ctx context.Context) error {
|
||||
_, err := db.conn.ExecContext(ctx, schemaSQL)
|
||||
// collectMigrations reads the embedded schema directory and returns
|
||||
// migration filenames sorted lexicographically.
|
||||
func collectMigrations() ([]string, error) {
|
||||
entries, err := schemaFS.ReadDir("schema")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read schema directory: %w", err)
|
||||
}
|
||||
|
||||
var migrations []string
|
||||
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".sql") {
|
||||
migrations = append(migrations, entry.Name())
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(migrations)
|
||||
|
||||
return migrations, nil
|
||||
}
|
||||
|
||||
// bootstrapMigrationsTable ensures the schema_migrations table exists
|
||||
// by applying 000.sql if the table is missing.
|
||||
func bootstrapMigrationsTable(ctx context.Context, db *sql.DB) error {
|
||||
var tableExists int
|
||||
|
||||
err := db.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'",
|
||||
).Scan(&tableExists)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check for migrations table: %w", err)
|
||||
}
|
||||
|
||||
if tableExists > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
content, err := schemaFS.ReadFile("schema/000.sql")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read bootstrap migration 000.sql: %w", err)
|
||||
}
|
||||
|
||||
log.Info("applying bootstrap migration", "version", bootstrapVersion)
|
||||
|
||||
_, err = db.ExecContext(ctx, string(content))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to apply bootstrap migration: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyMigrations applies all pending migrations to db. It first bootstraps
|
||||
// the schema_migrations table via 000.sql, then iterates through remaining
|
||||
// migration files in order.
|
||||
func applyMigrations(ctx context.Context, db *sql.DB) error {
|
||||
err := bootstrapMigrationsTable(ctx, db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
migrations, err := collectMigrations()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, migration := range migrations {
|
||||
version, parseErr := ParseMigrationVersion(migration)
|
||||
if parseErr != nil {
|
||||
return parseErr
|
||||
}
|
||||
|
||||
// Check if already applied.
|
||||
var count int
|
||||
|
||||
err := db.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM schema_migrations WHERE version = ?",
|
||||
version,
|
||||
).Scan(&count)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check migration status: %w", err)
|
||||
}
|
||||
|
||||
if count > 0 {
|
||||
log.Debug("migration already applied", "version", version)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// Read and apply migration.
|
||||
content, readErr := schemaFS.ReadFile(filepath.Join("schema", migration))
|
||||
if readErr != nil {
|
||||
return fmt.Errorf("failed to read migration %s: %w", migration, readErr)
|
||||
}
|
||||
|
||||
log.Info("applying migration", "version", version)
|
||||
|
||||
_, execErr := db.ExecContext(ctx, string(content))
|
||||
if execErr != nil {
|
||||
return fmt.Errorf("failed to apply migration %s: %w", migration, execErr)
|
||||
}
|
||||
|
||||
// Record migration as applied.
|
||||
_, recErr := db.ExecContext(ctx,
|
||||
"INSERT INTO schema_migrations (version) VALUES (?)",
|
||||
version,
|
||||
)
|
||||
if recErr != nil {
|
||||
return fmt.Errorf("failed to record migration %s: %w", migration, recErr)
|
||||
}
|
||||
|
||||
log.Info("migration applied successfully", "version", version)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewTestDB creates an in-memory SQLite database for testing purposes.
|
||||
// The database is automatically initialized with the schema and is ready for use.
|
||||
// Each call creates a new independent database instance.
|
||||
func NewTestDB() (*DB, error) {
|
||||
return New(context.Background(), ":memory:")
|
||||
}
|
||||
|
||||
// repeatPlaceholder generates a string of ", ?" repeated n times for IN clause construction.
|
||||
// For example, repeatPlaceholder(2) returns ", ?, ?".
|
||||
// repeatPlaceholder generates a string of ", ?" repeated n times for IN
|
||||
// clause construction. For example, repeatPlaceholder(2) returns ", ?, ?".
|
||||
func repeatPlaceholder(n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
return strings.Repeat(", ?", n)
|
||||
}
|
||||
|
||||
// LogSQL logs SQL queries and their arguments when debug mode is enabled.
|
||||
// Debug mode is activated by setting the GODEBUG environment variable to include "vaultik".
|
||||
// This is useful for troubleshooting database operations and understanding query patterns.
|
||||
// Debug mode is activated by setting the GODEBUG environment variable to
|
||||
// include "vaultik". This is useful for troubleshooting database operations
|
||||
// and understanding query patterns.
|
||||
//
|
||||
// The operation parameter describes the type of SQL operation (e.g., "Execute", "Query").
|
||||
// The query parameter is the SQL statement being executed.
|
||||
// The args parameter contains the query arguments that will be interpolated.
|
||||
func LogSQL(operation, query string, args ...interface{}) {
|
||||
// The operation parameter describes the type of SQL operation (e.g.,
|
||||
// "Execute", "Query"). The query parameter is the SQL statement being
|
||||
// executed. The args parameter contains the query arguments that will be
|
||||
// interpolated.
|
||||
func LogSQL(operation, query string, args ...any) {
|
||||
if strings.Contains(os.Getenv("GODEBUG"), "vaultik") {
|
||||
log.Debug(
|
||||
"SQL "+operation,
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
//nolint:testpackage // exercises unexported migration internals
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDatabase(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||
|
||||
@@ -15,8 +19,10 @@ func TestDatabase(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -26,16 +32,20 @@ func TestDatabase(t *testing.T) {
|
||||
t.Fatal("database connection is nil")
|
||||
}
|
||||
|
||||
// Test schema creation (already done in New)
|
||||
// Test schema creation (already done in New via migrations)
|
||||
// Verify tables exist
|
||||
tables := []string{
|
||||
"schema_migrations",
|
||||
"files", "file_chunks", "chunks", "blobs",
|
||||
"blob_chunks", "chunk_files", "snapshots",
|
||||
}
|
||||
|
||||
for _, table := range tables {
|
||||
var name string
|
||||
err := db.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name)
|
||||
|
||||
err := db.conn.QueryRowContext(ctx,
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?", table,
|
||||
).Scan(&name)
|
||||
if err != nil {
|
||||
t.Errorf("table %s does not exist: %v", table, err)
|
||||
}
|
||||
@@ -43,6 +53,8 @@ func TestDatabase(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDatabaseInvalidPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Test with invalid path
|
||||
@@ -53,6 +65,8 @@ func TestDatabaseInvalidPath(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDatabaseConcurrentAccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||
|
||||
@@ -60,8 +74,10 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -71,18 +87,20 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
|
||||
index int
|
||||
err error
|
||||
}
|
||||
|
||||
results := make(chan result, 10)
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
for i := range 10 {
|
||||
go func(i int) {
|
||||
_, err := db.ExecWithLog(ctx, "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)",
|
||||
_, err := db.ExecWithLog(ctx,
|
||||
"INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)",
|
||||
fmt.Sprintf("hash%d", i), i*1024)
|
||||
results <- result{index: i, err: err}
|
||||
}(i)
|
||||
}
|
||||
|
||||
// Wait for all goroutines and check results
|
||||
for i := 0; i < 10; i++ {
|
||||
for range 10 {
|
||||
r := <-results
|
||||
if r.err != nil {
|
||||
t.Fatalf("concurrent insert %d failed: %v", r.index, r.err)
|
||||
@@ -91,11 +109,196 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
|
||||
|
||||
// Verify all inserts succeeded
|
||||
var count int
|
||||
|
||||
err = db.conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM chunks").Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to count chunks: %v", err)
|
||||
}
|
||||
|
||||
if count != 10 {
|
||||
t.Errorf("expected 10 chunks, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMigrationVersion(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
filename string
|
||||
wantVer int
|
||||
wantError bool
|
||||
}{
|
||||
{name: "valid 000.sql", filename: "000.sql", wantVer: 0, wantError: false},
|
||||
{name: "valid 001.sql", filename: "001.sql", wantVer: 1, wantError: false},
|
||||
{name: "valid 099.sql", filename: "099.sql", wantVer: 99, wantError: false},
|
||||
{
|
||||
name: "valid with description", filename: "001_initial_schema.sql",
|
||||
wantVer: 1, wantError: false,
|
||||
},
|
||||
{
|
||||
name: "valid large version", filename: "123_big_migration.sql",
|
||||
wantVer: 123, wantError: false,
|
||||
},
|
||||
{name: "invalid alpha version", filename: "abc.sql", wantVer: 0, wantError: true},
|
||||
{name: "invalid mixed chars", filename: "12a.sql", wantVer: 0, wantError: true},
|
||||
{name: "invalid no extension", filename: "schema.sql", wantVer: 0, wantError: true},
|
||||
{name: "empty string", filename: "", wantVer: 0, wantError: true},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := ParseMigrationVersion(tc.filename)
|
||||
if tc.wantError {
|
||||
if err == nil {
|
||||
t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error",
|
||||
tc.filename, got)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("ParseMigrationVersion(%q) unexpected error: %v",
|
||||
tc.filename, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if got != tc.wantVer {
|
||||
t.Errorf("ParseMigrationVersion(%q) = %d; want %d",
|
||||
tc.filename, got, tc.wantVer)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyMigrations_Idempotent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open database: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err := conn.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
conn.SetMaxOpenConns(1)
|
||||
conn.SetMaxIdleConns(1)
|
||||
|
||||
// First run: apply all migrations.
|
||||
err = applyMigrations(ctx, conn)
|
||||
if err != nil {
|
||||
t.Fatalf("first applyMigrations failed: %v", err)
|
||||
}
|
||||
|
||||
// Count rows in schema_migrations after first run.
|
||||
var countBefore int
|
||||
|
||||
err = conn.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM schema_migrations",
|
||||
).Scan(&countBefore)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to count schema_migrations after first run: %v", err)
|
||||
}
|
||||
|
||||
// Second run: must be a no-op.
|
||||
err = applyMigrations(ctx, conn)
|
||||
if err != nil {
|
||||
t.Fatalf("second applyMigrations failed: %v", err)
|
||||
}
|
||||
|
||||
// Count rows in schema_migrations after second run — must be unchanged.
|
||||
var countAfter int
|
||||
|
||||
err = conn.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM schema_migrations",
|
||||
).Scan(&countAfter)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to count schema_migrations after second run: %v", err)
|
||||
}
|
||||
|
||||
if countBefore != countAfter {
|
||||
t.Errorf("schema_migrations row count changed: before=%d, after=%d",
|
||||
countBefore, countAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open database: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err := conn.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
conn.SetMaxOpenConns(1)
|
||||
conn.SetMaxIdleConns(1)
|
||||
|
||||
// Verify schema_migrations does NOT exist yet.
|
||||
var tableBefore int
|
||||
|
||||
err = conn.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'",
|
||||
).Scan(&tableBefore)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to check for table before bootstrap: %v", err)
|
||||
}
|
||||
|
||||
if tableBefore != 0 {
|
||||
t.Fatal("schema_migrations table should not exist before bootstrap")
|
||||
}
|
||||
|
||||
// Run bootstrap.
|
||||
err = bootstrapMigrationsTable(ctx, conn)
|
||||
if err != nil {
|
||||
t.Fatalf("bootstrapMigrationsTable failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify schema_migrations now exists.
|
||||
var tableAfter int
|
||||
|
||||
err = conn.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'",
|
||||
).Scan(&tableAfter)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to check for table after bootstrap: %v", err)
|
||||
}
|
||||
|
||||
if tableAfter != 1 {
|
||||
t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d",
|
||||
tableAfter)
|
||||
}
|
||||
|
||||
// Verify version 0 row exists.
|
||||
var version int
|
||||
|
||||
err = conn.QueryRowContext(ctx,
|
||||
"SELECT version FROM schema_migrations WHERE version = 0",
|
||||
).Scan(&version)
|
||||
if err != nil {
|
||||
t.Fatalf("version 0 row not found in schema_migrations: %v", err)
|
||||
}
|
||||
|
||||
if version != 0 {
|
||||
t.Errorf("expected version 0, got %d", version)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,20 +1,12 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Fatal prints an error message to stderr and exits with status 1
|
||||
func Fatal(format string, args ...interface{}) {
|
||||
// Fatalf prints an error message to stderr and exits with status 1
|
||||
func Fatalf(format string, args ...any) {
|
||||
fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// CloseRows closes rows and exits on error
|
||||
func CloseRows(rows *sql.Rows) {
|
||||
if err := rows.Close(); err != nil {
|
||||
Fatal("failed to close rows: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,19 +4,26 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// FileChunkRepository provides access to the file_chunks table, which maps
|
||||
// files to their ordered constituent chunks.
|
||||
type FileChunkRepository struct {
|
||||
db *DB
|
||||
}
|
||||
|
||||
// NewFileChunkRepository creates a FileChunkRepository backed by db.
|
||||
func NewFileChunkRepository(db *DB) *FileChunkRepository {
|
||||
return &FileChunkRepository{db: db}
|
||||
}
|
||||
|
||||
func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileChunk) error {
|
||||
// Create inserts a file_chunks row (idempotently), using tx when non-nil.
|
||||
func (r *FileChunkRepository) Create(
|
||||
ctx context.Context, tx *sql.Tx, fc *FileChunk,
|
||||
) error {
|
||||
query := `
|
||||
INSERT INTO file_chunks (file_id, idx, chunk_hash)
|
||||
VALUES (?, ?, ?)
|
||||
@@ -27,7 +34,8 @@ func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileCh
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
|
||||
_, err = r.db.ExecWithLog(ctx, query,
|
||||
fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -37,7 +45,10 @@ func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileCh
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *FileChunkRepository) GetByPath(ctx context.Context, path string) ([]*FileChunk, error) {
|
||||
// GetByPath returns the ordered chunks of the file at the given path.
|
||||
func (r *FileChunkRepository) GetByPath(
|
||||
ctx context.Context, path string,
|
||||
) ([]*FileChunk, error) {
|
||||
query := `
|
||||
SELECT fc.file_id, fc.idx, fc.chunk_hash
|
||||
FROM file_chunks fc
|
||||
@@ -50,13 +61,21 @@ func (r *FileChunkRepository) GetByPath(ctx context.Context, path string) ([]*Fi
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying file chunks: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return r.scanFileChunks(rows)
|
||||
}
|
||||
|
||||
// GetByFileID retrieves file chunks by file ID
|
||||
func (r *FileChunkRepository) GetByFileID(ctx context.Context, fileID types.FileID) ([]*FileChunk, error) {
|
||||
func (r *FileChunkRepository) GetByFileID(
|
||||
ctx context.Context, fileID types.FileID,
|
||||
) ([]*FileChunk, error) {
|
||||
query := `
|
||||
SELECT file_id, idx, chunk_hash
|
||||
FROM file_chunks
|
||||
@@ -68,13 +87,21 @@ func (r *FileChunkRepository) GetByFileID(ctx context.Context, fileID types.File
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying file chunks: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
return r.scanFileChunks(rows)
|
||||
}
|
||||
|
||||
// GetByPathTx retrieves file chunks within a transaction
|
||||
func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path string) ([]*FileChunk, error) {
|
||||
func (r *FileChunkRepository) GetByPathTx(
|
||||
ctx context.Context, tx *sql.Tx, path string,
|
||||
) ([]*FileChunk, error) {
|
||||
query := `
|
||||
SELECT fc.file_id, fc.idx, fc.chunk_hash
|
||||
FROM file_chunks fc
|
||||
@@ -84,40 +111,33 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path
|
||||
`
|
||||
|
||||
LogSQL("GetByPathTx", query, path)
|
||||
|
||||
rows, err := tx.QueryContext(ctx, query, path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying file chunks: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
fileChunks, err := r.scanFileChunks(rows)
|
||||
LogSQL("GetByPathTx", "Complete", path, "count", len(fileChunks))
|
||||
|
||||
return fileChunks, err
|
||||
}
|
||||
|
||||
// scanFileChunks is a helper that scans file chunk rows
|
||||
func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) {
|
||||
var fileChunks []*FileChunk
|
||||
for rows.Next() {
|
||||
var fc FileChunk
|
||||
var fileIDStr, chunkHashStr string
|
||||
err := rows.Scan(&fileIDStr, &fc.Idx, &chunkHashStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning file chunk: %w", err)
|
||||
}
|
||||
fc.FileID, err = types.ParseFileID(fileIDStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||
}
|
||||
fc.ChunkHash = types.ChunkHash(chunkHashStr)
|
||||
fileChunks = append(fileChunks, &fc)
|
||||
}
|
||||
|
||||
return fileChunks, rows.Err()
|
||||
}
|
||||
|
||||
func (r *FileChunkRepository) DeleteByPath(ctx context.Context, tx *sql.Tx, path string) error {
|
||||
query := `DELETE FROM file_chunks WHERE file_id = (SELECT id FROM files WHERE path = ?)`
|
||||
// DeleteByPath deletes all file_chunks rows for the file at the given path.
|
||||
func (r *FileChunkRepository) DeleteByPath(
|
||||
ctx context.Context, tx *sql.Tx, path string,
|
||||
) error {
|
||||
query := `
|
||||
DELETE FROM file_chunks
|
||||
WHERE file_id = (SELECT id FROM files WHERE path = ?)
|
||||
`
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
@@ -134,7 +154,9 @@ func (r *FileChunkRepository) DeleteByPath(ctx context.Context, tx *sql.Tx, path
|
||||
}
|
||||
|
||||
// DeleteByFileID deletes all chunks for a file by its UUID
|
||||
func (r *FileChunkRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fileID types.FileID) error {
|
||||
func (r *FileChunkRepository) DeleteByFileID(
|
||||
ctx context.Context, tx *sql.Tx, fileID types.FileID,
|
||||
) error {
|
||||
query := `DELETE FROM file_chunks WHERE file_id = ?`
|
||||
|
||||
var err error
|
||||
@@ -152,7 +174,11 @@ func (r *FileChunkRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fi
|
||||
}
|
||||
|
||||
// DeleteByFileIDs deletes all chunks for multiple files in a single statement.
|
||||
func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, fileIDs []types.FileID) error {
|
||||
//
|
||||
//nolint:dupl // symmetric implementation for a parallel association table
|
||||
func (r *FileChunkRepository) DeleteByFileIDs(
|
||||
ctx context.Context, tx *sql.Tx, fileIDs []types.FileID,
|
||||
) error {
|
||||
if len(fileIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -161,14 +187,15 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
const batchSize = 500
|
||||
|
||||
for i := 0; i < len(fileIDs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(fileIDs) {
|
||||
end = len(fileIDs)
|
||||
}
|
||||
end := min(i+batchSize, len(fileIDs))
|
||||
|
||||
batch := fileIDs[i:end]
|
||||
|
||||
query := "DELETE FROM file_chunks WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
|
||||
args := make([]interface{}, len(batch))
|
||||
//nolint:gosec // G202: concatenates constant SQL and "?" placeholders only
|
||||
query := "DELETE FROM file_chunks WHERE file_id IN (?" +
|
||||
repeatPlaceholder(len(batch)-1) + ")"
|
||||
|
||||
args := make([]any, len(batch))
|
||||
for j, id := range batch {
|
||||
args[j] = id.String()
|
||||
}
|
||||
@@ -179,6 +206,7 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch deleting file_chunks: %w", err)
|
||||
}
|
||||
@@ -189,32 +217,44 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
|
||||
// CreateBatch inserts multiple file_chunks in a single statement for efficiency.
|
||||
// Batches are automatically split to stay within SQLite's variable limit.
|
||||
func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs []FileChunk) error {
|
||||
func (r *FileChunkRepository) CreateBatch(
|
||||
ctx context.Context, tx *sql.Tx, fcs []FileChunk,
|
||||
) error {
|
||||
if len(fcs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SQLite has a limit on variables (typically 999 or 32766).
|
||||
// Each FileChunk has 3 values, so batch at 300 to be safe.
|
||||
// Each file_chunks row binds this many SQL variables.
|
||||
const fileChunkCols = 3
|
||||
|
||||
// SQLite has a limit on variables (typically 999 or 32766), so batch
|
||||
// at 300 rows to be safe.
|
||||
const batchSize = 300
|
||||
|
||||
for i := 0; i < len(fcs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(fcs) {
|
||||
end = len(fcs)
|
||||
}
|
||||
end := min(i+batchSize, len(fcs))
|
||||
|
||||
batch := fcs[i:end]
|
||||
|
||||
// Build the query with multiple value sets
|
||||
query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES "
|
||||
args := make([]interface{}, 0, len(batch)*3)
|
||||
|
||||
args := make([]any, 0, len(batch)*fileChunkCols)
|
||||
|
||||
var querySb211 strings.Builder
|
||||
|
||||
for j, fc := range batch {
|
||||
if j > 0 {
|
||||
query += ", "
|
||||
querySb211.WriteString(", ")
|
||||
}
|
||||
query += "(?, ?, ?)"
|
||||
|
||||
querySb211.WriteString("(?, ?, ?)")
|
||||
|
||||
args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
|
||||
}
|
||||
|
||||
query += querySb211.String() //nolint:gosec // G202: appends "?" placeholders only
|
||||
|
||||
query += " ON CONFLICT(file_id, idx) DO NOTHING"
|
||||
|
||||
var err error
|
||||
@@ -223,6 +263,7 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch inserting file_chunks: %w", err)
|
||||
}
|
||||
@@ -232,17 +273,50 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [
|
||||
}
|
||||
|
||||
// GetByFile is an alias for GetByPath for compatibility
|
||||
func (r *FileChunkRepository) GetByFile(ctx context.Context, path string) ([]*FileChunk, error) {
|
||||
func (r *FileChunkRepository) GetByFile(
|
||||
ctx context.Context, path string,
|
||||
) ([]*FileChunk, error) {
|
||||
LogSQL("GetByFile", "Starting", path)
|
||||
result, err := r.GetByPath(ctx, path)
|
||||
LogSQL("GetByFile", "Complete", path, "count", len(result))
|
||||
|
||||
return result, err
|
||||
}
|
||||
|
||||
// GetByFileTx retrieves file chunks within a transaction
|
||||
func (r *FileChunkRepository) GetByFileTx(ctx context.Context, tx *sql.Tx, path string) ([]*FileChunk, error) {
|
||||
func (r *FileChunkRepository) GetByFileTx(
|
||||
ctx context.Context, tx *sql.Tx, path string,
|
||||
) ([]*FileChunk, error) {
|
||||
LogSQL("GetByFileTx", "Starting", path)
|
||||
result, err := r.GetByPathTx(ctx, tx, path)
|
||||
LogSQL("GetByFileTx", "Complete", path, "count", len(result))
|
||||
|
||||
return result, err
|
||||
}
|
||||
|
||||
// scanFileChunks is a helper that scans file chunk rows
|
||||
func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) {
|
||||
var fileChunks []*FileChunk
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
fc FileChunk
|
||||
fileIDStr, chunkHashStr string
|
||||
)
|
||||
|
||||
err := rows.Scan(&fileIDStr, &fc.Idx, &chunkHashStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning file chunk: %w", err)
|
||||
}
|
||||
|
||||
fc.FileID, err = types.ParseFileID(fileIDStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||
}
|
||||
|
||||
fc.ChunkHash = types.ChunkHash(chunkHashStr)
|
||||
fileChunks = append(fileChunks, &fc)
|
||||
}
|
||||
|
||||
return fileChunks, rows.Err()
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package database
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -6,21 +6,25 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func TestFileChunkRepository(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewFileChunkRepository(db)
|
||||
fileRepo := NewFileRepository(db)
|
||||
repo := database.NewFileChunkRepository(db)
|
||||
fileRepo := database.NewFileRepository(db)
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Create test file first
|
||||
testTime := time.Now().Truncate(time.Second)
|
||||
file := &File{
|
||||
Path: "/test/file.txt",
|
||||
file := &database.File{
|
||||
Path: testFileTxt,
|
||||
MTime: testTime,
|
||||
Size: 3072,
|
||||
Mode: 0644,
|
||||
@@ -28,63 +32,51 @@ func TestFileChunkRepository(t *testing.T) {
|
||||
GID: 1000,
|
||||
LinkTarget: "",
|
||||
}
|
||||
err := fileRepo.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file: %v", err)
|
||||
}
|
||||
|
||||
// Create chunks first
|
||||
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
|
||||
chunkRepo := NewChunkRepository(db)
|
||||
for _, chunkHash := range chunks {
|
||||
chunk := &Chunk{
|
||||
ChunkHash: chunkHash,
|
||||
Size: 1024,
|
||||
}
|
||||
err = chunkRepo.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||
}
|
||||
}
|
||||
mustCreateFile(t, fileRepo, file)
|
||||
mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash)
|
||||
|
||||
// Test Create
|
||||
fc1 := &FileChunk{
|
||||
fc1 := &database.FileChunk{
|
||||
FileID: file.ID,
|
||||
Idx: 0,
|
||||
ChunkHash: types.ChunkHash("chunk1"),
|
||||
ChunkHash: types.ChunkHash(chunk1Hash),
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, fc1)
|
||||
err := repo.Create(ctx, nil, fc1)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file chunk: %v", err)
|
||||
}
|
||||
|
||||
// Add more chunks for the same file
|
||||
fc2 := &FileChunk{
|
||||
fc2 := &database.FileChunk{
|
||||
FileID: file.ID,
|
||||
Idx: 1,
|
||||
ChunkHash: types.ChunkHash("chunk2"),
|
||||
ChunkHash: types.ChunkHash(chunk2Hash),
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, fc2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create second file chunk: %v", err)
|
||||
}
|
||||
|
||||
fc3 := &FileChunk{
|
||||
fc3 := &database.FileChunk{
|
||||
FileID: file.ID,
|
||||
Idx: 2,
|
||||
ChunkHash: types.ChunkHash("chunk3"),
|
||||
ChunkHash: types.ChunkHash(chunk3Hash),
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, fc3)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create third file chunk: %v", err)
|
||||
}
|
||||
|
||||
// Test GetByFile
|
||||
fileChunks, err := repo.GetByFile(ctx, "/test/file.txt")
|
||||
fileChunks, err := repo.GetByFile(ctx, testFileTxt)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get file chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(fileChunks) != 3 {
|
||||
t.Errorf("expected 3 chunks, got %d", len(fileChunks))
|
||||
}
|
||||
@@ -101,6 +93,41 @@ func TestFileChunkRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create duplicate file chunk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileChunkRepositoryDeleteByFileID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := database.NewFileChunkRepository(db)
|
||||
fileRepo := database.NewFileRepository(db)
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
file := &database.File{
|
||||
Path: testFileTxt,
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
mustCreateFile(t, fileRepo, file)
|
||||
mustCreateChunks(t, repos, chunk1Hash)
|
||||
|
||||
fc := &database.FileChunk{
|
||||
FileID: file.ID,
|
||||
Idx: 0,
|
||||
ChunkHash: types.ChunkHash(chunk1Hash),
|
||||
}
|
||||
|
||||
err := repo.Create(ctx, nil, fc)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file chunk: %v", err)
|
||||
}
|
||||
|
||||
// Test DeleteByFileID
|
||||
err = repo.DeleteByFileID(ctx, nil, file.ID)
|
||||
@@ -108,30 +135,33 @@ func TestFileChunkRepository(t *testing.T) {
|
||||
t.Fatalf("failed to delete file chunks: %v", err)
|
||||
}
|
||||
|
||||
fileChunks, err = repo.GetByFileID(ctx, file.ID)
|
||||
fileChunks, err := repo.GetByFileID(ctx, file.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get deleted file chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(fileChunks) != 0 {
|
||||
t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewFileChunkRepository(db)
|
||||
fileRepo := NewFileRepository(db)
|
||||
repo := database.NewFileChunkRepository(db)
|
||||
fileRepo := database.NewFileRepository(db)
|
||||
|
||||
// Create test files
|
||||
testTime := time.Now().Truncate(time.Second)
|
||||
filePaths := []string{"/file1.txt", "/file2.txt", "/file3.txt"}
|
||||
files := make([]*File, len(filePaths))
|
||||
filePaths := []string{testFilePath1, testFilePath2, "/file3.txt"}
|
||||
files := make([]*database.File, len(filePaths))
|
||||
|
||||
for i, path := range filePaths {
|
||||
file := &File{
|
||||
file := &database.File{
|
||||
Path: types.FilePath(path),
|
||||
MTime: testTime,
|
||||
Size: 2048,
|
||||
@@ -140,22 +170,23 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
|
||||
GID: 1000,
|
||||
LinkTarget: "",
|
||||
}
|
||||
err := fileRepo.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file %s: %v", path, err)
|
||||
}
|
||||
|
||||
mustCreateFile(t, fileRepo, file)
|
||||
|
||||
files[i] = file
|
||||
}
|
||||
|
||||
// Create all chunks first
|
||||
chunkRepo := NewChunkRepository(db)
|
||||
chunkRepo := database.NewChunkRepository(db)
|
||||
|
||||
for i := range files {
|
||||
for j := 0; j < 2; j++ {
|
||||
for j := range 2 {
|
||||
chunkHash := types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j))
|
||||
chunk := &Chunk{
|
||||
chunk := &database.Chunk{
|
||||
ChunkHash: chunkHash,
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err := chunkRepo.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||
@@ -165,12 +196,13 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
|
||||
|
||||
// Create chunks for multiple files
|
||||
for i, file := range files {
|
||||
for j := 0; j < 2; j++ {
|
||||
fc := &FileChunk{
|
||||
for j := range 2 {
|
||||
fc := &database.FileChunk{
|
||||
FileID: file.ID,
|
||||
Idx: j,
|
||||
ChunkHash: types.ChunkHash(fmt.Sprintf("file%d_chunk%d", i, j)),
|
||||
}
|
||||
|
||||
err := repo.Create(ctx, nil, fc)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file chunk: %v", err)
|
||||
@@ -184,6 +216,7 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks for file %d: %v", i, err)
|
||||
}
|
||||
|
||||
if len(chunks) != 2 {
|
||||
t.Errorf("expected 2 chunks for file %d, got %d", i, len(chunks))
|
||||
}
|
||||
|
||||
+166
-101
@@ -3,21 +3,29 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// FileRepository provides access to the files table, which stores file
|
||||
// metadata (path, times, permissions, ownership, symlink targets).
|
||||
type FileRepository struct {
|
||||
db *DB
|
||||
}
|
||||
|
||||
// NewFileRepository creates a FileRepository backed by db.
|
||||
func NewFileRepository(db *DB) *FileRepository {
|
||||
return &FileRepository{db: db}
|
||||
}
|
||||
|
||||
// Create inserts or updates a file row (upsert on path), using tx when
|
||||
// non-nil. The file's ID is generated when zero and updated from the
|
||||
// database's RETURNING clause.
|
||||
func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) error {
|
||||
// Generate UUID if not provided
|
||||
if file.ID.IsZero() {
|
||||
@@ -38,13 +46,25 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
|
||||
RETURNING id
|
||||
`
|
||||
|
||||
var idStr string
|
||||
var err error
|
||||
var (
|
||||
idStr string
|
||||
err error
|
||||
)
|
||||
|
||||
if tx != nil {
|
||||
LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String())
|
||||
err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr)
|
||||
LogSQL("Execute", query,
|
||||
file.ID.String(), file.Path.String(), file.SourcePath.String(),
|
||||
file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
|
||||
file.LinkTarget.String())
|
||||
err = tx.QueryRowContext(ctx, query,
|
||||
file.ID.String(), file.Path.String(), file.SourcePath.String(),
|
||||
file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
|
||||
file.LinkTarget.String()).Scan(&idStr)
|
||||
} else {
|
||||
err = r.db.QueryRowWithLog(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr)
|
||||
err = r.db.QueryRowWithLog(ctx, query,
|
||||
file.ID.String(), file.Path.String(), file.SourcePath.String(),
|
||||
file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
|
||||
file.LinkTarget.String()).Scan(&idStr)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -60,6 +80,8 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetByPath returns the file at the given path, or nil if the path is not
|
||||
// in the index.
|
||||
func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, error) {
|
||||
query := `
|
||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
||||
@@ -68,9 +90,10 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err
|
||||
`
|
||||
|
||||
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, path))
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying file: %w", err)
|
||||
}
|
||||
@@ -87,9 +110,10 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e
|
||||
`
|
||||
|
||||
file, err := r.scanFile(r.db.conn.QueryRowContext(ctx, query, id.String()))
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying file: %w", err)
|
||||
}
|
||||
@@ -97,7 +121,11 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e
|
||||
return file, nil
|
||||
}
|
||||
|
||||
func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path string) (*File, error) {
|
||||
// GetByPathTx returns the file at the given path within a transaction, or
|
||||
// nil if the path is not in the index.
|
||||
func (r *FileRepository) GetByPathTx(
|
||||
ctx context.Context, tx *sql.Tx, path string,
|
||||
) (*File, error) {
|
||||
query := `
|
||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
||||
FROM files
|
||||
@@ -108,9 +136,10 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin
|
||||
file, err := r.scanFile(tx.QueryRowContext(ctx, query, path))
|
||||
LogSQL("GetByPathTx Scan complete", query, path)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying file: %w", err)
|
||||
}
|
||||
@@ -118,79 +147,16 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// scanFile is a helper that scans a single file row
|
||||
func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
|
||||
var file File
|
||||
var idStr, pathStr, sourcePathStr string
|
||||
var mtimeUnix int64
|
||||
var linkTarget sql.NullString
|
||||
|
||||
err := row.Scan(
|
||||
&idStr,
|
||||
&pathStr,
|
||||
&sourcePathStr,
|
||||
&mtimeUnix,
|
||||
&file.Size,
|
||||
&file.Mode,
|
||||
&file.UID,
|
||||
&file.GID,
|
||||
&linkTarget,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
file.ID, err = types.ParseFileID(idStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||
}
|
||||
file.Path = types.FilePath(pathStr)
|
||||
file.SourcePath = types.SourcePath(sourcePathStr)
|
||||
file.MTime = time.Unix(mtimeUnix, 0).UTC()
|
||||
if linkTarget.Valid {
|
||||
file.LinkTarget = types.FilePath(linkTarget.String)
|
||||
}
|
||||
|
||||
return &file, nil
|
||||
// fileRowScanner abstracts *sql.Row and *sql.Rows for scanning a file row.
|
||||
type fileRowScanner interface {
|
||||
Scan(dest ...any) error
|
||||
}
|
||||
|
||||
// scanFileRows is a helper that scans a file row from rows iterator
|
||||
func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) {
|
||||
var file File
|
||||
var idStr, pathStr, sourcePathStr string
|
||||
var mtimeUnix int64
|
||||
var linkTarget sql.NullString
|
||||
|
||||
err := rows.Scan(
|
||||
&idStr,
|
||||
&pathStr,
|
||||
&sourcePathStr,
|
||||
&mtimeUnix,
|
||||
&file.Size,
|
||||
&file.Mode,
|
||||
&file.UID,
|
||||
&file.GID,
|
||||
&linkTarget,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
file.ID, err = types.ParseFileID(idStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||
}
|
||||
file.Path = types.FilePath(pathStr)
|
||||
file.SourcePath = types.SourcePath(sourcePathStr)
|
||||
file.MTime = time.Unix(mtimeUnix, 0).UTC()
|
||||
if linkTarget.Valid {
|
||||
file.LinkTarget = types.FilePath(linkTarget.String)
|
||||
}
|
||||
|
||||
return &file, nil
|
||||
}
|
||||
|
||||
func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time) ([]*File, error) {
|
||||
// ListModifiedSince returns all files whose recorded mtime is at or after
|
||||
// since, ordered by path.
|
||||
func (r *FileRepository) ListModifiedSince(
|
||||
ctx context.Context, since time.Time,
|
||||
) ([]*File, error) {
|
||||
query := `
|
||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
||||
FROM files
|
||||
@@ -202,20 +168,29 @@ func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying files: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
var files []*File
|
||||
|
||||
for rows.Next() {
|
||||
file, err := r.scanFileRows(rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning file: %w", err)
|
||||
}
|
||||
|
||||
files = append(files, file)
|
||||
}
|
||||
|
||||
return files, rows.Err()
|
||||
}
|
||||
|
||||
// Delete removes the file row at the given path, using tx when non-nil.
|
||||
func (r *FileRepository) Delete(ctx context.Context, tx *sql.Tx, path string) error {
|
||||
query := `DELETE FROM files WHERE path = ?`
|
||||
|
||||
@@ -234,7 +209,9 @@ func (r *FileRepository) Delete(ctx context.Context, tx *sql.Tx, path string) er
|
||||
}
|
||||
|
||||
// DeleteByID deletes a file by its UUID
|
||||
func (r *FileRepository) DeleteByID(ctx context.Context, tx *sql.Tx, id types.FileID) error {
|
||||
func (r *FileRepository) DeleteByID(
|
||||
ctx context.Context, tx *sql.Tx, id types.FileID,
|
||||
) error {
|
||||
query := `DELETE FROM files WHERE id = ?`
|
||||
|
||||
var err error
|
||||
@@ -251,7 +228,11 @@ func (r *FileRepository) DeleteByID(ctx context.Context, tx *sql.Tx, id types.Fi
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*File, error) {
|
||||
// ListByPrefix returns all files whose path starts with prefix, ordered by
|
||||
// path.
|
||||
func (r *FileRepository) ListByPrefix(
|
||||
ctx context.Context, prefix string,
|
||||
) ([]*File, error) {
|
||||
query := `
|
||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
||||
FROM files
|
||||
@@ -263,14 +244,22 @@ func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*Fi
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying files: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
var files []*File
|
||||
|
||||
for rows.Next() {
|
||||
file, err := r.scanFileRows(rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning file: %w", err)
|
||||
}
|
||||
|
||||
files = append(files, file)
|
||||
}
|
||||
|
||||
@@ -289,14 +278,22 @@ func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying files: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
var files []*File
|
||||
|
||||
for rows.Next() {
|
||||
file, err := r.scanFileRows(rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning file: %w", err)
|
||||
}
|
||||
|
||||
files = append(files, file)
|
||||
}
|
||||
|
||||
@@ -305,30 +302,47 @@ func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
|
||||
|
||||
// CreateBatch inserts or updates multiple files in a single statement for efficiency.
|
||||
// File IDs must be pre-generated before calling this method.
|
||||
func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*File) error {
|
||||
func (r *FileRepository) CreateBatch(
|
||||
ctx context.Context, tx *sql.Tx, files []*File,
|
||||
) error {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Each File has 9 values, so batch at 100 to be safe with SQLite's variable limit
|
||||
// Each files row binds this many SQL variables.
|
||||
const fileCols = 9
|
||||
|
||||
// Batch at 100 rows to be safe with SQLite's variable limit.
|
||||
const batchSize = 100
|
||||
|
||||
for i := 0; i < len(files); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(files) {
|
||||
end = len(files)
|
||||
}
|
||||
end := min(i+batchSize, len(files))
|
||||
|
||||
batch := files[i:end]
|
||||
|
||||
query := `INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target) VALUES `
|
||||
args := make([]interface{}, 0, len(batch)*9)
|
||||
query := `INSERT INTO files
|
||||
(id, path, source_path, mtime, size, mode, uid, gid, link_target)
|
||||
VALUES `
|
||||
|
||||
args := make([]any, 0, len(batch)*fileCols)
|
||||
|
||||
var querySb325 strings.Builder
|
||||
|
||||
for j, f := range batch {
|
||||
if j > 0 {
|
||||
query += ", "
|
||||
querySb325.WriteString(", ")
|
||||
}
|
||||
query += "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
||||
args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String())
|
||||
|
||||
querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
||||
|
||||
args = append(args,
|
||||
f.ID.String(), f.Path.String(), f.SourcePath.String(),
|
||||
f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID,
|
||||
f.LinkTarget.String())
|
||||
}
|
||||
|
||||
query += querySb325.String() //nolint:gosec // G202: appends "?" placeholders only
|
||||
|
||||
query += ` ON CONFLICT(path) DO UPDATE SET
|
||||
source_path = excluded.source_path,
|
||||
mtime = excluded.mtime,
|
||||
@@ -344,6 +358,7 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch inserting files: %w", err)
|
||||
}
|
||||
@@ -374,3 +389,53 @@ func (r *FileRepository) DeleteOrphaned(ctx context.Context) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// scanFile is a helper that scans a single file row
|
||||
func (r *FileRepository) scanFile(row *sql.Row) (*File, error) {
|
||||
return r.scanFileFrom(row)
|
||||
}
|
||||
|
||||
// scanFileRows is a helper that scans a file row from rows iterator
|
||||
func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) {
|
||||
return r.scanFileFrom(rows)
|
||||
}
|
||||
|
||||
// scanFileFrom scans one file row from any row scanner.
|
||||
func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) {
|
||||
var (
|
||||
file File
|
||||
idStr, pathStr, sourcePathStr string
|
||||
mtimeUnix int64
|
||||
linkTarget sql.NullString
|
||||
)
|
||||
|
||||
err := row.Scan(
|
||||
&idStr,
|
||||
&pathStr,
|
||||
&sourcePathStr,
|
||||
&mtimeUnix,
|
||||
&file.Size,
|
||||
&file.Mode,
|
||||
&file.UID,
|
||||
&file.GID,
|
||||
&linkTarget,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
file.ID, err = types.ParseFileID(idStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||
}
|
||||
|
||||
file.Path = types.FilePath(pathStr)
|
||||
file.SourcePath = types.SourcePath(sourcePathStr)
|
||||
|
||||
file.MTime = time.Unix(mtimeUnix, 0).UTC()
|
||||
if linkTarget.Valid {
|
||||
file.LinkTarget = types.FilePath(linkTarget.String)
|
||||
}
|
||||
|
||||
return &file, nil
|
||||
}
|
||||
|
||||
@@ -1,43 +1,32 @@
|
||||
package database
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
)
|
||||
|
||||
func setupTestDB(t *testing.T) (*DB, func()) {
|
||||
ctx := context.Background()
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||
|
||||
db, err := New(ctx, dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
if err := db.Close(); err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return db, cleanup
|
||||
}
|
||||
// errTestRollback is the sentinel returned from transaction bodies to
|
||||
// force a rollback in tests.
|
||||
var errTestRollback = errors.New("test rollback")
|
||||
|
||||
func TestFileRepository(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewFileRepository(db)
|
||||
repo := database.NewFileRepository(db)
|
||||
|
||||
// Test Create
|
||||
file := &File{
|
||||
Path: "/test/file.txt",
|
||||
file := &database.File{
|
||||
Path: testFileTxt,
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
@@ -56,18 +45,23 @@ func TestFileRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get file: %v", err)
|
||||
}
|
||||
|
||||
if retrieved == nil {
|
||||
t.Fatal("expected file, got nil")
|
||||
}
|
||||
|
||||
if retrieved.Path != file.Path {
|
||||
t.Errorf("path mismatch: got %s, want %s", retrieved.Path, file.Path)
|
||||
}
|
||||
|
||||
if !retrieved.MTime.Equal(file.MTime) {
|
||||
t.Errorf("mtime mismatch: got %v, want %v", retrieved.MTime, file.MTime)
|
||||
}
|
||||
|
||||
if retrieved.Size != file.Size {
|
||||
t.Errorf("size mismatch: got %d, want %d", retrieved.Size, file.Size)
|
||||
}
|
||||
|
||||
if retrieved.Mode != file.Mode {
|
||||
t.Errorf("mode mismatch: got %o, want %o", retrieved.Mode, file.Mode)
|
||||
}
|
||||
@@ -75,6 +69,7 @@ func TestFileRepository(t *testing.T) {
|
||||
// Test Update (upsert)
|
||||
file.Size = 2048
|
||||
file.MTime = time.Now().Truncate(time.Second)
|
||||
|
||||
err = repo.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to update file: %v", err)
|
||||
@@ -84,15 +79,41 @@ func TestFileRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get updated file: %v", err)
|
||||
}
|
||||
|
||||
if retrieved.Size != 2048 {
|
||||
t.Errorf("size not updated: got %d, want %d", retrieved.Size, 2048)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileRepositoryListDelete(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := database.NewFileRepository(db)
|
||||
|
||||
file := &database.File{
|
||||
Path: testFileTxt,
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err := repo.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file: %v", err)
|
||||
}
|
||||
|
||||
// Test ListModifiedSince
|
||||
files, err := repo.ListModifiedSince(ctx, time.Now().Add(-1*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to list files: %v", err)
|
||||
}
|
||||
|
||||
if len(files) != 1 {
|
||||
t.Errorf("expected 1 file, got %d", len(files))
|
||||
}
|
||||
@@ -103,24 +124,27 @@ func TestFileRepository(t *testing.T) {
|
||||
t.Fatalf("failed to delete file: %v", err)
|
||||
}
|
||||
|
||||
retrieved, err = repo.GetByPath(ctx, file.Path.String())
|
||||
retrieved, err := repo.GetByPath(ctx, file.Path.String())
|
||||
if err != nil {
|
||||
t.Fatalf("error getting deleted file: %v", err)
|
||||
}
|
||||
|
||||
if retrieved != nil {
|
||||
t.Error("expected nil for deleted file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileRepositorySymlink(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewFileRepository(db)
|
||||
repo := database.NewFileRepository(db)
|
||||
|
||||
// Test symlink
|
||||
symlink := &File{
|
||||
symlink := &database.File{
|
||||
Path: "/test/link",
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
Size: 0,
|
||||
@@ -139,25 +163,30 @@ func TestFileRepositorySymlink(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get symlink: %v", err)
|
||||
}
|
||||
|
||||
if !retrieved.IsSymlink() {
|
||||
t.Error("expected IsSymlink() to be true")
|
||||
}
|
||||
|
||||
if retrieved.LinkTarget != symlink.LinkTarget {
|
||||
t.Errorf("link target mismatch: got %s, want %s", retrieved.LinkTarget, symlink.LinkTarget)
|
||||
t.Errorf("link target mismatch: got %s, want %s",
|
||||
retrieved.LinkTarget, symlink.LinkTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileRepositoryTransaction(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Test transaction rollback
|
||||
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
file := &File{
|
||||
Path: "/test/tx_file.txt",
|
||||
file := &database.File{
|
||||
Path: testTxFile,
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
@@ -165,23 +194,24 @@ func TestFileRepositoryTransaction(t *testing.T) {
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
if err := repos.Files.Create(ctx, tx, file); err != nil {
|
||||
err := repos.Files.Create(ctx, tx, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Return error to trigger rollback
|
||||
return fmt.Errorf("test rollback")
|
||||
return errTestRollback
|
||||
})
|
||||
|
||||
if err == nil || err.Error() != "test rollback" {
|
||||
if !errors.Is(err, errTestRollback) {
|
||||
t.Fatalf("expected rollback error, got: %v", err)
|
||||
}
|
||||
|
||||
// Verify file was not created
|
||||
retrieved, err := repos.Files.GetByPath(ctx, "/test/tx_file.txt")
|
||||
retrieved, err := repos.Files.GetByPath(ctx, testTxFile)
|
||||
if err != nil {
|
||||
t.Fatalf("error checking for file: %v", err)
|
||||
}
|
||||
|
||||
if retrieved != nil {
|
||||
t.Error("file should not exist after rollback")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// Common fixture values shared by the internal repository tests.
|
||||
const (
|
||||
internalTestHost = "test-host"
|
||||
internalTestSnapshotID = "test-snapshot"
|
||||
internalTestFilePath = "/test.txt"
|
||||
internalTestFile1 = "/file1.txt"
|
||||
internalTestFile2 = "/file2.txt"
|
||||
|
||||
// countFilesQuery counts the rows of the files table.
|
||||
countFilesQuery = "SELECT COUNT(*) FROM files"
|
||||
)
|
||||
|
||||
// mustCreateFileRow inserts the file row, failing the test on error.
|
||||
func mustCreateFileRow(t *testing.T, repos *Repositories, file *File) {
|
||||
t.Helper()
|
||||
|
||||
err := repos.Files.Create(context.Background(), nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file %s: %v", file.Path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// mustAddFileToSnapshot associates a file with a snapshot, failing the
|
||||
// test on error.
|
||||
func mustAddFileToSnapshot(
|
||||
t *testing.T, repos *Repositories, snapshotID string, fileID types.FileID,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
err := repos.Snapshots.AddFileByID(context.Background(), nil, snapshotID, fileID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// setupTestDB creates an on-disk test database in a per-test temp
|
||||
// directory and returns it along with a cleanup func that closes it.
|
||||
func setupTestDB(t *testing.T) (*DB, func()) {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||
|
||||
db, err := New(ctx, dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return db, cleanup
|
||||
}
|
||||
|
||||
// countRow runs a single-integer COUNT-style query and returns the value.
|
||||
func countRow(t *testing.T, db *DB, query string, args ...any) int {
|
||||
t.Helper()
|
||||
|
||||
var count int
|
||||
|
||||
err := db.conn.QueryRowContext(context.Background(), query, args...).Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return count
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
)
|
||||
|
||||
// Common fixture values shared by the repository tests.
|
||||
const (
|
||||
testFilePath1 = "/file1.txt"
|
||||
testFilePath2 = "/file2.txt"
|
||||
testFileTxt = "/test/file.txt"
|
||||
testTxFile = "/test/tx_file.txt"
|
||||
testHostname = "test-host"
|
||||
testVersion = "1.0.0"
|
||||
)
|
||||
|
||||
// mustCreateFile inserts the given file row, failing the test on error.
|
||||
func mustCreateFile(t *testing.T, repo *database.FileRepository, file *database.File) {
|
||||
t.Helper()
|
||||
|
||||
err := repo.Create(context.Background(), nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file %s: %v", file.Path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// setupTestDB creates an on-disk test database in a per-test temp
|
||||
// directory and returns it along with a cleanup func that closes it.
|
||||
func setupTestDB(t *testing.T) (*database.DB, func()) {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return db, cleanup
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// LocalMetaKeyStorageURL is the key under which the destination store's
|
||||
// URL is recorded when a mutating command first binds the local index
|
||||
// to a specific backup destination.
|
||||
const LocalMetaKeyStorageURL = "storage_url"
|
||||
|
||||
// LocalMetaRepository provides keyed access to host-local settings
|
||||
// stored in the local_meta table.
|
||||
type LocalMetaRepository struct {
|
||||
db *DB
|
||||
}
|
||||
|
||||
// NewLocalMetaRepository creates a LocalMetaRepository backed by db.
|
||||
func NewLocalMetaRepository(db *DB) *LocalMetaRepository {
|
||||
return &LocalMetaRepository{db: db}
|
||||
}
|
||||
|
||||
// Get returns the value stored at key, or the empty string if the key
|
||||
// is not set. A missing key is not an error — the caller distinguishes
|
||||
// "unset" (bind on first use) from "set to something" (compare).
|
||||
func (r *LocalMetaRepository) Get(ctx context.Context, key string) (string, error) {
|
||||
var value string
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx,
|
||||
"SELECT value FROM local_meta WHERE key = ?", key,
|
||||
).Scan(&value)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("reading local_meta %q: %w", key, err)
|
||||
}
|
||||
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// Set writes key=value, replacing any prior value.
|
||||
func (r *LocalMetaRepository) Set(ctx context.Context, key, value string) error {
|
||||
_, err := r.db.ExecWithLog(ctx,
|
||||
`INSERT INTO local_meta (key, value) VALUES (?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value`,
|
||||
key, value,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing local_meta %q: %w", key, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
)
|
||||
|
||||
func TestLocalMetaEmptyOnFresh(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, err := database.NewTestDB()
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
got, err := repos.LocalMeta.Get(context.Background(), database.LocalMetaKeyStorageURL)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, got, "fresh DB must return empty for unset keys, not error")
|
||||
}
|
||||
|
||||
func TestLocalMetaSetGetRoundTrip(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, err := database.NewTestDB()
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, repos.LocalMeta.Set(
|
||||
ctx, database.LocalMetaKeyStorageURL, "file:///mnt/backups"))
|
||||
|
||||
got, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "file:///mnt/backups", got)
|
||||
}
|
||||
|
||||
func TestLocalMetaSetOverwrites(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, err := database.NewTestDB()
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, repos.LocalMeta.Set(
|
||||
ctx, database.LocalMetaKeyStorageURL, "s3://old"))
|
||||
require.NoError(t, repos.LocalMeta.Set(
|
||||
ctx, database.LocalMetaKeyStorageURL, "s3://new"))
|
||||
|
||||
got, err := repos.LocalMeta.Get(ctx, database.LocalMetaKeyStorageURL)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "s3://new", got)
|
||||
}
|
||||
@@ -1,11 +1,9 @@
|
||||
// Package database provides data models and repository interfaces for the Vaultik backup system.
|
||||
// It includes types for files, chunks, blobs, snapshots, and their relationships.
|
||||
package database
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// File represents a file or directory in the backup system.
|
||||
@@ -15,7 +13,10 @@ import (
|
||||
type File struct {
|
||||
ID types.FileID // UUID primary key
|
||||
Path types.FilePath // Absolute path of the file
|
||||
SourcePath types.SourcePath // The source directory this file came from (for restore path stripping)
|
||||
|
||||
// SourcePath is the source directory this file came from (used for
|
||||
// restore path stripping).
|
||||
SourcePath types.SourcePath
|
||||
MTime time.Time
|
||||
Size int64
|
||||
Mode uint32
|
||||
@@ -56,7 +57,10 @@ type Chunk struct {
|
||||
// -> encrypted with age -> hashed -> uploaded to S3 with the hash as filename.
|
||||
type Blob struct {
|
||||
ID types.BlobID // UUID assigned when blob creation starts
|
||||
Hash types.BlobHash // SHA256 of final compressed+encrypted content (empty until finalized)
|
||||
|
||||
// Hash is the SHA256 of the final compressed+encrypted content
|
||||
// (empty until finalized).
|
||||
Hash types.BlobHash
|
||||
CreatedTS time.Time // When blob creation started
|
||||
FinishedTS *time.Time // When blob was finalized (nil if still packing)
|
||||
UncompressedSize int64 // Total size of raw chunks before compression
|
||||
@@ -75,9 +79,10 @@ type BlobChunk struct {
|
||||
Length int64
|
||||
}
|
||||
|
||||
// ChunkFile represents the reverse mapping showing which files contain a specific chunk.
|
||||
// This is used during deduplication to identify all files that share a chunk,
|
||||
// which is important for garbage collection and integrity verification.
|
||||
// ChunkFile represents the reverse mapping showing which files contain a
|
||||
// specific chunk. This is used during deduplication to identify all files
|
||||
// that share a chunk, which is important for garbage collection and
|
||||
// integrity verification.
|
||||
type ChunkFile struct {
|
||||
ChunkHash types.ChunkHash
|
||||
FileID types.FileID
|
||||
@@ -97,7 +102,10 @@ type Snapshot struct {
|
||||
ChunkCount int64
|
||||
BlobCount int64
|
||||
TotalSize int64 // Total size of all referenced files
|
||||
BlobSize int64 // Total size of all referenced blobs (compressed and encrypted)
|
||||
|
||||
// BlobSize is the total size of all referenced blobs (compressed and
|
||||
// encrypted).
|
||||
BlobSize int64
|
||||
BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs
|
||||
CompressionRatio float64 // Compression ratio (BlobSize / BlobUncompressedSize)
|
||||
CompressionLevel int // Compression level used for this snapshot
|
||||
|
||||
@@ -6,12 +6,18 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
// indexDirPerm restricts the local index directory to the owning user;
|
||||
// the index describes the backed-up file tree and must stay private.
|
||||
const indexDirPerm = 0o700
|
||||
|
||||
// Module provides database dependencies
|
||||
//
|
||||
//nolint:gochecknoglobals // fx module definitions are package globals by convention
|
||||
var Module = fx.Module("database",
|
||||
fx.Provide(
|
||||
provideDatabase,
|
||||
@@ -22,7 +28,9 @@ var Module = fx.Module("database",
|
||||
func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) {
|
||||
// Ensure the index directory exists
|
||||
indexDir := filepath.Dir(cfg.IndexPath)
|
||||
if err := os.MkdirAll(indexDir, 0700); err != nil {
|
||||
|
||||
err := os.MkdirAll(indexDir, indexDirPerm)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating index directory: %w", err)
|
||||
}
|
||||
|
||||
@@ -32,13 +40,18 @@ func provideDatabase(lc fx.Lifecycle, cfg *config.Config) (*DB, error) {
|
||||
}
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
OnStop: func(ctx context.Context) error {
|
||||
OnStop: func(_ context.Context) error {
|
||||
log.Debug("Database module OnStop hook called")
|
||||
if err := db.Close(); err != nil {
|
||||
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
log.Error("Failed to close database in OnStop hook", "error", err)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debug("Database closed successfully in OnStop hook")
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
@@ -19,6 +19,7 @@ type Repositories struct {
|
||||
ChunkFiles *ChunkFileRepository
|
||||
Snapshots *SnapshotRepository
|
||||
Uploads *UploadRepository
|
||||
LocalMeta *LocalMetaRepository
|
||||
}
|
||||
|
||||
// NewRepositories creates a new Repositories instance with all repository types.
|
||||
@@ -34,6 +35,7 @@ func NewRepositories(db *DB) *Repositories {
|
||||
ChunkFiles: NewChunkFileRepository(db),
|
||||
Snapshots: NewSnapshotRepository(db),
|
||||
Uploads: NewUploadRepository(db.conn),
|
||||
LocalMeta: NewLocalMetaRepository(db),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,21 +50,26 @@ type TxFunc func(ctx context.Context, tx *sql.Tx) error
|
||||
// This method should be used for all write operations to ensure atomicity.
|
||||
func (r *Repositories) WithTx(ctx context.Context, fn TxFunc) error {
|
||||
LogSQL("WithTx", "Beginning transaction", "")
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("beginning transaction: %w", err)
|
||||
}
|
||||
|
||||
LogSQL("WithTx", "Transaction started", "")
|
||||
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||
Fatal("failed to rollback transaction: %v", rollbackErr)
|
||||
rollbackErr := tx.Rollback()
|
||||
if rollbackErr != nil {
|
||||
Fatalf("failed to rollback transaction: %v", rollbackErr)
|
||||
}
|
||||
|
||||
panic(p)
|
||||
} else if err != nil {
|
||||
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||
Fatal("failed to rollback transaction: %v", rollbackErr)
|
||||
rollbackErr := tx.Rollback()
|
||||
if rollbackErr != nil {
|
||||
Fatalf("failed to rollback transaction: %v", rollbackErr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
@@ -88,6 +95,7 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error {
|
||||
opts := &sql.TxOptions{
|
||||
ReadOnly: true,
|
||||
}
|
||||
|
||||
tx, err := r.db.BeginTx(ctx, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("beginning read transaction: %w", err)
|
||||
@@ -95,13 +103,16 @@ func (r *Repositories) WithReadTx(ctx context.Context, fn TxFunc) error {
|
||||
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||
Fatal("failed to rollback transaction: %v", rollbackErr)
|
||||
rollbackErr := tx.Rollback()
|
||||
if rollbackErr != nil {
|
||||
Fatalf("failed to rollback transaction: %v", rollbackErr)
|
||||
}
|
||||
|
||||
panic(p)
|
||||
} else if err != nil {
|
||||
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||
Fatal("failed to rollback transaction: %v", rollbackErr)
|
||||
rollbackErr := tx.Rollback()
|
||||
if rollbackErr != nil {
|
||||
Fatalf("failed to rollback transaction: %v", rollbackErr)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -1,124 +1,162 @@
|
||||
package database
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func TestRepositoriesTransaction(t *testing.T) {
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
// errIntentionalRollback forces a transaction rollback in tests.
|
||||
var errIntentionalRollback = errors.New("intentional rollback")
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
// Test successful transaction with multiple operations
|
||||
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
// Create a file
|
||||
file := &File{
|
||||
Path: "/test/tx_file.txt",
|
||||
// createTxTestData returns a transaction body that creates a file with
|
||||
// two chunks packed into one blob.
|
||||
func createTxTestData(
|
||||
repos *database.Repositories,
|
||||
) func(context.Context, *sql.Tx) error {
|
||||
return func(ctx context.Context, tx *sql.Tx) error {
|
||||
file := &database.File{
|
||||
Path: testTxFile,
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
if err := repos.Files.Create(ctx, tx, file); err != nil {
|
||||
|
||||
err := repos.Files.Create(ctx, tx, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = createTxFileChunks(ctx, tx, repos, file.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return createTxBlob(ctx, tx, repos)
|
||||
}
|
||||
}
|
||||
|
||||
// createTxFileChunks creates the two test chunks and maps them to the file.
|
||||
func createTxFileChunks(
|
||||
ctx context.Context, tx *sql.Tx,
|
||||
repos *database.Repositories, fileID types.FileID,
|
||||
) error {
|
||||
// Create chunks
|
||||
chunk1 := &Chunk{
|
||||
chunk1 := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash("tx_chunk1"),
|
||||
Size: 512,
|
||||
}
|
||||
if err := repos.Chunks.Create(ctx, tx, chunk1); err != nil {
|
||||
|
||||
err := repos.Chunks.Create(ctx, tx, chunk1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chunk2 := &Chunk{
|
||||
chunk2 := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash("tx_chunk2"),
|
||||
Size: 512,
|
||||
}
|
||||
if err := repos.Chunks.Create(ctx, tx, chunk2); err != nil {
|
||||
|
||||
err = repos.Chunks.Create(ctx, tx, chunk2)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Map chunks to file
|
||||
fc1 := &FileChunk{
|
||||
FileID: file.ID,
|
||||
fc1 := &database.FileChunk{
|
||||
FileID: fileID,
|
||||
Idx: 0,
|
||||
ChunkHash: chunk1.ChunkHash,
|
||||
}
|
||||
if err := repos.FileChunks.Create(ctx, tx, fc1); err != nil {
|
||||
|
||||
err = repos.FileChunks.Create(ctx, tx, fc1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fc2 := &FileChunk{
|
||||
FileID: file.ID,
|
||||
fc2 := &database.FileChunk{
|
||||
FileID: fileID,
|
||||
Idx: 1,
|
||||
ChunkHash: chunk2.ChunkHash,
|
||||
}
|
||||
if err := repos.FileChunks.Create(ctx, tx, fc2); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create blob
|
||||
blob := &Blob{
|
||||
return repos.FileChunks.Create(ctx, tx, fc2)
|
||||
}
|
||||
|
||||
// createTxBlob creates the test blob and maps both chunks into it.
|
||||
func createTxBlob(
|
||||
ctx context.Context, tx *sql.Tx, repos *database.Repositories,
|
||||
) error {
|
||||
blob := &database.Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("tx_blob1"),
|
||||
CreatedTS: time.Now().Truncate(time.Second),
|
||||
}
|
||||
if err := repos.Blobs.Create(ctx, tx, blob); err != nil {
|
||||
|
||||
err := repos.Blobs.Create(ctx, tx, blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Map chunks to blob
|
||||
bc1 := &BlobChunk{
|
||||
bc1 := &database.BlobChunk{
|
||||
BlobID: blob.ID,
|
||||
ChunkHash: chunk1.ChunkHash,
|
||||
ChunkHash: types.ChunkHash("tx_chunk1"),
|
||||
Offset: 0,
|
||||
Length: 512,
|
||||
}
|
||||
if err := repos.BlobChunks.Create(ctx, tx, bc1); err != nil {
|
||||
|
||||
err = repos.BlobChunks.Create(ctx, tx, bc1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bc2 := &BlobChunk{
|
||||
bc2 := &database.BlobChunk{
|
||||
BlobID: blob.ID,
|
||||
ChunkHash: chunk2.ChunkHash,
|
||||
ChunkHash: types.ChunkHash("tx_chunk2"),
|
||||
Offset: 512,
|
||||
Length: 512,
|
||||
}
|
||||
if err := repos.BlobChunks.Create(ctx, tx, bc2); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
return repos.BlobChunks.Create(ctx, tx, bc2)
|
||||
}
|
||||
|
||||
func TestRepositoriesTransaction(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
err := repos.WithTx(ctx, createTxTestData(repos))
|
||||
if err != nil {
|
||||
t.Fatalf("transaction failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify all data was committed
|
||||
file, err := repos.Files.GetByPath(ctx, "/test/tx_file.txt")
|
||||
file, err := repos.Files.GetByPath(ctx, testTxFile)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get file: %v", err)
|
||||
}
|
||||
|
||||
if file == nil {
|
||||
t.Error("expected file after transaction")
|
||||
}
|
||||
|
||||
chunks, err := repos.FileChunks.GetByFile(ctx, "/test/tx_file.txt")
|
||||
chunks, err := repos.FileChunks.GetByFile(ctx, testTxFile)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get file chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(chunks) != 2 {
|
||||
t.Errorf("expected 2 file chunks, got %d", len(chunks))
|
||||
}
|
||||
@@ -127,22 +165,25 @@ func TestRepositoriesTransaction(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob: %v", err)
|
||||
}
|
||||
|
||||
if blob == nil {
|
||||
t.Error("expected blob after transaction")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoriesTransactionRollback(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Test transaction rollback
|
||||
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
// Create a file
|
||||
file := &File{
|
||||
file := &database.File{
|
||||
Path: "/test/rollback_file.txt",
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
Size: 1024,
|
||||
@@ -150,24 +191,27 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
if err := repos.Files.Create(ctx, tx, file); err != nil {
|
||||
|
||||
err := repos.Files.Create(ctx, tx, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create a chunk
|
||||
chunk := &Chunk{
|
||||
chunk := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash("rollback_chunk"),
|
||||
Size: 1024,
|
||||
}
|
||||
if err := repos.Chunks.Create(ctx, tx, chunk); err != nil {
|
||||
|
||||
err = repos.Chunks.Create(ctx, tx, chunk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Return error to trigger rollback
|
||||
return fmt.Errorf("intentional rollback")
|
||||
return errIntentionalRollback
|
||||
})
|
||||
|
||||
if err == nil || err.Error() != "intentional rollback" {
|
||||
if !errors.Is(err, errIntentionalRollback) {
|
||||
t.Fatalf("expected rollback error, got: %v", err)
|
||||
}
|
||||
|
||||
@@ -176,6 +220,7 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error checking for file: %v", err)
|
||||
}
|
||||
|
||||
if file != nil {
|
||||
t.Error("file should not exist after rollback")
|
||||
}
|
||||
@@ -184,20 +229,23 @@ func TestRepositoriesTransactionRollback(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error checking for chunk: %v", err)
|
||||
}
|
||||
|
||||
if chunk != nil {
|
||||
t.Error("chunk should not exist after rollback")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepositoriesReadTransaction(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// First, create some data
|
||||
file := &File{
|
||||
file := &database.File{
|
||||
Path: "/test/read_file.txt",
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
Size: 1024,
|
||||
@@ -205,22 +253,25 @@ func TestRepositoriesReadTransaction(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err := repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file: %v", err)
|
||||
}
|
||||
|
||||
// Test read-only transaction
|
||||
var retrievedFile *File
|
||||
var retrievedFile *database.File
|
||||
|
||||
err = repos.WithReadTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
var err error
|
||||
|
||||
retrievedFile, err = repos.Files.GetByPathTx(ctx, tx, "/test/read_file.txt")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Try to write in read-only transaction (should fail)
|
||||
_ = repos.Files.Create(ctx, tx, &File{
|
||||
_ = repos.Files.Create(ctx, tx, &database.File{
|
||||
Path: "/test/should_fail.txt",
|
||||
MTime: time.Now(),
|
||||
Size: 0,
|
||||
@@ -232,7 +283,6 @@ func TestRepositoriesReadTransaction(t *testing.T) {
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("read transaction failed: %v", err)
|
||||
}
|
||||
|
||||
@@ -1,17 +1,24 @@
|
||||
//nolint:testpackage // inspects the unexported database connection
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// errTxIntentionalRollback forces a transaction rollback in tests.
|
||||
var errTxIntentionalRollback = errors.New("intentional rollback")
|
||||
|
||||
// TestFileRepositoryUUIDGeneration tests that files get unique UUIDs
|
||||
func TestFileRepositoryUUIDGeneration(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
@@ -21,7 +28,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
|
||||
// Create multiple files
|
||||
files := []*File{
|
||||
{
|
||||
Path: "/file1.txt",
|
||||
Path: internalTestFile1,
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
@@ -29,7 +36,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
|
||||
GID: 1000,
|
||||
},
|
||||
{
|
||||
Path: "/file2.txt",
|
||||
Path: internalTestFile2,
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
Size: 2048,
|
||||
Mode: 0644,
|
||||
@@ -39,6 +46,7 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
|
||||
}
|
||||
|
||||
uuids := make(map[string]bool)
|
||||
|
||||
for _, file := range files {
|
||||
err := repo.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
@@ -54,12 +62,15 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) {
|
||||
if uuids[file.ID.String()] {
|
||||
t.Errorf("duplicate UUID generated: %s", file.ID)
|
||||
}
|
||||
|
||||
uuids[file.ID.String()] = true
|
||||
}
|
||||
}
|
||||
|
||||
// TestFileRepositoryGetByID tests retrieving files by UUID
|
||||
func TestFileRepositoryGetByID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
@@ -68,7 +79,7 @@ func TestFileRepositoryGetByID(t *testing.T) {
|
||||
|
||||
// Create a file
|
||||
file := &File{
|
||||
Path: "/test.txt",
|
||||
Path: internalTestFilePath,
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
@@ -90,16 +101,20 @@ func TestFileRepositoryGetByID(t *testing.T) {
|
||||
if retrieved.ID != file.ID {
|
||||
t.Errorf("ID mismatch: expected %s, got %s", file.ID, retrieved.ID)
|
||||
}
|
||||
|
||||
if retrieved.Path != file.Path {
|
||||
t.Errorf("Path mismatch: expected %s, got %s", file.Path, retrieved.Path)
|
||||
}
|
||||
|
||||
// Test non-existent ID
|
||||
nonExistentID := types.NewFileID() // Generate a new UUID that won't exist in the database
|
||||
// Test non-existent ID: generate a new UUID that won't exist in the
|
||||
// database.
|
||||
nonExistentID := types.NewFileID()
|
||||
|
||||
nonExistent, err := repo.GetByID(ctx, nonExistentID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetByID should not return error for non-existent ID: %v", err)
|
||||
}
|
||||
|
||||
if nonExistent != nil {
|
||||
t.Error("expected nil for non-existent ID")
|
||||
}
|
||||
@@ -107,6 +122,8 @@ func TestFileRepositoryGetByID(t *testing.T) {
|
||||
|
||||
// TestOrphanedFileCleanup tests the cleanup of orphaned files
|
||||
func TestOrphanedFileCleanup(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
@@ -135,6 +152,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file1: %v", err)
|
||||
}
|
||||
|
||||
err = repos.Files.Create(ctx, nil, file2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file2: %v", err)
|
||||
@@ -142,20 +160,18 @@ func TestOrphanedFileCleanup(t *testing.T) {
|
||||
|
||||
// Create a snapshot and reference only file2
|
||||
snapshot := &Snapshot{
|
||||
ID: "test-snapshot",
|
||||
Hostname: "test-host",
|
||||
ID: internalTestSnapshotID,
|
||||
Hostname: internalTestHost,
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
err = repos.Snapshots.Create(ctx, nil, snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create snapshot: %v", err)
|
||||
}
|
||||
|
||||
// Add file2 to snapshot
|
||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file2.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to add file to snapshot: %v", err)
|
||||
}
|
||||
mustAddFileToSnapshot(t, repos, snapshot.ID.String(), file2.ID)
|
||||
|
||||
// Run orphaned cleanup
|
||||
err = repos.Files.DeleteOrphaned(ctx)
|
||||
@@ -168,6 +184,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting file: %v", err)
|
||||
}
|
||||
|
||||
if orphanedFile != nil {
|
||||
t.Error("orphaned file should have been deleted")
|
||||
}
|
||||
@@ -177,6 +194,7 @@ func TestOrphanedFileCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting file: %v", err)
|
||||
}
|
||||
|
||||
if referencedFile == nil {
|
||||
t.Error("referenced file should not have been deleted")
|
||||
}
|
||||
@@ -184,6 +202,8 @@ func TestOrphanedFileCleanup(t *testing.T) {
|
||||
|
||||
// TestOrphanedChunkCleanup tests the cleanup of orphaned chunks
|
||||
func TestOrphanedChunkCleanup(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
@@ -204,6 +224,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk1: %v", err)
|
||||
}
|
||||
|
||||
err = repos.Chunks.Create(ctx, nil, chunk2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk2: %v", err)
|
||||
@@ -211,13 +232,14 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
||||
|
||||
// Create a file and reference only chunk2
|
||||
file := &File{
|
||||
Path: "/test.txt",
|
||||
Path: internalTestFilePath,
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err = repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file: %v", err)
|
||||
@@ -229,6 +251,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
||||
Idx: 0,
|
||||
ChunkHash: chunk2.ChunkHash,
|
||||
}
|
||||
|
||||
err = repos.FileChunks.Create(ctx, nil, fc)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file chunk: %v", err)
|
||||
@@ -245,6 +268,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting chunk: %v", err)
|
||||
}
|
||||
|
||||
if orphanedChunk != nil {
|
||||
t.Error("orphaned chunk should have been deleted")
|
||||
}
|
||||
@@ -254,6 +278,7 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting chunk: %v", err)
|
||||
}
|
||||
|
||||
if referencedChunk == nil {
|
||||
t.Error("referenced chunk should not have been deleted")
|
||||
}
|
||||
@@ -261,6 +286,8 @@ func TestOrphanedChunkCleanup(t *testing.T) {
|
||||
|
||||
// TestOrphanedBlobCleanup tests the cleanup of orphaned blobs
|
||||
func TestOrphanedBlobCleanup(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
@@ -283,6 +310,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob1: %v", err)
|
||||
}
|
||||
|
||||
err = repos.Blobs.Create(ctx, nil, blob2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob2: %v", err)
|
||||
@@ -290,10 +318,11 @@ func TestOrphanedBlobCleanup(t *testing.T) {
|
||||
|
||||
// Create a snapshot and reference only blob2
|
||||
snapshot := &Snapshot{
|
||||
ID: "test-snapshot",
|
||||
Hostname: "test-host",
|
||||
ID: internalTestSnapshotID,
|
||||
Hostname: internalTestHost,
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
err = repos.Snapshots.Create(ctx, nil, snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create snapshot: %v", err)
|
||||
@@ -316,6 +345,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting blob: %v", err)
|
||||
}
|
||||
|
||||
if orphanedBlob != nil {
|
||||
t.Error("orphaned blob should have been deleted")
|
||||
}
|
||||
@@ -325,6 +355,7 @@ func TestOrphanedBlobCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting blob: %v", err)
|
||||
}
|
||||
|
||||
if referencedBlob == nil {
|
||||
t.Error("referenced blob should not have been deleted")
|
||||
}
|
||||
@@ -332,6 +363,8 @@ func TestOrphanedBlobCleanup(t *testing.T) {
|
||||
|
||||
// TestFileChunkRepositoryWithUUIDs tests file-chunk relationships with UUIDs
|
||||
func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
@@ -340,17 +373,15 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
||||
|
||||
// Create a file
|
||||
file := &File{
|
||||
Path: "/test.txt",
|
||||
Path: internalTestFilePath,
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
Size: 3072,
|
||||
Mode: 0644,
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
err := repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file: %v", err)
|
||||
}
|
||||
|
||||
mustCreateFileRow(t, repos, file)
|
||||
|
||||
// Create chunks
|
||||
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
|
||||
@@ -359,7 +390,8 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
||||
ChunkHash: chunkHash,
|
||||
Size: 1024,
|
||||
}
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
|
||||
err := repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk: %v", err)
|
||||
}
|
||||
@@ -370,6 +402,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
||||
Idx: i,
|
||||
ChunkHash: chunkHash,
|
||||
}
|
||||
|
||||
err = repos.FileChunks.Create(ctx, nil, fc)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file chunk: %v", err)
|
||||
@@ -381,6 +414,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get file chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(fileChunks) != 3 {
|
||||
t.Errorf("expected 3 chunks, got %d", len(fileChunks))
|
||||
}
|
||||
@@ -395,6 +429,7 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get file chunks after delete: %v", err)
|
||||
}
|
||||
|
||||
if len(fileChunks) != 0 {
|
||||
t.Errorf("expected 0 chunks after delete, got %d", len(fileChunks))
|
||||
}
|
||||
@@ -402,6 +437,8 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) {
|
||||
|
||||
// TestChunkFileRepositoryWithUUIDs tests chunk-file relationships with UUIDs
|
||||
func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
@@ -410,7 +447,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
||||
|
||||
// Create files
|
||||
file1 := &File{
|
||||
Path: "/file1.txt",
|
||||
Path: internalTestFile1,
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
@@ -418,7 +455,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
||||
GID: 1000,
|
||||
}
|
||||
file2 := &File{
|
||||
Path: "/file2.txt",
|
||||
Path: internalTestFile2,
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
@@ -426,21 +463,16 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err := repos.Files.Create(ctx, nil, file1)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file1: %v", err)
|
||||
}
|
||||
err = repos.Files.Create(ctx, nil, file2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file2: %v", err)
|
||||
}
|
||||
mustCreateFileRow(t, repos, file1)
|
||||
mustCreateFileRow(t, repos, file2)
|
||||
|
||||
// Create a chunk that appears in both files (deduplication)
|
||||
chunk := &Chunk{
|
||||
ChunkHash: types.ChunkHash("shared-chunk"),
|
||||
Size: 1024,
|
||||
}
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
|
||||
err := repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk: %v", err)
|
||||
}
|
||||
@@ -463,6 +495,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk file 1: %v", err)
|
||||
}
|
||||
|
||||
err = repos.ChunkFiles.Create(ctx, nil, cf2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk file 2: %v", err)
|
||||
@@ -473,6 +506,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunk files: %v", err)
|
||||
}
|
||||
|
||||
if len(chunkFiles) != 2 {
|
||||
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
|
||||
}
|
||||
@@ -482,6 +516,7 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks by file ID: %v", err)
|
||||
}
|
||||
|
||||
if len(chunkFiles) != 1 {
|
||||
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
|
||||
}
|
||||
@@ -489,6 +524,8 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) {
|
||||
|
||||
// TestSnapshotRepositoryExtendedFields tests snapshot with version and git revision
|
||||
func TestSnapshotRepositoryExtendedFields(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
@@ -498,7 +535,7 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) {
|
||||
// Create snapshot with extended fields
|
||||
snapshot := &Snapshot{
|
||||
ID: "test-20250722-120000Z",
|
||||
Hostname: "test-host",
|
||||
Hostname: internalTestHost,
|
||||
VaultikVersion: "0.0.1",
|
||||
VaultikGitRevision: "abc123def456",
|
||||
StartedAt: time.Now(),
|
||||
@@ -526,31 +563,39 @@ func TestSnapshotRepositoryExtendedFields(t *testing.T) {
|
||||
}
|
||||
|
||||
if retrieved.VaultikVersion != snapshot.VaultikVersion {
|
||||
t.Errorf("version mismatch: expected %s, got %s", snapshot.VaultikVersion, retrieved.VaultikVersion)
|
||||
t.Errorf("version mismatch: expected %s, got %s",
|
||||
snapshot.VaultikVersion, retrieved.VaultikVersion)
|
||||
}
|
||||
|
||||
if retrieved.VaultikGitRevision != snapshot.VaultikGitRevision {
|
||||
t.Errorf("git revision mismatch: expected %s, got %s", snapshot.VaultikGitRevision, retrieved.VaultikGitRevision)
|
||||
t.Errorf("git revision mismatch: expected %s, got %s",
|
||||
snapshot.VaultikGitRevision, retrieved.VaultikGitRevision)
|
||||
}
|
||||
|
||||
if retrieved.CompressionLevel != snapshot.CompressionLevel {
|
||||
t.Errorf("compression level mismatch: expected %d, got %d", snapshot.CompressionLevel, retrieved.CompressionLevel)
|
||||
t.Errorf("compression level mismatch: expected %d, got %d",
|
||||
snapshot.CompressionLevel, retrieved.CompressionLevel)
|
||||
}
|
||||
|
||||
if retrieved.BlobUncompressedSize != snapshot.BlobUncompressedSize {
|
||||
t.Errorf("uncompressed size mismatch: expected %d, got %d", snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize)
|
||||
t.Errorf("uncompressed size mismatch: expected %d, got %d",
|
||||
snapshot.BlobUncompressedSize, retrieved.BlobUncompressedSize)
|
||||
}
|
||||
|
||||
if retrieved.UploadDurationMs != snapshot.UploadDurationMs {
|
||||
t.Errorf("upload duration mismatch: expected %d, got %d", snapshot.UploadDurationMs, retrieved.UploadDurationMs)
|
||||
t.Errorf("upload duration mismatch: expected %d, got %d",
|
||||
snapshot.UploadDurationMs, retrieved.UploadDurationMs)
|
||||
}
|
||||
}
|
||||
|
||||
// TestComplexOrphanedDataScenario tests a complex scenario with multiple relationships
|
||||
func TestComplexOrphanedDataScenario(t *testing.T) {
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
// createOrphanScenarioFixtures creates two snapshots and three files for
|
||||
// the orphaned-data cleanup scenario.
|
||||
func createOrphanScenarioFixtures(
|
||||
ctx context.Context, t *testing.T, repos *Repositories,
|
||||
) (*Snapshot, *Snapshot, []*File) {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
// Create snapshots
|
||||
snapshot1 := &Snapshot{
|
||||
ID: "snapshot1",
|
||||
Hostname: "host1",
|
||||
@@ -566,6 +611,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create snapshot1: %v", err)
|
||||
}
|
||||
|
||||
err = repos.Snapshots.Create(ctx, nil, snapshot2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create snapshot2: %v", err)
|
||||
@@ -582,40 +628,44 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err = repos.Files.Create(ctx, nil, files[i])
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file%d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
return snapshot1, snapshot2, files
|
||||
}
|
||||
|
||||
func TestComplexOrphanedDataScenario(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
snapshot1, snapshot2, files := createOrphanScenarioFixtures(ctx, t, repos)
|
||||
|
||||
// Add files to snapshots
|
||||
// Snapshot1: file0, file1
|
||||
// Snapshot2: file1, file2
|
||||
// file0: only in snapshot1
|
||||
// file1: in both snapshots
|
||||
// file2: only in snapshot2
|
||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[0].ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot1.ID.String(), files[1].ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[1].ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot2.ID.String(), files[2].ID)
|
||||
mustAddFileToSnapshot(t, repos, snapshot1.ID.String(), files[0].ID)
|
||||
mustAddFileToSnapshot(t, repos, snapshot1.ID.String(), files[1].ID)
|
||||
mustAddFileToSnapshot(t, repos, snapshot2.ID.String(), files[1].ID)
|
||||
mustAddFileToSnapshot(t, repos, snapshot2.ID.String(), files[2].ID)
|
||||
|
||||
// Delete snapshot1
|
||||
err := repos.Snapshots.DeleteSnapshotFiles(ctx, snapshot1.ID.String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Delete snapshot1
|
||||
err = repos.Snapshots.DeleteSnapshotFiles(ctx, snapshot1.ID.String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = repos.Snapshots.Delete(ctx, snapshot1.ID.String())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -633,6 +683,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting file0: %v", err)
|
||||
}
|
||||
|
||||
if file0 != nil {
|
||||
t.Error("file0 should have been deleted")
|
||||
}
|
||||
@@ -642,6 +693,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting file1: %v", err)
|
||||
}
|
||||
|
||||
if file1 == nil {
|
||||
t.Error("file1 should still exist")
|
||||
}
|
||||
@@ -651,6 +703,7 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting file2: %v", err)
|
||||
}
|
||||
|
||||
if file2 == nil {
|
||||
t.Error("file2 should still exist")
|
||||
}
|
||||
@@ -658,6 +711,8 @@ func TestComplexOrphanedDataScenario(t *testing.T) {
|
||||
|
||||
// TestCascadeDelete tests that cascade deletes work properly
|
||||
func TestCascadeDelete(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
@@ -673,17 +728,19 @@ func TestCascadeDelete(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err := repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file: %v", err)
|
||||
}
|
||||
|
||||
// Create chunks and file-chunk mappings
|
||||
for i := 0; i < 3; i++ {
|
||||
for i := range 3 {
|
||||
chunk := &Chunk{
|
||||
ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)),
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk: %v", err)
|
||||
@@ -694,6 +751,7 @@ func TestCascadeDelete(t *testing.T) {
|
||||
Idx: i,
|
||||
ChunkHash: chunk.ChunkHash,
|
||||
}
|
||||
|
||||
err = repos.FileChunks.Create(ctx, nil, fc)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file chunk: %v", err)
|
||||
@@ -705,6 +763,7 @@ func TestCascadeDelete(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(fileChunks) != 3 {
|
||||
t.Errorf("expected 3 file chunks, got %d", len(fileChunks))
|
||||
}
|
||||
@@ -720,6 +779,7 @@ func TestCascadeDelete(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(fileChunks) != 0 {
|
||||
t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks))
|
||||
}
|
||||
@@ -727,6 +787,8 @@ func TestCascadeDelete(t *testing.T) {
|
||||
|
||||
// TestTransactionIsolation tests that transactions properly isolate changes
|
||||
func TestTransactionIsolation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
@@ -744,6 +806,7 @@ func TestTransactionIsolation(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err := repos.Files.Create(ctx, tx, file)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -754,9 +817,8 @@ func TestTransactionIsolation(t *testing.T) {
|
||||
// For now, we'll just test that rollback works
|
||||
|
||||
// Return an error to trigger rollback
|
||||
return fmt.Errorf("intentional rollback")
|
||||
return errTxIntentionalRollback
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected error from transaction")
|
||||
}
|
||||
@@ -766,37 +828,22 @@ func TestTransactionIsolation(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(files) != 0 {
|
||||
t.Error("file should not exist after rollback")
|
||||
}
|
||||
}
|
||||
|
||||
// TestConcurrentOrphanedCleanup tests that concurrent cleanup operations don't interfere
|
||||
func TestConcurrentOrphanedCleanup(t *testing.T) {
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
// TestConcurrentOrphanedCleanup tests that concurrent cleanup operations
|
||||
// don't interfere.
|
||||
// createConcurrentCleanupFiles creates 20 files and associates the
|
||||
// even-numbered ones with the snapshot, leaving the rest orphaned.
|
||||
func createConcurrentCleanupFiles(
|
||||
ctx context.Context, t *testing.T, repos *Repositories, snapshotID string,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
// Set a 5-second busy timeout to handle concurrent operations
|
||||
if _, err := db.conn.Exec("PRAGMA busy_timeout = 5000"); err != nil {
|
||||
t.Fatalf("failed to set busy timeout: %v", err)
|
||||
}
|
||||
|
||||
// Create a snapshot
|
||||
snapshot := &Snapshot{
|
||||
ID: "concurrent-test",
|
||||
Hostname: "test-host",
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
err := repos.Snapshots.Create(ctx, nil, snapshot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create many files, some orphaned
|
||||
for i := 0; i < 20; i++ {
|
||||
for i := range 20 {
|
||||
file := &File{
|
||||
Path: types.FilePath(fmt.Sprintf("/concurrent-%d.txt", i)),
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
@@ -805,31 +852,63 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
err = repos.Files.Create(ctx, nil, file)
|
||||
|
||||
err := repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Add even-numbered files to snapshot
|
||||
if i%2 == 0 {
|
||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file.ID)
|
||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshotID, file.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentOrphanedCleanup(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
// Set a 5-second busy timeout to handle concurrent operations
|
||||
_, err := db.conn.ExecContext(ctx, "PRAGMA busy_timeout = 5000")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to set busy timeout: %v", err)
|
||||
}
|
||||
|
||||
// Create a snapshot
|
||||
snapshot := &Snapshot{
|
||||
ID: "concurrent-test",
|
||||
Hostname: internalTestHost,
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
err = repos.Snapshots.Create(ctx, nil, snapshot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
createConcurrentCleanupFiles(ctx, t, repos, snapshot.ID.String())
|
||||
|
||||
// Run multiple cleanup operations concurrently
|
||||
// Note: SQLite has limited support for concurrent writes, so we expect some to fail
|
||||
done := make(chan error, 3)
|
||||
for i := 0; i < 3; i++ {
|
||||
|
||||
for range 3 {
|
||||
go func() {
|
||||
done <- repos.Files.DeleteOrphaned(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait for all to complete
|
||||
for i := 0; i < 3; i++ {
|
||||
for i := range 3 {
|
||||
err := <-done
|
||||
if err != nil {
|
||||
t.Errorf("cleanup %d failed: %v", i, err)
|
||||
@@ -850,10 +929,12 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
|
||||
// Verify all remaining files are even-numbered
|
||||
for _, file := range files {
|
||||
var num int
|
||||
|
||||
_, err := fmt.Sscanf(file.Path.String(), "/concurrent-%d.txt", &num)
|
||||
if err != nil {
|
||||
t.Logf("failed to parse file number from %s: %v", file.Path, err)
|
||||
}
|
||||
|
||||
if num%2 != 0 {
|
||||
t.Errorf("odd-numbered file %s should have been deleted", file.Path)
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
//nolint:testpackage // inspects the unexported database connection
|
||||
package database
|
||||
|
||||
import (
|
||||
@@ -6,15 +7,51 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestOrphanedFileCleanupDebug tests orphaned file cleanup with debug output
|
||||
func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
// logSnapshotFileIDs logs every file_id present in snapshot_files.
|
||||
func logSnapshotFileIDs(t *testing.T, db *DB) {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
// Create files
|
||||
rows, err := db.conn.QueryContext(ctx, "SELECT file_id FROM snapshot_files")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
t.Logf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
t.Log("Files in snapshot_files:")
|
||||
|
||||
for rows.Next() {
|
||||
var fileID string
|
||||
|
||||
err := rows.Scan(&fileID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf(" - %s", fileID)
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOrphanedFileCleanupDebug tests orphaned file cleanup with debug output
|
||||
// createOrphanDebugFixtures creates one orphaned file, one referenced
|
||||
// file, and the snapshot that will reference the latter.
|
||||
func createOrphanDebugFixtures(
|
||||
ctx context.Context, t *testing.T, repos *Repositories,
|
||||
) (*File, *File, *Snapshot) {
|
||||
t.Helper()
|
||||
|
||||
file1 := &File{
|
||||
Path: "/orphaned.txt",
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
@@ -36,72 +73,65 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file1: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Created file1 with ID: %s", file1.ID)
|
||||
|
||||
err = repos.Files.Create(ctx, nil, file2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file2: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Created file2 with ID: %s", file2.ID)
|
||||
|
||||
// Create a snapshot and reference only file2
|
||||
snapshot := &Snapshot{
|
||||
ID: "test-snapshot",
|
||||
Hostname: "test-host",
|
||||
ID: internalTestSnapshotID,
|
||||
Hostname: internalTestHost,
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
err = repos.Snapshots.Create(ctx, nil, snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create snapshot: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Created snapshot: %s", snapshot.ID)
|
||||
|
||||
return file1, file2, snapshot
|
||||
}
|
||||
|
||||
func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
file1, file2, snapshot := createOrphanDebugFixtures(ctx, t, repos)
|
||||
|
||||
// Check snapshot_files before adding
|
||||
var count int
|
||||
err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
count := countRow(t, db, "SELECT COUNT(*) FROM snapshot_files")
|
||||
t.Logf("snapshot_files count before add: %d", count)
|
||||
|
||||
// Add file2 to snapshot
|
||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file2.ID)
|
||||
err := repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file2.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to add file to snapshot: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Added file2 to snapshot")
|
||||
|
||||
// Check snapshot_files after adding
|
||||
err = db.conn.QueryRow("SELECT COUNT(*) FROM snapshot_files").Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
count = countRow(t, db, "SELECT COUNT(*) FROM snapshot_files")
|
||||
t.Logf("snapshot_files count after add: %d", count)
|
||||
|
||||
// Check which files are referenced
|
||||
rows, err := db.conn.Query("SELECT file_id FROM snapshot_files")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := rows.Close(); err != nil {
|
||||
t.Logf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
t.Log("Files in snapshot_files:")
|
||||
for rows.Next() {
|
||||
var fileID string
|
||||
if err := rows.Scan(&fileID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf(" - %s", fileID)
|
||||
}
|
||||
logSnapshotFileIDs(t, db)
|
||||
|
||||
// Check files before cleanup
|
||||
err = db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
count = countRow(t, db, countFilesQuery)
|
||||
t.Logf("Files count before cleanup: %d", count)
|
||||
|
||||
// Run orphaned cleanup
|
||||
@@ -109,13 +139,11 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to delete orphaned files: %v", err)
|
||||
}
|
||||
|
||||
t.Log("Ran orphaned cleanup")
|
||||
|
||||
// Check files after cleanup
|
||||
err = db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
count = countRow(t, db, countFilesQuery)
|
||||
t.Logf("Files count after cleanup: %d", count)
|
||||
|
||||
// List remaining files
|
||||
@@ -123,7 +151,9 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Log("Remaining files:")
|
||||
|
||||
for _, f := range files {
|
||||
t.Logf(" - ID: %s, Path: %s", f.ID, f.Path)
|
||||
}
|
||||
@@ -133,19 +163,16 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting file: %v", err)
|
||||
}
|
||||
|
||||
if orphanedFile != nil {
|
||||
t.Error("orphaned file should have been deleted")
|
||||
// Let's check why it wasn't deleted
|
||||
var exists bool
|
||||
err = db.conn.QueryRow(`
|
||||
stillReferenced := countRow(t, db, `
|
||||
SELECT EXISTS(
|
||||
SELECT 1 FROM snapshot_files
|
||||
WHERE file_id = ?
|
||||
)`, file1.ID).Scan(&exists)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("File1 exists in snapshot_files: %v", exists)
|
||||
)`, file1.ID)
|
||||
t.Logf("File1 exists in snapshot_files: %v", stillReferenced != 0)
|
||||
} else {
|
||||
t.Log("Orphaned file was correctly deleted")
|
||||
}
|
||||
@@ -155,6 +182,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("error getting file: %v", err)
|
||||
}
|
||||
|
||||
if referencedFile == nil {
|
||||
t.Error("referenced file should not have been deleted")
|
||||
} else {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
//nolint:testpackage // inspects the unexported database connection
|
||||
package database
|
||||
|
||||
import (
|
||||
@@ -7,23 +8,20 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// TestFileRepositoryEdgeCases tests edge cases for file repository
|
||||
func TestFileRepositoryEdgeCases(t *testing.T) {
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewFileRepository(db)
|
||||
|
||||
tests := []struct {
|
||||
// fileEdgeCase describes one Create edge-case scenario.
|
||||
type fileEdgeCase struct {
|
||||
name string
|
||||
file *File
|
||||
wantErr bool
|
||||
errMsg string
|
||||
}{
|
||||
}
|
||||
|
||||
// fileEdgeCases returns the Create edge-case table.
|
||||
func fileEdgeCases() []fileEdgeCase {
|
||||
return []fileEdgeCase{
|
||||
{
|
||||
name: "empty path",
|
||||
file: &File{
|
||||
@@ -51,6 +49,7 @@ func TestFileRepositoryEdgeCases(t *testing.T) {
|
||||
{
|
||||
name: "path with special characters",
|
||||
file: &File{
|
||||
//nolint:gosmopolitan // non-ASCII path is deliberate test data
|
||||
Path: "/test/file with spaces and 特殊文字.txt",
|
||||
MTime: time.Now(),
|
||||
Size: 1024,
|
||||
@@ -86,18 +85,33 @@ func TestFileRepositoryEdgeCases(t *testing.T) {
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
for i, tt := range tests {
|
||||
// TestFileRepositoryEdgeCases tests edge cases for file repository
|
||||
func TestFileRepositoryEdgeCases(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
t.Cleanup(cleanup)
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewFileRepository(db)
|
||||
|
||||
for i, tt := range fileEdgeCases() {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Add a unique suffix to paths to avoid UNIQUE constraint violations
|
||||
if tt.file.Path != "" {
|
||||
tt.file.Path = types.FilePath(fmt.Sprintf("%s_%d_%d", tt.file.Path, i, time.Now().UnixNano()))
|
||||
tt.file.Path = types.FilePath(fmt.Sprintf("%s_%d_%d",
|
||||
tt.file.Path, i, time.Now().UnixNano()))
|
||||
}
|
||||
|
||||
err := repo.Create(ctx, nil, tt.file)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Create() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
|
||||
if err != nil && tt.errMsg != "" && !strings.Contains(err.Error(), tt.errMsg) {
|
||||
t.Errorf("Create() error = %v, want error containing %q", err, tt.errMsg)
|
||||
}
|
||||
@@ -105,16 +119,12 @@ func TestFileRepositoryEdgeCases(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestDuplicateHandling tests handling of duplicate entries
|
||||
func TestDuplicateHandling(t *testing.T) {
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
// testDuplicateFilePaths exercises the UPSERT behavior for duplicate paths.
|
||||
func testDuplicateFilePaths(t *testing.T, repos *Repositories) {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
// Test duplicate file paths - Create uses UPSERT logic
|
||||
t.Run("duplicate file paths", func(t *testing.T) {
|
||||
file1 := &File{
|
||||
Path: "/duplicate.txt",
|
||||
MTime: time.Now(),
|
||||
@@ -136,6 +146,7 @@ func TestDuplicateHandling(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file1: %v", err)
|
||||
}
|
||||
|
||||
originalID := file1.ID
|
||||
|
||||
// Create with same path should update the existing record (UPSERT behavior)
|
||||
@@ -157,31 +168,17 @@ func TestDuplicateHandling(t *testing.T) {
|
||||
|
||||
// ID might be different due to the UPSERT
|
||||
if retrievedFile.ID != file2.ID {
|
||||
t.Logf("File ID changed from %s to %s during upsert", originalID, retrievedFile.ID)
|
||||
t.Logf("File ID changed from %s to %s during upsert",
|
||||
originalID, retrievedFile.ID)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Test duplicate chunk hashes
|
||||
t.Run("duplicate chunk hashes", func(t *testing.T) {
|
||||
chunk := &Chunk{
|
||||
ChunkHash: types.ChunkHash("duplicate-chunk"),
|
||||
Size: 1024,
|
||||
}
|
||||
// testDuplicateFileChunks exercises idempotent file-chunk mapping creation.
|
||||
func testDuplicateFileChunks(t *testing.T, repos *Repositories) {
|
||||
t.Helper()
|
||||
|
||||
err := repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk: %v", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
// Creating the same chunk again should be idempotent (ON CONFLICT DO NOTHING)
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Errorf("duplicate chunk creation should be idempotent, got error: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Test duplicate file-chunk mappings
|
||||
t.Run("duplicate file-chunk mappings", func(t *testing.T) {
|
||||
file := &File{
|
||||
Path: "/test-dup-fc.txt",
|
||||
MTime: time.Now(),
|
||||
@@ -190,6 +187,7 @@ func TestDuplicateHandling(t *testing.T) {
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
|
||||
err := repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -199,6 +197,7 @@ func TestDuplicateHandling(t *testing.T) {
|
||||
ChunkHash: types.ChunkHash("test-chunk-dup"),
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -220,19 +219,66 @@ func TestDuplicateHandling(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Error("file-chunk creation should be idempotent")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDuplicateHandling tests handling of duplicate entries
|
||||
func TestDuplicateHandling(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
t.Cleanup(cleanup)
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
// Test duplicate file paths - Create uses UPSERT logic
|
||||
t.Run("duplicate file paths", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
testDuplicateFilePaths(t, repos)
|
||||
})
|
||||
|
||||
// Test duplicate chunk hashes
|
||||
t.Run("duplicate chunk hashes", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
chunk := &Chunk{
|
||||
ChunkHash: types.ChunkHash("duplicate-chunk"),
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err := repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk: %v", err)
|
||||
}
|
||||
|
||||
// Creating the same chunk again should be idempotent (ON CONFLICT DO NOTHING)
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Errorf("duplicate chunk creation should be idempotent, got error: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Test duplicate file-chunk mappings
|
||||
t.Run("duplicate file-chunk mappings", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
testDuplicateFileChunks(t, repos)
|
||||
})
|
||||
}
|
||||
|
||||
// TestNullHandling tests handling of NULL values
|
||||
func TestNullHandling(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
t.Cleanup(cleanup)
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
// Test file with no link target
|
||||
t.Run("file without link target", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
file := &File{
|
||||
Path: "/regular.txt",
|
||||
MTime: time.Now(),
|
||||
@@ -260,9 +306,11 @@ func TestNullHandling(t *testing.T) {
|
||||
|
||||
// Test snapshot with NULL completed_at
|
||||
t.Run("incomplete snapshot", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
snapshot := &Snapshot{
|
||||
ID: "incomplete-test",
|
||||
Hostname: "test-host",
|
||||
Hostname: internalTestHost,
|
||||
StartedAt: time.Now(),
|
||||
CompletedAt: nil, // Should remain NULL until completed
|
||||
}
|
||||
@@ -284,6 +332,18 @@ func TestNullHandling(t *testing.T) {
|
||||
|
||||
// Test blob with NULL uploaded_ts
|
||||
t.Run("blob not uploaded", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
verifyBlobNullUploadTS(ctx, t, repos)
|
||||
})
|
||||
}
|
||||
|
||||
// verifyBlobNullUploadTS checks that a blob created without an upload
|
||||
// timestamp round-trips with UploadedTS nil.
|
||||
func verifyBlobNullUploadTS(
|
||||
ctx context.Context, t *testing.T, repos *Repositories,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
blob := &Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("test-hash"),
|
||||
@@ -304,11 +364,54 @@ func TestNullHandling(t *testing.T) {
|
||||
if retrieved.UploadedTS != nil {
|
||||
t.Error("expected nil UploadedTS for non-uploaded blob")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// createLargeDatasetFiles creates fileCount files and adds every other
|
||||
// one to the snapshot.
|
||||
func createLargeDatasetFiles(
|
||||
t *testing.T,
|
||||
repos *Repositories,
|
||||
snapshotID string,
|
||||
fileCount int,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
start := time.Now()
|
||||
|
||||
for i := range fileCount {
|
||||
file := &File{
|
||||
Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)),
|
||||
MTime: time.Now(),
|
||||
Size: int64(i * 1024),
|
||||
Mode: 0644,
|
||||
UID: uint32(1000 + (i % 10)),
|
||||
GID: uint32(1000 + (i % 10)),
|
||||
}
|
||||
|
||||
err := repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file %d: %v", i, err)
|
||||
}
|
||||
|
||||
// Add half to snapshot
|
||||
if i%2 == 0 {
|
||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshotID, file.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("Created %d files in %v", fileCount, time.Since(start))
|
||||
}
|
||||
|
||||
// TestLargeDatasets tests operations with large amounts of data
|
||||
//
|
||||
//nolint:tparallel // subtests share one database and are order-dependent
|
||||
func TestLargeDatasets(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if testing.Short() {
|
||||
t.Skip("skipping large dataset test in short mode")
|
||||
}
|
||||
@@ -322,9 +425,10 @@ func TestLargeDatasets(t *testing.T) {
|
||||
// Create a snapshot
|
||||
snapshot := &Snapshot{
|
||||
ID: "large-dataset-test",
|
||||
Hostname: "test-host",
|
||||
Hostname: internalTestHost,
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
|
||||
err := repos.Snapshots.Create(ctx, nil, snapshot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -332,56 +436,39 @@ func TestLargeDatasets(t *testing.T) {
|
||||
|
||||
// Create many files
|
||||
const fileCount = 1000
|
||||
fileIDs := make([]types.FileID, fileCount)
|
||||
|
||||
//nolint:paralleltest // phases share one database and are order-dependent
|
||||
t.Run("create many files", func(t *testing.T) {
|
||||
start := time.Now()
|
||||
for i := 0; i < fileCount; i++ {
|
||||
file := &File{
|
||||
Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)),
|
||||
MTime: time.Now(),
|
||||
Size: int64(i * 1024),
|
||||
Mode: 0644,
|
||||
UID: uint32(1000 + (i % 10)),
|
||||
GID: uint32(1000 + (i % 10)),
|
||||
}
|
||||
err := repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file %d: %v", i, err)
|
||||
}
|
||||
fileIDs[i] = file.ID
|
||||
|
||||
// Add half to snapshot
|
||||
if i%2 == 0 {
|
||||
err = repos.Snapshots.AddFileByID(ctx, nil, snapshot.ID.String(), file.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Logf("Created %d files in %v", fileCount, time.Since(start))
|
||||
createLargeDatasetFiles(t, repos, snapshot.ID.String(), fileCount)
|
||||
})
|
||||
|
||||
// Test ListByPrefix performance
|
||||
//nolint:paralleltest // phases share one database and are order-dependent
|
||||
t.Run("list by prefix performance", func(t *testing.T) {
|
||||
start := time.Now()
|
||||
|
||||
files, err := repos.Files.ListByPrefix(ctx, "/large/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(files) != fileCount {
|
||||
t.Errorf("expected %d files, got %d", fileCount, len(files))
|
||||
}
|
||||
|
||||
t.Logf("Listed %d files in %v", len(files), time.Since(start))
|
||||
})
|
||||
|
||||
// Test orphaned cleanup performance
|
||||
//nolint:paralleltest // phases share one database and are order-dependent
|
||||
t.Run("orphaned cleanup performance", func(t *testing.T) {
|
||||
start := time.Now()
|
||||
|
||||
err := repos.Files.DeleteOrphaned(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("Cleaned up orphaned files in %v", time.Since(start))
|
||||
|
||||
// Verify correct number remain
|
||||
@@ -389,26 +476,33 @@ func TestLargeDatasets(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(files) != fileCount/2 {
|
||||
t.Errorf("expected %d files after cleanup, got %d", fileCount/2, len(files))
|
||||
t.Errorf("expected %d files after cleanup, got %d",
|
||||
fileCount/2, len(files))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestErrorPropagation tests that errors are properly propagated
|
||||
func TestErrorPropagation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
t.Cleanup(cleanup)
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
// Test GetByID with non-existent ID
|
||||
t.Run("GetByID non-existent", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
file, err := repos.Files.GetByID(ctx, types.NewFileID())
|
||||
if err != nil {
|
||||
t.Errorf("GetByID should not return error for non-existent ID, got: %v", err)
|
||||
}
|
||||
|
||||
if file != nil {
|
||||
t.Error("expected nil file for non-existent ID")
|
||||
}
|
||||
@@ -416,10 +510,14 @@ func TestErrorPropagation(t *testing.T) {
|
||||
|
||||
// Test GetByPath with non-existent path
|
||||
t.Run("GetByPath non-existent", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
file, err := repos.Files.GetByPath(ctx, "/non/existent/path.txt")
|
||||
if err != nil {
|
||||
t.Errorf("GetByPath should not return error for non-existent path, got: %v", err)
|
||||
t.Errorf("GetByPath should not return error for non-existent path, got: %v",
|
||||
err)
|
||||
}
|
||||
|
||||
if file != nil {
|
||||
t.Error("expected nil file for non-existent path")
|
||||
}
|
||||
@@ -427,15 +525,19 @@ func TestErrorPropagation(t *testing.T) {
|
||||
|
||||
// Test invalid foreign key reference
|
||||
t.Run("invalid foreign key", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fc := &FileChunk{
|
||||
FileID: types.NewFileID(),
|
||||
Idx: 0,
|
||||
ChunkHash: types.ChunkHash("some-chunk"),
|
||||
}
|
||||
|
||||
err := repos.FileChunks.Create(ctx, nil, fc)
|
||||
if err == nil {
|
||||
t.Error("expected error for invalid foreign key")
|
||||
}
|
||||
|
||||
if !strings.Contains(err.Error(), "FOREIGN KEY") {
|
||||
t.Errorf("expected foreign key error, got: %v", err)
|
||||
}
|
||||
@@ -444,8 +546,10 @@ func TestErrorPropagation(t *testing.T) {
|
||||
|
||||
// TestQueryInjection tests that the system is safe from SQL injection
|
||||
func TestQueryInjection(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
t.Cleanup(cleanup)
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
@@ -460,6 +564,8 @@ func TestQueryInjection(t *testing.T) {
|
||||
|
||||
for _, injection := range injectionTests {
|
||||
t.Run("injection attempt", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Try injection in file path
|
||||
file := &File{
|
||||
Path: types.FilePath(injection),
|
||||
@@ -475,7 +581,8 @@ func TestQueryInjection(t *testing.T) {
|
||||
|
||||
// Verify tables still exist
|
||||
var count int
|
||||
err := db.conn.QueryRow("SELECT COUNT(*) FROM files").Scan(&count)
|
||||
|
||||
err := db.conn.QueryRowContext(ctx, countFilesQuery).Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatal("files table was damaged by injection")
|
||||
}
|
||||
@@ -485,6 +592,8 @@ func TestQueryInjection(t *testing.T) {
|
||||
|
||||
// TestTimezoneHandling tests that times are properly handled in UTC
|
||||
func TestTimezoneHandling(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
-- Migration 000: Schema migrations tracking table
|
||||
-- Applied as a bootstrap step before the normal migration loop.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version INTEGER PRIMARY KEY,
|
||||
applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);
|
||||
@@ -1,6 +1,5 @@
|
||||
-- Vaultik Database Schema
|
||||
-- Note: This database does not support migrations. If the schema changes,
|
||||
-- delete the local database and perform a full backup to recreate it.
|
||||
-- Migration 001: Initial Vaultik schema
|
||||
-- All core tables for tracking files, chunks, blobs, snapshots, and uploads.
|
||||
|
||||
-- Files table: stores metadata about files in the filesystem
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
@@ -134,3 +133,17 @@ CREATE TABLE IF NOT EXISTS uploads (
|
||||
|
||||
-- Index for efficient snapshot lookups
|
||||
CREATE INDEX IF NOT EXISTS idx_uploads_snapshot_id ON uploads(snapshot_id);
|
||||
|
||||
-- Local metadata: keyed, host-local settings that bind the state of the
|
||||
-- local index database to external context. The primary use is
|
||||
-- storage_url: once a backup writes blobs to a destination, the local
|
||||
-- index is only valid against that destination — if the configured
|
||||
-- storage_url later changes, the scanner would silently think already-
|
||||
-- known chunks are still on the new (empty) destination and skip
|
||||
-- uploading them, corrupting future snapshots. On every mutating
|
||||
-- command startup, we compare the configured storage_url to the stored
|
||||
-- one and refuse to proceed on mismatch.
|
||||
CREATE TABLE IF NOT EXISTS local_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
@@ -1,11 +0,0 @@
|
||||
-- Track blob upload metrics
|
||||
CREATE TABLE IF NOT EXISTS uploads (
|
||||
blob_hash TEXT PRIMARY KEY,
|
||||
uploaded_at TIMESTAMP NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
duration_ms INTEGER NOT NULL,
|
||||
FOREIGN KEY (blob_hash) REFERENCES blobs(blob_hash)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_uploads_uploaded_at ON uploads(uploaded_at);
|
||||
CREATE INDEX idx_uploads_duration ON uploads(duration_ms);
|
||||
+369
-167
@@ -3,43 +3,60 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// SnapshotRepository provides access to the snapshots table and its
|
||||
// snapshot_files / snapshot_blobs association tables.
|
||||
type SnapshotRepository struct {
|
||||
db *DB
|
||||
}
|
||||
|
||||
// NewSnapshotRepository creates a SnapshotRepository backed by db.
|
||||
func NewSnapshotRepository(db *DB) *SnapshotRepository {
|
||||
return &SnapshotRepository{db: db}
|
||||
}
|
||||
|
||||
func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *Snapshot) error {
|
||||
// Create inserts a snapshot row, using tx when non-nil.
|
||||
func (r *SnapshotRepository) Create(
|
||||
ctx context.Context, tx *sql.Tx, snapshot *Snapshot,
|
||||
) error {
|
||||
query := `
|
||||
INSERT INTO snapshots (id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at,
|
||||
file_count, chunk_count, blob_count, total_size, blob_size, blob_uncompressed_size,
|
||||
compression_ratio, compression_level, upload_bytes, upload_duration_ms)
|
||||
INSERT INTO snapshots (id, hostname, vaultik_version,
|
||||
vaultik_git_revision, started_at, completed_at,
|
||||
file_count, chunk_count, blob_count, total_size, blob_size,
|
||||
blob_uncompressed_size, compression_ratio, compression_level,
|
||||
upload_bytes, upload_duration_ms)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
var completedAt *int64
|
||||
|
||||
if snapshot.CompletedAt != nil {
|
||||
ts := snapshot.CompletedAt.Unix()
|
||||
completedAt = &ts
|
||||
}
|
||||
|
||||
args := []any{
|
||||
snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion,
|
||||
snapshot.VaultikGitRevision, snapshot.StartedAt.Unix(),
|
||||
completedAt, snapshot.FileCount, snapshot.ChunkCount,
|
||||
snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize,
|
||||
snapshot.BlobUncompressedSize, snapshot.CompressionRatio,
|
||||
snapshot.CompressionLevel, snapshot.UploadBytes,
|
||||
snapshot.UploadDurationMs,
|
||||
}
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, snapshot.VaultikGitRevision, snapshot.StartedAt.Unix(),
|
||||
completedAt, snapshot.FileCount, snapshot.ChunkCount, snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize, snapshot.BlobUncompressedSize,
|
||||
snapshot.CompressionRatio, snapshot.CompressionLevel, snapshot.UploadBytes, snapshot.UploadDurationMs)
|
||||
_, err = tx.ExecContext(ctx, query, args...)
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, snapshot.ID, snapshot.Hostname, snapshot.VaultikVersion, snapshot.VaultikGitRevision, snapshot.StartedAt.Unix(),
|
||||
completedAt, snapshot.FileCount, snapshot.ChunkCount, snapshot.BlobCount, snapshot.TotalSize, snapshot.BlobSize, snapshot.BlobUncompressedSize,
|
||||
snapshot.CompressionRatio, snapshot.CompressionLevel, snapshot.UploadBytes, snapshot.UploadDurationMs)
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -49,7 +66,14 @@ func (r *SnapshotRepository) Create(ctx context.Context, tx *sql.Tx, snapshot *S
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snapshotID string, fileCount, chunkCount, blobCount, totalSize, blobSize int64) error {
|
||||
// UpdateCounts updates a snapshot's file/chunk/blob counters and sizes,
|
||||
// recomputing the compression ratio, using tx when non-nil.
|
||||
func (r *SnapshotRepository) UpdateCounts(
|
||||
ctx context.Context,
|
||||
tx *sql.Tx,
|
||||
snapshotID string,
|
||||
fileCount, chunkCount, blobCount, totalSize, blobSize int64,
|
||||
) error {
|
||||
compressionRatio := 1.0
|
||||
if totalSize > 0 {
|
||||
compressionRatio = float64(blobSize) / float64(totalSize)
|
||||
@@ -68,9 +92,13 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, fileCount, chunkCount, blobCount, totalSize, blobSize, compressionRatio, snapshotID)
|
||||
_, err = tx.ExecContext(ctx, query,
|
||||
fileCount, chunkCount, blobCount, totalSize, blobSize,
|
||||
compressionRatio, snapshotID)
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, fileCount, chunkCount, blobCount, totalSize, blobSize, compressionRatio, snapshotID)
|
||||
_, err = r.db.ExecWithLog(ctx, query,
|
||||
fileCount, chunkCount, blobCount, totalSize, blobSize,
|
||||
compressionRatio, snapshotID)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -81,27 +109,19 @@ func (r *SnapshotRepository) UpdateCounts(ctx context.Context, tx *sql.Tx, snaps
|
||||
}
|
||||
|
||||
// UpdateExtendedStats updates extended statistics for a snapshot
|
||||
func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx, snapshotID string, blobUncompressedSize int64, compressionLevel int, uploadDurationMs int64) error {
|
||||
// Calculate compression ratio based on uncompressed vs compressed sizes
|
||||
var compressionRatio float64
|
||||
if blobUncompressedSize > 0 {
|
||||
// Get current blob_size from DB to calculate ratio
|
||||
var blobSize int64
|
||||
queryGet := `SELECT blob_size FROM snapshots WHERE id = ?`
|
||||
if tx != nil {
|
||||
err := tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
|
||||
func (r *SnapshotRepository) UpdateExtendedStats(
|
||||
ctx context.Context,
|
||||
tx *sql.Tx,
|
||||
snapshotID string,
|
||||
blobUncompressedSize int64,
|
||||
compressionLevel int,
|
||||
uploadDurationMs int64,
|
||||
) error {
|
||||
compressionRatio, err := r.extendedCompressionRatio(
|
||||
ctx, tx, snapshotID, blobUncompressedSize,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting blob size: %w", err)
|
||||
}
|
||||
} else {
|
||||
err := r.db.conn.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting blob size: %w", err)
|
||||
}
|
||||
}
|
||||
compressionRatio = float64(blobSize) / float64(blobUncompressedSize)
|
||||
} else {
|
||||
compressionRatio = 1.0
|
||||
return err
|
||||
}
|
||||
|
||||
query := `
|
||||
@@ -114,20 +134,28 @@ func (r *SnapshotRepository) UpdateExtendedStats(ctx context.Context, tx *sql.Tx
|
||||
WHERE id = ?
|
||||
`
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, blobUncompressedSize, compressionRatio, compressionLevel, uploadDurationMs, snapshotID)
|
||||
_, err = tx.ExecContext(ctx, query,
|
||||
blobUncompressedSize, compressionRatio, compressionLevel,
|
||||
uploadDurationMs, snapshotID)
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, blobUncompressedSize, compressionRatio, compressionLevel, uploadDurationMs, snapshotID)
|
||||
_, err = r.db.ExecWithLog(ctx, query,
|
||||
blobUncompressedSize, compressionRatio, compressionLevel,
|
||||
uploadDurationMs, snapshotID)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("updating extended stats: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*Snapshot, error) {
|
||||
// GetByID returns the snapshot with the given ID, or nil if no such
|
||||
// snapshot exists.
|
||||
func (r *SnapshotRepository) GetByID(
|
||||
ctx context.Context, snapshotID string,
|
||||
) (*Snapshot, error) {
|
||||
query := `
|
||||
SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at,
|
||||
file_count, chunk_count, blob_count, total_size, blob_size, blob_uncompressed_size,
|
||||
@@ -136,9 +164,11 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
|
||||
WHERE id = ?
|
||||
`
|
||||
|
||||
var snapshot Snapshot
|
||||
var startedAtUnix int64
|
||||
var completedAtUnix *int64
|
||||
var (
|
||||
snapshot Snapshot
|
||||
startedAtUnix int64
|
||||
completedAtUnix *int64
|
||||
)
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(
|
||||
&snapshot.ID,
|
||||
@@ -159,9 +189,10 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
|
||||
&snapshot.UploadDurationMs,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying snapshot: %w", err)
|
||||
}
|
||||
@@ -175,9 +206,14 @@ func (r *SnapshotRepository) GetByID(ctx context.Context, snapshotID string) (*S
|
||||
return &snapshot, nil
|
||||
}
|
||||
|
||||
func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snapshot, error) {
|
||||
// ListRecent returns up to limit snapshots, most recently started first.
|
||||
func (r *SnapshotRepository) ListRecent(
|
||||
ctx context.Context, limit int,
|
||||
) ([]*Snapshot, error) {
|
||||
query := `
|
||||
SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio
|
||||
SELECT id, hostname, vaultik_version, vaultik_git_revision,
|
||||
started_at, completed_at, file_count, chunk_count, blob_count,
|
||||
total_size, blob_size, compression_ratio
|
||||
FROM snapshots
|
||||
ORDER BY started_at DESC
|
||||
LIMIT ?
|
||||
@@ -187,46 +223,21 @@ func (r *SnapshotRepository) ListRecent(ctx context.Context, limit int) ([]*Snap
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying snapshots: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
var snapshots []*Snapshot
|
||||
for rows.Next() {
|
||||
var snapshot Snapshot
|
||||
var startedAtUnix int64
|
||||
var completedAtUnix *int64
|
||||
|
||||
err := rows.Scan(
|
||||
&snapshot.ID,
|
||||
&snapshot.Hostname,
|
||||
&snapshot.VaultikVersion,
|
||||
&snapshot.VaultikGitRevision,
|
||||
&startedAtUnix,
|
||||
&completedAtUnix,
|
||||
&snapshot.FileCount,
|
||||
&snapshot.ChunkCount,
|
||||
&snapshot.BlobCount,
|
||||
&snapshot.TotalSize,
|
||||
&snapshot.BlobSize,
|
||||
&snapshot.CompressionRatio,
|
||||
)
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning snapshot: %w", err)
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
snapshot.StartedAt = time.Unix(startedAtUnix, 0)
|
||||
if completedAtUnix != nil {
|
||||
t := time.Unix(*completedAtUnix, 0)
|
||||
snapshot.CompletedAt = &t
|
||||
}
|
||||
|
||||
snapshots = append(snapshots, &snapshot)
|
||||
}
|
||||
|
||||
return snapshots, rows.Err()
|
||||
return r.scanSnapshotRows(rows)
|
||||
}
|
||||
|
||||
// MarkComplete marks a snapshot as completed with the current timestamp
|
||||
func (r *SnapshotRepository) MarkComplete(ctx context.Context, tx *sql.Tx, snapshotID string) error {
|
||||
func (r *SnapshotRepository) MarkComplete(
|
||||
ctx context.Context, tx *sql.Tx, snapshotID string,
|
||||
) error {
|
||||
query := `
|
||||
UPDATE snapshots
|
||||
SET completed_at = ?
|
||||
@@ -250,7 +261,9 @@ func (r *SnapshotRepository) MarkComplete(ctx context.Context, tx *sql.Tx, snaps
|
||||
}
|
||||
|
||||
// AddFile adds a file to a snapshot
|
||||
func (r *SnapshotRepository) AddFile(ctx context.Context, tx *sql.Tx, snapshotID string, filePath string) error {
|
||||
func (r *SnapshotRepository) AddFile(
|
||||
ctx context.Context, tx *sql.Tx, snapshotID string, filePath string,
|
||||
) error {
|
||||
query := `
|
||||
INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id)
|
||||
SELECT ?, id FROM files WHERE path = ?
|
||||
@@ -271,7 +284,9 @@ func (r *SnapshotRepository) AddFile(ctx context.Context, tx *sql.Tx, snapshotID
|
||||
}
|
||||
|
||||
// AddFileByID adds a file to a snapshot by file ID
|
||||
func (r *SnapshotRepository) AddFileByID(ctx context.Context, tx *sql.Tx, snapshotID string, fileID types.FileID) error {
|
||||
func (r *SnapshotRepository) AddFileByID(
|
||||
ctx context.Context, tx *sql.Tx, snapshotID string, fileID types.FileID,
|
||||
) error {
|
||||
query := `
|
||||
INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id)
|
||||
VALUES (?, ?)
|
||||
@@ -292,37 +307,49 @@ func (r *SnapshotRepository) AddFileByID(ctx context.Context, tx *sql.Tx, snapsh
|
||||
}
|
||||
|
||||
// AddFilesByIDBatch adds multiple files to a snapshot in batched inserts
|
||||
func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx, snapshotID string, fileIDs []types.FileID) error {
|
||||
func (r *SnapshotRepository) AddFilesByIDBatch(
|
||||
ctx context.Context, tx *sql.Tx, snapshotID string, fileIDs []types.FileID,
|
||||
) error {
|
||||
if len(fileIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Each entry has 2 values, so batch at 400 to be safe
|
||||
// Each snapshot_files row binds this many SQL variables.
|
||||
const snapshotFileCols = 2
|
||||
|
||||
// Batch at 400 rows to be safe with SQLite's variable limit.
|
||||
const batchSize = 400
|
||||
|
||||
for i := 0; i < len(fileIDs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(fileIDs) {
|
||||
end = len(fileIDs)
|
||||
}
|
||||
end := min(i+batchSize, len(fileIDs))
|
||||
|
||||
batch := fileIDs[i:end]
|
||||
|
||||
query := "INSERT OR IGNORE INTO snapshot_files (snapshot_id, file_id) VALUES "
|
||||
args := make([]interface{}, 0, len(batch)*2)
|
||||
|
||||
args := make([]any, 0, len(batch)*snapshotFileCols)
|
||||
|
||||
var querySb312 strings.Builder
|
||||
|
||||
for j, fileID := range batch {
|
||||
if j > 0 {
|
||||
query += ", "
|
||||
querySb312.WriteString(", ")
|
||||
}
|
||||
query += "(?, ?)"
|
||||
|
||||
querySb312.WriteString("(?, ?)")
|
||||
|
||||
args = append(args, snapshotID, fileID.String())
|
||||
}
|
||||
|
||||
query += querySb312.String() //nolint:gosec // G202: appends "?" placeholders only
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, args...)
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch adding files to snapshot: %w", err)
|
||||
}
|
||||
@@ -331,8 +358,57 @@ func (r *SnapshotRepository) AddFilesByIDBatch(ctx context.Context, tx *sql.Tx,
|
||||
return nil
|
||||
}
|
||||
|
||||
// PopulateReferencedBlobs ensures snapshot_blobs contains an entry for
|
||||
// every blob that holds a chunk referenced by any file in the snapshot.
|
||||
// This is necessary because the AddBlob hook only runs when a blob is
|
||||
// newly uploaded during a snapshot — fully-deduplicated snapshots (where
|
||||
// every chunk already exists in storage from a prior run) would otherwise
|
||||
// have an empty snapshot_blobs set and be impossible to restore.
|
||||
//
|
||||
// Returns the number of rows inserted (i.e. blobs that were previously
|
||||
// referenced indirectly via file_chunks but not yet recorded in
|
||||
// snapshot_blobs for this snapshot).
|
||||
func (r *SnapshotRepository) PopulateReferencedBlobs(
|
||||
ctx context.Context, tx *sql.Tx, snapshotID string,
|
||||
) (int64, error) {
|
||||
query := `
|
||||
INSERT OR IGNORE INTO snapshot_blobs (snapshot_id, blob_id, blob_hash)
|
||||
SELECT DISTINCT ?, blobs.id, blobs.blob_hash
|
||||
FROM blobs
|
||||
JOIN blob_chunks ON blob_chunks.blob_id = blobs.id
|
||||
JOIN file_chunks ON file_chunks.chunk_hash = blob_chunks.chunk_hash
|
||||
JOIN snapshot_files ON snapshot_files.file_id = file_chunks.file_id
|
||||
WHERE snapshot_files.snapshot_id = ?
|
||||
AND blobs.blob_hash IS NOT NULL
|
||||
`
|
||||
|
||||
var (
|
||||
result sql.Result
|
||||
err error
|
||||
)
|
||||
if tx != nil {
|
||||
result, err = tx.ExecContext(ctx, query, snapshotID, snapshotID)
|
||||
} else {
|
||||
result, err = r.db.ExecWithLog(ctx, query, snapshotID, snapshotID)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("populating referenced blobs: %w", err)
|
||||
}
|
||||
|
||||
n, _ := result.RowsAffected()
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// AddBlob adds a blob to a snapshot
|
||||
func (r *SnapshotRepository) AddBlob(ctx context.Context, tx *sql.Tx, snapshotID string, blobID types.BlobID, blobHash types.BlobHash) error {
|
||||
func (r *SnapshotRepository) AddBlob(
|
||||
ctx context.Context,
|
||||
tx *sql.Tx,
|
||||
snapshotID string,
|
||||
blobID types.BlobID,
|
||||
blobHash types.BlobHash,
|
||||
) error {
|
||||
query := `
|
||||
INSERT OR IGNORE INTO snapshot_blobs (snapshot_id, blob_id, blob_hash)
|
||||
VALUES (?, ?, ?)
|
||||
@@ -353,7 +429,9 @@ func (r *SnapshotRepository) AddBlob(ctx context.Context, tx *sql.Tx, snapshotID
|
||||
}
|
||||
|
||||
// GetBlobHashes returns all blob hashes for a snapshot
|
||||
func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID string) ([]string, error) {
|
||||
func (r *SnapshotRepository) GetBlobHashes(
|
||||
ctx context.Context, snapshotID string,
|
||||
) ([]string, error) {
|
||||
query := `
|
||||
SELECT sb.blob_hash
|
||||
FROM snapshot_blobs sb
|
||||
@@ -365,22 +443,35 @@ func (r *SnapshotRepository) GetBlobHashes(ctx context.Context, snapshotID strin
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying blob hashes: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
var blobs []string
|
||||
|
||||
for rows.Next() {
|
||||
var blobHash string
|
||||
if err := rows.Scan(&blobHash); err != nil {
|
||||
|
||||
err := rows.Scan(&blobHash)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning blob hash: %w", err)
|
||||
}
|
||||
|
||||
blobs = append(blobs, blobHash)
|
||||
}
|
||||
|
||||
return blobs, rows.Err()
|
||||
}
|
||||
|
||||
// GetSnapshotTotalCompressedSize returns the total compressed size of all blobs referenced by a snapshot
|
||||
func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context, snapshotID string) (int64, error) {
|
||||
// GetSnapshotTotalCompressedSize returns the total compressed size of all
|
||||
// blobs referenced by a snapshot.
|
||||
func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(
|
||||
ctx context.Context, snapshotID string,
|
||||
) (int64, error) {
|
||||
query := `
|
||||
SELECT COALESCE(SUM(b.compressed_size), 0)
|
||||
FROM snapshot_blobs sb
|
||||
@@ -389,6 +480,7 @@ func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context,
|
||||
`
|
||||
|
||||
var totalSize int64
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("querying total compressed size: %w", err)
|
||||
@@ -397,10 +489,81 @@ func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(ctx context.Context,
|
||||
return totalSize, nil
|
||||
}
|
||||
|
||||
// GetIncompleteSnapshots returns all snapshots that haven't been completed
|
||||
func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Snapshot, error) {
|
||||
// GetSnapshotUncompressedChunkSize returns the sum of plaintext sizes of all unique
|
||||
// chunks referenced by a snapshot (via snapshot_files → file_chunks → chunks).
|
||||
func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(
|
||||
ctx context.Context, snapshotID string,
|
||||
) (int64, error) {
|
||||
query := `
|
||||
SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio
|
||||
SELECT COALESCE(SUM(c.size), 0)
|
||||
FROM (
|
||||
SELECT DISTINCT fc.chunk_hash
|
||||
FROM snapshot_files sf
|
||||
JOIN file_chunks fc ON sf.file_id = fc.file_id
|
||||
WHERE sf.snapshot_id = ?
|
||||
) sc
|
||||
JOIN chunks c ON sc.chunk_hash = c.chunk_hash
|
||||
`
|
||||
|
||||
var totalSize int64
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("querying uncompressed chunk size: %w", err)
|
||||
}
|
||||
|
||||
return totalSize, nil
|
||||
}
|
||||
|
||||
// GetSnapshotNewChunkSize returns the sum of plaintext sizes of chunks that are
|
||||
// referenced by this snapshot but not by any earlier completed snapshot known to
|
||||
// the local database. The result is the marginal uncompressed data this snapshot
|
||||
// added to the dedup pool — i.e., the delta from prior snapshots.
|
||||
func (r *SnapshotRepository) GetSnapshotNewChunkSize(
|
||||
ctx context.Context, snapshotID string,
|
||||
) (int64, error) {
|
||||
query := `
|
||||
WITH this_snap_chunks AS (
|
||||
SELECT DISTINCT fc.chunk_hash
|
||||
FROM snapshot_files sf
|
||||
JOIN file_chunks fc ON sf.file_id = fc.file_id
|
||||
WHERE sf.snapshot_id = ?
|
||||
),
|
||||
prior_chunks AS (
|
||||
SELECT DISTINCT fc.chunk_hash
|
||||
FROM snapshots s
|
||||
JOIN snapshot_files sf ON sf.snapshot_id = s.id
|
||||
JOIN file_chunks fc ON fc.file_id = sf.file_id
|
||||
WHERE s.completed_at IS NOT NULL
|
||||
AND s.id != ?
|
||||
AND s.started_at < (SELECT started_at FROM snapshots WHERE id = ?)
|
||||
)
|
||||
SELECT COALESCE(SUM(c.size), 0)
|
||||
FROM chunks c
|
||||
JOIN this_snap_chunks t ON c.chunk_hash = t.chunk_hash
|
||||
WHERE c.chunk_hash NOT IN (SELECT chunk_hash FROM prior_chunks)
|
||||
`
|
||||
|
||||
var totalSize int64
|
||||
|
||||
err := r.db.conn.QueryRowContext(
|
||||
ctx, query, snapshotID, snapshotID, snapshotID,
|
||||
).Scan(&totalSize)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("querying new chunk size: %w", err)
|
||||
}
|
||||
|
||||
return totalSize, nil
|
||||
}
|
||||
|
||||
// GetIncompleteSnapshots returns all snapshots that haven't been completed
|
||||
func (r *SnapshotRepository) GetIncompleteSnapshots(
|
||||
ctx context.Context,
|
||||
) ([]*Snapshot, error) {
|
||||
query := `
|
||||
SELECT id, hostname, vaultik_version, vaultik_git_revision,
|
||||
started_at, completed_at, file_count, chunk_count, blob_count,
|
||||
total_size, blob_size, compression_ratio
|
||||
FROM snapshots
|
||||
WHERE completed_at IS NULL
|
||||
ORDER BY started_at DESC
|
||||
@@ -410,48 +573,25 @@ func (r *SnapshotRepository) GetIncompleteSnapshots(ctx context.Context) ([]*Sna
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying incomplete snapshots: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
var snapshots []*Snapshot
|
||||
for rows.Next() {
|
||||
var snapshot Snapshot
|
||||
var startedAtUnix int64
|
||||
var completedAtUnix *int64
|
||||
|
||||
err := rows.Scan(
|
||||
&snapshot.ID,
|
||||
&snapshot.Hostname,
|
||||
&snapshot.VaultikVersion,
|
||||
&snapshot.VaultikGitRevision,
|
||||
&startedAtUnix,
|
||||
&completedAtUnix,
|
||||
&snapshot.FileCount,
|
||||
&snapshot.ChunkCount,
|
||||
&snapshot.BlobCount,
|
||||
&snapshot.TotalSize,
|
||||
&snapshot.BlobSize,
|
||||
&snapshot.CompressionRatio,
|
||||
)
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning snapshot: %w", err)
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
snapshot.StartedAt = time.Unix(startedAtUnix, 0)
|
||||
if completedAtUnix != nil {
|
||||
t := time.Unix(*completedAtUnix, 0)
|
||||
snapshot.CompletedAt = &t
|
||||
}
|
||||
|
||||
snapshots = append(snapshots, &snapshot)
|
||||
}
|
||||
|
||||
return snapshots, rows.Err()
|
||||
return r.scanSnapshotRows(rows)
|
||||
}
|
||||
|
||||
// GetIncompleteByHostname returns all incomplete snapshots for a specific hostname
|
||||
func (r *SnapshotRepository) GetIncompleteByHostname(ctx context.Context, hostname string) ([]*Snapshot, error) {
|
||||
func (r *SnapshotRepository) GetIncompleteByHostname(
|
||||
ctx context.Context, hostname string,
|
||||
) ([]*Snapshot, error) {
|
||||
query := `
|
||||
SELECT id, hostname, vaultik_version, vaultik_git_revision, started_at, completed_at, file_count, chunk_count, blob_count, total_size, blob_size, compression_ratio
|
||||
SELECT id, hostname, vaultik_version, vaultik_git_revision,
|
||||
started_at, completed_at, file_count, chunk_count, blob_count,
|
||||
total_size, blob_size, compression_ratio
|
||||
FROM snapshots
|
||||
WHERE completed_at IS NULL AND hostname = ?
|
||||
ORDER BY started_at DESC
|
||||
@@ -461,42 +601,17 @@ func (r *SnapshotRepository) GetIncompleteByHostname(ctx context.Context, hostna
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying incomplete snapshots: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
var snapshots []*Snapshot
|
||||
for rows.Next() {
|
||||
var snapshot Snapshot
|
||||
var startedAtUnix int64
|
||||
var completedAtUnix *int64
|
||||
|
||||
err := rows.Scan(
|
||||
&snapshot.ID,
|
||||
&snapshot.Hostname,
|
||||
&snapshot.VaultikVersion,
|
||||
&snapshot.VaultikGitRevision,
|
||||
&startedAtUnix,
|
||||
&completedAtUnix,
|
||||
&snapshot.FileCount,
|
||||
&snapshot.ChunkCount,
|
||||
&snapshot.BlobCount,
|
||||
&snapshot.TotalSize,
|
||||
&snapshot.BlobSize,
|
||||
&snapshot.CompressionRatio,
|
||||
)
|
||||
defer func() {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning snapshot: %w", err)
|
||||
Fatalf("failed to close rows: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
snapshot.StartedAt = time.Unix(startedAtUnix, 0).UTC()
|
||||
if completedAtUnix != nil {
|
||||
t := time.Unix(*completedAtUnix, 0).UTC()
|
||||
snapshot.CompletedAt = &t
|
||||
}
|
||||
|
||||
snapshots = append(snapshots, &snapshot)
|
||||
}
|
||||
|
||||
return snapshots, rows.Err()
|
||||
// Same column set as every other multi-row snapshot query, so the
|
||||
// shared scanner applies — including its timestamp normalization.
|
||||
return r.scanSnapshotRows(rows)
|
||||
}
|
||||
|
||||
// Delete removes a snapshot record
|
||||
@@ -512,7 +627,9 @@ func (r *SnapshotRepository) Delete(ctx context.Context, snapshotID string) erro
|
||||
}
|
||||
|
||||
// DeleteSnapshotFiles removes all snapshot_files entries for a snapshot
|
||||
func (r *SnapshotRepository) DeleteSnapshotFiles(ctx context.Context, snapshotID string) error {
|
||||
func (r *SnapshotRepository) DeleteSnapshotFiles(
|
||||
ctx context.Context, snapshotID string,
|
||||
) error {
|
||||
query := `DELETE FROM snapshot_files WHERE snapshot_id = ?`
|
||||
|
||||
_, err := r.db.ExecWithLog(ctx, query, snapshotID)
|
||||
@@ -524,7 +641,9 @@ func (r *SnapshotRepository) DeleteSnapshotFiles(ctx context.Context, snapshotID
|
||||
}
|
||||
|
||||
// DeleteSnapshotBlobs removes all snapshot_blobs entries for a snapshot
|
||||
func (r *SnapshotRepository) DeleteSnapshotBlobs(ctx context.Context, snapshotID string) error {
|
||||
func (r *SnapshotRepository) DeleteSnapshotBlobs(
|
||||
ctx context.Context, snapshotID string,
|
||||
) error {
|
||||
query := `DELETE FROM snapshot_blobs WHERE snapshot_id = ?`
|
||||
|
||||
_, err := r.db.ExecWithLog(ctx, query, snapshotID)
|
||||
@@ -536,7 +655,9 @@ func (r *SnapshotRepository) DeleteSnapshotBlobs(ctx context.Context, snapshotID
|
||||
}
|
||||
|
||||
// DeleteSnapshotUploads removes all uploads entries for a snapshot
|
||||
func (r *SnapshotRepository) DeleteSnapshotUploads(ctx context.Context, snapshotID string) error {
|
||||
func (r *SnapshotRepository) DeleteSnapshotUploads(
|
||||
ctx context.Context, snapshotID string,
|
||||
) error {
|
||||
query := `DELETE FROM uploads WHERE snapshot_id = ?`
|
||||
|
||||
_, err := r.db.ExecWithLog(ctx, query, snapshotID)
|
||||
@@ -546,3 +667,84 @@ func (r *SnapshotRepository) DeleteSnapshotUploads(ctx context.Context, snapshot
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// extendedCompressionRatio computes the compression ratio for a snapshot
|
||||
// from its stored blob_size and the given uncompressed size. Returns 1.0
|
||||
// when the uncompressed size is zero.
|
||||
func (r *SnapshotRepository) extendedCompressionRatio(
|
||||
ctx context.Context,
|
||||
tx *sql.Tx,
|
||||
snapshotID string,
|
||||
blobUncompressedSize int64,
|
||||
) (float64, error) {
|
||||
if blobUncompressedSize <= 0 {
|
||||
return 1.0, nil
|
||||
}
|
||||
|
||||
// Get current blob_size from DB to calculate ratio
|
||||
var blobSize int64
|
||||
|
||||
queryGet := `SELECT blob_size FROM snapshots WHERE id = ?`
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
err = tx.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
|
||||
} else {
|
||||
err = r.db.conn.QueryRowContext(ctx, queryGet, snapshotID).Scan(&blobSize)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("getting blob size: %w", err)
|
||||
}
|
||||
|
||||
return float64(blobSize) / float64(blobUncompressedSize), nil
|
||||
}
|
||||
|
||||
// scanSnapshotRows scans the standard snapshot column set from a rows
|
||||
// iterator into Snapshot records.
|
||||
func (r *SnapshotRepository) scanSnapshotRows(rows *sql.Rows) ([]*Snapshot, error) {
|
||||
var snapshots []*Snapshot
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
snapshot Snapshot
|
||||
startedAtUnix int64
|
||||
completedAtUnix *int64
|
||||
)
|
||||
|
||||
err := rows.Scan(
|
||||
&snapshot.ID,
|
||||
&snapshot.Hostname,
|
||||
&snapshot.VaultikVersion,
|
||||
&snapshot.VaultikGitRevision,
|
||||
&startedAtUnix,
|
||||
&completedAtUnix,
|
||||
&snapshot.FileCount,
|
||||
&snapshot.ChunkCount,
|
||||
&snapshot.BlobCount,
|
||||
&snapshot.TotalSize,
|
||||
&snapshot.BlobSize,
|
||||
&snapshot.CompressionRatio,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning snapshot: %w", err)
|
||||
}
|
||||
|
||||
// UTC, matching every other snapshot scanner in this file. The
|
||||
// column holds a bare Unix second, so the zone is a decode
|
||||
// choice rather than stored data, and callers render these
|
||||
// timestamps through zone-less format strings alongside
|
||||
// timestamps read from remote manifests. Decoding in the host's
|
||||
// local zone here would put two different wall clocks in one
|
||||
// column.
|
||||
snapshot.StartedAt = time.Unix(startedAtUnix, 0).UTC()
|
||||
if completedAtUnix != nil {
|
||||
t := time.Unix(*completedAtUnix, 0).UTC()
|
||||
snapshot.CompletedAt = &t
|
||||
}
|
||||
|
||||
snapshots = append(snapshots, &snapshot)
|
||||
}
|
||||
|
||||
return snapshots, rows.Err()
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package database
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -7,7 +7,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -21,17 +22,19 @@ const (
|
||||
)
|
||||
|
||||
func TestSnapshotRepository(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewSnapshotRepository(db)
|
||||
repo := database.NewSnapshotRepository(db)
|
||||
|
||||
// Test Create
|
||||
snapshot := &Snapshot{
|
||||
snapshot := &database.Snapshot{
|
||||
ID: "2024-01-01T12:00:00Z",
|
||||
Hostname: "test-host",
|
||||
VaultikVersion: "1.0.0",
|
||||
Hostname: testHostname,
|
||||
VaultikVersion: testVersion,
|
||||
StartedAt: time.Now().Truncate(time.Second),
|
||||
CompletedAt: nil,
|
||||
FileCount: 100,
|
||||
@@ -52,62 +55,118 @@ func TestSnapshotRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get snapshot: %v", err)
|
||||
}
|
||||
|
||||
if retrieved == nil {
|
||||
t.Fatal("expected snapshot, got nil")
|
||||
}
|
||||
|
||||
if retrieved.ID != snapshot.ID {
|
||||
t.Errorf("ID mismatch: got %s, want %s", retrieved.ID, snapshot.ID)
|
||||
}
|
||||
|
||||
if retrieved.Hostname != snapshot.Hostname {
|
||||
t.Errorf("hostname mismatch: got %s, want %s", retrieved.Hostname, snapshot.Hostname)
|
||||
t.Errorf("hostname mismatch: got %s, want %s",
|
||||
retrieved.Hostname, snapshot.Hostname)
|
||||
}
|
||||
|
||||
if retrieved.FileCount != snapshot.FileCount {
|
||||
t.Errorf("file count mismatch: got %d, want %d", retrieved.FileCount, snapshot.FileCount)
|
||||
t.Errorf("file count mismatch: got %d, want %d",
|
||||
retrieved.FileCount, snapshot.FileCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotRepositoryUpdateCounts(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := database.NewSnapshotRepository(db)
|
||||
|
||||
snapshot := &database.Snapshot{
|
||||
ID: "2024-01-02T12:00:00Z",
|
||||
Hostname: testHostname,
|
||||
VaultikVersion: testVersion,
|
||||
StartedAt: time.Now().Truncate(time.Second),
|
||||
CompletedAt: nil,
|
||||
FileCount: 100,
|
||||
ChunkCount: 500,
|
||||
BlobCount: 10,
|
||||
TotalSize: oneHundredMebibytes,
|
||||
BlobSize: fortyMebibytes,
|
||||
CompressionRatio: compressionRatioPoint4,
|
||||
}
|
||||
|
||||
err := repo.Create(ctx, nil, snapshot)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create snapshot: %v", err)
|
||||
}
|
||||
|
||||
// Test UpdateCounts
|
||||
err = repo.UpdateCounts(ctx, nil, snapshot.ID.String(), 200, 1000, 20, twoHundredMebibytes, sixtyMebibytes)
|
||||
err = repo.UpdateCounts(ctx, nil, snapshot.ID.String(),
|
||||
200, 1000, 20, twoHundredMebibytes, sixtyMebibytes)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to update counts: %v", err)
|
||||
}
|
||||
|
||||
retrieved, err = repo.GetByID(ctx, snapshot.ID.String())
|
||||
retrieved, err := repo.GetByID(ctx, snapshot.ID.String())
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get updated snapshot: %v", err)
|
||||
}
|
||||
|
||||
if retrieved.FileCount != 200 {
|
||||
t.Errorf("file count not updated: got %d, want %d", retrieved.FileCount, 200)
|
||||
}
|
||||
|
||||
if retrieved.ChunkCount != 1000 {
|
||||
t.Errorf("chunk count not updated: got %d, want %d", retrieved.ChunkCount, 1000)
|
||||
t.Errorf("chunk count not updated: got %d, want %d",
|
||||
retrieved.ChunkCount, 1000)
|
||||
}
|
||||
|
||||
if retrieved.BlobCount != 20 {
|
||||
t.Errorf("blob count not updated: got %d, want %d", retrieved.BlobCount, 20)
|
||||
}
|
||||
|
||||
if retrieved.TotalSize != twoHundredMebibytes {
|
||||
t.Errorf("total size not updated: got %d, want %d", retrieved.TotalSize, twoHundredMebibytes)
|
||||
}
|
||||
if retrieved.BlobSize != sixtyMebibytes {
|
||||
t.Errorf("blob size not updated: got %d, want %d", retrieved.BlobSize, sixtyMebibytes)
|
||||
}
|
||||
expectedRatio := compressionRatioPoint3 // 0.3
|
||||
if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 {
|
||||
t.Errorf("compression ratio not updated: got %f, want %f", retrieved.CompressionRatio, expectedRatio)
|
||||
t.Errorf("total size not updated: got %d, want %d",
|
||||
retrieved.TotalSize, twoHundredMebibytes)
|
||||
}
|
||||
|
||||
// Test ListRecent
|
||||
// Add more snapshots
|
||||
for i := 2; i <= 5; i++ {
|
||||
s := &Snapshot{
|
||||
if retrieved.BlobSize != sixtyMebibytes {
|
||||
t.Errorf("blob size not updated: got %d, want %d",
|
||||
retrieved.BlobSize, sixtyMebibytes)
|
||||
}
|
||||
|
||||
expectedRatio := compressionRatioPoint3 // 0.3
|
||||
if math.Abs(retrieved.CompressionRatio-expectedRatio) > 0.001 {
|
||||
t.Errorf("compression ratio not updated: got %f, want %f",
|
||||
retrieved.CompressionRatio, expectedRatio)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotRepositoryListRecent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := database.NewSnapshotRepository(db)
|
||||
|
||||
// Add snapshots
|
||||
for i := 1; i <= 5; i++ {
|
||||
s := &database.Snapshot{
|
||||
ID: types.SnapshotID(fmt.Sprintf("2024-01-0%dT12:00:00Z", i)),
|
||||
Hostname: "test-host",
|
||||
VaultikVersion: "1.0.0",
|
||||
Hostname: testHostname,
|
||||
VaultikVersion: testVersion,
|
||||
StartedAt: time.Now().Add(time.Duration(i) * time.Hour).Truncate(time.Second),
|
||||
CompletedAt: nil,
|
||||
FileCount: int64(100 * i),
|
||||
ChunkCount: int64(500 * i),
|
||||
BlobCount: int64(10 * i),
|
||||
}
|
||||
|
||||
err := repo.Create(ctx, nil, s)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create snapshot %d: %v", i, err)
|
||||
@@ -119,36 +178,154 @@ func TestSnapshotRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to list recent snapshots: %v", err)
|
||||
}
|
||||
|
||||
if len(recent) != 3 {
|
||||
t.Errorf("expected 3 recent snapshots, got %d", len(recent))
|
||||
}
|
||||
|
||||
// Verify order (most recent first)
|
||||
for i := 0; i < len(recent)-1; i++ {
|
||||
for i := range len(recent) - 1 {
|
||||
if recent[i].StartedAt.Before(recent[i+1].StartedAt) {
|
||||
t.Error("snapshots not in descending order")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotRepositoryNotFound(t *testing.T) {
|
||||
// TestSnapshotTimestampsDecodeAsUTC pins the zone every snapshot reader
|
||||
// returns. started_at and completed_at are stored as bare Unix seconds,
|
||||
// so the zone is a decode choice, and callers (notably `snapshot list`)
|
||||
// render these timestamps through zone-less format strings in the same
|
||||
// column as timestamps read from remote manifests, which are always
|
||||
// UTC. If one reader decodes in the host's local zone, that column
|
||||
// silently shows two different wall clocks for the same instant.
|
||||
//
|
||||
// The assertions compare *time.Location pointers, so this fails on a
|
||||
// UTC host too: time.Unix returns time.Local, which is never the same
|
||||
// Location value as time.UTC no matter what the host's offset is.
|
||||
func TestSnapshotTimestampsDecodeAsUTC(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewSnapshotRepository(db)
|
||||
repo := database.NewSnapshotRepository(db)
|
||||
|
||||
startedAt := time.Date(2026, 3, 1, 10, 0, 0, 0, time.UTC)
|
||||
completedAt := startedAt.Add(time.Minute)
|
||||
|
||||
completed := &database.Snapshot{
|
||||
ID: types.SnapshotID("testhost_home_2026-03-01T10:00:00Z"),
|
||||
Hostname: testHostname,
|
||||
VaultikVersion: testVersion,
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: &completedAt,
|
||||
}
|
||||
|
||||
err := repo.Create(ctx, nil, completed)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create completed snapshot: %v", err)
|
||||
}
|
||||
|
||||
// An incomplete row as well, so the scanner shared by the two
|
||||
// GetIncomplete* readers is covered with a nil completed_at too.
|
||||
incomplete := &database.Snapshot{
|
||||
ID: types.SnapshotID("testhost_home_2026-03-02T10:00:00Z"),
|
||||
Hostname: testHostname,
|
||||
VaultikVersion: testVersion,
|
||||
StartedAt: startedAt.Add(time.Hour),
|
||||
CompletedAt: nil,
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, incomplete)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create incomplete snapshot: %v", err)
|
||||
}
|
||||
|
||||
byID, err := repo.GetByID(ctx, completed.ID.String())
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get snapshot by id: %v", err)
|
||||
}
|
||||
|
||||
recent, err := repo.ListRecent(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to list recent snapshots: %v", err)
|
||||
}
|
||||
|
||||
incompletes, err := repo.GetIncompleteSnapshots(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to list incomplete snapshots: %v", err)
|
||||
}
|
||||
|
||||
byHost, err := repo.GetIncompleteByHostname(ctx, testHostname)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to list incomplete snapshots by hostname: %v", err)
|
||||
}
|
||||
|
||||
read := make([]*database.Snapshot, 0,
|
||||
1+len(recent)+len(incompletes)+len(byHost))
|
||||
read = append(read, byID)
|
||||
read = append(read, recent...)
|
||||
read = append(read, incompletes...)
|
||||
read = append(read, byHost...)
|
||||
|
||||
if len(read) < 5 {
|
||||
t.Fatalf("expected every reader to return rows, got %d", len(read))
|
||||
}
|
||||
|
||||
assertTimestampsAreUTC(t, read)
|
||||
|
||||
// And the wall clock is the UTC one, not the host's rendering of it.
|
||||
rendered := byID.StartedAt.Format("2006-01-02 15:04:05")
|
||||
if rendered != "2026-03-01 10:00:00" {
|
||||
t.Errorf("started_at rendered as %q, want the UTC wall clock", rendered)
|
||||
}
|
||||
}
|
||||
|
||||
// assertTimestampsAreUTC fails for any snapshot whose timestamps did not
|
||||
// decode in UTC. It compares *time.Location pointers rather than
|
||||
// offsets, so it is equally strict on a host whose local zone happens to
|
||||
// be UTC: time.Unix returns time.Local, which is never the same Location
|
||||
// value as time.UTC.
|
||||
func assertTimestampsAreUTC(t *testing.T, snapshots []*database.Snapshot) {
|
||||
t.Helper()
|
||||
|
||||
for _, snapshot := range snapshots {
|
||||
if snapshot.StartedAt.Location() != time.UTC {
|
||||
t.Errorf("snapshot %s: started_at decoded in %s, want UTC",
|
||||
snapshot.ID, snapshot.StartedAt.Location())
|
||||
}
|
||||
|
||||
if snapshot.CompletedAt != nil &&
|
||||
snapshot.CompletedAt.Location() != time.UTC {
|
||||
t.Errorf("snapshot %s: completed_at decoded in %s, want UTC",
|
||||
snapshot.ID, snapshot.CompletedAt.Location())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotRepositoryNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := database.NewSnapshotRepository(db)
|
||||
|
||||
// Test GetByID with non-existent ID
|
||||
snapshot, err := repo.GetByID(ctx, "nonexistent")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if snapshot != nil {
|
||||
t.Error("expected nil for non-existent snapshot")
|
||||
}
|
||||
|
||||
// Test UpdateCounts on non-existent snapshot
|
||||
err = repo.UpdateCounts(ctx, nil, "nonexistent", 100, 200, 10, oneHundredMebibytes, fortyMebibytes)
|
||||
err = repo.UpdateCounts(ctx, nil, "nonexistent",
|
||||
100, 200, 10, oneHundredMebibytes, fortyMebibytes)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
@@ -156,16 +333,18 @@ func TestSnapshotRepositoryNotFound(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSnapshotRepositoryDuplicate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewSnapshotRepository(db)
|
||||
repo := database.NewSnapshotRepository(db)
|
||||
|
||||
snapshot := &Snapshot{
|
||||
snapshot := &database.Snapshot{
|
||||
ID: "2024-01-01T12:00:00Z",
|
||||
Hostname: "test-host",
|
||||
VaultikVersion: "1.0.0",
|
||||
Hostname: testHostname,
|
||||
VaultikVersion: testVersion,
|
||||
StartedAt: time.Now().Truncate(time.Second),
|
||||
CompletedAt: nil,
|
||||
FileCount: 100,
|
||||
|
||||
@@ -3,9 +3,10 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
// Upload represents a blob upload record
|
||||
@@ -28,7 +29,9 @@ func NewUploadRepository(conn *sql.DB) *UploadRepository {
|
||||
}
|
||||
|
||||
// Create inserts a new upload record
|
||||
func (r *UploadRepository) Create(ctx context.Context, tx *sql.Tx, upload *Upload) error {
|
||||
func (r *UploadRepository) Create(
|
||||
ctx context.Context, tx *sql.Tx, upload *Upload,
|
||||
) error {
|
||||
query := `
|
||||
INSERT INTO uploads (blob_hash, snapshot_id, uploaded_at, size, duration_ms)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
@@ -36,16 +39,22 @@ func (r *UploadRepository) Create(ctx context.Context, tx *sql.Tx, upload *Uploa
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, upload.BlobHash, upload.SnapshotID, upload.UploadedAt, upload.Size, upload.DurationMs)
|
||||
_, err = tx.ExecContext(ctx, query,
|
||||
upload.BlobHash, upload.SnapshotID, upload.UploadedAt,
|
||||
upload.Size, upload.DurationMs)
|
||||
} else {
|
||||
_, err = r.conn.ExecContext(ctx, query, upload.BlobHash, upload.SnapshotID, upload.UploadedAt, upload.Size, upload.DurationMs)
|
||||
_, err = r.conn.ExecContext(ctx, query,
|
||||
upload.BlobHash, upload.SnapshotID, upload.UploadedAt,
|
||||
upload.Size, upload.DurationMs)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// GetByBlobHash retrieves an upload record by blob hash
|
||||
func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (*Upload, error) {
|
||||
func (r *UploadRepository) GetByBlobHash(
|
||||
ctx context.Context, blobHash string,
|
||||
) (*Upload, error) {
|
||||
query := `
|
||||
SELECT blob_hash, uploaded_at, size, duration_ms
|
||||
FROM uploads
|
||||
@@ -53,6 +62,7 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
|
||||
`
|
||||
|
||||
var upload Upload
|
||||
|
||||
err := r.conn.QueryRowContext(ctx, query, blobHash).Scan(
|
||||
&upload.BlobHash,
|
||||
&upload.UploadedAt,
|
||||
@@ -60,9 +70,10 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
|
||||
&upload.DurationMs,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -71,7 +82,9 @@ func (r *UploadRepository) GetByBlobHash(ctx context.Context, blobHash string) (
|
||||
}
|
||||
|
||||
// GetRecentUploads retrieves recent uploads ordered by upload time
|
||||
func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]*Upload, error) {
|
||||
func (r *UploadRepository) GetRecentUploads(
|
||||
ctx context.Context, limit int,
|
||||
) ([]*Upload, error) {
|
||||
query := `
|
||||
SELECT blob_hash, uploaded_at, size, duration_ms
|
||||
FROM uploads
|
||||
@@ -83,18 +96,26 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]*
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := rows.Close(); err != nil {
|
||||
err := rows.Close()
|
||||
if err != nil {
|
||||
log.Error("failed to close rows", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
var uploads []*Upload
|
||||
|
||||
for rows.Next() {
|
||||
var upload Upload
|
||||
if err := rows.Scan(&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs); err != nil {
|
||||
|
||||
err := rows.Scan(
|
||||
&upload.BlobHash, &upload.UploadedAt, &upload.Size, &upload.DurationMs,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
uploads = append(uploads, &upload)
|
||||
}
|
||||
|
||||
@@ -102,7 +123,9 @@ func (r *UploadRepository) GetRecentUploads(ctx context.Context, limit int) ([]*
|
||||
}
|
||||
|
||||
// GetUploadStats returns aggregate statistics for uploads
|
||||
func (r *UploadRepository) GetUploadStats(ctx context.Context, since time.Time) (*UploadStats, error) {
|
||||
func (r *UploadRepository) GetUploadStats(
|
||||
ctx context.Context, since time.Time,
|
||||
) (*UploadStats, error) {
|
||||
query := `
|
||||
SELECT
|
||||
COUNT(*) as count,
|
||||
@@ -115,6 +138,7 @@ func (r *UploadRepository) GetUploadStats(ctx context.Context, since time.Time)
|
||||
`
|
||||
|
||||
var stats UploadStats
|
||||
|
||||
err := r.conn.QueryRowContext(ctx, query, since).Scan(
|
||||
&stats.Count,
|
||||
&stats.TotalSize,
|
||||
@@ -136,12 +160,17 @@ type UploadStats struct {
|
||||
}
|
||||
|
||||
// GetCountBySnapshot returns the count of uploads for a specific snapshot
|
||||
func (r *UploadRepository) GetCountBySnapshot(ctx context.Context, snapshotID string) (int64, error) {
|
||||
func (r *UploadRepository) GetCountBySnapshot(
|
||||
ctx context.Context, snapshotID string,
|
||||
) (int64, error) {
|
||||
query := `SELECT COUNT(*) FROM uploads WHERE snapshot_id = ?`
|
||||
|
||||
var count int64
|
||||
|
||||
err := r.conn.QueryRowContext(ctx, query, snapshotID).Scan(&count)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
@@ -1,31 +1,88 @@
|
||||
// Package globals holds application-wide metadata (name, version,
|
||||
// commit) that is populated at build time via linker flags.
|
||||
package globals
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Appname is the application name, populated from main().
|
||||
var Appname string = "vaultik"
|
||||
var Appname = "vaultik" //nolint:gochecknoglobals // set via -ldflags at build time
|
||||
|
||||
// DevVersion is the version a binary reports when it was not built
|
||||
// from a tagged commit. script/version emits either this exact string
|
||||
// (outside a git checkout) or this string followed by "-" and the
|
||||
// commit it was built from, and goreleaser's snapshot template matches
|
||||
// that shape. It is deliberately not a number: a build that is not a
|
||||
// release must not name itself like one.
|
||||
const DevVersion = "dev"
|
||||
|
||||
// Version is the application version, populated from main().
|
||||
var Version string = "dev"
|
||||
var Version = DevVersion //nolint:gochecknoglobals // set via -ldflags at build time
|
||||
|
||||
// Commit is the git commit hash, populated from main().
|
||||
var Commit string = "unknown"
|
||||
var Commit = "unknown" //nolint:gochecknoglobals // set via -ldflags at build time
|
||||
|
||||
// CommitDate is the ISO-8601 date of the commit, populated from main().
|
||||
var CommitDate = "unknown" //nolint:gochecknoglobals // set via -ldflags at build time
|
||||
|
||||
// Author identifies the upstream author of vaultik.
|
||||
const Author = "Jeffrey Paul <sneak@sneak.berlin>"
|
||||
|
||||
// Homepage is the canonical URL for vaultik.
|
||||
const Homepage = "https://sneak.berlin/go/vaultik"
|
||||
|
||||
// ReleasesURL is where tagged release artifacts are published.
|
||||
const ReleasesURL = "https://git.eeqj.de/sneak/vaultik/releases"
|
||||
|
||||
// License is the SPDX identifier for the project license.
|
||||
const License = "MIT"
|
||||
|
||||
// Globals contains application-wide configuration and metadata.
|
||||
type Globals struct {
|
||||
Appname string
|
||||
Version string
|
||||
Commit string
|
||||
CommitDate string
|
||||
StartTime time.Time
|
||||
}
|
||||
|
||||
// New creates and returns a new Globals instance initialized with the package-level variables.
|
||||
// New creates and returns a new Globals instance initialized with the
|
||||
// package-level variables.
|
||||
func New() (*Globals, error) {
|
||||
return &Globals{
|
||||
Appname: Appname,
|
||||
Version: Version,
|
||||
Commit: Commit,
|
||||
CommitDate: CommitDate,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// IsDevVersion reports whether v names a development build rather than
|
||||
// a release. Both "dev" and "dev-<sha>" (and its "-dirty" variant)
|
||||
// count: a caller that compares against "dev" exactly would treat every
|
||||
// commit-stamped development build as a release.
|
||||
//
|
||||
// The empty string counts too. Nothing that knows its version reports
|
||||
// no version, so an empty Version means the stamping failed, and the
|
||||
// safe reading of "we could not establish that this is a release" is
|
||||
// that it is not one. The Makefile refuses to build at all in that
|
||||
// case; this is the second line of defence, for a binary linked by
|
||||
// something other than the Makefile.
|
||||
func IsDevVersion(v string) bool {
|
||||
return v == "" || v == DevVersion || strings.HasPrefix(v, DevVersion+"-")
|
||||
}
|
||||
|
||||
// shortCommitLen is the number of commit-hash characters ShortCommit keeps.
|
||||
const shortCommitLen = 12
|
||||
|
||||
// ShortCommit returns the first 12 chars of the commit hash, or the
|
||||
// whole string if it's shorter (e.g. "unknown").
|
||||
func (g *Globals) ShortCommit() string {
|
||||
if len(g.Commit) > shortCommitLen {
|
||||
return g.Commit[:shortCommitLen]
|
||||
}
|
||||
|
||||
return g.Commit
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user