Compare commits
328
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1ecfe12e78 | ||
|
|
1aea50d254 | ||
|
|
83981783b0 | ||
|
|
79fccfecce | ||
|
|
252db3259a | ||
|
|
a52a8a781c | ||
|
|
b6f8e860e1 | ||
|
|
64c31f890b | ||
|
|
8f34718a0f | ||
|
|
8654d9b7f5 | ||
|
|
63f8d4df59 | ||
|
|
b6a2c61f82 | ||
|
|
43c9b2c39b | ||
|
|
de9ef62073 | ||
|
|
5abacbc5c1 | ||
|
|
ff50a137dd | ||
|
|
e9b17cc5fd | ||
|
|
850e2de25a | ||
|
|
6a57750a7b | ||
|
|
2590f49583 | ||
|
|
5b8d84e38a | ||
|
|
7a950ea2aa | ||
|
|
16b03dfda5 | ||
|
|
2125a9d451 | ||
|
|
a0de8bfdb7 | ||
|
|
291c39174f | ||
|
|
65036bba1b | ||
|
|
54ad279dd6 | ||
|
|
d156a02c49 | ||
|
|
aeac897805 | ||
|
|
d8f4f62f41 | ||
|
|
9fd88a34c7 | ||
|
|
5db2e920e5 | ||
|
|
66ea7f5f45 | ||
|
|
dd3667c8e9 | ||
|
|
30285c7a8c | ||
|
|
8d5e6f3399 | ||
|
|
abad8161bc | ||
|
|
05a2c86d65 | ||
|
|
d8f78a1f45 | ||
|
|
2c087be577 | ||
|
|
37d525c322 | ||
|
|
f89fac34f7 | ||
|
|
5d8c515192 | ||
|
|
42eb50b151 | ||
|
|
25ee9211d3 | ||
|
|
3361e9ebd9 | ||
|
|
fb8abcd049 | ||
|
|
6aceaaf086 | ||
|
|
3b2b9cec84 | ||
|
|
dfa3bbc9e1 | ||
|
|
740f474c59 | ||
|
|
27e9798da5 | ||
|
|
4193ced1f6 | ||
|
|
e48bafccef | ||
|
|
bffd4578fc | ||
|
|
f4ab1ba5ae | ||
|
|
005927a7fc | ||
|
|
09a0c01c43 | ||
|
|
8c54e0ff75 | ||
|
|
d1213ecb34 | ||
|
|
5f6455ba17 | ||
|
|
0288735aaf | ||
|
|
35eed79b1a | ||
|
|
94412d9ef3 | ||
|
|
167c0a0746 | ||
|
|
07c957f84b | ||
|
|
c9a519c233 | ||
|
|
fa56165fe4 | ||
|
|
34f61fb408 | ||
|
|
05bf73c48a | ||
|
|
79a73fa122 | ||
|
|
0d0368df81 | ||
|
|
c06c4d201b | ||
|
|
70f008a21a | ||
|
|
fce253fa39 | ||
|
|
e459a66099 | ||
|
|
32c4a46b49 | ||
|
|
b5389a62b5 | ||
|
|
d87202fb70 | ||
|
|
e161343eac | ||
|
|
d53202eb86 | ||
|
|
8b22ae8d42 | ||
|
|
3fc8a8f2f4 | ||
|
|
7696f83258 | ||
|
|
5d1118d143 | ||
|
|
b57ce2277d | ||
|
|
8496404d8b | ||
|
|
cdc60c4dfa | ||
|
|
49eed7a3e5 | ||
|
|
85d4ef118d | ||
|
|
b06f992152 | ||
|
|
5d685f03ce | ||
|
|
f59086c0e5 | ||
|
|
d276d891da | ||
|
|
5aa5ba5544 | ||
|
|
315b6483b8 | ||
|
|
14fc4c9893 | ||
|
|
66c80a70e3 | ||
|
|
81f83b29f6 | ||
|
|
c4adb72d80 | ||
|
|
4a167e153a | ||
|
|
ea72697992 | ||
|
|
713be502bd | ||
|
|
35cf985c18 | ||
|
|
070090124a | ||
|
|
584444b619 | ||
|
|
b30e79ee45 | ||
|
|
d886a9026f | ||
|
|
6e1f499048 | ||
|
|
15e6506e5d | ||
|
|
d24f5dc33c | ||
|
|
eed117fe25 | ||
|
|
dd7a610c23 | ||
|
|
1548c0f933 | ||
|
|
c3bec7d3aa | ||
|
|
1244c9e48d | ||
|
|
82c51a5337 | ||
|
|
d88ed64489 | ||
|
|
bd9656dbd4 | ||
|
|
7e611b95db | ||
|
|
4f27608560 | ||
|
|
ae6aaaa388 | ||
|
|
f788668287 | ||
|
|
238ce3985f | ||
|
|
548a7ae156 | ||
|
|
3a58377127 | ||
|
|
a6434de57f | ||
|
|
b4654f8e52 | ||
|
|
39aef1c47c | ||
|
|
96ebcd40d7 | ||
|
|
d9f0220f94 | ||
|
|
4c83e82543 | ||
|
|
86361c8b50 | ||
|
|
d77663d039 | ||
|
|
3abe9cbd9e | ||
|
|
76a6917a35 | ||
|
|
38ebfd843a | ||
|
|
6b7517a4dc | ||
|
|
994e5de613 | ||
|
|
42f4e648d7 | ||
|
|
343129f891 | ||
|
|
a50e3fa038 | ||
|
|
6fcd8e1668 | ||
|
|
aab6a87f8c | ||
|
|
c355ef4d25 | ||
|
|
5927e1aa3d | ||
|
|
9ca962969a | ||
|
|
89ebfc78e2 | ||
|
|
07ef3a1c78 | ||
|
|
c423d13191 | ||
|
|
75a10d3a22 | ||
|
|
3d56dd7eb0 | ||
|
|
bdce350041 | ||
|
|
753bc3ef60 | ||
|
|
d2a0510cb4 | ||
|
|
583f65040a | ||
|
|
d257f8f658 | ||
|
|
696ed9ab4d | ||
|
|
f21e7c9e70 | ||
|
|
c16ef476a9 | ||
|
|
e3f407b440 | ||
|
|
b6e4a218a3 | ||
|
|
c51f693527 | ||
|
|
3f9c2e5033 | ||
|
|
50816b7415 | ||
|
|
c3bb3b5580 | ||
|
|
3bcdbcfd83 | ||
|
|
50e20b460e | ||
|
|
af607e3597 | ||
|
|
e496aa334b | ||
|
|
cc58583130 | ||
|
|
b87b72d4b9 | ||
|
|
e7b49d58ab | ||
|
|
919229f224 | ||
|
|
68cffba35d | ||
|
|
26cbb63749 | ||
|
|
bf1d3c6bad | ||
|
|
dca3c50cd2 | ||
|
|
1ee0d291ca | ||
|
|
2f7e37153c | ||
|
|
26909b058a | ||
|
|
2dfdc5f095 | ||
|
|
217d60eeaa | ||
|
|
f7ba056814 | ||
|
|
82eb352eb5 | ||
|
|
0296e26210 | ||
|
|
1e05fa0dd7 | ||
|
|
bec964fc20 | ||
|
|
b1451bb17e | ||
|
|
ee83f50281 | ||
|
|
76f79af733 | ||
|
|
070a8a5447 | ||
|
|
40516d1263 | ||
|
|
a66e1f9844 | ||
|
|
5e4df7d04f | ||
|
|
d34868a9c4 | ||
|
|
04fce150bc | ||
|
|
23d22a0f19 | ||
|
|
c9c72ef29d | ||
|
|
928c389a5a | ||
|
|
2aaeeb4966 | ||
|
|
43346e62db | ||
|
|
df975bb8f0 | ||
|
|
fc56b0cb30 | ||
|
|
1f32820607 | ||
|
|
34a4d163f2 | ||
|
|
9497a31d0f | ||
|
|
d330f9f031 | ||
|
|
fda6d7a7eb | ||
|
|
1a97a80a81 | ||
|
|
d492f34e8d | ||
|
|
c17426b556 | ||
|
|
c06d53ad45 | ||
|
|
b39d765374 | ||
|
|
017ad7d3a6 | ||
|
|
fd759a921a | ||
|
|
a84b911155 | ||
|
|
5ce1dfa39e | ||
|
|
aa3e8f081b | ||
|
|
1f22b9c603 | ||
|
|
60abeb636a | ||
|
|
7ae49a1b2c | ||
|
|
a92b1a82ad | ||
|
|
39d5d21d48 | ||
|
|
44c9008e7e | ||
|
|
8036d93914 | ||
|
|
b55d5763ad | ||
|
|
53febb48d2 | ||
|
|
d55ddc5914 | ||
|
|
d9319dc0fb | ||
|
|
af330f2777 | ||
|
|
683fb0b103 | ||
|
|
cf8a527d35 | ||
|
|
a63c729fbc | ||
|
|
a1065d4f1f | ||
|
|
0e9c96c8b5 | ||
|
|
cafae65f61 | ||
|
|
7a0d5bfd73 | ||
|
|
8d1c8982d7 | ||
|
|
e75367c594 | ||
|
|
64c69cd8e3 | ||
|
|
132f7149ca | ||
|
|
f1ce085972 | ||
|
|
d8edf90fac | ||
|
|
301ea217e8 | ||
|
|
9f537b9c4c | ||
|
|
cf5b643bee | ||
|
|
3113014b58 | ||
|
|
706284d590 | ||
|
|
75564a504e | ||
|
|
edd3e5fdb2 | ||
|
|
d5796bd6c1 | ||
|
|
90e855ef99 | ||
|
|
2185421c01 | ||
|
|
ce0d7b45a1 | ||
|
|
1266a263fc | ||
|
|
70632e4353 | ||
|
|
77b9d943e4 | ||
|
|
fc4d0d6dc7 | ||
|
|
22227aa0c5 | ||
|
|
9cb14d143d | ||
|
|
00d4b36e35 | ||
|
|
8de8f8e5cc | ||
|
|
6e6e107243 | ||
|
|
6bb6f7c8a8 | ||
|
|
8e55d2f970 | ||
|
|
b0747657e3 | ||
|
|
2a9718855c | ||
|
|
485f3296d9 | ||
|
|
adf73c5413 | ||
|
|
8959741c90 | ||
|
|
e534746cf3 | ||
|
|
5397b37c13 | ||
|
|
2df2792a75 | ||
|
|
4fe568f803 | ||
|
|
27e85f01f2 | ||
|
|
d479bfcd52 | ||
|
|
cb16d6869f | ||
|
|
ff85f1e4f8 | ||
|
|
b2e160944f | ||
|
|
307867f59e | ||
|
|
9d12d500fa | ||
|
|
2e2bf01130 | ||
|
|
e9687c68b7 | ||
|
|
a8970a87fc | ||
|
|
e6ee488d9d | ||
|
|
2e2b02a056 | ||
|
|
0b95cb4308 | ||
|
|
4a3e61f8e1 | ||
|
|
6fbcac0cd8 | ||
|
|
34f73f72d8 | ||
|
|
ee240faa32 | ||
|
|
f719ab3adc | ||
|
|
1a8baf7491 | ||
|
|
7d5d3fa598 | ||
|
|
ac5d2f4a0d | ||
|
|
b250ddfa94 | ||
|
|
fe3ad13a91 | ||
|
|
ebd6619638 | ||
|
|
20d3a9ac8c | ||
|
|
0889cf2804 | ||
|
|
f9ebb4bf25 | ||
|
|
9f2d722734 | ||
|
|
6821215b0e | ||
|
|
f97a1dc2eb | ||
|
|
18c14d1507 | ||
|
|
65da291ddf | ||
|
|
dcf3ec399a | ||
|
|
495dede1bc | ||
|
|
1c72a37bc8 | ||
|
|
60b6746db9 | ||
|
|
f28c8a73b7 | ||
|
|
1c0f5b8eb2 | ||
|
|
689109a2b8 | ||
|
|
ac2f21a89d | ||
|
|
8c59f55096 | ||
|
|
c24e7e6360 | ||
|
|
7a5943958d | ||
|
|
d8a51804d2 | ||
|
|
76f4421eb3 | ||
|
|
53ac868c5d | ||
|
|
8c4ea2b870 | ||
|
|
597b560398 | ||
|
|
1e2eced092 | ||
|
|
76e047bbb2 | ||
|
|
ddc23f8057 | ||
|
|
cafb3d45b8 |
@@ -0,0 +1,79 @@
|
||||
# .dockerignore does NOT use .gitignore semantics. Docker matches with
|
||||
# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross
|
||||
# `/` and an unprefixed pattern is anchored at the context root. Every
|
||||
# depth-independent pattern therefore needs `**/`, or `config/.env` and
|
||||
# `certs/server.key` still ship while this file reads as solved. Only
|
||||
# genuinely root-anchored entries go unprefixed. Never transplant these
|
||||
# into .gitignore, where `**/` is wrong.
|
||||
#
|
||||
# Matching is case-sensitive, so secrets use character ranges rather
|
||||
# than an ALL-CAPS twin, which would still miss `Server.Key`.
|
||||
#
|
||||
# Extend with this repo's own host-built artifacts, written anchored:
|
||||
# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and
|
||||
# deletes the package directory from the context.
|
||||
|
||||
# .git is sent without its config. Without a VERSION build argument the
|
||||
# stage that compiles runs `git describe --tags --always` on .git, which
|
||||
# does not need .git/config; that file can hold a credential, such as a
|
||||
# password in a remote URL or the token the CI checkout step stores there.
|
||||
# Each submodule keeps a config with the same exposure in its git directory
|
||||
# under .git/modules/, nested again for a submodule's own submodules, or in
|
||||
# its own .git directory when it keeps one.
|
||||
# KNOWN GAP: a submodule whose name has a `config` segment (`config`,
|
||||
# `deploy/config`, `config/lib`) loses its whole git directory, because
|
||||
# `**/.git/modules/**/config` also matches that segment's directory
|
||||
# under .git/modules/. Go's version stamping then fails the build;
|
||||
# nothing leaks. Name such a submodule without that segment:
|
||||
# `git submodule add --name`.
|
||||
**/.git/config
|
||||
**/.git/modules/**/config
|
||||
|
||||
# Agent scratch: one full checkout of the repo per in-flight agent.
|
||||
# Anchored because it occurs once where agents run at the repo root.
|
||||
# KNOWN GAP: a repo running agents in subdirectories still ships
|
||||
# `services/api/.claude/` and must add its own anchored entry.
|
||||
.claude
|
||||
|
||||
# Environment files. `*.env` covers bare `.env` and the `prod.env`
|
||||
# convention. Re-include a committed template with a negation if the
|
||||
# build needs one: `!docs/example.env`.
|
||||
**/*.[eE][nN][vV]
|
||||
**/.[eE][nN][vV].*
|
||||
**/.[eE][nN][vV][rR][cC]
|
||||
|
||||
# Private keys and the bundles carrying them. Public certificates
|
||||
# (*.crt, *.cer) are deliberately absent: they are legitimate inputs.
|
||||
**/*.[pP][eE][mM]
|
||||
**/*.[kK][eE][yY]
|
||||
**/*.[pP]12
|
||||
**/*.[pP][fF][xX]
|
||||
**/[iI][dD]_[rR][sS][aA]
|
||||
**/[iI][dD]_[dD][sS][aA]
|
||||
**/[iI][dD]_[eE][cC][dD][sS][aA]
|
||||
**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
|
||||
**/[iI][dD]_[eE][dD]25519
|
||||
**/[iI][dD]_[eE][dD]25519_[sS][kK]
|
||||
|
||||
# Dependencies: restored inside the image, never copied in.
|
||||
**/node_modules
|
||||
|
||||
# OS metadata.
|
||||
**/.DS_Store
|
||||
**/Thumbs.db
|
||||
|
||||
# Editor state: never a build input, and it churns COPY.
|
||||
**/*.swp
|
||||
**/*.swo
|
||||
**/*~
|
||||
**/*.bak
|
||||
**/.idea
|
||||
**/.vscode
|
||||
**/*.sublime-*
|
||||
|
||||
# This repo's own host-built artifacts.
|
||||
/vaultik
|
||||
/dist
|
||||
/.tool
|
||||
/coverage.out
|
||||
/coverage.html
|
||||
@@ -0,0 +1,15 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
end_of_line = lf
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
|
||||
[*.go]
|
||||
indent_style = tab
|
||||
@@ -0,0 +1,9 @@
|
||||
name: check
|
||||
on: [push]
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# actions/checkout v4.2.2, 2026-02-22
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
- run: script/cibuild
|
||||
@@ -0,0 +1,54 @@
|
||||
name: release
|
||||
on:
|
||||
push:
|
||||
tags: ["v*"]
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# actions/checkout v4, 2024-09-16
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
with:
|
||||
# goreleaser needs the tags and the full history: the version
|
||||
# it stamps comes from the tag, and the changelog comes from
|
||||
# the commits since the previous one. A shallow checkout
|
||||
# silently produces a mislabelled release.
|
||||
fetch-depth: 0
|
||||
# goreleaser is not a compiler: it shells out to `go` for the
|
||||
# `before:` hook and for every one of the four cross-compiles.
|
||||
# Without this step the release either fails at the before-hook or,
|
||||
# worse, ships binaries built by whatever Go the runner happens to
|
||||
# carry. check.yml's runner gets the same Go through
|
||||
# script/bootstrap, which calls script/install-go, and uses it only
|
||||
# for `go mod download` and gofmt; it compiles inside the
|
||||
# digest-pinned Dockerfile images.
|
||||
#
|
||||
# actions/setup-go would pin the action by commit sha, but the Go
|
||||
# tarball it downloads at runtime is verified against no value in
|
||||
# this repo, and the action exposes no checksum input.
|
||||
# REPO_POLICIES.md requires every external reference to be pinned
|
||||
# by hash with no exceptions, and this is the compiler that
|
||||
# produces the published binaries -- the input where a substituted
|
||||
# artifact matters most. So Go is installed the way goreleaser is:
|
||||
# script/install-go downloads the exact archive for go.mod's `go`
|
||||
# directive and refuses it unless its sha256 matches the value
|
||||
# committed in the script, then puts .tool/go/bin on PATH for the
|
||||
# steps below.
|
||||
- name: Install Go
|
||||
run: script/install-go
|
||||
- name: Install goreleaser
|
||||
run: script/install-goreleaser
|
||||
- name: Release
|
||||
run: script/release
|
||||
env:
|
||||
# RELEASE_TOKEN is a repository Actions secret: a Gitea access
|
||||
# token with write access to this repository's releases (scope
|
||||
# write:repository), owned by an account that can publish here.
|
||||
# It is deliberately not the runner's automatic token, which is
|
||||
# not guaranteed to carry that scope.
|
||||
GITEA_TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||
# Build with the toolchain install-go just verified, never a
|
||||
# different one auto-downloaded from a `toolchain` directive:
|
||||
# the point of the hash pin is that this exact compiler makes
|
||||
# the release.
|
||||
GOTOOLCHAIN: local
|
||||
+57
-17
@@ -1,22 +1,62 @@
|
||||
# Binary
|
||||
vaultik
|
||||
|
||||
# Test artifacts
|
||||
*.out
|
||||
*.test
|
||||
coverage.html
|
||||
coverage.out
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Local config for development
|
||||
# Editors
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
*.bak
|
||||
.idea/
|
||||
.vscode/
|
||||
*.sublime-*
|
||||
|
||||
# Agent scratch (worktrees of this repo, created and destroyed by
|
||||
# in-flight tooling). Unanchored: .gitignore patterns already match at
|
||||
# every depth, so no prefix is wanted here. This is not a .dockerignore
|
||||
# entry and must not be given a `**/` prefix on the way into one.
|
||||
.claude/
|
||||
|
||||
# Node
|
||||
node_modules/
|
||||
|
||||
# Secrets. Unanchored like every entry above, so each matches at every
|
||||
# depth. Matching is case-sensitive on Linux, so names use character
|
||||
# ranges rather than a lowercase form that misses `Server.Key`.
|
||||
|
||||
# Environment files. `*.env` covers bare `.env` and the `prod.env`
|
||||
# convention. Only the templates `example.env` and `sample.env` are
|
||||
# re-included below. A repository that commits any other template adds
|
||||
# its own negation after these lines, for example `!.env.example`.
|
||||
*.[eE][nN][vV]
|
||||
.[eE][nN][vV].*
|
||||
.[eE][nN][vV][rR][cC]
|
||||
!example.env
|
||||
!sample.env
|
||||
|
||||
# Private keys and the bundles carrying them.
|
||||
*.[pP][eE][mM]
|
||||
*.[kK][eE][yY]
|
||||
*.[pP]12
|
||||
*.[pP][fF][xX]
|
||||
[iI][dD]_[rR][sS][aA]
|
||||
[iI][dD]_[dD][sS][aA]
|
||||
[iI][dD]_[eE][cC][dD][sS][aA]
|
||||
[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
|
||||
[iI][dD]_[eE][dD]25519
|
||||
[iI][dD]_[eE][dD]25519_[sS][kK]
|
||||
|
||||
# Go build and test output.
|
||||
*.log
|
||||
*.out
|
||||
*.test
|
||||
coverage.html
|
||||
|
||||
# This repo's own host-built artifacts.
|
||||
/vaultik
|
||||
/dist/
|
||||
/.tool/
|
||||
|
||||
# Local configs for development; they hold storage credentials.
|
||||
local-config.yaml
|
||||
dev-config.yaml
|
||||
dev-config.yaml
|
||||
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
version: "2"
|
||||
|
||||
# Config schema uses the golangci-lint v2 layout (settings live under
|
||||
# linters.settings, not top-level linters-settings) so that the
|
||||
# thresholds below are actually applied by golangci-lint >= v2.
|
||||
|
||||
run:
|
||||
timeout: 5m
|
||||
modules-download-mode: readonly
|
||||
|
||||
linters:
|
||||
default: all
|
||||
enable:
|
||||
# Successor to the deprecated gomodguard. Named explicitly, rather than
|
||||
# left to `default: all`, because it carries the module policy below.
|
||||
- gomodguard_v2
|
||||
disable:
|
||||
# Genuinely incompatible with project patterns
|
||||
- exhaustruct # Requires all struct fields
|
||||
- exhaustruct_v5 # Requires all struct fields (successor to exhaustruct)
|
||||
- godot # Requires comments to end with periods
|
||||
- wrapcheck # Too verbose for internal packages
|
||||
- varnamelen # Short names like db, id are idiomatic Go
|
||||
# Deprecated: the warning is attached to the old name, so it is
|
||||
# silenced by disabling that name, not by enabling the successor.
|
||||
- wsl # Deprecated, replaced by wsl_v5
|
||||
- gomodguard # Deprecated, replaced by gomodguard_v2
|
||||
settings:
|
||||
lll:
|
||||
line-length: 88
|
||||
funlen:
|
||||
lines: 80
|
||||
statements: 50
|
||||
cyclop:
|
||||
max-complexity: 15
|
||||
dupl:
|
||||
threshold: 100
|
||||
depguard:
|
||||
# Test-support code must not be compiled into the shipped binary. A
|
||||
# test-support package exists to hand a test privileges the program
|
||||
# itself must never have, so a file that is not a test must not import
|
||||
# one. Test files, and the files inside a package whose directory name
|
||||
# ends in `test`, are where that code belongs, and are exempt.
|
||||
#
|
||||
# The deny list below is the one part of this file a repository is
|
||||
# expected to extend, and the only part it may. depguard matches an
|
||||
# import path against a list of prefixes, so it cannot be told "any path
|
||||
# whose last segment ends in test"; a repository's own test-support
|
||||
# packages have to be named here one at a time, by full import path,
|
||||
# under a module path that differs from repository to repository. Add
|
||||
# them; change nothing else.
|
||||
rules:
|
||||
test-support:
|
||||
list-mode: lax
|
||||
files:
|
||||
- "$all"
|
||||
- "!$test"
|
||||
- "!**/*test/**"
|
||||
deny:
|
||||
- pkg: net/http/httptest
|
||||
desc: >-
|
||||
Test-support code belongs in test files and in packages whose
|
||||
directory name ends in test, not in the shipped binary.
|
||||
- pkg: sneak.berlin/go/vaultik/internal/storage/faultstore
|
||||
desc: >-
|
||||
Test-support code belongs in test files and in packages whose
|
||||
directory name ends in test, not in the shipped binary.
|
||||
# Only decisions already recorded in the Go package defaults are
|
||||
# listed here. Every entry matches the module path exactly.
|
||||
gomodguard_v2:
|
||||
blocked:
|
||||
- module: github.com/rs/zerolog
|
||||
recommendations:
|
||||
- log/slog
|
||||
reason: "Structured logging is stdlib log/slog."
|
||||
# One entry per pre-fork module path, because the later releases
|
||||
# are separate paths. A prefix match would be shorter but would
|
||||
# also reach github.com/go-redis/redismock, the test double for
|
||||
# the successor these entries recommend.
|
||||
- module: github.com/go-redis/redis
|
||||
recommendations:
|
||||
- github.com/redis/go-redis/v9
|
||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||
- module: github.com/go-redis/redis/v7
|
||||
recommendations:
|
||||
- github.com/redis/go-redis/v9
|
||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||
- module: github.com/go-redis/redis/v8
|
||||
recommendations:
|
||||
- github.com/redis/go-redis/v9
|
||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||
- module: github.com/sergi/go-diff
|
||||
recommendations:
|
||||
- github.com/aymanbagabas/go-udiff
|
||||
reason: "No unified diff output; use go-udiff."
|
||||
- module: github.com/hexops/gotextdiff
|
||||
recommendations:
|
||||
- github.com/aymanbagabas/go-udiff
|
||||
reason: "Unmaintained fork; use go-udiff."
|
||||
|
||||
issues:
|
||||
max-issues-per-linter: 0
|
||||
max-same-issues: 0
|
||||
@@ -0,0 +1,67 @@
|
||||
version: 2
|
||||
|
||||
project_name: vaultik
|
||||
|
||||
# This repo lives on Gitea, not GitHub. Without this block goreleaser
|
||||
# talks to the GitHub API by default and a `goreleaser release` either
|
||||
# fails outright or publishes somewhere nobody is looking.
|
||||
gitea_urls:
|
||||
api: https://git.eeqj.de/api/v1
|
||||
download: https://git.eeqj.de
|
||||
|
||||
before:
|
||||
hooks:
|
||||
- go mod tidy
|
||||
|
||||
builds:
|
||||
- id: vaultik
|
||||
main: ./cmd/vaultik
|
||||
binary: vaultik
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
goos:
|
||||
- linux
|
||||
- darwin
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
ldflags:
|
||||
- -s -w
|
||||
- -X 'sneak.berlin/go/vaultik/internal/globals.Version={{ .Version }}'
|
||||
- -X 'sneak.berlin/go/vaultik/internal/globals.Commit={{ .Commit }}'
|
||||
- -X 'sneak.berlin/go/vaultik/internal/globals.CommitDate={{ slice .CommitDate 0 10 }}'
|
||||
|
||||
archives:
|
||||
- id: default
|
||||
name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
|
||||
formats:
|
||||
- tar.gz
|
||||
files:
|
||||
- LICENSE
|
||||
- README.md
|
||||
|
||||
checksum:
|
||||
name_template: "checksums.txt"
|
||||
algorithm: sha256
|
||||
|
||||
# A snapshot is not a release and must not name itself like one. The
|
||||
# previous `{{ incpatch .Version }}-next` derived a plausible-looking
|
||||
# release number from the last tag -- and with no tags in the repo at
|
||||
# all, from goreleaser's fabricated v0.0.0.
|
||||
snapshot:
|
||||
version_template: "dev-{{ slice .FullCommit 0 12 }}"
|
||||
|
||||
changelog:
|
||||
sort: asc
|
||||
use: git
|
||||
filters:
|
||||
exclude:
|
||||
- "^docs:"
|
||||
- "^test:"
|
||||
- "^chore:"
|
||||
- "Merge pull request"
|
||||
- "Merge branch"
|
||||
|
||||
release:
|
||||
draft: true
|
||||
prerelease: auto
|
||||
@@ -38,10 +38,9 @@ Version: 2025-06-08
|
||||
1. Before committing, tests must pass (`make test`), linting must pass
|
||||
(`make lint`), and code must be formatted (`make fmt`). For go, those
|
||||
makefile targets should use `go fmt` and `go test -v ./...` and
|
||||
`golangci-lint run`. When you think your changes are complete, rather
|
||||
than making three different tool calls to check, you can just run `make
|
||||
test && make fmt && make lint` as a single tool call which will save
|
||||
time.
|
||||
`golangci-lint run`. Each Makefile target does exactly one thing — to
|
||||
run lint + fmt-check + test together (the standard pre-commit gate),
|
||||
use `make check`.
|
||||
|
||||
2. Always write a `Makefile` with the default target being `test`, and with
|
||||
a `fmt` target that formats the code. The `test` target should run all
|
||||
@@ -84,8 +83,8 @@ Version: 2025-06-08
|
||||
possible to mock or stub these side-effects in tests.
|
||||
|
||||
9. Always use structured logging. Log any relevant state/context with the
|
||||
messages (but do not log secrets). If stdout is not a terminal, output
|
||||
the structured logs in jsonl format.
|
||||
messages (but do not log secrets). If the log stream is not a terminal,
|
||||
output the structured logs in jsonl format.
|
||||
|
||||
10. Avoid using bare strings or numbers in code, especially if they appear
|
||||
anywhere more than once. Always define a constant (usually at the top
|
||||
@@ -103,3 +102,29 @@ Version: 2025-06-08
|
||||
build files are acceptable in the root, but source code and other files
|
||||
should be organized in appropriate subdirectories.
|
||||
|
||||
13. Pre-1.0: NEVER add a database migration. Migrations are supported, but
|
||||
nothing is installed anywhere yet, so there is nothing to migrate. To
|
||||
change the schema, edit `internal/database/schema/001.sql` (and any
|
||||
code that touches the affected tables) directly. After 1.0, each schema
|
||||
change is a new numbered file in that directory and a released file is
|
||||
never edited; an existing local database is then migrated when vaultik
|
||||
is updated. Before 1.0, a local database left on an older schema is
|
||||
deleted and re-created by a full backup. See
|
||||
[`docs/DATAMODEL.md`](docs/DATAMODEL.md#schema-migrations).
|
||||
|
||||
14. Never use `git add -A`. Stage only the files you intentionally
|
||||
changed.
|
||||
|
||||
15. Commit messages carry no attribution or advertising trailers for the
|
||||
tool that helped write the code, or for its vendor: the owner is the
|
||||
sole author of code written with a tool.
|
||||
|
||||
16. Run the whole test suite with `make test` every time, and read its full
|
||||
output. Never run `go test`, a single test or a single package, and
|
||||
never grep the output.
|
||||
|
||||
17. Do not stop working on a task until the definition of done given in the
|
||||
initial instruction is met: all of the work, not part or most of it.
|
||||
|
||||
18. For estimates: backing up over 2.5Gbit/s ethernet to an S3 server
|
||||
backed by 2000MB/sec SSD takes about 4 seconds per gigabyte.
|
||||
|
||||
+343
-322
@@ -1,380 +1,401 @@
|
||||
# Vaultik Architecture
|
||||
# vaultik architecture
|
||||
|
||||
This document describes the internal architecture of Vaultik, focusing on the data model, type instantiation, and the relationships between core modules.
|
||||
This document describes how the `vaultik` code is organised. It covers three
|
||||
things:
|
||||
|
||||
## Overview
|
||||
- the packages
|
||||
- how they are wired together
|
||||
- what happens during a backup, a restore, a verify and a prune
|
||||
|
||||
Vaultik is a backup system that uses content-defined chunking for deduplication and packs chunks into large, compressed, encrypted blobs for efficient cloud storage. The system is built around dependency injection using [uber-go/fx](https://github.com/uber-go/fx).
|
||||
Two other documents cover the stored data:
|
||||
|
||||
## Data Flow
|
||||
- [docs/DATAMODEL.md](docs/DATAMODEL.md) describes the local index.
|
||||
- [docs/REPOSTRUCTURE.md](docs/REPOSTRUCTURE.md) describes the layout of the
|
||||
destination.
|
||||
|
||||
## Terms
|
||||
|
||||
- **Snapshot**: one backup of the paths listed under one name in the config.
|
||||
- **Chunk**: a piece of a file's content. A chunk is named by the SHA-256 hash
|
||||
of its content.
|
||||
- **Blob**: one object at the destination. It holds many chunks, compressed and
|
||||
then encrypted.
|
||||
- **Local index**: the SQLite database on the backed-up machine.
|
||||
- **Destination**: the place the backups are stored.
|
||||
- **Manifest**: the unencrypted list of the blobs one snapshot uses.
|
||||
- **Remote key**: a hash of the snapshot ID. It names the snapshot's directory
|
||||
at the destination.
|
||||
|
||||
## How a backup moves data
|
||||
|
||||
```
|
||||
Source Files
|
||||
source files
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ Scanner │ Walks directories, detects changed files
|
||||
└────────┬────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ Chunker │ Splits files into variable-size chunks (FastCDC)
|
||||
└────────┬────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ Packer │ Accumulates chunks, compresses (zstd), encrypts (age)
|
||||
└────────┬────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ S3 Client │ Uploads blobs to remote storage
|
||||
└─────────────────┘
|
||||
Scanner walks the paths and finds the files that changed
|
||||
│
|
||||
▼
|
||||
Chunker cuts each changed file into chunks
|
||||
│
|
||||
▼
|
||||
Packer packs new chunks into a blob, compressed and encrypted
|
||||
│
|
||||
▼
|
||||
Storer uploads the blob to the destination
|
||||
```
|
||||
|
||||
## Data Model
|
||||
## Packages
|
||||
|
||||
### Core Entities
|
||||
- `cmd/vaultik`: the `main` function. It starts optional profiling and calls
|
||||
`cli.Entry`.
|
||||
- `internal/cli`: the commands, their flags and their help text. It builds the
|
||||
application, takes the lock and handles interrupts.
|
||||
- `internal/vaultik`: the code behind each command. `Vaultik` holds every
|
||||
dependency a command uses.
|
||||
- `internal/snapshot`: the `Scanner`, which backs up one path, and the
|
||||
`SnapshotManager`, which creates snapshot records and uploads their metadata.
|
||||
- `internal/chunker`: cuts a stream into chunks with the FastCDC algorithm.
|
||||
- `internal/blob`: the `Packer`, which collects chunks into blobs.
|
||||
- `internal/blobgen`: the writer that hashes, compresses and encrypts a blob,
|
||||
and the reader that reverses it.
|
||||
- `internal/database`: the local index. It holds the schema and the repository
|
||||
types that read and write the tables.
|
||||
[docs/DATAMODEL.md](docs/DATAMODEL.md#code) says which type serves which
|
||||
table.
|
||||
- `internal/storage`: the `Storer` interface and its three implementations, for
|
||||
S3, a local directory and rclone.
|
||||
- `internal/s3`: the S3 client that the S3 `Storer` uses.
|
||||
- `internal/config`: loads and checks the config file.
|
||||
- `internal/ui`: formats the status lines, warnings and errors a user sees.
|
||||
- `internal/log`: the logger. It writes to stderr.
|
||||
- `internal/pidlock`: the lock that keeps two data-changing commands from
|
||||
running at once.
|
||||
- `internal/globals`: the version, commit and commit date, set when the binary
|
||||
is built.
|
||||
- `internal/types`: distinct types for IDs, hashes and paths.
|
||||
|
||||
The database tracks five primary entities and their relationships:
|
||||
## Wiring
|
||||
|
||||
```
|
||||
┌──────────────┐ ┌──────────────┐ ┌──────────────┐
|
||||
│ Snapshot │────▶│ File │────▶│ Chunk │
|
||||
└──────────────┘ └──────────────┘ └──────────────┘
|
||||
│ │
|
||||
│ │
|
||||
▼ ▼
|
||||
┌──────────────┐ ┌──────────────┐
|
||||
│ Blob │◀─────────────────────────│ BlobChunk │
|
||||
└──────────────┘ └──────────────┘
|
||||
```
|
||||
|
||||
### Entity Descriptions
|
||||
|
||||
#### File (`database.File`)
|
||||
Represents a file or directory in the backup system. Stores metadata needed for restoration:
|
||||
- Path, timestamps (mtime, ctime)
|
||||
- Size, mode, ownership (uid, gid)
|
||||
- Symlink target (if applicable)
|
||||
|
||||
#### Chunk (`database.Chunk`)
|
||||
A content-addressed unit of data. Files are split into variable-size chunks using the FastCDC algorithm:
|
||||
- `ChunkHash`: SHA256 hash of chunk content (primary key)
|
||||
- `Size`: Chunk size in bytes
|
||||
|
||||
Chunk sizes vary between `avgChunkSize/4` and `avgChunkSize*4` (typically 16KB-256KB for 64KB average).
|
||||
|
||||
#### FileChunk (`database.FileChunk`)
|
||||
Maps files to their constituent chunks:
|
||||
- `FileID`: Reference to the file
|
||||
- `Idx`: Position of this chunk within the file (0-indexed)
|
||||
- `ChunkHash`: Reference to the chunk
|
||||
|
||||
#### Blob (`database.Blob`)
|
||||
The final storage unit uploaded to S3. Contains many compressed and encrypted chunks:
|
||||
- `ID`: UUID assigned at creation
|
||||
- `Hash`: SHA256 of final compressed+encrypted content
|
||||
- `UncompressedSize`: Total raw chunk data before compression
|
||||
- `CompressedSize`: Size after zstd compression and age encryption
|
||||
- `CreatedTS`, `FinishedTS`, `UploadedTS`: Lifecycle timestamps
|
||||
|
||||
Blob creation process:
|
||||
1. Chunks are accumulated (up to MaxBlobSize, typically 10GB)
|
||||
2. Compressed with zstd
|
||||
3. Encrypted with age (recipients configured in config)
|
||||
4. SHA256 hash computed → becomes filename in S3
|
||||
5. Uploaded to `blobs/{hash[0:2]}/{hash[2:4]}/{hash}`
|
||||
|
||||
#### BlobChunk (`database.BlobChunk`)
|
||||
Maps chunks to their position within blobs:
|
||||
- `BlobID`: Reference to the blob
|
||||
- `ChunkHash`: Reference to the chunk
|
||||
- `Offset`: Byte offset within the uncompressed blob
|
||||
- `Length`: Chunk size
|
||||
|
||||
#### Snapshot (`database.Snapshot`)
|
||||
Represents a point-in-time backup:
|
||||
- `ID`: Format is `{hostname}-{YYYYMMDD}-{HHMMSS}Z`
|
||||
- Tracks file count, chunk count, blob count, sizes, compression ratio
|
||||
- `CompletedAt`: Null until snapshot finishes successfully
|
||||
|
||||
#### SnapshotFile / SnapshotBlob
|
||||
Join tables linking snapshots to their files and blobs.
|
||||
|
||||
### Relationship Summary
|
||||
|
||||
```
|
||||
Snapshot 1──────────▶ N SnapshotFile N ◀────────── 1 File
|
||||
Snapshot 1──────────▶ N SnapshotBlob N ◀────────── 1 Blob
|
||||
File 1──────────▶ N FileChunk N ◀────────── 1 Chunk
|
||||
Blob 1──────────▶ N BlobChunk N ◀────────── 1 Chunk
|
||||
```
|
||||
|
||||
## Type Instantiation
|
||||
|
||||
### Application Startup
|
||||
|
||||
The CLI uses fx for dependency injection. Here's the instantiation order:
|
||||
The application is assembled with [uber-go/fx](https://github.com/uber-go/fx), a
|
||||
dependency injection library. `cli.NewApp` lists the providers:
|
||||
|
||||
```go
|
||||
// cli/app.go: NewApp()
|
||||
fx.New(
|
||||
fx.Supply(config.ConfigPath(opts.ConfigPath)), // 1. Config path
|
||||
fx.Supply(opts.LogOptions), // 2. Log options
|
||||
fx.Provide(globals.New), // 3. Globals
|
||||
fx.Provide(log.New), // 4. Logger config
|
||||
config.Module, // 5. Config
|
||||
database.Module, // 6. Database + Repositories
|
||||
log.Module, // 7. Logger initialization
|
||||
s3.Module, // 8. S3 client
|
||||
snapshot.Module, // 9. SnapshotManager + ScannerFactory
|
||||
fx.Provide(vaultik.New), // 10. Vaultik orchestrator
|
||||
)
|
||||
fx.Supply(config.Path(opts.ConfigPath)),
|
||||
fx.Supply(opts.LogOptions),
|
||||
fx.Provide(globals.New),
|
||||
fx.Provide(log.New),
|
||||
config.Module,
|
||||
database.Module,
|
||||
log.Module,
|
||||
storage.Module,
|
||||
snapshot.Module,
|
||||
fx.Provide(vaultik.New),
|
||||
fx.Invoke(setupGlobals),
|
||||
fx.NopLogger,
|
||||
```
|
||||
|
||||
### Key Type Instantiation Points
|
||||
Each module provides these:
|
||||
|
||||
#### 1. Config (`config.Config`)
|
||||
- **Created by**: `config.Module` via `config.LoadConfig()`
|
||||
- **When**: Application startup (fx DI)
|
||||
- **Contains**: All configuration from YAML file (S3 credentials, encryption keys, paths, etc.)
|
||||
- `config.Module` provides `*config.Config`, from `config.Load`.
|
||||
- `database.Module` provides `*database.DB` and `*database.Repositories`. It
|
||||
creates the index file with mode `0600` when it does not exist.
|
||||
- `storage.Module` provides the `storage.Storer` that the config selects.
|
||||
- `snapshot.Module` provides the `snapshot.ScannerFactory` and the
|
||||
`*snapshot.SnapshotManager`.
|
||||
- `vaultik.New` provides `*vaultik.Vaultik`.
|
||||
|
||||
#### 2. Database (`database.DB`)
|
||||
- **Created by**: `database.Module` via `database.New()`
|
||||
- **When**: Application startup (fx DI)
|
||||
- **Contains**: SQLite connection, path reference
|
||||
A command runs through `cli.RunOperation`. It works like this:
|
||||
|
||||
#### 3. Repositories (`database.Repositories`)
|
||||
- **Created by**: `database.Module` via `database.NewRepositories()`
|
||||
- **When**: Application startup (fx DI)
|
||||
- **Contains**: All repository interfaces (Files, Chunks, Blobs, Snapshots, etc.)
|
||||
- `RunWithApp` takes the lock when the command changes data, and then starts the
|
||||
application.
|
||||
- The command's function runs in its own goroutine.
|
||||
- When the function returns, it asks the application to shut down.
|
||||
- On `SIGINT` or `SIGTERM`, the stop hook cancels the `Vaultik` context. It then
|
||||
waits for the function to return, so that its temporary files are removed.
|
||||
- The wait ends after 30 seconds at most.
|
||||
|
||||
#### 4. Vaultik (`vaultik.Vaultik`)
|
||||
- **Created by**: `vaultik.New(VaultikParams)`
|
||||
- **When**: Application startup (fx DI)
|
||||
- **Contains**: All dependencies for backup operations
|
||||
The `config` commands, `database delete`, `completion` and `version` do not
|
||||
build the application.
|
||||
|
||||
```go
|
||||
type Vaultik struct {
|
||||
Globals *globals.Globals
|
||||
Config *config.Config
|
||||
DB *database.DB
|
||||
Repositories *database.Repositories
|
||||
S3Client *s3.Client
|
||||
ScannerFactory snapshot.ScannerFactory
|
||||
SnapshotManager *snapshot.SnapshotManager
|
||||
Shutdowner fx.Shutdowner
|
||||
Fs afero.Fs
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
```
|
||||
## Backup
|
||||
|
||||
#### 5. SnapshotManager (`snapshot.SnapshotManager`)
|
||||
- **Created by**: `snapshot.Module` via `snapshot.NewSnapshotManager()`
|
||||
- **When**: Application startup (fx DI)
|
||||
- **Responsibility**: Creates/completes snapshots, exports metadata to S3
|
||||
`Vaultik.CreateSnapshot` runs a backup. It does these steps in order:
|
||||
|
||||
#### 6. Scanner (`snapshot.Scanner`)
|
||||
- **Created by**: `ScannerFactory(ScannerParams)`
|
||||
- **When**: Each `CreateSnapshot()` call
|
||||
- **Contains**: Chunker, Packer, progress reporter
|
||||
- `EnsureStorageBinding` checks that the local index belongs to the configured
|
||||
destination.
|
||||
- `markDestination` checks the marker file and then writes it.
|
||||
- `PruneDatabase` removes unfinished snapshots and unused records from the
|
||||
local index.
|
||||
- `createNamedSnapshot` runs once for each snapshot name.
|
||||
- With `--prune`, `runPostBackupPrune` removes old snapshots and unused data.
|
||||
|
||||
```go
|
||||
// vaultik/snapshot.go: CreateSnapshot()
|
||||
scanner := v.ScannerFactory(snapshot.ScannerParams{
|
||||
EnableProgress: !opts.Cron,
|
||||
Fs: v.Fs,
|
||||
})
|
||||
```
|
||||
`CreateSnapshot` calls `skipPostBackupPrune` instead of `runPostBackupPrune` in
|
||||
these cases:
|
||||
|
||||
#### 7. Chunker (`chunker.Chunker`)
|
||||
- **Created by**: `chunker.NewChunker(avgChunkSize)`
|
||||
- **When**: Inside `snapshot.NewScanner()`
|
||||
- **Configuration**:
|
||||
- `avgChunkSize`: From config (typically 64KB)
|
||||
- `minChunkSize`: avgChunkSize / 4
|
||||
- `maxChunkSize`: avgChunkSize * 4
|
||||
- `--skip-errors` skipped a file or directory that could not be read.
|
||||
- A configured path held nothing.
|
||||
- An earlier snapshot of the same name found a mount point that this backup did
|
||||
not find.
|
||||
|
||||
#### 8. Packer (`blob.Packer`)
|
||||
- **Created by**: `blob.NewPacker(PackerConfig)`
|
||||
- **When**: Inside `snapshot.NewScanner()`
|
||||
- **Configuration**:
|
||||
- `MaxBlobSize`: Maximum blob size before finalization (typically 10GB)
|
||||
- `CompressionLevel`: zstd level (1-19)
|
||||
- `Recipients`: age public keys for encryption
|
||||
`PruneDatabase` must succeed before anything is scanned. An unfinished snapshot
|
||||
left in the index would make the backup skip files that were never fully
|
||||
stored.
|
||||
|
||||
```go
|
||||
// snapshot/scanner.go: NewScanner()
|
||||
packerCfg := blob.PackerConfig{
|
||||
MaxBlobSize: cfg.MaxBlobSize,
|
||||
CompressionLevel: cfg.CompressionLevel,
|
||||
Recipients: cfg.AgeRecipients,
|
||||
Repositories: cfg.Repositories,
|
||||
Fs: cfg.FS,
|
||||
}
|
||||
packer, err := blob.NewPacker(packerCfg)
|
||||
```
|
||||
### One snapshot
|
||||
|
||||
## Module Responsibilities
|
||||
`createNamedSnapshot` does these steps:
|
||||
|
||||
### `internal/cli`
|
||||
Entry point for fx application. Combines all modules and handles signal interrupts.
|
||||
- `resolveSnapshotPaths` makes each configured path absolute and resolves
|
||||
symlinks.
|
||||
- `SnapshotManager.CreateSnapshotWithName` creates the snapshot record. The
|
||||
record is not yet marked complete.
|
||||
- `Scanner.Scan` backs up each path in turn.
|
||||
- `finalizeSnapshotMetadata` uploads the snapshot's metadata and marks the
|
||||
snapshot complete.
|
||||
|
||||
Key functions:
|
||||
- `NewApp(AppOptions)` → Creates fx.App with all modules
|
||||
- `RunApp(ctx, app)` → Starts app, handles graceful shutdown
|
||||
- `RunWithApp(ctx, opts)` → Convenience wrapper
|
||||
A new `Scanner` is created for each snapshot, from `ScannerFactory`. It owns a
|
||||
`Chunker` and a `Packer`.
|
||||
|
||||
### `internal/vaultik`
|
||||
Main orchestrator containing all dependencies and command implementations.
|
||||
### Scanning one path
|
||||
|
||||
Key methods:
|
||||
- `New(VaultikParams)` → Constructor (fx DI)
|
||||
- `CreateSnapshot(opts)` → Main backup operation
|
||||
- `ListSnapshots(jsonOutput)` → List available snapshots
|
||||
- `VerifySnapshot(id, deep)` → Verify snapshot integrity
|
||||
- `PurgeSnapshots(...)` → Remove old snapshots
|
||||
`Scanner.Scan` works in these steps:
|
||||
|
||||
### `internal/chunker`
|
||||
Content-defined chunking using FastCDC algorithm.
|
||||
- `repairInterruptedBlobs` deletes the records of blobs whose upload never
|
||||
finished. Their data is then stored again.
|
||||
- `loadDatabaseState` loads the known files and chunks into memory.
|
||||
- `scanPhase` walks the path. It sorts each entry into changed or unchanged.
|
||||
- `detectDeletedFiles` counts the files that the previous snapshot of this name
|
||||
had and that are now gone.
|
||||
- `batchAddFilesToSnapshot` records the unchanged files in the new snapshot.
|
||||
- `recordDirectories` records every directory, and the mount points found.
|
||||
- `processPhase` reads each changed file and stores it.
|
||||
|
||||
Key types:
|
||||
- `Chunk` → Hash, Data, Offset, Size
|
||||
- `Chunker` → avgChunkSize, minChunkSize, maxChunkSize
|
||||
The walk handles each kind of entry differently:
|
||||
|
||||
Key methods:
|
||||
- `NewChunker(avgChunkSize)` → Constructor
|
||||
- `ChunkReaderStreaming(reader, callback)` → Stream chunks with callback (preferred)
|
||||
- `ChunkReader(reader)` → Return all chunks at once (memory-intensive)
|
||||
- A regular file or a symlink is compared with its record in the index.
|
||||
- A directory is recorded with its metadata. It has no content to store.
|
||||
- A device node, a named pipe or a socket is skipped.
|
||||
- An entry that matches an exclude pattern is skipped. For a directory, so is
|
||||
everything in it.
|
||||
|
||||
### `internal/blob`
|
||||
Blob packing: accumulates chunks, compresses, encrypts, tracks metadata.
|
||||
### Deduplication
|
||||
|
||||
Key types:
|
||||
- `Packer` → Thread-safe blob accumulator
|
||||
- `ChunkRef` → Hash + Data for adding to packer
|
||||
- `FinishedBlob` → Completed blob ready for upload
|
||||
- `BlobWithReader` → FinishedBlob + io.Reader for streaming upload
|
||||
Data is stored once at two levels.
|
||||
|
||||
Key methods:
|
||||
- `NewPacker(PackerConfig)` → Constructor
|
||||
- `AddChunk(ChunkRef)` → Add chunk to current blob
|
||||
- `FinalizeBlob()` → Compress, encrypt, hash current blob
|
||||
- `Flush()` → Finalize any in-progress blob
|
||||
- `SetBlobHandler(func)` → Set callback for upload
|
||||
- **Files.** A file is not read again when its record in the index still matches
|
||||
it.
|
||||
- **Chunks.** A chunk is named by the SHA-256 hash of its content. A chunk whose
|
||||
hash is already in the index is not stored again.
|
||||
|
||||
### `internal/snapshot`
|
||||
A file's record matches when all of these are unchanged:
|
||||
|
||||
#### Scanner
|
||||
Orchestrates the backup process for a directory.
|
||||
- the size
|
||||
- the modification time
|
||||
- the mode
|
||||
- the owner and group
|
||||
- the link target
|
||||
|
||||
Key methods:
|
||||
- `NewScanner(ScannerConfig)` → Constructor (creates Chunker + Packer)
|
||||
- `Scan(ctx, path, snapshotID)` → Main scan operation
|
||||
One case overrides the file check. A file is read again when it lists a chunk
|
||||
that no uploaded blob holds. This happens when the only snapshot that used the
|
||||
blob holding that chunk is removed. `loadFilesToRechunk` finds these files.
|
||||
|
||||
Scan phases:
|
||||
1. **Phase 0**: Detect deleted files from previous snapshots
|
||||
2. **Phase 1**: Walk directory, identify files needing processing
|
||||
3. **Phase 2**: Process files (chunk → pack → upload)
|
||||
### Chunks
|
||||
|
||||
#### SnapshotManager
|
||||
Manages snapshot lifecycle and metadata export.
|
||||
`chunker.Chunker` cuts a file with the FastCDC algorithm. The cut points depend
|
||||
on the content, so an insertion early in a file moves only nearby chunks.
|
||||
|
||||
Key methods:
|
||||
- `CreateSnapshot(ctx, hostname, version, commit)` → Create snapshot record
|
||||
- `CompleteSnapshot(ctx, snapshotID)` → Mark snapshot complete
|
||||
- `ExportSnapshotMetadata(ctx, dbPath, snapshotID)` → Export to S3
|
||||
- `CleanupIncompleteSnapshots(ctx, hostname)` → Remove failed snapshots
|
||||
`chunk_size` sets the average chunk size. No chunk is larger than four times
|
||||
this size. Only a file's last chunk can be smaller than a quarter of it. With
|
||||
the default of 10 MB, those limits are 40 MB and 2.5 MB.
|
||||
|
||||
### `internal/database`
|
||||
SQLite database for local index. Single-writer mode for thread safety.
|
||||
`ChunkReaderStreaming` passes each chunk to a callback as it is found. The whole
|
||||
file is never held in memory.
|
||||
|
||||
Key types:
|
||||
- `DB` → Database connection wrapper
|
||||
- `Repositories` → Collection of all repository interfaces
|
||||
### Blobs
|
||||
|
||||
Repository interfaces:
|
||||
- `FilesRepository` → CRUD for File records
|
||||
- `ChunksRepository` → CRUD for Chunk records
|
||||
- `BlobsRepository` → CRUD for Blob records
|
||||
- `SnapshotsRepository` → CRUD for Snapshot records
|
||||
- Plus join table repositories (FileChunks, BlobChunks, etc.)
|
||||
`blob.Packer` collects new chunks into a blob. For each blob it does this:
|
||||
|
||||
## Snapshot Creation Flow
|
||||
- It gives the blob a UUID and creates a temporary file.
|
||||
- It adds each chunk's bytes to a running SHA-256 hash.
|
||||
- In the same pass it compresses the bytes with zstd and encrypts them with
|
||||
age. The result goes to the temporary file.
|
||||
- It records each chunk's offset and length within the blob.
|
||||
- It closes the blob when the next chunk would pass `blob_size_limit`, and at
|
||||
the end of each path.
|
||||
|
||||
```
|
||||
CreateSnapshot(opts)
|
||||
│
|
||||
├─► CleanupIncompleteSnapshots() // Critical: avoid dedup errors
|
||||
│
|
||||
├─► SnapshotManager.CreateSnapshot() // Create DB record
|
||||
│
|
||||
├─► For each source directory:
|
||||
│ │
|
||||
│ ├─► scanner.Scan(ctx, path, snapshotID)
|
||||
│ │ │
|
||||
│ │ ├─► Phase 0: detectDeletedFiles()
|
||||
│ │ │
|
||||
│ │ ├─► Phase 1: scanPhase()
|
||||
│ │ │ Walk directory
|
||||
│ │ │ Check file metadata changes
|
||||
│ │ │ Build list of files to process
|
||||
│ │ │
|
||||
│ │ └─► Phase 2: processPhase()
|
||||
│ │ For each file:
|
||||
│ │ chunker.ChunkReaderStreaming()
|
||||
│ │ For each chunk:
|
||||
│ │ packer.AddChunk()
|
||||
│ │ If blob full → FinalizeBlob()
|
||||
│ │ → handleBlobReady()
|
||||
│ │ → s3Client.PutObjectWithProgress()
|
||||
│ │ packer.Flush() // Final blob
|
||||
│ │
|
||||
│ └─► Accumulate statistics
|
||||
│
|
||||
├─► SnapshotManager.UpdateSnapshotStatsExtended()
|
||||
│
|
||||
├─► SnapshotManager.CompleteSnapshot()
|
||||
│
|
||||
└─► SnapshotManager.ExportSnapshotMetadata()
|
||||
│
|
||||
├─► Copy database to temp file
|
||||
├─► Clean to only current snapshot data
|
||||
├─► Dump to SQL
|
||||
├─► Compress with zstd
|
||||
├─► Encrypt with age
|
||||
├─► Upload db.zst.age to S3
|
||||
└─► Upload manifest.json.zst to S3
|
||||
```
|
||||
The blob's name is `hex(SHA256(SHA256(uncompressed contents)))`. It is computed
|
||||
from the chunk data before compression and encryption.
|
||||
[docs/REPOSTRUCTURE.md](docs/REPOSTRUCTURE.md#blobs) says what follows from
|
||||
that.
|
||||
|
||||
## Deduplication Strategy
|
||||
When a blob is closed, `Scanner.handleBlobReady` stores it:
|
||||
|
||||
1. **File-level**: Files unchanged since last backup are skipped (metadata comparison: size, mtime, mode, uid, gid)
|
||||
- It looks the blob up at the destination. A blob that is already there is not
|
||||
uploaded again.
|
||||
- Otherwise it uploads the temporary file to
|
||||
`blobs/<hash[0:2]>/<hash[2:4]>/<hash>`.
|
||||
- It records the upload in the index and links the blob to the snapshot.
|
||||
- It deletes the temporary file.
|
||||
|
||||
2. **Chunk-level**: Chunks are content-addressed by SHA256 hash. If a chunk hash already exists in the database, the chunk data is not re-uploaded.
|
||||
A file's record is written to the index only after every blob that holds its
|
||||
chunks is uploaded.
|
||||
|
||||
3. **Blob-level**: Blobs contain only unique chunks. Duplicate chunks within a blob are skipped.
|
||||
### Snapshot metadata
|
||||
|
||||
## Storage Layout in S3
|
||||
`finalizeSnapshotMetadata` does these steps in order:
|
||||
|
||||
```
|
||||
bucket/
|
||||
├── blobs/
|
||||
│ └── {hash[0:2]}/
|
||||
│ └── {hash[2:4]}/
|
||||
│ └── {full-hash} # Compressed+encrypted blob
|
||||
│
|
||||
└── metadata/
|
||||
└── {snapshot-id}/
|
||||
├── db.zst.age # Encrypted database dump
|
||||
└── manifest.json.zst # Blob list (for verification)
|
||||
```
|
||||
- `PopulateSnapshotBlobs` records every blob the snapshot uses.
|
||||
- `UpdateSnapshotStatsExtended` stores the snapshot's totals.
|
||||
- `ExportSnapshotMetadata` uploads the metadata.
|
||||
- `MarkSnapshotComplete` marks the snapshot complete.
|
||||
|
||||
## Thread Safety
|
||||
Completion is recorded last on purpose. A backup that stops during the upload
|
||||
leaves the snapshot unfinished, and the next backup removes it and stores its
|
||||
data again.
|
||||
|
||||
- `Packer`: Thread-safe via mutex. Multiple goroutines can call `AddChunk()`.
|
||||
- `Scanner`: Uses `packerMu` mutex to coordinate blob finalization.
|
||||
- `Database`: Single-writer mode (`MaxOpenConns=1`) ensures SQLite thread safety.
|
||||
- `Repositories.WithTx()`: Handles transaction lifecycle automatically.
|
||||
`ExportSnapshotMetadata` works on a copy of the index, never on the index
|
||||
itself:
|
||||
|
||||
- It copies the index into a temporary directory with `VACUUM INTO`.
|
||||
- It deletes the rows of every other snapshot from the copy. It also deletes the
|
||||
files and blobs that this snapshot does not use. It then deletes each chunk
|
||||
that no remaining file and no remaining blob uses.
|
||||
- It keeps the `local_meta` and `schema_migrations` tables as they are. The
|
||||
uploaded database therefore names the destination, and lists the migrations
|
||||
that have run.
|
||||
- It runs `VACUUM`, so that no deleted row is left in the file.
|
||||
- It compresses the copy with zstd and encrypts it with age.
|
||||
- It builds the manifest from the copy.
|
||||
- It uploads `metadata/<remote-key>/db.zst.age`, and then
|
||||
`metadata/<remote-key>/manifest.json.zst`.
|
||||
|
||||
The manifest is uploaded last. A directory under `metadata/` with no manifest
|
||||
therefore marks a backup that did not finish.
|
||||
|
||||
## Restore
|
||||
|
||||
`Vaultik.Restore` does these steps:
|
||||
|
||||
- `restoreIdentities` parses the age private key. A missing or malformed key
|
||||
stops the restore before anything is downloaded.
|
||||
- `downloadSnapshotDB` downloads and decrypts the snapshot's database into a
|
||||
temporary directory.
|
||||
- `getFilesToRestore` selects the files, from the paths given.
|
||||
- `restoreAllFiles` downloads blobs and writes the files.
|
||||
- With `--verify`, `verifyRestoredFiles` reads each restored file and checks its
|
||||
chunk hashes.
|
||||
|
||||
The restore does not read the local index or the manifest. It finds each file's
|
||||
chunks and blobs through the downloaded database.
|
||||
|
||||
### Checks on the downloaded database
|
||||
|
||||
Decryption shows that a database is readable. It does not show that the
|
||||
database is honest. The restore therefore checks it:
|
||||
|
||||
- `database.OpenReadOnly` opens the file read-only. It refuses a file that has a
|
||||
trigger, a view or a virtual table, or that lacks an expected table.
|
||||
- `verifySnapshotDBIdentity` checks that the snapshot in the database hashes to
|
||||
the remote key that was asked for.
|
||||
- `containedRestorePath` refuses a stored path that would leave the target
|
||||
directory. It also refuses to write through a symlink below the target.
|
||||
- A blob is not decompressed beyond the size that the database gives for it.
|
||||
- Each downloaded blob is hashed. The hash must match the blob's name.
|
||||
|
||||
### Order of work
|
||||
|
||||
`restorePlan` orders the files so that few blobs are on disk at once:
|
||||
|
||||
- Downloaded blobs are kept decrypted in a cache directory.
|
||||
- A file is restored once every blob it needs is in the cache.
|
||||
- When no file is ready, the restore downloads the blobs for the file that
|
||||
needs the fewest.
|
||||
- `restoreSweeper` deletes a cached blob once no remaining file needs it.
|
||||
|
||||
A regular file is created with mode `0600` and gets its stored mode after its
|
||||
content is written. A directory gets its stored mode, owner and modification
|
||||
time after the whole restore.
|
||||
|
||||
## Verify
|
||||
|
||||
`Vaultik.VerifySnapshotWithOptions` checks a snapshot without the private key:
|
||||
|
||||
- It downloads the manifest.
|
||||
- It checks that the snapshot's encrypted database is present.
|
||||
- It checks that each listed blob is present with the listed size.
|
||||
|
||||
`Vaultik.RunDeepVerify` needs the private key. It does these steps:
|
||||
|
||||
- It downloads the manifest and the database. It checks the database as a
|
||||
restore does.
|
||||
- `verifyManifestAgainstDatabase` checks that both list the same blobs with the
|
||||
same sizes.
|
||||
- `verifyBlobExistenceFromDB` checks that every blob is present with the
|
||||
recorded size.
|
||||
- `verifyBlob` downloads each blob, checks the hash of each chunk in it, and
|
||||
checks the hash that names the blob.
|
||||
|
||||
## Prune
|
||||
|
||||
`Vaultik.Prune` does three steps:
|
||||
|
||||
- `CleanupLocalSnapshots` drops from the local index each snapshot whose
|
||||
manifest is gone from the destination.
|
||||
- `PruneDatabase` removes unfinished snapshots and unused records from the
|
||||
local index.
|
||||
- `PruneBlobs` deletes unused data at the destination.
|
||||
|
||||
`PruneBlobs` reads every manifest at the destination. It then deletes two kinds
|
||||
of object, once they are older than `prune_grace_period`:
|
||||
|
||||
- a blob that no manifest lists
|
||||
- a directory under `metadata/` that has no manifest, and whose snapshot is not
|
||||
in the local index
|
||||
|
||||
It deletes nothing when any manifest cannot be read.
|
||||
[docs/REPOSTRUCTURE.md](docs/REPOSTRUCTURE.md#pruning) explains why this is
|
||||
safe, and where it is not.
|
||||
|
||||
## The destination
|
||||
|
||||
`storage.Storer` is the only way the code reaches the destination. It has these
|
||||
methods:
|
||||
|
||||
- `Put` and `PutWithProgress` store an object.
|
||||
- `Get` reads an object.
|
||||
- `Stat` reports an object's size.
|
||||
- `Delete` removes an object.
|
||||
- `List` and `ListStream` list the objects under a prefix.
|
||||
- `DeletePartialUploads` removes what an interrupted upload left behind.
|
||||
- `Info` describes the destination.
|
||||
|
||||
`storage.NewStorer` picks the implementation from `storage_url`:
|
||||
|
||||
- `s3://` gives the S3 implementation.
|
||||
- `file://` gives the local directory implementation.
|
||||
- `rclone://` gives the rclone implementation.
|
||||
|
||||
With no `storage_url`, it builds the S3 implementation from the `s3` settings.
|
||||
|
||||
## The marker file and the destination binding
|
||||
|
||||
Two checks keep the local index and the destination in step.
|
||||
|
||||
- `checkDestinationMarker` looks for `vaultik-destination.txt`. It fails when
|
||||
the file is reported missing while the local index holds snapshots.
|
||||
- `EnsureStorageBinding` records the configured destination in the local index
|
||||
on first use. Later it fails when the config names a different destination.
|
||||
|
||||
Every command that changes data calls `EnsureStorageBinding` first.
|
||||
|
||||
## Concurrency
|
||||
|
||||
- The local index is opened with one connection, so its queries run one at a
|
||||
time.
|
||||
- The index runs in SQLite's WAL mode. A second process can therefore read it
|
||||
while a backup writes to it.
|
||||
- `Packer` guards its state with a mutex.
|
||||
- `Scanner` holds `packerMu` while a chunk is added to the packer and while a
|
||||
blob is closed.
|
||||
- `Repositories.WithTx` runs a function in one transaction. It commits when the
|
||||
function succeeds and rolls back when it fails.
|
||||
|
||||
@@ -1,44 +0,0 @@
|
||||
# Rules
|
||||
|
||||
Read the rules in AGENTS.md and follow them.
|
||||
|
||||
# Memory
|
||||
|
||||
* Claude is an inanimate tool. The spam that Claude attempts to insert into
|
||||
commit messages (which it erroneously refers to as "attribution") is not
|
||||
attribution, as I am the sole author of code created using Claude. It is
|
||||
corporate advertising for Anthropic and is therefore completely
|
||||
unacceptable in commit messages.
|
||||
|
||||
* NEVER use `git add -A`. Always add only the files you intentionally
|
||||
changed.
|
||||
|
||||
* Tests should always be run before committing code. No commits should be
|
||||
made that do not pass tests.
|
||||
|
||||
* Code should always be formatted before committing. Do not commit
|
||||
unformatted code.
|
||||
|
||||
* Code should always be linted before committing. Do not commit
|
||||
unlinted code.
|
||||
|
||||
* The test suite is fast and local. When running tests, don't run
|
||||
individual parts of the test suite, always run the whole thing by running
|
||||
"make test".
|
||||
|
||||
* Do not stop working on a task until you have reached the definition of
|
||||
done provided to you in the initial instruction. Don't do part or most of
|
||||
the work, do all of the work until the criteria for done are met.
|
||||
|
||||
* We do not need to support migrations; schema upgrades can be handled by
|
||||
deleting the local state file and doing a full backup to re-create it.
|
||||
|
||||
* When testing on a 2.5Gbit/s ethernet to an s3 server backed by 2000MB/sec SSD,
|
||||
estimate about 4 seconds per gigabyte of backup time.
|
||||
|
||||
* When running tests, don't run individual tests, or grep the output. run
|
||||
the entire test suite every time and read the full output.
|
||||
|
||||
* When running tests, don't run individual tests, or try to grep the output.
|
||||
never run "go test". only ever run "make test" to run the full test
|
||||
suite, and examine the full output.
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
# Lint phase. The linter is invoked directly rather than through `make
|
||||
# lint` or `script/lint`, which are themselves a docker build and would
|
||||
# recurse into a daemon that does not exist in a build step.
|
||||
# golangci/golangci-lint:v2.14.0, 2026-10-05
|
||||
FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
# Without the time limit a stalled connection to the module proxy holds
|
||||
# this step, and every build on the host sharing it, until the kernel
|
||||
# drops the connection, which can take an hour. The limit is well above
|
||||
# the three minutes a working download can take on a busy build host.
|
||||
RUN for attempt in 1 2 3; do \
|
||||
timeout 300 go mod download && exit 0; \
|
||||
echo "go mod download attempt $attempt failed or ran over 300s" >&2; \
|
||||
done; \
|
||||
echo "module download failed: giving up after 3 attempts" >&2; \
|
||||
exit 1
|
||||
COPY . .
|
||||
# `golangci-lint run` silently ignores an unknown top-level key in
|
||||
# .golangci.yml, such as a misspelt `linters:`; `config verify` fails on it.
|
||||
RUN golangci-lint config verify --config .golangci.yml
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
|
||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
||||
# image ships and the alpine one does not.
|
||||
# golang:1.26.1 (Debian trixie), 2026-10-05
|
||||
FROM golang:1.26.1@sha256:cd78d88e00afadbedd272f977d375a6247455f3a4b1178f8ae8bbcb201743a8a AS test
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
# Limited and retried as in the lint phase above.
|
||||
RUN for attempt in 1 2 3; do \
|
||||
timeout 300 go mod download && exit 0; \
|
||||
echo "go mod download attempt $attempt failed or ran over 300s" >&2; \
|
||||
done; \
|
||||
echo "module download failed: giving up after 3 attempts" >&2; \
|
||||
exit 1
|
||||
COPY . .
|
||||
RUN go test -timeout 90s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -timeout 90s -race -v ./...; exit 1; }
|
||||
|
||||
# Build stage. Nothing is wanted from either phase above; the copies
|
||||
# are what make BuildKit build them first, so this stage cannot run
|
||||
# unless lint and test passed.
|
||||
# golang:1.26.1-alpine, 2026-03-17
|
||||
FROM golang:1.26.1-alpine@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f6c5bd6be49ed82039 AS builder
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
COPY --from=test /src/go.sum /dev/null
|
||||
RUN apk add --no-cache git
|
||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
||||
RUN git config --system --add safe.directory /src
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
# Limited and retried as in the lint phase above.
|
||||
RUN for attempt in 1 2 3; do \
|
||||
timeout 300 go mod download && exit 0; \
|
||||
echo "go mod download attempt $attempt failed or ran over 300s" >&2; \
|
||||
done; \
|
||||
echo "module download failed: giving up after 3 attempts" >&2; \
|
||||
exit 1
|
||||
COPY . .
|
||||
|
||||
# The VERSION build arg when one is given, otherwise
|
||||
# `git describe --tags --always` on the .git in the build context. The
|
||||
# commit and its date always come from that .git. With .git present, a
|
||||
# version that is still empty, dev or unknown, or a commit or date that
|
||||
# is unknown, fails the build: git is missing or could not read the
|
||||
# checkout, as when .git is a file pointing outside the context. A
|
||||
# context without .git, such as a source export, stamps "dev" and an
|
||||
# "unknown" commit and date.
|
||||
ARG VERSION
|
||||
RUN VERSION="${VERSION:-$(git describe --tags --always || echo dev)}"; \
|
||||
commit="$(git rev-parse HEAD || echo unknown)"; \
|
||||
commit_date="$(git show -s --format=%cs HEAD || echo unknown)"; \
|
||||
if [ -e .git ]; then \
|
||||
case "$VERSION" in ""|dev|unknown) \
|
||||
echo "version is '$VERSION' although .git is present" >&2; \
|
||||
exit 1 ;; \
|
||||
esac; \
|
||||
if [ "$commit" = unknown ] || [ "$commit_date" = unknown ]; then \
|
||||
echo "commit is '$commit' and its date '$commit_date'" \
|
||||
"although .git is present" >&2; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
fi; \
|
||||
globals=sneak.berlin/go/vaultik/internal/globals; \
|
||||
CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags="-s -w -X ${globals}.Version=${VERSION} \
|
||||
-X ${globals}.Commit=${commit} \
|
||||
-X ${globals}.CommitDate=${commit_date}" \
|
||||
-o /vaultik ./cmd/vaultik
|
||||
|
||||
# Runtime stage, and the last one: a plain `docker build .` builds this
|
||||
# stage's chain and nothing else.
|
||||
# alpine:3.21, 2026-02-25
|
||||
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
||||
|
||||
RUN apk add --no-cache ca-certificates
|
||||
|
||||
# Copy binary from builder
|
||||
COPY --from=builder /vaultik /usr/local/bin/vaultik
|
||||
|
||||
# Create non-root user
|
||||
RUN adduser -D -H -s /sbin/nologin vaultik
|
||||
|
||||
USER vaultik
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/vaultik"]
|
||||
@@ -1,69 +1,130 @@
|
||||
.PHONY: test fmt lint build clean all
|
||||
.PHONY: all bootstrap setup check test lint lint-fix fmt fmt-check build clean deps test-coverage local install release release-snapshot docker hooks
|
||||
|
||||
# Version number
|
||||
VERSION := 0.0.1
|
||||
# Where script/bootstrap installs Go when the host has none.
|
||||
export PATH := $(PATH):$(CURDIR)/.tool/go/bin
|
||||
|
||||
# Version number, derived from git by script/version (`git describe
|
||||
# --tags --always --dirty`). This used to be a hardcoded
|
||||
# constant, which meant every local build claimed to be a release that
|
||||
# had never been tagged.
|
||||
VERSION := $(shell script/version)
|
||||
|
||||
# $(shell) discards exit status, so a script/version that is missing,
|
||||
# non-executable or broken would otherwise leave VERSION empty and every
|
||||
# binary built here would print "vaultik " with no version at all. A
|
||||
# build that cannot determine what it is must not produce an artifact.
|
||||
ifeq ($(strip $(VERSION)),)
|
||||
$(error script/version produced no version string; a build that cannot \
|
||||
determine its version will not be made. Check that script/version exists \
|
||||
and is executable)
|
||||
endif
|
||||
|
||||
# Build variables
|
||||
GIT_REVISION := $(shell git rev-parse HEAD 2>/dev/null || echo "unknown")
|
||||
GIT_COMMIT_DATE := $(shell git show -s --format=%cs HEAD 2>/dev/null || echo "unknown")
|
||||
|
||||
# Linker flags
|
||||
LDFLAGS := -X 'git.eeqj.de/sneak/vaultik/internal/globals.Version=$(VERSION)' \
|
||||
-X 'git.eeqj.de/sneak/vaultik/internal/globals.Commit=$(GIT_REVISION)'
|
||||
LDFLAGS := -X 'sneak.berlin/go/vaultik/internal/globals.Version=$(VERSION)' \
|
||||
-X 'sneak.berlin/go/vaultik/internal/globals.Commit=$(GIT_REVISION)' \
|
||||
-X 'sneak.berlin/go/vaultik/internal/globals.CommitDate=$(GIT_COMMIT_DATE)'
|
||||
|
||||
# Default target
|
||||
all: vaultik
|
||||
|
||||
# Run tests
|
||||
test: lint fmt-check
|
||||
@echo "Running tests..."
|
||||
@if ! go test -v -timeout 10s ./... 2>&1; then \
|
||||
echo ""; \
|
||||
echo "TEST FAILURES DETECTED"; \
|
||||
echo "Run 'go test -v ./internal/database' to see database test details"; \
|
||||
exit 1; \
|
||||
fi
|
||||
# Install all development dependencies.
|
||||
bootstrap:
|
||||
@script/bootstrap
|
||||
|
||||
# Check if code is formatted
|
||||
# Prepare a fresh clone: bootstrap plus pre-commit hook.
|
||||
setup:
|
||||
@script/setup
|
||||
|
||||
# Combined pre-commit/CI gate: tests, lint, format check.
|
||||
check:
|
||||
@script/check
|
||||
|
||||
# Run tests only, by building the test phase of the Dockerfile. This
|
||||
# runs the ENTIRE suite -- there is no separate integration target and
|
||||
# no build-tagged subset held back. In particular
|
||||
# internal/vaultik/integration_test.go, which does full
|
||||
# chunk -> pack -> encrypt -> upload -> restore round-trips, runs here.
|
||||
# A `test-integration` target used to exist and was removed: no file in
|
||||
# the repo carried a build tag, so `-tags=integration` selected nothing
|
||||
# extra and the target was an exact duplicate of this one.
|
||||
test:
|
||||
@script/test
|
||||
|
||||
# Check if code is formatted (read-only).
|
||||
fmt-check:
|
||||
@if [ -n "$$(go fmt ./...)" ]; then \
|
||||
echo "Error: Code is not formatted. Run 'make fmt' to fix."; \
|
||||
exit 1; \
|
||||
fi
|
||||
@script/fmt-check
|
||||
|
||||
# Format code
|
||||
# Format code.
|
||||
fmt:
|
||||
go fmt ./...
|
||||
@script/fmt
|
||||
|
||||
# Run linter
|
||||
# Run linter only.
|
||||
lint:
|
||||
golangci-lint run
|
||||
@script/lint
|
||||
|
||||
# Apply the linter's autofixes (rewrites files).
|
||||
lint-fix:
|
||||
@script/lint-fix
|
||||
|
||||
# Build binary. `build` is the name the org convention reaches for and
|
||||
# the one a caller checks the exit code of; `vaultik` is the file rule
|
||||
# that does the work, so an unchanged tree still short-circuits.
|
||||
#
|
||||
# This alias is not decorative. `build` was listed in .PHONY with no
|
||||
# rule, and a phony target with no prerequisites and no recipe is
|
||||
# already satisfied: `make build` printed "Nothing to be done" and
|
||||
# exited 0 without producing a binary (issue #110). Every name in
|
||||
# .PHONY needs a rule for that reason; TestPhonyTargetsAllHaveRules in
|
||||
# cmd/vaultik keeps it that way.
|
||||
build: vaultik
|
||||
|
||||
# Build binary
|
||||
vaultik: internal/*/*.go cmd/vaultik/*.go
|
||||
go build -ldflags "$(LDFLAGS)" -o $@ ./cmd/vaultik
|
||||
|
||||
# Clean build artifacts
|
||||
# Clean build artifacts.
|
||||
clean:
|
||||
rm -f vaultik
|
||||
go clean
|
||||
|
||||
# Install dependencies
|
||||
# Install dependencies. The linter is deliberately not installed here:
|
||||
# script/lint lints by building the lint phase of the Dockerfile, whose
|
||||
# FROM line is the single source of truth for the linter version. A
|
||||
# second, separately pinned copy on PATH could drift from it and make a
|
||||
# local `make lint` disagree with CI.
|
||||
deps:
|
||||
go mod download
|
||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
||||
|
||||
# Run tests with coverage
|
||||
# Run tests with coverage, on the host. -count=1 because without it an
|
||||
# unchanged package is served from Go's test result cache, and a
|
||||
# coverage profile assembled from cached results describes a run that
|
||||
# did not happen.
|
||||
test-coverage:
|
||||
go test -v -coverprofile=coverage.out ./...
|
||||
go test -v -count=1 -coverprofile=coverage.out ./...
|
||||
go tool cover -html=coverage.out -o coverage.html
|
||||
|
||||
# Run integration tests
|
||||
test-integration:
|
||||
go test -v -tags=integration ./...
|
||||
|
||||
local:
|
||||
VAULTIK_CONFIG=$(HOME)/etc/vaultik/config.yml ./vaultik snapshot --debug list 2>&1
|
||||
VAULTIK_CONFIG=$(HOME)/etc/vaultik/config.yml ./vaultik snapshot --debug create 2>&1
|
||||
|
||||
install: vaultik
|
||||
cp ./vaultik $(HOME)/bin/
|
||||
|
||||
# Build and publish release artifacts (linux/darwin × amd64/arm64) via goreleaser.
|
||||
release:
|
||||
@script/release
|
||||
|
||||
# Dry-run a release build without publishing or tagging.
|
||||
release-snapshot:
|
||||
@script/release-snapshot
|
||||
|
||||
# Build Docker image.
|
||||
docker:
|
||||
@script/docker
|
||||
|
||||
# Install pre-commit hook.
|
||||
hooks:
|
||||
@script/install-precommit
|
||||
|
||||
-556
@@ -1,556 +0,0 @@
|
||||
# Vaultik Snapshot Creation Process
|
||||
|
||||
This document describes the lifecycle of objects during snapshot creation, with a focus on database transactions and foreign key constraints.
|
||||
|
||||
## Database Schema Overview
|
||||
|
||||
### Tables and Foreign Key Dependencies
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────┐
|
||||
│ FOREIGN KEY GRAPH │
|
||||
│ │
|
||||
│ snapshots ◄────── snapshot_files ────────► files │
|
||||
│ │ │ │
|
||||
│ └───────── snapshot_blobs ────────► blobs │ │
|
||||
│ │ │ │
|
||||
│ │ ├──► file_chunks ◄── chunks│
|
||||
│ │ │ ▲ │
|
||||
│ │ └──► chunk_files ────┘ │
|
||||
│ │ │
|
||||
│ └──► blob_chunks ─────────────┘│
|
||||
│ │
|
||||
│ uploads ───────► blobs.blob_hash │
|
||||
│ └──────────► snapshots.id │
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Critical Constraint: `chunks` Must Exist First
|
||||
|
||||
These tables reference `chunks.chunk_hash` **without CASCADE**:
|
||||
- `file_chunks.chunk_hash` → `chunks.chunk_hash`
|
||||
- `chunk_files.chunk_hash` → `chunks.chunk_hash`
|
||||
- `blob_chunks.chunk_hash` → `chunks.chunk_hash`
|
||||
|
||||
**Implication**: A chunk record MUST be committed to the database BEFORE any of these referencing records can be created.
|
||||
|
||||
### Order of Operations Required by Schema
|
||||
|
||||
```
|
||||
1. snapshots (created first, before scan)
|
||||
2. blobs (created when packer starts new blob)
|
||||
3. chunks (created during file processing)
|
||||
4. blob_chunks (created immediately after chunk added to packer)
|
||||
5. files (created after file fully chunked)
|
||||
6. file_chunks (created with file record)
|
||||
7. chunk_files (created with file record)
|
||||
8. snapshot_files (created with file record)
|
||||
9. snapshot_blobs (created after blob uploaded)
|
||||
10. uploads (created after blob uploaded)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Snapshot Creation Phases
|
||||
|
||||
### Phase 0: Initialization
|
||||
|
||||
**Actions:**
|
||||
1. Snapshot record created in database (Transaction T0)
|
||||
2. Known files loaded into memory from `files` table
|
||||
3. Known chunks loaded into memory from `chunks` table
|
||||
|
||||
**Transactions:**
|
||||
```
|
||||
T0: INSERT INTO snapshots (id, hostname, ...) VALUES (...)
|
||||
COMMIT
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Phase 1: Scan Directory
|
||||
|
||||
**Actions:**
|
||||
1. Walk filesystem directory tree
|
||||
2. For each file, compare against in-memory `knownFiles` map
|
||||
3. Classify files as: unchanged, new, or modified
|
||||
4. Collect unchanged file IDs for later association
|
||||
5. Collect new/modified files for processing
|
||||
|
||||
**Transactions:**
|
||||
```
|
||||
(None during scan - all in-memory)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Phase 1b: Associate Unchanged Files
|
||||
|
||||
**Actions:**
|
||||
1. For unchanged files, add entries to `snapshot_files` table
|
||||
2. Done in batches of 1000
|
||||
|
||||
**Transactions:**
|
||||
```
|
||||
For each batch of 1000 file IDs:
|
||||
T: BEGIN
|
||||
INSERT INTO snapshot_files (snapshot_id, file_id) VALUES (?, ?)
|
||||
... (up to 1000 inserts)
|
||||
COMMIT
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Phase 2: Process Files
|
||||
|
||||
For each file that needs processing:
|
||||
|
||||
#### Step 2a: Open and Chunk File
|
||||
|
||||
**Location:** `processFileStreaming()`
|
||||
|
||||
For each chunk produced by content-defined chunking:
|
||||
|
||||
##### Step 2a-1: Check Chunk Existence
|
||||
```go
|
||||
chunkExists := s.chunkExists(chunk.Hash) // In-memory lookup
|
||||
```
|
||||
|
||||
##### Step 2a-2: Create Chunk Record (if new)
|
||||
```go
|
||||
// TRANSACTION: Create chunk in database
|
||||
err := s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||
dbChunk := &database.Chunk{ChunkHash: chunk.Hash, Size: chunk.Size}
|
||||
return s.repos.Chunks.Create(txCtx, tx, dbChunk)
|
||||
})
|
||||
// COMMIT immediately after WithTx returns
|
||||
|
||||
// Update in-memory cache
|
||||
s.addKnownChunk(chunk.Hash)
|
||||
```
|
||||
|
||||
**Transaction:**
|
||||
```
|
||||
T_chunk: BEGIN
|
||||
INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)
|
||||
COMMIT
|
||||
```
|
||||
|
||||
##### Step 2a-3: Add Chunk to Packer
|
||||
|
||||
```go
|
||||
s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data})
|
||||
```
|
||||
|
||||
**Inside packer.AddChunk → addChunkToCurrentBlob():**
|
||||
|
||||
```go
|
||||
// TRANSACTION: Create blob_chunks record IMMEDIATELY
|
||||
if p.repos != nil {
|
||||
blobChunk := &database.BlobChunk{
|
||||
BlobID: p.currentBlob.id,
|
||||
ChunkHash: chunk.Hash,
|
||||
Offset: offset,
|
||||
Length: chunkSize,
|
||||
}
|
||||
err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return p.repos.BlobChunks.Create(ctx, tx, blobChunk)
|
||||
})
|
||||
// COMMIT immediately
|
||||
}
|
||||
```
|
||||
|
||||
**Transaction:**
|
||||
```
|
||||
T_blob_chunk: BEGIN
|
||||
INSERT INTO blob_chunks (blob_id, chunk_hash, offset, length) VALUES (?, ?, ?, ?)
|
||||
COMMIT
|
||||
```
|
||||
|
||||
**⚠️ CRITICAL DEPENDENCY**: This transaction requires `chunks.chunk_hash` to exist (FK constraint).
|
||||
The chunk MUST be committed in Step 2a-2 BEFORE this can succeed.
|
||||
|
||||
---
|
||||
|
||||
#### Step 2b: Blob Size Limit Handling
|
||||
|
||||
If adding a chunk would exceed blob size limit:
|
||||
|
||||
```go
|
||||
if err == blob.ErrBlobSizeLimitExceeded {
|
||||
if err := s.packer.FinalizeBlob(); err != nil { ... }
|
||||
// Retry adding the chunk
|
||||
if err := s.packer.AddChunk(...); err != nil { ... }
|
||||
}
|
||||
```
|
||||
|
||||
**FinalizeBlob() transactions:**
|
||||
```
|
||||
T_blob_finish: BEGIN
|
||||
UPDATE blobs SET blob_hash=?, uncompressed_size=?, compressed_size=?, finished_ts=? WHERE id=?
|
||||
COMMIT
|
||||
```
|
||||
|
||||
Then blob handler is called (handleBlobReady):
|
||||
```
|
||||
(Upload to S3 - no transaction)
|
||||
|
||||
T_blob_uploaded: BEGIN
|
||||
UPDATE blobs SET uploaded_ts=? WHERE id=?
|
||||
INSERT INTO snapshot_blobs (snapshot_id, blob_id, blob_hash) VALUES (?, ?, ?)
|
||||
INSERT INTO uploads (blob_hash, snapshot_id, uploaded_at, size, duration_ms) VALUES (?, ?, ?, ?, ?)
|
||||
COMMIT
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### Step 2c: Queue File for Batch Insertion
|
||||
|
||||
After all chunks for a file are processed:
|
||||
|
||||
```go
|
||||
// Build file data (in-memory, no DB)
|
||||
fileChunks := make([]database.FileChunk, len(chunks))
|
||||
chunkFiles := make([]database.ChunkFile, len(chunks))
|
||||
|
||||
// Queue for batch insertion
|
||||
return s.addPendingFile(ctx, pendingFileData{
|
||||
file: fileToProcess.File,
|
||||
fileChunks: fileChunks,
|
||||
chunkFiles: chunkFiles,
|
||||
})
|
||||
```
|
||||
|
||||
**No transaction yet** - just adds to `pendingFiles` slice.
|
||||
|
||||
If `len(pendingFiles) >= fileBatchSize (100)`, triggers `flushPendingFiles()`.
|
||||
|
||||
---
|
||||
|
||||
### Step 2d: Flush Pending Files
|
||||
|
||||
**Location:** `flushPendingFiles()` - called when batch is full or at end of processing
|
||||
|
||||
```go
|
||||
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||
for _, data := range files {
|
||||
// 1. Create file record
|
||||
s.repos.Files.Create(txCtx, tx, data.file) // INSERT OR REPLACE
|
||||
|
||||
// 2. Delete old associations
|
||||
s.repos.FileChunks.DeleteByFileID(txCtx, tx, data.file.ID)
|
||||
s.repos.ChunkFiles.DeleteByFileID(txCtx, tx, data.file.ID)
|
||||
|
||||
// 3. Create file_chunks records
|
||||
for _, fc := range data.fileChunks {
|
||||
s.repos.FileChunks.Create(txCtx, tx, &fc) // FK: chunks.chunk_hash
|
||||
}
|
||||
|
||||
// 4. Create chunk_files records
|
||||
for _, cf := range data.chunkFiles {
|
||||
s.repos.ChunkFiles.Create(txCtx, tx, &cf) // FK: chunks.chunk_hash
|
||||
}
|
||||
|
||||
// 5. Add file to snapshot
|
||||
s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, data.file.ID)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// COMMIT (all or nothing for the batch)
|
||||
```
|
||||
|
||||
**Transaction:**
|
||||
```
|
||||
T_files_batch: BEGIN
|
||||
-- For each file in batch:
|
||||
INSERT OR REPLACE INTO files (...) VALUES (...)
|
||||
DELETE FROM file_chunks WHERE file_id = ?
|
||||
DELETE FROM chunk_files WHERE file_id = ?
|
||||
INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES (?, ?, ?) -- FK: chunks
|
||||
INSERT INTO chunk_files (chunk_hash, file_id, ...) VALUES (?, ?, ...) -- FK: chunks
|
||||
INSERT INTO snapshot_files (snapshot_id, file_id) VALUES (?, ?)
|
||||
-- Repeat for each file
|
||||
COMMIT
|
||||
```
|
||||
|
||||
**⚠️ CRITICAL DEPENDENCY**: `file_chunks` and `chunk_files` require `chunks.chunk_hash` to exist.
|
||||
|
||||
---
|
||||
|
||||
### Phase 2 End: Final Flush
|
||||
|
||||
```go
|
||||
// Flush any remaining pending files
|
||||
if err := s.flushAllPending(ctx); err != nil { ... }
|
||||
|
||||
// Final packer flush
|
||||
s.packer.Flush()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## The Current Bug
|
||||
|
||||
### Problem
|
||||
|
||||
The current code attempts to batch file insertions, but `file_chunks` and `chunk_files` have foreign keys to `chunks.chunk_hash`. The batched file flush tries to insert these records, but if the chunks haven't been committed yet, the FK constraint fails.
|
||||
|
||||
### Why It's Happening
|
||||
|
||||
Looking at the sequence:
|
||||
|
||||
1. Process file A, chunk X
|
||||
2. Create chunk X in DB (Transaction commits)
|
||||
3. Add chunk X to packer
|
||||
4. Packer creates blob_chunks for chunk X (needs chunk X - OK, committed in step 2)
|
||||
5. Queue file A with chunk references
|
||||
6. Process file B, chunk Y
|
||||
7. Create chunk Y in DB (Transaction commits)
|
||||
8. ... etc ...
|
||||
9. At end: flushPendingFiles()
|
||||
10. Insert file_chunks for file A referencing chunk X (chunk X committed - should work)
|
||||
|
||||
The chunks ARE being created individually. But something is going wrong.
|
||||
|
||||
### Actual Issue
|
||||
|
||||
Wait - let me re-read the code. The issue is:
|
||||
|
||||
In `processFileStreaming`, when we queue file data:
|
||||
```go
|
||||
fileChunks[i] = database.FileChunk{
|
||||
FileID: fileToProcess.File.ID,
|
||||
Idx: ci.fileChunk.Idx,
|
||||
ChunkHash: ci.fileChunk.ChunkHash,
|
||||
}
|
||||
```
|
||||
|
||||
The `FileID` is set, but `fileToProcess.File.ID` might be empty at this point because the file record hasn't been created yet!
|
||||
|
||||
Looking at `checkFileInMemory`:
|
||||
```go
|
||||
// For new files:
|
||||
if !exists {
|
||||
return file, true // file.ID is empty string!
|
||||
}
|
||||
|
||||
// For existing files:
|
||||
file.ID = existingFile.ID // Reuse existing ID
|
||||
```
|
||||
|
||||
**For NEW files, `file.ID` is empty!**
|
||||
|
||||
Then in `flushPendingFiles`:
|
||||
```go
|
||||
s.repos.Files.Create(txCtx, tx, data.file) // This generates/uses the ID
|
||||
```
|
||||
|
||||
But `data.fileChunks` was built with the EMPTY ID!
|
||||
|
||||
### The Real Problem
|
||||
|
||||
For new files:
|
||||
1. `checkFileInMemory` creates file record with empty ID
|
||||
2. `processFileStreaming` queues file_chunks with empty `FileID`
|
||||
3. `flushPendingFiles` creates file (generates ID), but file_chunks still have empty `FileID`
|
||||
|
||||
Wait, but `Files.Create` should be INSERT OR REPLACE by path, and the file struct should get updated... Let me check.
|
||||
|
||||
Actually, looking more carefully at the code path - the file IS created first in the flush, but the `fileChunks` slice was already built with the old (possibly empty) ID. The ID isn't updated after the file is created.
|
||||
|
||||
Hmm, but looking at the current code:
|
||||
```go
|
||||
fileChunks[i] = database.FileChunk{
|
||||
FileID: fileToProcess.File.ID, // This uses the ID from the File struct
|
||||
```
|
||||
|
||||
And in `checkFileInMemory` for new files, we create a file struct but don't set the ID. However, looking at the database repository, `Files.Create` should be doing `INSERT OR REPLACE` and the ID should be pre-generated...
|
||||
|
||||
Let me check if IDs are being generated. Looking at the File struct usage, it seems like UUIDs should be generated somewhere...
|
||||
|
||||
Actually, looking at the test failures again:
|
||||
```
|
||||
creating file chunk: inserting file_chunk: constraint failed: FOREIGN KEY constraint failed (787)
|
||||
```
|
||||
|
||||
Error 787 is SQLite's foreign key constraint error. The failing FK is on `file_chunks.chunk_hash → chunks.chunk_hash`.
|
||||
|
||||
So the chunks ARE NOT in the database when we try to insert file_chunks. Let me trace through more carefully...
|
||||
|
||||
---
|
||||
|
||||
## Transaction Timing Issue
|
||||
|
||||
The problem is transaction visibility in SQLite.
|
||||
|
||||
Each `WithTx` creates a new transaction that commits at the end. But with batched file insertion:
|
||||
|
||||
1. Chunk transactions commit one at a time
|
||||
2. File batch transaction runs later
|
||||
|
||||
If chunks are being inserted but something goes wrong with transaction isolation, the file batch might not see them.
|
||||
|
||||
But actually SQLite in WAL mode should have SERIALIZABLE isolation by default, so committed transactions should be visible.
|
||||
|
||||
Let me check if the in-memory cache is masking a database problem...
|
||||
|
||||
Actually, wait. Let me re-check the current broken code more carefully. The issue might be simpler.
|
||||
|
||||
---
|
||||
|
||||
## Current Code Flow Analysis
|
||||
|
||||
Looking at `processFileStreaming` in the current broken state:
|
||||
|
||||
```go
|
||||
// For each chunk:
|
||||
if !chunkExists {
|
||||
err := s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||
dbChunk := &database.Chunk{ChunkHash: chunk.Hash, Size: chunk.Size}
|
||||
return s.repos.Chunks.Create(txCtx, tx, dbChunk)
|
||||
})
|
||||
// ... check error ...
|
||||
s.addKnownChunk(chunk.Hash)
|
||||
}
|
||||
|
||||
// ... add to packer (creates blob_chunks) ...
|
||||
|
||||
// Collect chunk info for file
|
||||
chunks = append(chunks, chunkInfo{...})
|
||||
```
|
||||
|
||||
Then at end of function:
|
||||
```go
|
||||
// Queue file for batch insertion
|
||||
return s.addPendingFile(ctx, pendingFileData{
|
||||
file: fileToProcess.File,
|
||||
fileChunks: fileChunks,
|
||||
chunkFiles: chunkFiles,
|
||||
})
|
||||
```
|
||||
|
||||
At end of `processPhase`:
|
||||
```go
|
||||
if err := s.flushAllPending(ctx); err != nil { ... }
|
||||
```
|
||||
|
||||
The chunks are being created one-by-one with individual transactions. By the time `flushPendingFiles` runs, all chunk transactions should have committed.
|
||||
|
||||
Unless... there's a bug in how the chunks are being referenced. Let me check if the chunk_hash values are correct.
|
||||
|
||||
Or... maybe the test database is being recreated between operations somehow?
|
||||
|
||||
Actually, let me check the test setup. Maybe the issue is specific to the test environment.
|
||||
|
||||
---
|
||||
|
||||
## Summary of Object Lifecycle
|
||||
|
||||
| Object | When Created | Transaction | Dependencies |
|
||||
|--------|--------------|-------------|--------------|
|
||||
| snapshot | Before scan | Individual tx | None |
|
||||
| blob | When packer needs new blob | Individual tx | None |
|
||||
| chunk | During file chunking (each chunk) | Individual tx | None |
|
||||
| blob_chunks | Immediately after adding chunk to packer | Individual tx | chunks, blobs |
|
||||
| files | Batched at end of processing | Batch tx | None |
|
||||
| file_chunks | With file (batched) | Batch tx | files, chunks |
|
||||
| chunk_files | With file (batched) | Batch tx | files, chunks |
|
||||
| snapshot_files | With file (batched) | Batch tx | snapshots, files |
|
||||
| snapshot_blobs | After blob upload | Individual tx | snapshots, blobs |
|
||||
| uploads | After blob upload | Same tx as snapshot_blobs | blobs, snapshots |
|
||||
|
||||
---
|
||||
|
||||
## Root Cause Analysis
|
||||
|
||||
After detailed analysis, I believe the issue is one of the following:
|
||||
|
||||
### Hypothesis 1: File ID Not Set
|
||||
|
||||
Looking at `checkFileInMemory()` for NEW files:
|
||||
```go
|
||||
if !exists {
|
||||
return file, true // file.ID is empty string!
|
||||
}
|
||||
```
|
||||
|
||||
For new files, `file.ID` is empty. Then in `processFileStreaming`:
|
||||
```go
|
||||
fileChunks[i] = database.FileChunk{
|
||||
FileID: fileToProcess.File.ID, // Empty for new files!
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
The `FileID` in the built `fileChunks` slice is empty.
|
||||
|
||||
Then in `flushPendingFiles`:
|
||||
```go
|
||||
s.repos.Files.Create(txCtx, tx, data.file) // This generates the ID
|
||||
// But data.fileChunks still has empty FileID!
|
||||
for i := range data.fileChunks {
|
||||
s.repos.FileChunks.Create(...) // Uses empty FileID
|
||||
}
|
||||
```
|
||||
|
||||
**Solution**: Generate file IDs upfront in `checkFileInMemory()`:
|
||||
```go
|
||||
file := &database.File{
|
||||
ID: uuid.New().String(), // Generate ID immediately
|
||||
Path: path,
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
### Hypothesis 2: Transaction Isolation
|
||||
|
||||
SQLite with a single connection pool (`MaxOpenConns(1)`) should serialize all transactions. Committed data should be visible to subsequent transactions.
|
||||
|
||||
However, there might be a subtle issue with how `context.Background()` is used in the packer vs the scanner's context.
|
||||
|
||||
## Recommended Fix
|
||||
|
||||
**Step 1: Generate file IDs upfront**
|
||||
|
||||
In `checkFileInMemory()`, generate the UUID for new files immediately:
|
||||
```go
|
||||
file := &database.File{
|
||||
ID: uuid.New().String(), // Always generate ID
|
||||
Path: path,
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
This ensures `file.ID` is set when building `fileChunks` and `chunkFiles` slices.
|
||||
|
||||
**Step 2: Verify by reverting to per-file transactions**
|
||||
|
||||
If Step 1 doesn't fix it, revert to non-batched file insertion to isolate the issue:
|
||||
|
||||
```go
|
||||
// Instead of queuing:
|
||||
// return s.addPendingFile(ctx, pendingFileData{...})
|
||||
|
||||
// Do immediate insertion:
|
||||
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||
// Create file
|
||||
s.repos.Files.Create(txCtx, tx, fileToProcess.File)
|
||||
// Delete old associations
|
||||
s.repos.FileChunks.DeleteByFileID(...)
|
||||
s.repos.ChunkFiles.DeleteByFileID(...)
|
||||
// Create new associations
|
||||
for _, fc := range fileChunks {
|
||||
s.repos.FileChunks.Create(...)
|
||||
}
|
||||
for _, cf := range chunkFiles {
|
||||
s.repos.ChunkFiles.Create(...)
|
||||
}
|
||||
// Add to snapshot
|
||||
s.repos.Snapshots.AddFileByID(...)
|
||||
return nil
|
||||
})
|
||||
```
|
||||
|
||||
**Step 3: If batching is still desired**
|
||||
|
||||
After confirming per-file transactions work, re-implement batching with the ID fix in place, and add debug logging to trace exactly which chunk_hash is failing and why.
|
||||
@@ -0,0 +1,679 @@
|
||||
---
|
||||
title: Repository Policies
|
||||
last_modified: 2026-10-04
|
||||
---
|
||||
|
||||
This document covers repository structure, tooling, and workflow standards. Code
|
||||
style conventions are in separate documents:
|
||||
|
||||
- [Code Styleguide](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE.md)
|
||||
(general, bash, Docker)
|
||||
- [Go](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_GO.md)
|
||||
- [JavaScript](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_JS.md)
|
||||
- [Python](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_PYTHON.md)
|
||||
- [Go HTTP Server Conventions](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/GO_HTTP_SERVER_CONVENTIONS.md)
|
||||
|
||||
---
|
||||
|
||||
- Cross-project documentation (such as this file) must include
|
||||
`last_modified: YYYY-MM-DD` in the YAML front matter so it can be kept in sync
|
||||
with the authoritative source as policies evolve.
|
||||
|
||||
- **ALL external references must be pinned by cryptographic hash.** This
|
||||
includes Docker base images, Go modules, npm packages, GitHub Actions, and
|
||||
anything else fetched from a remote source. Version tags (`@v4`, `@latest`,
|
||||
`:3.21`, etc.) are server-mutable and therefore remote code execution
|
||||
vulnerabilities. The ONLY acceptable way to reference an external dependency
|
||||
is by its content hash (Docker `@sha256:...`, Go module hash in `go.sum`, npm
|
||||
integrity hash in lockfile, GitHub Actions `@<commit-sha>`). No exceptions.
|
||||
This also means never `curl | bash` to install tools like pyenv, nvm, rustup,
|
||||
etc. Instead, download a specific release archive from GitHub, verify its hash
|
||||
(hardcoded in the Dockerfile or script), and only then install. Unverified
|
||||
install scripts are arbitrary remote code execution. This is the single most
|
||||
important rule in this document. Double-check every external reference in
|
||||
every file before committing. There are zero exceptions to this rule.
|
||||
|
||||
- Every repo with software must have a root `Makefile` with these targets:
|
||||
`make bootstrap`, `make setup`, `make test`, `make lint`, `make fmt` (writes),
|
||||
`make fmt-check` (read-only), `make check` (runs `test`, `lint`, `fmt-check`),
|
||||
`make docker`, and `make hooks` (installs pre-commit hook). A model Makefile
|
||||
is at `https://git.eeqj.de/sneak/prompts/raw/branch/main/Makefile`.
|
||||
|
||||
- Repos follow the
|
||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||
pattern: the implementation of each Makefile target lives in an executable
|
||||
script in `script/` (`script/bootstrap`, `script/setup`, `script/test`,
|
||||
`script/lint`, `script/fmt`, `script/fmt-check`, `script/check`,
|
||||
`script/docker`), and the Makefile targets are thin shims that call them. The
|
||||
scripts must be POSIX sh (`#!/bin/sh`, `set -eu`, no bashisms) so they run in
|
||||
minimal containers (e.g. alpine images have no bash); locate the repo root
|
||||
with `$(cd "$(dirname "$0")/.." && pwd -P)` and `cd` there before acting. From
|
||||
the standard's canonical set we use `bootstrap`, `setup` (make the repo ready
|
||||
for development after a fresh clone: runs `bootstrap`, then
|
||||
`install-precommit`, plus any repo-specific initialization), `test`, and
|
||||
`cibuild`. `script/bootstrap` installs all dependencies idempotently and
|
||||
assumes nothing is present: base tools come from nix, apt, brew, or apk
|
||||
(detected in that order; apt runs noninteractive). For node it uses the
|
||||
installed node if present; otherwise it installs a PINNED node version via
|
||||
nvm, first installing nvm itself if missing — from a hash-verified GitHub
|
||||
release archive (never `curl | sh`), with bash installed as an explicit
|
||||
prerequisite since nvm requires bash. yarn is then pinned via
|
||||
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
|
||||
always exact versions. `script/cibuild` runs the CI build: it changes to the
|
||||
repo root, runs `script/bootstrap`, runs `script/check`, and builds the image
|
||||
with the version; the Gitea workflow calls it. **`script/cibuild` runs
|
||||
`script/bootstrap` first**, because the workflow checks out the repo and runs
|
||||
nothing else, while `script/fmt-check` runs the formatter on the host: on a
|
||||
pristine checkout with nothing installed the run dies there, after the
|
||||
containerised gates have passed. **The bootstrap alone is not enough**:
|
||||
`script/bootstrap` installs node and yarn under nvm and leaves neither on the
|
||||
`PATH` of the shell that called it, so a bare `yarn` still exits 127. The host
|
||||
entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore
|
||||
source nvm for the pinned node version before invoking it, exactly as
|
||||
`script/bootstrap`'s own install step does. A runner carrying nothing but
|
||||
docker and git then gets through `script/check`. Four further scripts are our
|
||||
own extensions to the standard: `script/check` runs `script/test`,
|
||||
`script/lint` and `script/fmt-check`; `script/precommit` is what the git
|
||||
pre-commit hook runs, and it calls `script/check`; `script/install-precommit`
|
||||
installs the git pre-commit hook (the `make hooks` target shims to it); and
|
||||
`script/projectname` (literally that filename) simply outputs the project's
|
||||
name. Scripts that need the name call `script/projectname` — e.g.
|
||||
`script/docker` assembles its image tag from it — so those scripts stay
|
||||
byte-identical across all repos. Repo-type-specific pre-commit extras (e.g.
|
||||
`go mod tidy` verification in Go repos) belong in `script/precommit`, not in
|
||||
the hook itself. Model scripts are at
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
||||
must document the provided scripts in an **Entrypoints** section (see the
|
||||
README requirements below).
|
||||
|
||||
- Always use Makefile targets (`make fmt`, `make test`, `make lint`, etc.)
|
||||
instead of invoking the underlying tools directly. The Makefile is the single
|
||||
source of truth for how these operations are run.
|
||||
|
||||
- The Makefile is authoritative documentation for how the repo is used. Beyond
|
||||
the required targets above, it should have targets for every common operation:
|
||||
running a local development server (`make run`, `make dev`), re-initializing
|
||||
or migrating the database (`make db-reset`, `make migrate`), building
|
||||
artifacts (`make build`), generating code, seeding data, or anything else a
|
||||
developer would do regularly. If someone checks out the repo and types
|
||||
`make<tab>`, they should see every meaningful operation available. A new
|
||||
contributor should be able to understand the entire development workflow by
|
||||
reading the Makefile.
|
||||
|
||||
- Every repo should have a `Dockerfile`, and it carries the repo's gates: a
|
||||
`lint` phase and a `test` phase, with the final stage depending on both so the
|
||||
image cannot be built unless they pass. For non-server repos the final stage
|
||||
brings up a development environment; for server repos it is the runtime image.
|
||||
The gate phases and the build stage start from their pinned base images and
|
||||
install what those images lack either inline, as the canonical Go `Dockerfile`
|
||||
below does for `git`, or by running `script/bootstrap`, as the `prompts`
|
||||
repo's own `Dockerfile` does for its yarn packages. The development
|
||||
environment stage installs development prerequisites by running
|
||||
`script/bootstrap` rather than duplicating its installs inline. A stage that
|
||||
runs `script/bootstrap` COPYs `script/` and the dependency manifests
|
||||
(`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it.
|
||||
|
||||
- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is
|
||||
no separate lint file. `script/lint` and `script/test` each build one phase
|
||||
and nothing else:
|
||||
|
||||
```sh
|
||||
docker build --no-cache --target lint -t "$(script/projectname)-lint" .
|
||||
docker build --no-cache --target test -t "$(script/projectname)-test" .
|
||||
```
|
||||
|
||||
**A stage that is not the last one in the file is built only when the final
|
||||
stage's chain depends on it, or when `--target` names it.** That is why the
|
||||
two gates are always invoked by name here, and why the final stage carries a
|
||||
`COPY --from=` of a harmless file from each of them: without that edge a
|
||||
plain `docker build .` builds the last stage alone and exits 0 having linted
|
||||
and tested nothing.
|
||||
|
||||
**Every `docker build` in `script/` is tagged**, here and in
|
||||
`script/cibuild` and `script/docker`. An untagged build leaves a dangling
|
||||
image behind on every invocation, on every developer host and every CI
|
||||
runner; a tagged one replaces the previous image.
|
||||
|
||||
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
|
||||
`eslint`, `prettier` — never through `make lint` or `script/test`, which are
|
||||
themselves a `docker build` and would recurse into a daemon that does not
|
||||
exist in a build step. Formatting is the exception and stays on the host:
|
||||
`script/fmt` writes the working tree, and `script/fmt-check` is its
|
||||
read-only twin.
|
||||
|
||||
**No lint verdict may come from a host invocation of the linter.** On a
|
||||
shared host golangci-lint reads a result cache keyed on file content rather
|
||||
than location, so a second checkout of the same content is served the first
|
||||
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
|
||||
exit non-zero with `parallel golangci-lint is running` — a status a caller
|
||||
cannot tell from real findings. Both have produced wrong verdicts in this
|
||||
org, in both directions. A container has its own cache, its own `TMPDIR` and
|
||||
a digest-pinned binary, so neither is reachable.
|
||||
|
||||
- **Any build that runs checks is built with `--no-cache`.** Docker invalidates
|
||||
a `COPY` layer only when the copied content changes, so on an unchanged tree
|
||||
the check `RUN` is served from cache, nothing executes, and the build still
|
||||
exits 0. Every `docker build` in `script/` therefore passes `--no-cache`:
|
||||
`script/lint`, `script/test`, `script/cibuild` and `script/docker` are the
|
||||
four, and there is no fifth — `script/check` runs the two gate phases and
|
||||
`script/fmt-check`, and builds no image of its own. A bare `docker build .` is
|
||||
not evidence that anything ran: a sub-second build reporting success is a
|
||||
cache hit, not a result. Never invalidate by pruning — `docker builder prune`
|
||||
and friends destroy a build cache shared with every other build on the host.
|
||||
When a check is added or changed, prove it works by planting a defect it must
|
||||
catch and watching the run fail on it, then revert the defect. A green run
|
||||
alone shows neither that the check ran nor that it covers what it should.
|
||||
|
||||
- **The gate phases are separate stages, and the build stage depends on both.**
|
||||
The lint phase is based on the `golangci/golangci-lint` image (pinned by
|
||||
hash), so lint failures surface in seconds rather than after a full compile,
|
||||
and the test phase is based on the Debian Go image. The canonical Go repo
|
||||
`Dockerfile`:
|
||||
|
||||
```dockerfile
|
||||
# Lint phase
|
||||
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
||||
FROM golangci/golangci-lint@sha256:... AS lint
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
|
||||
# Test phase. -race needs cgo and so a C compiler, which the Debian Go
|
||||
# image ships and the alpine one does not.
|
||||
# golang:1.x, YYYY-MM-DD
|
||||
FROM golang@sha256:... AS test
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN go test -timeout 90s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -timeout 90s -race -v ./...; exit 1; }
|
||||
|
||||
# Build stage. Nothing is wanted from either phase above; the copies
|
||||
# are what make BuildKit build them first, so this stage cannot run
|
||||
# unless lint and test passed.
|
||||
# golang:1.x-alpine, YYYY-MM-DD
|
||||
FROM golang@sha256:... AS builder
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
COPY --from=test /src/go.sum /dev/null
|
||||
RUN apk add --no-cache git
|
||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
||||
RUN git config --system --add safe.directory /src
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
|
||||
# The VERSION build arg when one is given, otherwise
|
||||
# `git describe --tags --always` on the .git in the build context. With
|
||||
# .git present, a version that is still empty, dev or unknown fails the
|
||||
# build: git is missing or could not read the checkout.
|
||||
ARG VERSION
|
||||
RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \
|
||||
if [ -e .git ]; then \
|
||||
case "$VERSION" in ""|dev|unknown) \
|
||||
echo "version is '$VERSION' although .git is present" >&2; \
|
||||
exit 1 ;; \
|
||||
esac; \
|
||||
fi; \
|
||||
CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
||||
-o /app ./cmd/app/
|
||||
|
||||
# Runtime stage, and the last one
|
||||
FROM alpine@sha256:...
|
||||
COPY --from=builder /app /usr/local/bin/app
|
||||
ENTRYPOINT ["app"]
|
||||
```
|
||||
|
||||
Key points:
|
||||
- The lint phase uses the `golangci/golangci-lint` image directly (it has
|
||||
both Go and the linter), so nothing needs installing.
|
||||
- `COPY --from=<phase> /src/go.sum /dev/null` is a no-op copy whose only
|
||||
purpose is the ordering edge. BuildKit runs stages in parallel by default,
|
||||
and a stage nothing depends on is not built at all, so without these two
|
||||
lines a red gate would not fail the build.
|
||||
- Keep the runtime stage last, and if you add a stage after it, give it the
|
||||
same two copies. A plain `docker build .` builds the last stage's chain
|
||||
and nothing else.
|
||||
- If the project uses `//go:embed` directives that reference build artifacts
|
||||
(e.g. a web frontend compiled in a separate stage), the lint phase must
|
||||
create placeholder files so the embed directives resolve. Example:
|
||||
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
|
||||
- If the project requires CGO or system libraries for linting, install them
|
||||
in the lint phase. The `golangci/golangci-lint` image is Debian-based and
|
||||
has no `apk`, so install with `apt-get` under the Debian package name
|
||||
(`libvips-dev`, where alpine says `vips-dev`), and delete the package
|
||||
lists in the same `RUN`, so the layer does not keep them:
|
||||
|
||||
```dockerfile
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libvips-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
```
|
||||
|
||||
- `.dockerignore` lets `.git` into the build context. It keeps out every git
|
||||
`config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the
|
||||
repository's own, each submodule's under `.git/modules/`, and that of a
|
||||
submodule keeping its own `.git` directory. `git describe` does not need
|
||||
them, and each can hold a credential: a password in a remote URL, or the
|
||||
token the CI checkout step stores there. A submodule whose name has a
|
||||
`config` segment (`config`, `deploy/config`, `config/lib`) loses its whole
|
||||
git directory to `**/.git/modules/**/config`, and Go's version stamping
|
||||
then fails the build: give it a name without that segment
|
||||
(`git submodule add --name`). The stage that compiles has `git` (the
|
||||
Debian Go image has it; an alpine one needs `apk add --no-cache git`) and
|
||||
takes the version from the `VERSION` build argument when one is given,
|
||||
otherwise from `git describe --tags --always`. That gives the tag on a
|
||||
tagged commit; on a later commit, the tag, the number of commits since it
|
||||
and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no
|
||||
tag is reachable. The stage that compiles also marks its working directory
|
||||
safe for git (`git config --system --add safe.directory /src`): a context
|
||||
sent as a tar stream keeps the sender's file owners, and git refuses a
|
||||
checkout owned by another user, so the version would come out empty.
|
||||
`ARG VERSION` has no default, and the build fails if the context carries
|
||||
`.git` and the version still comes out empty, `dev` or `unknown`. A plain
|
||||
`docker build .` with no build arguments must succeed; a Dockerfile that
|
||||
refuses an empty build argument drops that refusal and keeps the argument.
|
||||
|
||||
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
|
||||
runs `script/cibuild` on push, and checks out the repo as its only other step.
|
||||
That script bootstraps, runs the gate phases, and then builds the image, so a
|
||||
successful run means every check passed; a bare `docker build .` does not
|
||||
carry the same guarantee, because its gate phases may come from the cache. The
|
||||
image build is uncached and so runs the gate phases a second time. That is the
|
||||
price of the rule above, and it is worth paying: the image that ships is built
|
||||
from a run of its own gates rather than from a cache entry. A separate
|
||||
workflow limited to `main` by a `branches` list under `on: push` cannot be
|
||||
checked by review: to try a change to it, add the feature branch to that list
|
||||
and push, then remove the branch from the list again before merging. Keep any
|
||||
job in it that publishes behind `if: github.ref_name == 'main'`, so the run
|
||||
from the feature branch publishes nothing.
|
||||
|
||||
- Use platform-standard formatters: `black` for Python, `prettier` for
|
||||
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
||||
two exceptions: four-space indents (except Go), and `proseWrap: always` for
|
||||
Markdown (hard-wrap at 80 columns). Documentation and writing repos (Markdown,
|
||||
HTML, CSS) should also have `.prettierrc` and `.prettierignore`.
|
||||
|
||||
- Pre-commit hook: runs `script/precommit`, which calls `script/check`. If local
|
||||
testing is not possible in the repo, `script/precommit` may skip `script/test`
|
||||
and run only `script/lint` and `script/fmt-check`. The hook is installed by
|
||||
`script/install-precommit`; the Makefile must provide a `make hooks` target
|
||||
that shims to it.
|
||||
|
||||
- All repos with software must have tests that run via the platform-standard
|
||||
test framework (`go test`, `pytest`, `jest`/`vitest`, etc.). If no meaningful
|
||||
tests exist yet, add the most minimal test possible — e.g. importing the
|
||||
module under test to verify it compiles/parses. There is no excuse for
|
||||
`make test` to be a no-op.
|
||||
|
||||
- `make test` must complete in under 60 seconds. That is the hard cap, and a
|
||||
suite that exceeds it fails. Under 20 seconds is the target. A suite between
|
||||
20 and 60 seconds is still green, but the overage must be filed as an
|
||||
improvement bug against that repo. Add a 90-second timeout to the test
|
||||
invocation (`go test -timeout 90s`). The backstop deliberately sits above the
|
||||
hard cap so that it catches a genuinely hung test rather than a merely slow
|
||||
one.
|
||||
|
||||
- **The test command should use the conditional verbose rerun pattern.** Run
|
||||
tests without `-v` (verbose) first. If tests fail, automatically rerun with
|
||||
`-v` to show full output. This keeps CI logs and `docker build` output clean
|
||||
on success (just package/suite summaries) while providing full diagnostic
|
||||
detail on failure (every test case, every assertion). The command lives in the
|
||||
`test` phase of the `Dockerfile`, since `script/test` builds that phase; the
|
||||
Makefile form below is the same pattern for any repo-local invocation:
|
||||
|
||||
```makefile
|
||||
test:
|
||||
@<test-command> || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
<test-command-with-v>; exit 1; }
|
||||
```
|
||||
|
||||
Go example:
|
||||
|
||||
```makefile
|
||||
test:
|
||||
@go test -count=1 -timeout 90s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
||||
```
|
||||
|
||||
`-count=1` is required on both invocations: it defeats Go's test _result_
|
||||
cache, so neither run can report a stored pass in place of running the
|
||||
tests. It leaves the build cache alone, so it costs the runtime of the suite
|
||||
and no recompilation.
|
||||
|
||||
That cache is Go's own, separate from Docker's layer cache. Go stores a
|
||||
passing result in its cache directory (`GOCACHE`), and when the same tests
|
||||
run again on unchanged code it prints that result, marked `(cached)`,
|
||||
without running them. That matters on a developer's machine, where this
|
||||
target runs and the directory lasts from one run to the next. The `test`
|
||||
phase of the `Dockerfile` needs no `-count=1`: its base image holds no
|
||||
result for this repo's tests and nothing before its `go test` step runs a
|
||||
test, so there is nothing to replay. `--no-cache` (above) is what makes that
|
||||
step run on an unchanged tree.
|
||||
|
||||
Python example:
|
||||
|
||||
```makefile
|
||||
test:
|
||||
@python -m pytest || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
python -m pytest -v; exit 1; }
|
||||
```
|
||||
|
||||
The `exit 1` ensures the target always fails after a rerun — the first run
|
||||
already proved the tests are broken, so the build must not pass even if a
|
||||
flaky test happens to succeed on the second attempt. The rerun exists solely
|
||||
for diagnostic output.
|
||||
|
||||
- Docker builds must complete in under 5 minutes.
|
||||
|
||||
- `make check` must not modify any files in the repo. Tests may use temporary
|
||||
directories.
|
||||
|
||||
- `main` must always pass `make check`, no exceptions.
|
||||
|
||||
- Never commit secrets. `.env` files, credentials, API keys, and private keys
|
||||
must be in `.gitignore`. No exceptions.
|
||||
|
||||
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
|
||||
editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`),
|
||||
language build artifacts, and `node_modules/`. Fetch the standard `.gitignore`
|
||||
from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when
|
||||
setting up a new repo. These patterns are written to `.gitignore`'s own
|
||||
semantics, in which an unanchored pattern already matches at every depth; they
|
||||
are not a `.dockerignore` and must not be transplanted into one unmodified.
|
||||
|
||||
- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns
|
||||
across unmodified leaves secrets in the build context.** Docker matches with
|
||||
`moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so
|
||||
`*` does not cross `/` and a pattern without a leading `**/` is anchored at
|
||||
the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key`
|
||||
therefore excludes only the copies at the repository root, while `config/.env`
|
||||
and `certs/server.key` still reach the context and can land in an image layer
|
||||
— which is more dangerous than a short file with no secret patterns at all,
|
||||
because it reads as solved and stops anyone looking. Give every
|
||||
depth-independent pattern the `**/` prefix and leave only genuinely
|
||||
root-anchored entries unprefixed: `.claude`, and the repo's own host-built
|
||||
binary, written `/myapp` and never `**/myapp`, which would also match
|
||||
`cmd/myapp/` and delete the package directory from the context. Matching is
|
||||
case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so
|
||||
secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`,
|
||||
and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern
|
||||
also catches something the build needs, re-include it with a negation
|
||||
(`!docs/example.env`); deleting the pattern reopens the exposure for every
|
||||
other file it covers. Fetch the standard `.dockerignore` from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend
|
||||
it with the repo's own artifacts.
|
||||
|
||||
- **In-repo agent scratch belongs in both files, written to each file's own
|
||||
semantics.** `.claude/` holds one worktree per in-flight agent — an entire
|
||||
additional checkout of the repo — so under `COPY . .` the build context
|
||||
inflates by a multiple of the repo and another session's unreviewed work can
|
||||
be copied into an image layer. In `.gitignore` the entry is `.claude/`,
|
||||
unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/`
|
||||
prefix, because the prefixed form would also delete any nested directory of
|
||||
that name from the build. Anchoring carries a known gap that the canonical
|
||||
`.dockerignore` states in its own comment, since consuming repos receive the
|
||||
file and not the tracker: the directory is created in the agent's working
|
||||
directory, so a repo running agents in subdirectories still ships
|
||||
`services/api/.claude/` and must add its own anchored entry there.
|
||||
|
||||
- **A plain `docker build .` of a clone stamps the version that
|
||||
`git describe --tags --always` gives**, derived from the `.git` in the build
|
||||
context as the canonical `Dockerfile` above shows. Without its failure check,
|
||||
a missing `git` or an unreadable checkout would leave `-X main.Version=` empty
|
||||
and the build would still exit 0. `script/docker` and `script/cibuild` pass
|
||||
the version they compute on the host; it takes precedence. They do this
|
||||
byte-identically across repos:
|
||||
|
||||
```sh
|
||||
# Own line: a failing command substitution inside an argument does not
|
||||
# trip `set -e`, so the inline form degrades to an empty constant.
|
||||
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
|
||||
[ -n "$version" ] || version="unknown"
|
||||
docker build --no-cache \
|
||||
--build-arg VERSION="$version" \
|
||||
-t "$(script/projectname)" .
|
||||
```
|
||||
|
||||
`--always` makes an untagged repo yield an abbreviated commit hash rather
|
||||
than failing, and the `[ -n "$version" ]` line is the single place the
|
||||
fallback is applied — a live check that fires on a build from an export with
|
||||
no `.git` and on a repository with no commits yet. Do not fold it into the
|
||||
substitution as `|| echo unknown`, which makes the guard unreachable. The
|
||||
Dockerfile's side is `ARG VERSION` in the stage that compiles, declared
|
||||
there because `ARG` is stage-scoped; passing `VERSION` to a repo whose
|
||||
Dockerfile declares no such `ARG` is ignored and costs nothing, which is why
|
||||
the scripts stay byte-identical. One consequence for CI: the standard
|
||||
checkout action clones shallow and fetches no tags, so a repo that embeds a
|
||||
tag-derived version must set `fetch-depth: 0` on its checkout step.
|
||||
|
||||
- **Verify `.dockerignore` by enumerating the image, not by reading the
|
||||
patterns.** Plant files at the root _and_ at least two directories deep, build
|
||||
a probe image that does `COPY . .`, and list what actually landed
|
||||
(`docker run --rm --entrypoint find IMAGE /app`). The `transferring context`
|
||||
size is not a substitute: a nested secret is a few bytes, and BuildKit
|
||||
transfers only the delta from the previous build.
|
||||
|
||||
- **No build artifacts in version control.** Code-derived data (compiled
|
||||
bundles, minified output, generated assets) must never be committed to the
|
||||
repository if it can be avoided. The build process (e.g. Dockerfile, Makefile)
|
||||
should generate these at build time. Notable exception: Go protobuf generated
|
||||
files (`.pb.go`) ARE committed because repos need to work with `go get`, which
|
||||
downloads code but does not execute code generation.
|
||||
|
||||
- Never use `git add -A` or `git add .`. Always stage files explicitly by name.
|
||||
|
||||
- Never force-push to `main`.
|
||||
|
||||
- Make all changes on a feature branch. You can do whatever you want on a
|
||||
feature branch.
|
||||
|
||||
- `.golangci.yml` is standardized. The vendored copy in a consuming repo must
|
||||
_NEVER_ be modified by an agent: fetch it from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it
|
||||
byte-identical, so that no repo can quietly loosen its own linting. Linter
|
||||
configuration changes are made to the canonical copy in the `prompts` repo and
|
||||
reach consuming repos by re-vendoring; an agent may open a PR against
|
||||
canonical, which only the user merges. One list is exempt from byte-identity,
|
||||
because it cannot be written once for every repo: the `deny` list of the
|
||||
`test-support` depguard rule, where a repo names its own test-support packages
|
||||
by full import path. A repo adds entries there and changes nothing else, and a
|
||||
re-vendor carries its entries forward. The canonical golangci-lint version is
|
||||
v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base
|
||||
image
|
||||
(`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`,
|
||||
which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go`
|
||||
directive must not name a newer Go minor version than the one golangci-lint
|
||||
was built with, or golangci-lint refuses to lint it: this release lints
|
||||
`go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo
|
||||
installs golangci-lint on the host. A repo sets the lint phase digest to the
|
||||
one named here and re-vendors `.golangci.yml` in the same commit, whichever of
|
||||
the two prompted the change: the canonical copy can name linters that an older
|
||||
golangci-lint rejects, and a newer golangci-lint can add linters that
|
||||
`default: all` switches on until the canonical copy disables them.
|
||||
|
||||
- **`script/bootstrap` installs a pinned tool by comparing versions, never by
|
||||
testing presence.** An `if ! command -v <tool>; then install; fi` guard tests
|
||||
`PATH` only, so on an already-provisioned machine the pin is inert and a
|
||||
version bump is a silent no-op — while the Dockerfile, installing into a clean
|
||||
image, gets the pinned version, so a local `make check` and `make docker` can
|
||||
disagree about what the tool even is. The canonical form:
|
||||
- compares the installed version against the pin over the **whole** version
|
||||
token; a parser that stops at the first `-` reports `2.12.2` for a host
|
||||
running `2.12.2-rc1` and skips the install;
|
||||
- treats absent, non-zero, empty or unrecognised `--version` output as a
|
||||
mismatch, so the failure direction is a redundant install and never a
|
||||
skipped one;
|
||||
- after installing, re-resolves the binary the way callers do — `hash -r`,
|
||||
then through `PATH`, not through the directory the installer wrote to —
|
||||
and fails naming the resolved path, since an install that a shadowing
|
||||
binary hides succeeds while changing nothing any caller sees;
|
||||
- is actually called, and prints the version on both success paths: a
|
||||
function defined and never invoked has the same exit status and the same
|
||||
empty output as one that worked.
|
||||
|
||||
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
|
||||
|
||||
A Go tool a repo needs on the host is installed with `go install` pinned to
|
||||
a commit hash (`go install <package>@<commit hash>`). It is never tracked as
|
||||
a `go.mod` tool dependency or through a `tools.go` file, either of which
|
||||
pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`.
|
||||
|
||||
- When pinning images or packages by hash, add a comment above the reference
|
||||
with the version and date (YYYY-MM-DD).
|
||||
|
||||
- Use `yarn`, not `npm`.
|
||||
|
||||
- Write all dates as YYYY-MM-DD (ISO 8601).
|
||||
|
||||
- Simple projects should be configured with environment variables.
|
||||
|
||||
- Dockerized web services listen on port 8080 by default, overridable with
|
||||
`PORT`.
|
||||
|
||||
- **HTTP/web services must be hardened for production internet exposure before
|
||||
tagging 1.0.** This means full compliance with security best practices
|
||||
including, without limitation, all of the following:
|
||||
- **Security headers** on every response:
|
||||
- `Strict-Transport-Security` (HSTS) with `max-age` of at least one year
|
||||
and `includeSubDomains`.
|
||||
- `Content-Security-Policy` (CSP) with a restrictive default policy
|
||||
(`default-src 'self'` as a baseline, tightened per-resource as
|
||||
needed). Never use `unsafe-inline` or `unsafe-eval` unless
|
||||
unavoidable, and document the reason.
|
||||
- `X-Frame-Options: DENY` (or `SAMEORIGIN` if framing is required).
|
||||
Prefer the `frame-ancestors` CSP directive as the primary control.
|
||||
- `X-Content-Type-Options: nosniff`.
|
||||
- `Referrer-Policy: strict-origin-when-cross-origin` (or stricter).
|
||||
- `Permissions-Policy` restricting access to browser features the
|
||||
application does not use (camera, microphone, geolocation, etc.).
|
||||
- **Request and response limits:**
|
||||
- Maximum request body size enforced on all endpoints (e.g. Go
|
||||
`http.MaxBytesReader`). Choose a sane default per-route; never accept
|
||||
unbounded input.
|
||||
- Maximum response body size where applicable (e.g. paginated APIs).
|
||||
- `ReadTimeout` and `ReadHeaderTimeout` on the `http.Server` to defend
|
||||
against slowloris attacks.
|
||||
- `WriteTimeout` on the `http.Server`.
|
||||
- `IdleTimeout` on the `http.Server`.
|
||||
- Per-handler execution time limits via `context.WithTimeout` or
|
||||
chi/stdlib `middleware.Timeout`.
|
||||
- **Authentication and session security:**
|
||||
- Rate limiting on password-based authentication endpoints. API keys are
|
||||
high-entropy and not susceptible to brute force, so they are exempt.
|
||||
- CSRF tokens on all state-mutating HTML forms. API endpoints
|
||||
authenticated via `Authorization` header (Bearer token, API key) are
|
||||
exempt because the browser does not attach these automatically.
|
||||
- Passwords stored using bcrypt, scrypt, or argon2 — never plain-text,
|
||||
MD5, or SHA.
|
||||
- Session cookies set with `HttpOnly`, `Secure`, and `SameSite=Lax` (or
|
||||
`Strict`) attributes.
|
||||
- **Reverse proxy awareness:**
|
||||
- True client IP detection when behind a reverse proxy
|
||||
(`X-Forwarded-For`, `X-Real-IP`). The application must accept
|
||||
forwarded headers only from a configured set of trusted proxy
|
||||
addresses — never trust `X-Forwarded-For` unconditionally.
|
||||
- **CORS:**
|
||||
- Authenticated endpoints must restrict `Access-Control-Allow-Origin` to
|
||||
an explicit allowlist of known origins. Wildcard (`*`) is acceptable
|
||||
only for public, unauthenticated read-only APIs.
|
||||
- **Error handling:**
|
||||
- Internal errors must never leak stack traces, SQL queries, file paths,
|
||||
or other implementation details to the client. Return generic error
|
||||
messages in production; detailed errors only when `DEBUG` is enabled.
|
||||
- **TLS:**
|
||||
- Services never terminate TLS directly. They are always deployed behind
|
||||
a TLS-terminating reverse proxy. The service itself listens on plain
|
||||
HTTP. However, HSTS headers and `Secure` cookie flags must still be
|
||||
set by the application so that the browser enforces HTTPS end-to-end.
|
||||
|
||||
This list is non-exhaustive. Apply defense-in-depth: if a standard security
|
||||
hardening measure exists for HTTP services and is not listed here, it is
|
||||
still expected. When in doubt, harden.
|
||||
|
||||
- `README.md` is the primary documentation. Required sections:
|
||||
- **Description**: First line must include the project name, purpose,
|
||||
category (web server, SPA, CLI tool, etc.), license, and author. Example:
|
||||
"µPaaS is an MIT-licensed Go web application by @sneak that receives
|
||||
git-frontend webhooks and deploys applications via Docker in realtime."
|
||||
- **Getting Started**: Copy-pasteable install/usage code block.
|
||||
- **Entrypoints**: Opens by stating that the repo adheres to the
|
||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||
standard (with that link), then documents each provided `script/`
|
||||
entrypoint and its purpose.
|
||||
- **Rationale**: Why does this exist?
|
||||
- **Design**: How is the program structured?
|
||||
- **TODO**: Update meticulously, even between commits. When planning, put
|
||||
the todo list in the README so a new agent can pick up where the last one
|
||||
left off.
|
||||
- **License**: MIT, GPL, or WTFPL. Ask the user for new projects. Include a
|
||||
`LICENSE` file in the repo root and a License section in the README.
|
||||
- **Author**: [@sneak](https://sneak.berlin).
|
||||
|
||||
- First commit of a new repo should contain only `README.md`.
|
||||
|
||||
- Go module root: `sneak.berlin/go/<name>`. Always run `go mod tidy` before
|
||||
committing.
|
||||
|
||||
- Use SemVer.
|
||||
|
||||
- Database migrations live in `internal/db/migrations/` and must be embedded in
|
||||
the binary.
|
||||
- `000_migration.sql` — contains ONLY the creation of the migrations
|
||||
tracking table itself. Nothing else.
|
||||
- `001_schema.sql` — the full application schema.
|
||||
- **Pre-1.0.0:** never add additional migration files (002, 003, etc.).
|
||||
There is no installed base to migrate. Edit `001_schema.sql` directly.
|
||||
- **Post-1.0.0:** add new numbered migration files for each schema change.
|
||||
Never edit existing migrations after release.
|
||||
|
||||
- All repos should have an `.editorconfig` enforcing the project's indentation
|
||||
settings.
|
||||
|
||||
- Avoid putting files in the repo root unless necessary. Root should contain
|
||||
only project-level config files (`README.md`, `AGENTS.md`, `Makefile`,
|
||||
`Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`,
|
||||
and language-specific config). Everything else goes in a subdirectory.
|
||||
Canonical subdirectory names:
|
||||
- `bin/` — executable scripts and tools
|
||||
- `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose
|
||||
body is a single call into `internal/` or `pkg/`, no project logic in
|
||||
`cmd/`
|
||||
- `configs/` — configuration templates and examples
|
||||
- `deploy/` — deployment manifests (k8s, compose, terraform)
|
||||
- `docs/` — documentation and markdown (README.md stays in root)
|
||||
- `internal/` — Go internal packages
|
||||
- `internal/db/migrations/` — database migrations
|
||||
- `pkg/` — Go library packages
|
||||
- `share/` — systemd units, data files
|
||||
- `static/` — static assets (images, fonts, etc.)
|
||||
- `web/` — web frontend source
|
||||
|
||||
- When setting up a new repo, files from the `prompts` repo may be used as
|
||||
templates. Fetch them from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/<path>`.
|
||||
|
||||
- New repos must contain at minimum:
|
||||
- `README.md`, `.git`, `.gitignore`, `.editorconfig`
|
||||
- `LICENSE`, `REPO_POLICIES.md` (copy from the `prompts` repo)
|
||||
- `Makefile`
|
||||
- `script/` entrypoints (`bootstrap`, `setup`, `projectname`, `test`,
|
||||
`lint`, `fmt`, `fmt-check`, `check`, `docker`, `cibuild`, `precommit`,
|
||||
`install-precommit`)
|
||||
- `Dockerfile`, `.dockerignore`
|
||||
- `.gitea/workflows/check.yml`
|
||||
- Go: `go.mod`, `go.sum`, `.golangci.yml`
|
||||
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
|
||||
- Python: `pyproject.toml`
|
||||
|
||||
- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It
|
||||
is never committed under a file or directory named after one agent tool, such
|
||||
as `CLAUDE.md` or `.claude/`, and never split into separate memory files.
|
||||
@@ -0,0 +1,45 @@
|
||||
package main_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestDockerfileBoundsEveryModuleDownload fails unless every
|
||||
// `go mod download` in the Dockerfile runs under a time limit, is tried
|
||||
// a fixed number of times, and then fails the build with a message that
|
||||
// names the module download (issue #351). Like the version tests, this
|
||||
// parses the Dockerfile; that a stalled download really is cut off and
|
||||
// retried was checked by hand.
|
||||
func TestDockerfileBoundsEveryModuleDownload(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var downloads []string
|
||||
|
||||
for _, instruction := range instructions(t, productDockerfile) {
|
||||
if strings.Contains(instruction, "go mod download") {
|
||||
downloads = append(downloads, instruction)
|
||||
}
|
||||
}
|
||||
|
||||
require.NotEmpty(t, downloads, "%s must download modules",
|
||||
productDockerfile)
|
||||
|
||||
for _, download := range downloads {
|
||||
assert.Contains(t, download, "timeout 300 go mod download",
|
||||
"%s: a module download must have a time limit: %s",
|
||||
productDockerfile, download)
|
||||
assert.Contains(t, download, "for attempt in 1 2 3",
|
||||
"%s: a module download must be retried: %s",
|
||||
productDockerfile, download)
|
||||
assert.Contains(t, download, "module download failed",
|
||||
"%s: a module download that never succeeds must say so: %s",
|
||||
productDockerfile, download)
|
||||
assert.True(t, strings.HasSuffix(download, "exit 1"),
|
||||
"%s: a module download that never succeeds must fail the"+
|
||||
" build: %s", productDockerfile, download)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package main_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// This file guards the version stamping of the product image (issue
|
||||
// #75). The failure it protects against is silent: the image still
|
||||
// builds and runs, but `vaultik version` inside it reports "commit:
|
||||
// unknown" or a version of "dev", so an operator cannot tell which
|
||||
// source produced a given backup. The build takes the version as a
|
||||
// build arg, which script/docker computes on the host, and otherwise
|
||||
// derives it from the .git in its context; the commit and its date
|
||||
// always come from that .git.
|
||||
//
|
||||
// These are parses of the committed files, because shelling out to
|
||||
// docker would nest a build inside `make test`. That `vaultik version`
|
||||
// in the built image really prints the host's version is verified by
|
||||
// hand.
|
||||
|
||||
// The files under guard, relative to the repository root.
|
||||
const (
|
||||
productDockerfile = "Dockerfile"
|
||||
dockerScript = "script/docker"
|
||||
)
|
||||
|
||||
// TestProductDockerfileTakesVersionAsBuildArg fails unless the build
|
||||
// declares ARG VERSION, with no default, and stamps it into the binary
|
||||
// whenever it is given, ahead of the value derived in the container.
|
||||
func TestProductDockerfileTakesVersionAsBuildArg(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
found := instructions(t, productDockerfile)
|
||||
|
||||
require.Contains(t, found, "ARG VERSION",
|
||||
"%s must declare `ARG VERSION`, with no default, so the host can"+
|
||||
" pass it in", productDockerfile)
|
||||
|
||||
assertLdflagReferences(t, found, "VERSION")
|
||||
}
|
||||
|
||||
// TestProductDockerfileDerivesVersionFromGit fails unless a build given
|
||||
// no VERSION, such as a plain `docker build .` of a clone, takes it from
|
||||
// `git describe` of the .git in its context, stamps "dev" when the
|
||||
// context has no .git, and fails rather than stamp "dev" when that .git
|
||||
// yields no version. The commit and its date come from the same .git,
|
||||
// and the build fails rather than stamp them "unknown" when it is
|
||||
// present.
|
||||
func TestProductDockerfileDerivesVersionFromGit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
found := instructions(t, productDockerfile)
|
||||
|
||||
buildAt := indexContaining(found, "go build")
|
||||
require.GreaterOrEqual(t, buildAt, 0, "%s must build", productDockerfile)
|
||||
|
||||
assert.Contains(t, found[buildAt], "git describe --tags --always || echo dev",
|
||||
"%s must derive the version from git when no VERSION is given,"+
|
||||
" and stamp dev when the context has no .git", productDockerfile)
|
||||
assert.Contains(t, found[buildAt], "[ -e .git ]",
|
||||
"%s must fail when the context carries .git but yields no version",
|
||||
productDockerfile)
|
||||
assert.Contains(t, found[buildAt], "git rev-parse HEAD",
|
||||
"%s must stamp the commit from git", productDockerfile)
|
||||
assert.Contains(t, found[buildAt], "git show -s --format=%cs HEAD",
|
||||
"%s must stamp the commit date from git", productDockerfile)
|
||||
assert.Contains(t, found[buildAt],
|
||||
`[ "$commit" = unknown ] || [ "$commit_date" = unknown ]`,
|
||||
"%s must fail when the context carries .git but yields no commit"+
|
||||
" or date", productDockerfile)
|
||||
}
|
||||
|
||||
// TestDockerScriptComputesVersionOnTheHost fails unless script/docker
|
||||
// passes the version it derives where .git exists as a build arg.
|
||||
func TestDockerScriptComputesVersionOnTheHost(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
script := readRepoFile(t, dockerScript)
|
||||
|
||||
assert.Contains(t, script, "--build-arg VERSION=",
|
||||
"%s must pass --build-arg VERSION to the build", dockerScript)
|
||||
}
|
||||
|
||||
// assertLdflagReferences fails unless the build instruction uses the
|
||||
// named ARG whenever it is given (a ${arg:- reference), so a value
|
||||
// passed in is not overridden by one derived inside the container.
|
||||
func assertLdflagReferences(t *testing.T, found []string, arg string) {
|
||||
t.Helper()
|
||||
|
||||
for _, instruction := range found {
|
||||
if strings.HasPrefix(instruction, "RUN ") &&
|
||||
strings.Contains(instruction, "go build") &&
|
||||
strings.Contains(instruction, "${"+arg+":-") {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
assert.Fail(t, "version arg is declared but never stamped",
|
||||
"the go build in %s must use ${%s:-...}, or the arg is passed and"+
|
||||
" discarded", productDockerfile, arg)
|
||||
}
|
||||
|
||||
// instructions returns the Dockerfile's instructions, one per element,
|
||||
// with comments and blank lines dropped and continuation lines joined,
|
||||
// so a multi-line RUN is one string.
|
||||
func instructions(t *testing.T, name string) []string {
|
||||
t.Helper()
|
||||
|
||||
var (
|
||||
out []string
|
||||
continued string
|
||||
isContinued bool
|
||||
)
|
||||
|
||||
for line := range strings.SplitSeq(readRepoFile(t, name), "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if !isContinued && (trimmed == "" || strings.HasPrefix(trimmed, "#")) {
|
||||
continue
|
||||
}
|
||||
|
||||
isContinued = strings.HasSuffix(trimmed, `\`)
|
||||
continued += strings.TrimSuffix(trimmed, `\`)
|
||||
|
||||
if isContinued {
|
||||
continue
|
||||
}
|
||||
|
||||
out = append(out, strings.Join(strings.Fields(continued), " "))
|
||||
continued = ""
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// indexContaining returns the position of the first instruction
|
||||
// containing want; -1 if there is none.
|
||||
func indexContaining(found []string, want string) int {
|
||||
for i, instruction := range found {
|
||||
if strings.Contains(instruction, want) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
+25
-6
@@ -1,3 +1,4 @@
|
||||
// Package main is the vaultik command-line entry point.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -5,37 +6,55 @@ import (
|
||||
"runtime"
|
||||
"runtime/pprof"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/cli"
|
||||
"sneak.berlin/go/vaultik/internal/cli"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(run())
|
||||
}
|
||||
|
||||
// run sets up optional profiling, runs the CLI, and returns the process
|
||||
// exit code. os.Exit lives in main so it fires only after run's deferred
|
||||
// profile writers have flushed. cli.Entry returns a status code rather
|
||||
// than calling os.Exit itself: an os.Exit from inside it would skip
|
||||
// these defers and truncate the profile of a failing command -- exactly
|
||||
// the command one most often wants to profile.
|
||||
func run() int {
|
||||
// CPU profiling: set VAULTIK_CPUPROFILE=/path/to/cpu.prof
|
||||
if cpuProfile := os.Getenv("VAULTIK_CPUPROFILE"); cpuProfile != "" {
|
||||
f, err := os.Create(cpuProfile)
|
||||
f, err := os.Create(cpuProfile) //nolint:gosec // G304: operator-set path
|
||||
if err != nil {
|
||||
panic("could not create CPU profile: " + err.Error())
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
if err := pprof.StartCPUProfile(f); err != nil {
|
||||
|
||||
err = pprof.StartCPUProfile(f)
|
||||
if err != nil {
|
||||
panic("could not start CPU profile: " + err.Error())
|
||||
}
|
||||
|
||||
defer pprof.StopCPUProfile()
|
||||
}
|
||||
|
||||
// Memory profiling: set VAULTIK_MEMPROFILE=/path/to/mem.prof
|
||||
if memProfile := os.Getenv("VAULTIK_MEMPROFILE"); memProfile != "" {
|
||||
defer func() {
|
||||
f, err := os.Create(memProfile)
|
||||
f, err := os.Create(memProfile) //nolint:gosec // G304: operator-set path
|
||||
if err != nil {
|
||||
panic("could not create memory profile: " + err.Error())
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
runtime.GC() // get up-to-date statistics
|
||||
if err := pprof.WriteHeapProfile(f); err != nil {
|
||||
|
||||
err = pprof.WriteHeapProfile(f)
|
||||
if err != nil {
|
||||
panic("could not write memory profile: " + err.Error())
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
cli.CLIEntry()
|
||||
return cli.Entry()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
package main_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// This file guards the Makefile that builds this program, which is why
|
||||
// it lives beside it rather than in a package of its own.
|
||||
//
|
||||
// Issue #110: `build` was listed in .PHONY with no `build:` rule
|
||||
// anywhere in the file. That combination is silently successful — make
|
||||
// considers a phony target with no prerequisites and no recipe already
|
||||
// satisfied, so `rm -f vaultik && make build` printed "Nothing to be
|
||||
// done for 'build'" and exited 0 with no binary produced. Declaring the
|
||||
// name phony is precisely what converts the "No rule to make target"
|
||||
// error into a green.
|
||||
//
|
||||
// The guard is a parse of the Makefile rather than an invocation of
|
||||
// make. `make test` is what runs these tests, so shelling back into
|
||||
// `make build` here would nest a build inside the test run and drop a
|
||||
// binary into the tree as a side effect of testing. The one property a
|
||||
// parse cannot establish — that the recipe still fails when the build
|
||||
// fails — is not testable from inside the build either; it is verified
|
||||
// by hand against a deliberately broken tree.
|
||||
|
||||
// phonyDirective introduces the list of phony target names.
|
||||
const phonyDirective = ".PHONY:"
|
||||
|
||||
// ruleLine matches a rule's target list: a target starts in column
|
||||
// zero, so recipe lines (tab-indented) and the continuation lines of a
|
||||
// variable assignment (space-indented) are excluded by construction.
|
||||
//
|
||||
// The trailing (?:[^=]|$) rejects `:=` assignments such as
|
||||
// `VERSION := $(shell script/version)`, which are not rules. Directives
|
||||
// and function calls (`.PHONY:`, `ifeq`, `$(error ...)`) do not match
|
||||
// because a target here must begin with a letter, digit or underscore.
|
||||
var ruleLine = regexp.MustCompile(`^([A-Za-z0-9_][A-Za-z0-9_./ -]*):(?:[^=]|$)`)
|
||||
|
||||
// TestPhonyTargetsAllHaveRules fails on any name in .PHONY that has no
|
||||
// rule in the Makefile. Such a name is not a build target at all: it is
|
||||
// a command that reports success without doing anything, which is worse
|
||||
// than one that does not exist, because a caller checking the exit code
|
||||
// cannot tell the difference.
|
||||
func TestPhonyTargetsAllHaveRules(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
makefile := readMakefile(t)
|
||||
|
||||
phony := phonyTargets(makefile)
|
||||
require.NotEmpty(t, phony, "no .PHONY names found; the parser is broken")
|
||||
|
||||
rules := declaredRules(makefile)
|
||||
|
||||
// Sanity check on the rule parser before trusting its verdict: a
|
||||
// parser that found nothing would pass this test by accident.
|
||||
require.Contains(t, rules, "vaultik",
|
||||
"the file rule that builds the binary must be recognized")
|
||||
|
||||
for _, target := range phony {
|
||||
assert.Contains(t, rules, target,
|
||||
"`.PHONY` lists %q but the Makefile declares no %q rule, so "+
|
||||
"`make %s` exits 0 without doing anything", target, target, target)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildTargetBuildsTheBinary pins the specific shape of issue #110:
|
||||
// `make build` has to reach the rule that produces the binary. The test
|
||||
// above would also pass if `build:` were given an empty recipe of its
|
||||
// own, which would be the same silent success under a different
|
||||
// spelling.
|
||||
func TestBuildTargetBuildsTheBinary(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
prerequisites := rulePrerequisites(readMakefile(t), "build")
|
||||
require.NotNil(t, prerequisites, "the Makefile declares no `build` rule")
|
||||
|
||||
assert.Contains(t, prerequisites, "vaultik",
|
||||
"`make build` must depend on the rule that builds the binary")
|
||||
}
|
||||
|
||||
// readMakefile returns the contents of the repository's Makefile.
|
||||
func readMakefile(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
return readRepoFile(t, "Makefile")
|
||||
}
|
||||
|
||||
// readRepoFile reads a file by its path relative to the repository
|
||||
// root.
|
||||
func readRepoFile(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
|
||||
//nolint:gosec // G304: the path is a constant relative to this repo
|
||||
contents, err := os.ReadFile(filepath.Join(repoRoot(t), name))
|
||||
require.NoError(t, err)
|
||||
|
||||
return string(contents)
|
||||
}
|
||||
|
||||
// repoRoot returns the repository root. The test binary runs with its
|
||||
// package directory as the working directory, so the root is found by
|
||||
// walking up until the module file appears.
|
||||
func repoRoot(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
dir, err := os.Getwd()
|
||||
require.NoError(t, err)
|
||||
|
||||
for {
|
||||
_, err = os.Stat(filepath.Join(dir, "go.mod"))
|
||||
if err == nil {
|
||||
return dir
|
||||
}
|
||||
|
||||
parent := filepath.Dir(dir)
|
||||
require.NotEqual(t, dir, parent,
|
||||
"walked to the filesystem root without finding a go.mod")
|
||||
|
||||
dir = parent
|
||||
}
|
||||
}
|
||||
|
||||
// phonyTargets returns every name declared phony, across all .PHONY
|
||||
// lines.
|
||||
func phonyTargets(makefile string) []string {
|
||||
var targets []string
|
||||
|
||||
for line := range strings.SplitSeq(makefile, "\n") {
|
||||
if !strings.HasPrefix(line, phonyDirective) {
|
||||
continue
|
||||
}
|
||||
|
||||
targets = append(targets,
|
||||
strings.Fields(strings.TrimPrefix(line, phonyDirective))...)
|
||||
}
|
||||
|
||||
return targets
|
||||
}
|
||||
|
||||
// declaredRules returns the set of target names that have a rule.
|
||||
func declaredRules(makefile string) map[string]bool {
|
||||
rules := make(map[string]bool)
|
||||
|
||||
for line := range strings.SplitSeq(makefile, "\n") {
|
||||
match := ruleLine.FindStringSubmatch(line)
|
||||
if match == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// One rule may name several targets: `a b: prereq`.
|
||||
for target := range strings.FieldsSeq(match[1]) {
|
||||
rules[target] = true
|
||||
}
|
||||
}
|
||||
|
||||
return rules
|
||||
}
|
||||
|
||||
// rulePrerequisites returns the prerequisites of the named rule, or nil
|
||||
// if no such rule exists. A rule with none returns an empty slice, so
|
||||
// "declared with nothing to do" is distinguishable from "not declared".
|
||||
func rulePrerequisites(makefile, target string) []string {
|
||||
for line := range strings.SplitSeq(makefile, "\n") {
|
||||
match := ruleLine.FindStringSubmatch(line)
|
||||
if match == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if !slices.Contains(strings.Fields(match[1]), target) {
|
||||
continue
|
||||
}
|
||||
|
||||
_, after, _ := strings.Cut(line, ":")
|
||||
|
||||
return append([]string{}, strings.Fields(after)...)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
+69
-31
@@ -1,15 +1,31 @@
|
||||
# vaultik configuration file example
|
||||
# This file shows all available configuration options with their default values
|
||||
# Copy this file and uncomment/modify the values you need
|
||||
# This file shows every configuration option. The values are examples, not
|
||||
# defaults; where an option has a default, its comment gives it.
|
||||
# Copy this file and change the values you need
|
||||
|
||||
# Age recipient public keys for encryption
|
||||
# This is REQUIRED - backups are encrypted to these public keys
|
||||
# Generate with: age-keygen | grep "public key"
|
||||
# Backups are encrypted to these public keys. snapshot create needs at least
|
||||
# one; listing, verifying and restoring do not
|
||||
# Generate a key pair, and show its public key, with:
|
||||
# age-keygen -o vaultik_backup_private_key.txt
|
||||
# grep 'public key' vaultik_backup_private_key.txt
|
||||
# Keep the private key file: restoring needs it. Do not pipe age-keygen
|
||||
# straight into grep, which would discard the private key
|
||||
age_recipients:
|
||||
- age1cj2k2addawy294f6k2gr2mf9gps9r3syplryxca3nvxj3daqm96qfp84tz
|
||||
|
||||
# Age private key for decryption
|
||||
# Only snapshot restore and snapshot verify --deep need it. The
|
||||
# VAULTIK_AGE_SECRET_KEY environment variable overrides this setting
|
||||
# Do not set it on a machine you back up: prefer the environment variable,
|
||||
# supplied only on the machine you restore on
|
||||
# Default: not set
|
||||
#age_secret_key: YOUR_AGE_PRIVATE_KEY
|
||||
|
||||
# Named snapshots - each snapshot can contain multiple paths
|
||||
# Each snapshot gets its own ID and can have snapshot-specific excludes
|
||||
# Required: at least one snapshot with at least one path. Only the config
|
||||
# commands, version and completion run without one
|
||||
snapshots:
|
||||
testing:
|
||||
paths:
|
||||
@@ -257,16 +273,21 @@ exclude:
|
||||
|
||||
# Storage URL - use either this OR the s3 section below
|
||||
# Supports: s3://bucket/prefix, file:///path, rclone://remote/path
|
||||
storage_url: "rclone://las1stor1//srv/pool.2024.04/backups/heraklion"
|
||||
# An s3:// URL also needs s3.access_key_id and s3.secret_access_key below
|
||||
# A file:// path must be absolute, and the directory must already exist:
|
||||
# vaultik never creates it
|
||||
# An rclone:// URL needs the remote to be defined in rclone's own config
|
||||
# (rclone config)
|
||||
storage_url: "rclone://myremote/path/to/backups"
|
||||
|
||||
# S3-compatible storage configuration
|
||||
#s3:
|
||||
# # S3-compatible endpoint URL
|
||||
# # Examples: https://s3.amazonaws.com, https://storage.googleapis.com
|
||||
# endpoint: http://10.100.205.122:8333
|
||||
# endpoint: https://s3.example.com
|
||||
#
|
||||
# # Bucket name where backups will be stored
|
||||
# bucket: testbucket
|
||||
# bucket: mybucket
|
||||
#
|
||||
# # Prefix (folder) within the bucket for this host's backups
|
||||
# # Useful for organizing backups from multiple hosts
|
||||
@@ -274,51 +295,54 @@ storage_url: "rclone://las1stor1//srv/pool.2024.04/backups/heraklion"
|
||||
# #prefix: "hosts/myserver/"
|
||||
#
|
||||
# # S3 access credentials
|
||||
# access_key_id: Z9GT22M9YFU08WRMC5D4
|
||||
# secret_access_key: Pi0tPKjFbN4rZlRhcA4zBtEkib04yy2WcIzI+AXk
|
||||
# access_key_id: YOUR_ACCESS_KEY
|
||||
# secret_access_key: YOUR_SECRET_KEY
|
||||
#
|
||||
# # S3 region
|
||||
# # Default: us-east-1
|
||||
# #region: us-east-1
|
||||
#
|
||||
# # Use SSL/TLS for S3 connections
|
||||
# # Only read when storage_url is not set and the endpoint has no scheme;
|
||||
# # with an s3:// storage_url, use ?ssl=false in the URL instead
|
||||
# # Default: true
|
||||
# #use_ssl: true
|
||||
#
|
||||
# # Part size for multipart uploads
|
||||
# # Minimum 5MB, affects memory usage during upload
|
||||
# # Supports: 5MB, 10M, 100MiB, etc.
|
||||
# # Default: 5MB
|
||||
# #part_size: 5MB
|
||||
|
||||
# How often to run backups in daemon mode
|
||||
# Format: 1h, 30m, 24h, etc
|
||||
# Default: 1h
|
||||
#backup_interval: 1h
|
||||
|
||||
# How often to do a full filesystem scan in daemon mode
|
||||
# Between full scans, inotify is used to detect changes
|
||||
# Default: 24h
|
||||
#full_scan_interval: 24h
|
||||
|
||||
# Minimum time between backup runs in daemon mode
|
||||
# Prevents backups from running too frequently
|
||||
# Default: 15m
|
||||
#min_time_between_run: 15m
|
||||
# # Minimum 5MiB, maximum 5GiB; affects memory usage during upload
|
||||
# # A blob too large for 10,000 parts of this size gets larger parts
|
||||
# # Supports: 10MB, 16MiB, 100MiB, etc. (5MB is below the minimum)
|
||||
# # Default: 5MiB
|
||||
# #part_size: 5MiB
|
||||
|
||||
# Path to local SQLite index database
|
||||
# This database tracks file state for incremental backups
|
||||
# Default: /var/lib/vaultik/index.sqlite
|
||||
#index_path: /var/lib/vaultik/index.sqlite
|
||||
# The VAULTIK_INDEX_PATH environment variable overrides this setting
|
||||
# Default: the platform data directory, e.g.
|
||||
# macOS: ~/Library/Application Support/vaultik/index.sqlite
|
||||
# Linux: ~/.local/share/vaultik/index.sqlite
|
||||
#index_path: /path/to/index.sqlite
|
||||
|
||||
# Average chunk size for content-defined chunking
|
||||
# Smaller chunks = better deduplication but more metadata
|
||||
# Supports: 10MB, 5M, 1GB, 500KB, 64MiB, etc.
|
||||
# Minimum 1MB
|
||||
# Supports: 10MB, 5M, 1GB, 64MiB, etc.
|
||||
# Default: 10MB
|
||||
#chunk_size: 10MB
|
||||
|
||||
# Maximum blob size
|
||||
# Multiple chunks are packed into blobs up to this size
|
||||
# Must be at least four times chunk_size (the largest chunk the chunker can
|
||||
# emit); a smaller limit would let a single-chunk blob exceed it.
|
||||
# Chunking uses no secret (the FastCDC parameters are fixed and public). At a
|
||||
# large limit a blob holds hundreds of chunks, so individual chunk lengths are
|
||||
# not visible in its size; lowering the limit toward chunk_size exposes them.
|
||||
# A backup needs free temporary space, because each blob is written whole to
|
||||
# a temporary file before it is uploaded (up to about blob_size_limit; an
|
||||
# rclone destination that cannot stream uploads needs about twice that) and
|
||||
# the metadata export writes copies of the local index. Temporary files go to
|
||||
# $TMPDIR (default /tmp); with TMPDIR unset, SQLite writes one of those copies
|
||||
# to /var/tmp.
|
||||
# Supports: 1GB, 10G, 500MB, 1GiB, etc.
|
||||
# Default: 10GB
|
||||
#blob_size_limit: 10GB
|
||||
@@ -327,6 +351,20 @@ storage_url: "rclone://las1stor1//srv/pool.2024.04/backups/heraklion"
|
||||
# Higher = better compression but slower
|
||||
# Default: 3
|
||||
compression_level: 5
|
||||
|
||||
# How old a blob that no snapshot lists must be before prune deletes it, and
|
||||
# how old every object in a directory under metadata/ with no manifest must
|
||||
# be before prune deletes that directory
|
||||
# A backup uploads its blobs and its db.zst.age before its manifest, so on a
|
||||
# destination shared between hosts or users this must be longer than the
|
||||
# longest backup run
|
||||
# It does not protect a blob a backup reuses from an earlier run that stopped
|
||||
# before its manifest: that blob keeps the time of its first upload, so a
|
||||
# prune can delete it
|
||||
# Supports: 36h, 90m, etc.; 0s deletes such blobs and directories at once
|
||||
# Default: 24h
|
||||
#prune_grace_period: 24h
|
||||
|
||||
# Hostname to use in backup metadata
|
||||
# Default: system hostname
|
||||
#hostname: myserver
|
||||
|
||||
+351
-201
@@ -1,268 +1,418 @@
|
||||
# Vaultik Data Model
|
||||
# vaultik data model
|
||||
|
||||
## Overview
|
||||
`vaultik` keeps a SQLite database on the backed-up machine, called the local
|
||||
index. This document describes its tables and how each command uses them.
|
||||
|
||||
Vaultik uses a local SQLite database to track file metadata, chunk mappings, and blob associations during the backup process. This database serves as an index for incremental backups and enables efficient deduplication.
|
||||
The local index has two jobs:
|
||||
|
||||
**Important Notes:**
|
||||
- **No Migration Support**: Vaultik does not support database schema migrations. If the schema changes, the local database must be deleted and recreated by performing a full backup.
|
||||
- **Version Compatibility**: In rare cases, you may need to use the same version of Vaultik to restore a backup as was used to create it. This ensures compatibility with the metadata format stored in S3.
|
||||
- It records which files, chunks and blobs are already stored. The next backup
|
||||
then does not store them again.
|
||||
- It is the source of each snapshot's own database. A backup uploads a trimmed,
|
||||
encrypted copy of the index with every snapshot.
|
||||
|
||||
## Database Tables
|
||||
A restore does not need the local index. It reads the copy that was uploaded
|
||||
with the snapshot.
|
||||
|
||||
### 1. `files`
|
||||
Stores metadata about files in the filesystem being backed up.
|
||||
## Terms
|
||||
|
||||
**Columns:**
|
||||
- `id` (TEXT PRIMARY KEY) - UUID for the file record
|
||||
- `path` (TEXT NOT NULL UNIQUE) - Absolute file path
|
||||
- `mtime` (INTEGER NOT NULL) - Modification time as Unix timestamp
|
||||
- `ctime` (INTEGER NOT NULL) - Change time as Unix timestamp
|
||||
- `size` (INTEGER NOT NULL) - File size in bytes
|
||||
- `mode` (INTEGER NOT NULL) - Unix file permissions and type
|
||||
- `uid` (INTEGER NOT NULL) - User ID of file owner
|
||||
- `gid` (INTEGER NOT NULL) - Group ID of file owner
|
||||
- `link_target` (TEXT) - Symlink target path (NULL for regular files)
|
||||
- **Snapshot**: one backup of the paths listed under one name in the config.
|
||||
- **Chunk**: a piece of a file's content, named by the SHA-256 hash of that
|
||||
content.
|
||||
- **Blob**: one object at the destination. It holds many chunks, compressed and
|
||||
then encrypted.
|
||||
- **Destination**: the place the backups are stored.
|
||||
- **Manifest**: the unencrypted list of the blobs one snapshot uses.
|
||||
- **Remote key**: a hash of the snapshot ID. It names the snapshot's directory
|
||||
at the destination.
|
||||
|
||||
**Indexes:**
|
||||
- `idx_files_path` on `path` for efficient lookups
|
||||
## Where the index is
|
||||
|
||||
**Purpose:** Tracks file metadata to detect changes between backup runs. Used for incremental backup decisions. The UUID primary key provides stable references that don't change if files are moved.
|
||||
The index is the file `index.sqlite` in the platform data directory. That is
|
||||
`~/.local/share/vaultik/` on Linux and `~/Library/Application Support/vaultik/`
|
||||
on macOS. The `index_path` setting and the `VAULTIK_INDEX_PATH` environment
|
||||
variable change the path.
|
||||
|
||||
### 2. `chunks`
|
||||
Stores information about content-defined chunks created from files.
|
||||
The file is readable only by its owner, because it lists every backed-up path.
|
||||
`vaultik` sets that mode each time it opens the index.
|
||||
|
||||
**Columns:**
|
||||
- `chunk_hash` (TEXT PRIMARY KEY) - SHA256 hash of chunk content
|
||||
- `size` (INTEGER NOT NULL) - Chunk size in bytes
|
||||
## Schema migrations
|
||||
|
||||
**Purpose:** Enables deduplication by tracking unique chunks across all files.
|
||||
`vaultik` supports schema migrations. They are the numbered SQL files in
|
||||
`internal/database/schema/`, which are built into the binary.
|
||||
|
||||
### 3. `file_chunks`
|
||||
Maps files to their constituent chunks in order.
|
||||
- `000.sql` creates the `schema_migrations` table. That table records each
|
||||
migration that has run.
|
||||
- `001.sql` creates every other table.
|
||||
|
||||
**Columns:**
|
||||
- `file_id` (TEXT) - File ID (FK to files.id)
|
||||
- `idx` (INTEGER) - Chunk index within file (0-based)
|
||||
- `chunk_hash` (TEXT) - Chunk hash (FK to chunks.chunk_hash)
|
||||
- PRIMARY KEY (`file_id`, `idx`)
|
||||
`database.New` opens the index. It then applies, in order, every migration that
|
||||
the index has not recorded.
|
||||
|
||||
**Purpose:** Allows reconstruction of files from chunks during restore.
|
||||
Before version 1.0, no migration is added, because nothing is installed anywhere
|
||||
yet. A schema change edits `001.sql` directly, with the code that uses the
|
||||
changed tables.
|
||||
|
||||
### 4. `chunk_files`
|
||||
Reverse mapping showing which files contain each chunk.
|
||||
This has one consequence. An index created before such a change has already
|
||||
recorded `001.sql` as run, so it keeps the old schema and can become unusable.
|
||||
To rebuild it, run `vaultik database delete` and then a full backup.
|
||||
|
||||
**Columns:**
|
||||
- `chunk_hash` (TEXT) - Chunk hash (FK to chunks.chunk_hash)
|
||||
- `file_id` (TEXT) - File ID (FK to files.id)
|
||||
- `file_offset` (INTEGER) - Byte offset of chunk within file
|
||||
- `length` (INTEGER) - Length of chunk in bytes
|
||||
- PRIMARY KEY (`chunk_hash`, `file_id`)
|
||||
After version 1.0, each schema change is a new numbered file. An existing index
|
||||
is then migrated the first time an updated `vaultik` opens it. A file that was
|
||||
part of a release is never edited.
|
||||
|
||||
**Purpose:** Supports efficient queries for chunk usage and deduplication statistics.
|
||||
Migrations apply to the local index only. A snapshot's database at the
|
||||
destination is read as it was written. Each snapshot records the `vaultik`
|
||||
version that wrote it.
|
||||
|
||||
### 5. `blobs`
|
||||
Stores information about packed, compressed, and encrypted blob files.
|
||||
## Tables
|
||||
|
||||
**Columns:**
|
||||
- `id` (TEXT PRIMARY KEY) - UUID assigned when blob creation starts
|
||||
- `blob_hash` (TEXT UNIQUE) - SHA256 hash of final blob (NULL until finalized)
|
||||
- `created_ts` (INTEGER NOT NULL) - Creation timestamp
|
||||
- `finished_ts` (INTEGER) - Finalization timestamp (NULL if in progress)
|
||||
- `uncompressed_size` (INTEGER NOT NULL DEFAULT 0) - Total size of chunks before compression
|
||||
- `compressed_size` (INTEGER NOT NULL DEFAULT 0) - Size after compression and encryption
|
||||
- `uploaded_ts` (INTEGER) - Upload completion timestamp (NULL if not uploaded)
|
||||
Times are whole seconds since the Unix epoch, in UTC. The two exceptions are
|
||||
noted at their columns.
|
||||
|
||||
**Purpose:** Tracks blob lifecycle from creation through upload. The UUID primary key allows immediate association of chunks with blobs.
|
||||
### `files`
|
||||
|
||||
### 6. `blob_chunks`
|
||||
Maps chunks to the blobs that contain them.
|
||||
One row for each backed-up path: a regular file, a symlink or a directory.
|
||||
|
||||
**Columns:**
|
||||
- `blob_id` (TEXT) - Blob ID (FK to blobs.id)
|
||||
- `chunk_hash` (TEXT) - Chunk hash (FK to chunks.chunk_hash)
|
||||
- `offset` (INTEGER) - Byte offset of chunk within blob (before compression)
|
||||
- `length` (INTEGER) - Length of chunk in bytes
|
||||
- PRIMARY KEY (`blob_id`, `chunk_hash`)
|
||||
- `id`: a UUID. Primary key.
|
||||
- `path`: the absolute path. Unique.
|
||||
- `source_path`: the configured path that the scan found this entry under. A
|
||||
restore does not read it.
|
||||
- `mtime`: the modification time.
|
||||
- `mtime_nsec`: the nanoseconds within that second.
|
||||
- `size`: the size in bytes.
|
||||
- `mode`: the permission bits and the file type.
|
||||
- `uid`: the owner's user ID.
|
||||
- `gid`: the owner's group ID.
|
||||
- `link_target`: the target of a symlink. It is `NULL` for other entries.
|
||||
|
||||
**Purpose:** Enables chunk retrieval from blobs during restore operations.
|
||||
`idx_files_path` indexes `path`.
|
||||
|
||||
### 7. `snapshots`
|
||||
Tracks backup snapshots.
|
||||
There is one row for each path, and it is updated when the file changes. The
|
||||
row is shared by every snapshot in the local index that lists the path. An
|
||||
older snapshot's own view of the file is in that snapshot's database at the
|
||||
destination.
|
||||
|
||||
**Columns:**
|
||||
- `id` (TEXT PRIMARY KEY) - Snapshot ID (format: hostname-YYYYMMDD-HHMMSSZ)
|
||||
- `hostname` (TEXT) - Hostname where backup was created
|
||||
- `vaultik_version` (TEXT) - Version of Vaultik used
|
||||
- `vaultik_git_revision` (TEXT) - Git revision of Vaultik used
|
||||
- `started_at` (INTEGER) - Start timestamp
|
||||
- `completed_at` (INTEGER) - Completion timestamp (NULL if in progress)
|
||||
- `file_count` (INTEGER) - Number of files in snapshot
|
||||
- `chunk_count` (INTEGER) - Number of unique chunks
|
||||
- `blob_count` (INTEGER) - Number of blobs referenced
|
||||
- `total_size` (INTEGER) - Total size of all files
|
||||
- `blob_size` (INTEGER) - Total size of all blobs (compressed)
|
||||
- `blob_uncompressed_size` (INTEGER) - Total uncompressed size of all referenced blobs
|
||||
- `compression_ratio` (REAL) - Compression ratio achieved
|
||||
- `compression_level` (INTEGER) - Compression level used for this snapshot
|
||||
- `upload_bytes` (INTEGER) - Total bytes uploaded during this snapshot
|
||||
- `upload_duration_ms` (INTEGER) - Total milliseconds spent uploading to S3
|
||||
### `chunks`
|
||||
|
||||
**Purpose:** Provides snapshot metadata and statistics including version tracking for compatibility.
|
||||
One row for each distinct chunk.
|
||||
|
||||
### 8. `snapshot_files`
|
||||
Maps snapshots to the files they contain.
|
||||
- `chunk_hash`: the SHA-256 hash of the chunk's content, in hex. Primary key.
|
||||
- `size`: the size in bytes.
|
||||
|
||||
**Columns:**
|
||||
- `snapshot_id` (TEXT) - Snapshot ID (FK to snapshots.id)
|
||||
- `file_id` (TEXT) - File ID (FK to files.id)
|
||||
- PRIMARY KEY (`snapshot_id`, `file_id`)
|
||||
### `file_chunks`
|
||||
|
||||
**Purpose:** Records which files are included in each snapshot.
|
||||
The chunks that make up each file, in order.
|
||||
|
||||
### 9. `snapshot_blobs`
|
||||
Maps snapshots to the blobs they reference.
|
||||
- `file_id`: the file. References `files.id`.
|
||||
- `idx`: the position of the chunk within the file, counted from 0.
|
||||
- `chunk_hash`: the chunk. References `chunks.chunk_hash`.
|
||||
|
||||
**Columns:**
|
||||
- `snapshot_id` (TEXT) - Snapshot ID (FK to snapshots.id)
|
||||
- `blob_id` (TEXT) - Blob ID (FK to blobs.id)
|
||||
- `blob_hash` (TEXT) - Denormalized blob hash for manifest generation
|
||||
- PRIMARY KEY (`snapshot_id`, `blob_id`)
|
||||
The primary key is (`file_id`, `idx`). `idx_file_chunks_chunk_hash` indexes
|
||||
`chunk_hash`.
|
||||
|
||||
**Purpose:** Tracks blob dependencies for snapshots and enables manifest generation.
|
||||
### `chunk_files`
|
||||
|
||||
### 10. `uploads`
|
||||
Tracks blob upload metrics.
|
||||
The reverse of `file_chunks`: the files that contain each chunk.
|
||||
|
||||
**Columns:**
|
||||
- `blob_hash` (TEXT PRIMARY KEY) - Hash of uploaded blob
|
||||
- `snapshot_id` (TEXT NOT NULL) - The snapshot that triggered this upload (FK to snapshots.id)
|
||||
- `uploaded_at` (INTEGER) - Upload timestamp
|
||||
- `size` (INTEGER) - Size of uploaded blob
|
||||
- `duration_ms` (INTEGER) - Upload duration in milliseconds
|
||||
- `chunk_hash`: the chunk. References `chunks.chunk_hash`.
|
||||
- `file_id`: the file. References `files.id`.
|
||||
- `file_offset`: the byte offset of the chunk within the file.
|
||||
- `length`: the length of the chunk in bytes.
|
||||
|
||||
**Purpose:** Performance monitoring and tracking which blobs were newly created (uploaded) during each snapshot.
|
||||
The primary key is (`chunk_hash`, `file_id`). `idx_chunk_files_file_id` indexes
|
||||
`file_id`.
|
||||
|
||||
## Data Flow and Operations
|
||||
### `blobs`
|
||||
|
||||
### 1. Backup Process
|
||||
One row for each blob.
|
||||
|
||||
1. **File Scanning**
|
||||
- `INSERT OR REPLACE INTO files` - Update file metadata
|
||||
- `SELECT * FROM files WHERE path = ?` - Check if file has changed
|
||||
- `INSERT INTO snapshot_files` - Add file to current snapshot
|
||||
- `id`: a UUID, given when packing of the blob starts. Primary key.
|
||||
- `blob_hash`: the name of the blob at the destination. Unique. Until the blob
|
||||
is closed, it holds a placeholder that starts with `temp-placeholder-`.
|
||||
- `created_ts`: when packing started.
|
||||
- `finished_ts`: when the blob was closed. It is `NULL` before that.
|
||||
- `uncompressed_size`: the total size of the blob's chunks.
|
||||
- `compressed_size`: the size after compression and encryption.
|
||||
- `uploaded_ts`: when the upload finished. It is `NULL` before that.
|
||||
|
||||
2. **Chunking** (for changed files)
|
||||
- `INSERT OR IGNORE INTO chunks` - Store new chunks
|
||||
- `INSERT INTO file_chunks` - Map chunks to file
|
||||
- `INSERT INTO chunk_files` - Create reverse mapping
|
||||
`blob_hash` is `hex(SHA256(SHA256(uncompressed contents)))`. See
|
||||
[REPOSTRUCTURE.md](REPOSTRUCTURE.md#blobs).
|
||||
|
||||
3. **Blob Packing**
|
||||
- `INSERT INTO blobs` - Create blob record with UUID (blob_hash NULL)
|
||||
- `INSERT INTO blob_chunks` - Associate chunks with blob immediately
|
||||
- `UPDATE blobs SET blob_hash = ?, finished_ts = ?` - Finalize blob after packing
|
||||
Once a blob is uploaded, `compressed_size` is the size of the object at the
|
||||
destination. A blob that was already there can have a different size from the
|
||||
local copy. That happens when it was written with another compression level or
|
||||
other recipients.
|
||||
|
||||
4. **Upload**
|
||||
- `UPDATE blobs SET uploaded_ts = ?` - Mark blob as uploaded
|
||||
- `INSERT INTO uploads` - Record upload metrics with snapshot_id
|
||||
- `INSERT INTO snapshot_blobs` - Associate blob with snapshot
|
||||
### `blob_chunks`
|
||||
|
||||
5. **Snapshot Completion**
|
||||
- `UPDATE snapshots SET completed_at = ?, stats...` - Finalize snapshot
|
||||
- Generate and upload blob manifest from `snapshot_blobs`
|
||||
The chunks that each blob holds.
|
||||
|
||||
### 2. Incremental Backup
|
||||
- `blob_id`: the blob. References `blobs.id`.
|
||||
- `chunk_hash`: the chunk. References `chunks.chunk_hash`.
|
||||
- `offset`: the byte offset of the chunk within the uncompressed blob.
|
||||
- `length`: the length of the chunk in bytes.
|
||||
|
||||
1. **Change Detection**
|
||||
- `SELECT * FROM files WHERE path = ?` - Get previous file metadata
|
||||
- Compare mtime, size, mode to detect changes
|
||||
- Skip unchanged files but still add to `snapshot_files`
|
||||
The primary key is (`blob_id`, `chunk_hash`). `idx_blob_chunks_chunk_hash`
|
||||
indexes `chunk_hash`.
|
||||
|
||||
2. **Chunk Reuse**
|
||||
- `SELECT * FROM blob_chunks WHERE chunk_hash = ?` - Find existing chunks
|
||||
- `INSERT INTO snapshot_blobs` - Reference existing blobs for unchanged files
|
||||
A restore finds the blob for each chunk through this table.
|
||||
|
||||
### 3. Snapshot Metadata Export
|
||||
### `snapshots`
|
||||
|
||||
After a snapshot is completed:
|
||||
1. Copy database to temporary file
|
||||
2. Clean temporary database to contain only current snapshot data
|
||||
3. Export to SQL dump using sqlite3
|
||||
4. Compress with zstd and encrypt with age
|
||||
5. Upload to S3 as `metadata/{snapshot-id}/db.zst.age`
|
||||
6. Generate blob manifest and upload as `metadata/{snapshot-id}/manifest.json.zst`
|
||||
One row for each snapshot.
|
||||
|
||||
### 4. Restore Process
|
||||
- `id`: the snapshot ID. Primary key.
|
||||
- `hostname`: the hostname of the machine that made the backup.
|
||||
- `vaultik_version`: the `vaultik` version that made it.
|
||||
- `vaultik_git_revision`: the git commit of that version.
|
||||
- `started_at`: when the backup started.
|
||||
- `completed_at`: when the backup finished. It is `NULL` until then.
|
||||
- `file_count`: the number of files and symlinks in the snapshot. Directories
|
||||
are not counted.
|
||||
- `chunk_count`: the number of chunks this snapshot stored that were not stored
|
||||
before.
|
||||
- `new_chunk_size`: the total size of those chunks before compression.
|
||||
- `blob_count`: the number of blobs this snapshot created.
|
||||
- `total_size`: the total size of all files in the snapshot.
|
||||
- `uncompressed_chunk_size`: the total size, before compression, of the
|
||||
distinct chunks the snapshot's files use.
|
||||
- `blob_size`: the total compressed size of the blobs the snapshot uses.
|
||||
- `blob_uncompressed_size`: the total uncompressed size of those blobs.
|
||||
- `compression_ratio`: the compression ratio of those blobs.
|
||||
- `compression_level`: the compression level the backup used.
|
||||
- `upload_bytes`: the bytes uploaded during the backup.
|
||||
- `upload_duration_ms`: the time spent uploading, in milliseconds.
|
||||
|
||||
The restore process doesn't use the local database. Instead:
|
||||
1. Downloads snapshot metadata from S3
|
||||
2. Downloads required blobs based on manifest
|
||||
3. Reconstructs files from decrypted and decompressed chunks
|
||||
The snapshot ID has the form `<hostname>_<snapshot-name>_<timestamp>`. An
|
||||
example is `server1_home_2025-06-01T12:00:00Z`. It has three parts:
|
||||
|
||||
### 5. Pruning
|
||||
- the hostname, up to its first `.`
|
||||
- the snapshot name
|
||||
- the start time, in UTC
|
||||
|
||||
1. **Identify Unreferenced Blobs**
|
||||
- Query blobs not referenced by any remaining snapshot
|
||||
- Delete from S3 and local database
|
||||
`snapshot list` shows `uncompressed_chunk_size` as `UNCOMPRESSED SIZE` and
|
||||
`new_chunk_size` as `NEW CHUNK SIZE`.
|
||||
|
||||
### 6. Incomplete Snapshot Cleanup
|
||||
A snapshot with `completed_at` set to `NULL` is unfinished. `snapshot list`
|
||||
does not show it. The next backup deletes it, and so does `prune`.
|
||||
|
||||
Before each backup:
|
||||
1. Query incomplete snapshots (where `completed_at IS NULL`)
|
||||
2. Check if metadata exists in S3
|
||||
3. If no metadata, delete snapshot and all associations
|
||||
4. Clean up orphaned files, chunks, and blobs
|
||||
### `snapshot_files`
|
||||
|
||||
## Repository Pattern
|
||||
The files in each snapshot.
|
||||
|
||||
Vaultik uses a repository pattern for database access:
|
||||
- `snapshot_id`: the snapshot. References `snapshots.id`.
|
||||
- `file_id`: the file. References `files.id`.
|
||||
|
||||
- `FileRepository` - CRUD operations for files and file metadata
|
||||
- `ChunkRepository` - CRUD operations for content chunks
|
||||
- `FileChunkRepository` - Manage file-to-chunk mappings
|
||||
- `ChunkFileRepository` - Manage chunk-to-file reverse mappings
|
||||
- `BlobRepository` - Manage blob lifecycle (creation, finalization, upload)
|
||||
- `BlobChunkRepository` - Manage blob-to-chunk associations
|
||||
- `SnapshotRepository` - Manage snapshots and their relationships
|
||||
- `UploadRepository` - Track blob upload metrics
|
||||
The primary key is (`snapshot_id`, `file_id`). `idx_snapshot_files_file_id`
|
||||
indexes `file_id`.
|
||||
|
||||
Each repository provides methods like:
|
||||
- `Create()` - Insert new record
|
||||
- `GetByID()` / `GetByPath()` / `GetByHash()` - Retrieve records
|
||||
- `Update()` - Update existing records
|
||||
- `Delete()` - Remove records
|
||||
- Specialized queries for each entity type (e.g., `DeleteOrphaned()`, `GetIncompleteByHostname()`)
|
||||
### `snapshot_blobs`
|
||||
|
||||
## Transaction Management
|
||||
The blobs that each snapshot uses.
|
||||
|
||||
All database operations that modify multiple tables are wrapped in transactions:
|
||||
- `snapshot_id`: the snapshot. References `snapshots.id`.
|
||||
- `blob_id`: the blob. References `blobs.id`.
|
||||
- `blob_hash`: a copy of the blob's hash.
|
||||
|
||||
The primary key is (`snapshot_id`, `blob_id`). `idx_snapshot_blobs_blob_id`
|
||||
indexes `blob_id`.
|
||||
|
||||
The manifest is built from this table.
|
||||
|
||||
### `snapshot_mount_points`
|
||||
|
||||
The mount points that a snapshot's backup found.
|
||||
|
||||
- `snapshot_id`: the snapshot. References `snapshots.id`.
|
||||
- `path`: the absolute path of the directory.
|
||||
|
||||
The primary key is (`snapshot_id`, `path`).
|
||||
|
||||
A directory is recorded here when all of these hold:
|
||||
|
||||
- It is at or below a configured path.
|
||||
- The backup found an entry in it that is not excluded.
|
||||
- It is on a different filesystem from its parent directory.
|
||||
|
||||
The device number is read after the directory has been listed. Listing an
|
||||
automount point mounts its volume, so the number is that of the volume. An
|
||||
automount point whose volume cannot be mounted is not recorded, because the
|
||||
backup finds no entry in it.
|
||||
|
||||
`snapshot create --prune` reads this table. It skips the prune when an earlier
|
||||
snapshot of the same name recorded a mount point that the new backup did not
|
||||
find. That happens when the volume is not mounted.
|
||||
|
||||
### `uploads`
|
||||
|
||||
One row for each blob that a backup uploaded.
|
||||
|
||||
- `blob_hash`: the blob. References `blobs.blob_hash`. Primary key.
|
||||
- `snapshot_id`: the snapshot whose backup uploaded it. References
|
||||
`snapshots.id`.
|
||||
- `uploaded_at`: when the upload started. It is written as a date and time, not
|
||||
as seconds.
|
||||
- `size`: the size of the uploaded blob.
|
||||
- `duration_ms`: the duration of the upload, in milliseconds.
|
||||
|
||||
`idx_uploads_snapshot_id` indexes `snapshot_id`.
|
||||
|
||||
A blob that was already at the destination gets no row here.
|
||||
|
||||
### `local_meta`
|
||||
|
||||
Settings that belong to this copy of the index.
|
||||
|
||||
- `key`: the name of the setting. Primary key.
|
||||
- `value`: its value.
|
||||
|
||||
The only key is `storage_url`. It holds the destination that the index was
|
||||
first used with. A command that changes data refuses to run when the config
|
||||
names a different destination.
|
||||
|
||||
### `schema_migrations`
|
||||
|
||||
The migrations that have run.
|
||||
|
||||
- `version`: the number of the migration. Primary key.
|
||||
- `applied_at`: when it ran. It is written as a date and time, not as seconds.
|
||||
|
||||
## How commands use the tables
|
||||
|
||||
### Backup
|
||||
|
||||
A backup works on the tables in this order:
|
||||
|
||||
- It deletes unfinished snapshots, and the rows that no snapshot uses.
|
||||
- It creates a row in `snapshots`, with `completed_at` set to `NULL`.
|
||||
- It deletes the `blobs` rows that were never uploaded.
|
||||
- It walks each path and compares every entry with its row in `files`.
|
||||
- It links each unchanged file to the snapshot in `snapshot_files`.
|
||||
- It records each mount point in `snapshot_mount_points`.
|
||||
- It reads each changed file and stores it. See below.
|
||||
- It fills `snapshot_blobs` with every blob the snapshot uses.
|
||||
- It stores the snapshot's totals in `snapshots`.
|
||||
- It uploads the snapshot's metadata.
|
||||
- It sets `completed_at`.
|
||||
|
||||
A file counts as changed when its row in `files` differs from the file on disk
|
||||
in any of these:
|
||||
|
||||
- size
|
||||
- modification time
|
||||
- mode
|
||||
- owner or group
|
||||
- link target
|
||||
|
||||
A file is also read again when it lists a chunk that no uploaded blob holds.
|
||||
|
||||
Storing a changed file writes these rows:
|
||||
|
||||
- A row in `blobs`, when packing of a blob starts.
|
||||
- When the blob is closed, in one transaction:
|
||||
- a row in `chunks` for each new chunk in the blob
|
||||
- the blob's rows in `blob_chunks`
|
||||
- the blob's `blob_hash`, sizes and `finished_ts`
|
||||
- When the blob is stored at the destination, in one transaction:
|
||||
- `uploaded_ts` and `compressed_size` in `blobs`
|
||||
- a row in `snapshot_blobs`
|
||||
- a row in `uploads`, unless the blob was already at the destination
|
||||
- The row in `files`, with its `file_chunks`, `chunk_files` and
|
||||
`snapshot_files` rows.
|
||||
|
||||
The row in `files` is written last. It is written only after every blob that
|
||||
holds the file's new chunks is stored.
|
||||
|
||||
### Snapshot metadata
|
||||
|
||||
The metadata of a snapshot is made from a copy of the index:
|
||||
|
||||
- The index is copied into a temporary directory with `VACUUM INTO`.
|
||||
- The rows of every other snapshot are deleted from the copy. So are the files
|
||||
and blobs that this snapshot does not use. A chunk is then deleted when no
|
||||
remaining file and no remaining blob uses it.
|
||||
- The `local_meta` and `schema_migrations` tables are kept as they are. The
|
||||
uploaded database therefore names the destination, and lists the migrations
|
||||
that have run.
|
||||
- `VACUUM` rewrites the copy, so that no deleted row is left in the file.
|
||||
- The copy is compressed with zstd and encrypted with age.
|
||||
- The manifest is built from the copy's `snapshot_blobs` table.
|
||||
- The copy is uploaded as `metadata/<remote-key>/db.zst.age`.
|
||||
- The manifest is uploaded as `metadata/<remote-key>/manifest.json.zst`.
|
||||
|
||||
The copy is made through SQLite, not by copying the file. The index runs in WAL
|
||||
mode, so rows that are already committed can still be in the `-wal` file beside
|
||||
it.
|
||||
|
||||
### Restore
|
||||
|
||||
A restore does not use the local index.
|
||||
|
||||
- It downloads and decrypts `db.zst.age` into a temporary directory.
|
||||
- It opens that file read-only. It applies no migration to it.
|
||||
- It reads the files to restore from `files`.
|
||||
- It finds each file's chunks in `file_chunks`.
|
||||
- It finds the blob and offset of each chunk in `blob_chunks`.
|
||||
- It finds the name of each blob in `blobs`.
|
||||
|
||||
It does not read the manifest.
|
||||
|
||||
### Prune
|
||||
|
||||
`prune` changes the local index in two ways:
|
||||
|
||||
- It deletes each snapshot whose manifest is gone from the destination.
|
||||
- It deletes unfinished snapshots, and the rows that no snapshot uses.
|
||||
|
||||
The second step removes unused rows in this order:
|
||||
|
||||
- files that are in no snapshot
|
||||
- blobs that are in no snapshot
|
||||
- `blob_chunks` rows whose chunk is gone
|
||||
- chunks that no file and no blob uses
|
||||
|
||||
`prune` then deletes unused data at the destination. That step reads the
|
||||
manifests, not the local index. It reads the local index only to keep a
|
||||
directory under `metadata/` whose snapshot is still recorded there.
|
||||
|
||||
## Code
|
||||
|
||||
Each repository type in `internal/database` serves one or more tables:
|
||||
|
||||
- `FileRepository`: `files`
|
||||
- `ChunkRepository`: `chunks`
|
||||
- `FileChunkRepository`: `file_chunks`
|
||||
- `ChunkFileRepository`: `chunk_files`
|
||||
- `BlobRepository`: `blobs`
|
||||
- `BlobChunkRepository`: `blob_chunks`
|
||||
- `SnapshotRepository`:
|
||||
- `snapshots`
|
||||
- `snapshot_files`
|
||||
- `snapshot_blobs`
|
||||
- `snapshot_mount_points`
|
||||
- `UploadRepository`: `uploads`
|
||||
- `LocalMetaRepository`: `local_meta`
|
||||
|
||||
No repository type serves `schema_migrations`. The code that applies the
|
||||
migrations reads and writes it.
|
||||
|
||||
`Repositories` holds one of each type.
|
||||
|
||||
`Repositories.WithTx` runs a function in one transaction:
|
||||
|
||||
```go
|
||||
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
// Multiple repository operations using tx
|
||||
// several repository calls that use tx
|
||||
})
|
||||
```
|
||||
|
||||
This ensures consistency, especially important for operations like:
|
||||
- Creating file-chunk mappings
|
||||
- Associating chunks with blobs
|
||||
- Updating snapshot statistics
|
||||
The transaction commits when the function returns no error. Otherwise it is
|
||||
rolled back.
|
||||
|
||||
## Performance Considerations
|
||||
## SQLite settings
|
||||
|
||||
1. **Indexes**:
|
||||
- Primary keys are automatically indexed
|
||||
- `idx_files_path` on `files(path)` for efficient file lookups
|
||||
|
||||
2. **Prepared Statements**: All queries use prepared statements for performance and security
|
||||
|
||||
3. **Batch Operations**: Where possible, operations are batched within transactions
|
||||
|
||||
4. **Write-Ahead Logging**: SQLite WAL mode is enabled for better concurrency
|
||||
|
||||
## Data Integrity
|
||||
|
||||
1. **Foreign Keys**: Enforced through CASCADE DELETE and application-level repository methods
|
||||
2. **Unique Constraints**: Chunk hashes, file paths, and blob hashes are unique
|
||||
3. **Null Handling**: Nullable fields clearly indicate in-progress operations
|
||||
4. **Timestamp Tracking**: All major operations record timestamps for auditing
|
||||
- The index runs in WAL mode. A command that only reads, such as
|
||||
`snapshot list`, can read the index while a backup writes to it.
|
||||
- A connection waits up to 10 seconds for a lock before it fails.
|
||||
- Foreign keys are enforced.
|
||||
- Each open database uses one connection, so its queries run one at a time.
|
||||
- Queries pass their values as parameters.
|
||||
|
||||
+375
-102
@@ -1,143 +1,416 @@
|
||||
# Vaultik S3 Repository Structure
|
||||
# vaultik destination layout
|
||||
|
||||
This document describes the structure and organization of data stored in the S3 bucket by Vaultik.
|
||||
This document describes what `vaultik` stores at the destination. It covers the
|
||||
layout, what the layout protects, and the risks it accepts.
|
||||
|
||||
## Overview
|
||||
The destination is the place the backups are stored. It is an S3 bucket and
|
||||
prefix, a local directory or an rclone remote. The layout is the same on all
|
||||
three.
|
||||
|
||||
Vaultik stores all backup data in an S3-compatible object store. The repository consists of two main components:
|
||||
1. **Blobs** - The actual backup data (content-addressed, encrypted)
|
||||
2. **Metadata** - Snapshot information and manifests (partially encrypted)
|
||||
## Terms
|
||||
|
||||
## Directory Structure
|
||||
- **Snapshot**: one backup of the paths listed under one name in the config.
|
||||
- **Snapshot ID**: the name of a snapshot, in the form
|
||||
`<hostname>_<snapshot-name>_<timestamp>`.
|
||||
- **Chunk**: a piece of a file's content, named by the SHA-256 hash of that
|
||||
content.
|
||||
- **Blob**: one object under `blobs/`. It holds many chunks.
|
||||
- **Local index**: the SQLite database on the backed-up machine.
|
||||
- **Recipient**: an age public key that backups are encrypted to.
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
<bucket>/<prefix>/
|
||||
<destination>/
|
||||
├── vaultik-destination.txt
|
||||
├── blobs/
|
||||
│ └── <hash[0:2]>/
|
||||
│ └── <hash[2:4]>/
|
||||
│ └── <full-hash>
|
||||
│ └── <hash>
|
||||
└── metadata/
|
||||
└── <snapshot-id>/
|
||||
└── <remote-key>/
|
||||
├── db.zst.age
|
||||
└── manifest.json.zst
|
||||
```
|
||||
|
||||
## Blobs Directory (`blobs/`)
|
||||
The destination holds three things:
|
||||
|
||||
### Structure
|
||||
- **Path format**: `blobs/<first-2-chars>/<next-2-chars>/<full-hash>`
|
||||
- **Example**: `blobs/ca/fe/cafebabe1234567890abcdef1234567890abcdef1234567890abcdef12345678`
|
||||
- **Sharding**: The two-level directory structure (using the first 4 characters of the hash) prevents any single directory from containing too many objects
|
||||
- `vaultik-destination.txt`, the marker file
|
||||
- `blobs/`, which holds the file contents
|
||||
- `metadata/`, which holds one directory for each snapshot
|
||||
|
||||
### Content
|
||||
- **What it contains**: Packed collections of content-defined chunks from files
|
||||
- **Format**: Zstandard compressed, then Age encrypted
|
||||
- **Encryption**: Always encrypted with Age using the configured recipients
|
||||
- **Naming**: Content-addressed using SHA256 hash of the encrypted blob
|
||||
## The marker file
|
||||
|
||||
### Why Encrypted
|
||||
Blobs contain the actual file data from backups and must be encrypted for security. The content-addressing ensures deduplication while the encryption ensures privacy.
|
||||
Every backup writes `vaultik-destination.txt` before it uploads anything else.
|
||||
The file holds one fixed line of text. It names no host, path or time.
|
||||
|
||||
## Metadata Directory (`metadata/`)
|
||||
The marker tells an emptied or missing destination apart from a new one. An
|
||||
unplugged volume leaves an empty directory at its mount point. Without the
|
||||
marker, that directory would look like a destination whose snapshots are all
|
||||
gone.
|
||||
|
||||
Each snapshot has its own subdirectory named with the snapshot ID.
|
||||
The check applies while the local index holds snapshots. When the destination
|
||||
then answers that the marker does not exist, commands treat the destination as
|
||||
not present:
|
||||
|
||||
### Snapshot ID Format
|
||||
- **Format**: `<hostname>-<YYYYMMDD>-<HHMMSSZ>`
|
||||
- **Example**: `laptop-20240115-143052Z`
|
||||
- **Components**:
|
||||
- Hostname (may contain hyphens)
|
||||
- Date in YYYYMMDD format
|
||||
- Time in HHMMSSZ format (Z indicates UTC)
|
||||
- `snapshot create` refuses to write to it.
|
||||
- `prune`, `snapshot purge` and `remote nuke` fail.
|
||||
- `snapshot list` shows the local index alone.
|
||||
- `snapshot remove` deletes nothing at the destination.
|
||||
|
||||
### Files in Each Snapshot Directory
|
||||
The check is skipped when the look-up fails for any other reason. S3 refuses
|
||||
the look-up to credentials that can only write. A machine with such credentials
|
||||
therefore does not notice an emptied destination. Its next backup would not
|
||||
upload data that its local index says is stored. After a destination is emptied,
|
||||
run `vaultik database delete` on such a machine before it backs up again.
|
||||
|
||||
#### `db.zst.age` - Encrypted Database Dump
|
||||
- **What it contains**: Complete SQLite database dump for this snapshot
|
||||
- **Format**: SQL dump → Zstandard compressed → Age encrypted
|
||||
- **Encryption**: Encrypted with Age
|
||||
- **Purpose**: Contains full file metadata, chunk mappings, and all relationships
|
||||
- **Why encrypted**: Contains sensitive metadata like file paths, permissions, and ownership
|
||||
`remote nuke` deletes the marker last.
|
||||
|
||||
## Blobs
|
||||
|
||||
A blob holds the content of many chunks, one after another.
|
||||
|
||||
- **Path**: `blobs/<first 2 characters>/<next 2 characters>/<full hash>`.
|
||||
- **Example**:
|
||||
`blobs/ca/fe/cafebabe1234567890abcdef1234567890abcdef1234567890abcdef12345678`.
|
||||
- **Format**: the chunk data, compressed with zstd and then encrypted with age.
|
||||
- **Recipients**: the public keys in `age_recipients` when the blob was written.
|
||||
|
||||
The two directory levels keep any one directory from holding too many objects.
|
||||
|
||||
### Blob names
|
||||
|
||||
A blob's name is `hex(SHA256(SHA256(uncompressed contents)))`. The contents are
|
||||
the chunk data, joined in order, before compression and encryption. The name is
|
||||
not a hash of the stored bytes.
|
||||
|
||||
This has two consequences:
|
||||
|
||||
- Checking a blob against its name needs the private key. The blob must be
|
||||
decrypted and decompressed first. `snapshot restore` and
|
||||
`snapshot verify --deep` do this.
|
||||
- Without the key, only a blob's presence and size can be checked.
|
||||
`snapshot verify` does this.
|
||||
|
||||
`blobgen.DoubleSHA256` in `internal/blobgen` computes the name.
|
||||
|
||||
### Uploads
|
||||
|
||||
A backup looks a blob up by name before it uploads it. It does not upload a
|
||||
blob that is already there.
|
||||
|
||||
When the destination refuses the look-up, the backup uploads the blob. An
|
||||
object with that name is then replaced by one that holds the same chunk data.
|
||||
|
||||
## Metadata
|
||||
|
||||
Each snapshot has one directory under `metadata/`. The directory is named with
|
||||
the snapshot's remote key, not with its snapshot ID.
|
||||
|
||||
### Snapshot IDs
|
||||
|
||||
A snapshot ID has the form `<hostname>_<snapshot-name>_<timestamp>`. An example
|
||||
is `laptop_home_2024-01-15T14:30:52Z`. It has three parts:
|
||||
|
||||
- the hostname, up to its first `.`
|
||||
- the snapshot name from the config
|
||||
- the time the backup started, in UTC
|
||||
|
||||
The snapshot ID is used on the command line, in log records and in the local
|
||||
index. It reveals the hostname, the snapshot name and the backup time. It is
|
||||
therefore never written to the destination without encryption.
|
||||
|
||||
### Remote keys
|
||||
|
||||
The remote key is `hex(SHA256(SHA256("vaultik|" + snapshot ID)))`. It is 64 hex
|
||||
characters long. `internal/snapshot/remotekey.go` computes it.
|
||||
|
||||
For example:
|
||||
|
||||
- Snapshot ID: `server1_home_2025-06-01T12:00:00Z`
|
||||
- Remote key: `17f97bcde958748af076b926af59823943db59e80ce7170b40f124dfa28f64aa`
|
||||
- Directory:
|
||||
`metadata/17f97bcde958748af076b926af59823943db59e80ce7170b40f124dfa28f64aa/`
|
||||
|
||||
A listing of the destination therefore shows no hostname and no snapshot name.
|
||||
|
||||
The hash cannot be reversed, but it uses no secret. Someone who guesses a
|
||||
snapshot ID can hash it the same way and see whether that snapshot exists. The
|
||||
remote key keeps names out of a listing. It does not hide them from a guess.
|
||||
|
||||
### `db.zst.age`
|
||||
|
||||
The snapshot's database. A restore needs it.
|
||||
|
||||
- **Contents**: a SQLite database. It is a copy of the local index without the
|
||||
rows of other snapshots. It lists every file in the snapshot with its
|
||||
metadata, its chunks, and the blob that holds each chunk.
|
||||
- **Format**: the database file, compressed with zstd and then encrypted with
|
||||
age.
|
||||
|
||||
It is encrypted because it holds file paths, permissions and owners.
|
||||
[DATAMODEL.md](DATAMODEL.md) describes its tables.
|
||||
|
||||
### `manifest.json.zst`
|
||||
|
||||
The list of the blobs the snapshot uses.
|
||||
|
||||
- **Format**: JSON, compressed with zstd. It is not encrypted.
|
||||
|
||||
#### `manifest.json.zst` - Unencrypted Blob Manifest
|
||||
- **What it contains**: JSON list of all blob hashes referenced by this snapshot
|
||||
- **Format**: JSON → Zstandard compressed (NOT encrypted)
|
||||
- **Encryption**: NOT encrypted
|
||||
- **Purpose**: Enables pruning operations without requiring decryption keys
|
||||
- **Structure**:
|
||||
```json
|
||||
{
|
||||
"snapshot_id": "laptop-20240115-143052Z",
|
||||
"timestamp": "2024-01-15T14:30:52Z",
|
||||
"blob_count": 42,
|
||||
"blobs": [
|
||||
"cafebabe1234567890abcdef1234567890abcdef1234567890abcdef12345678",
|
||||
"deadbeef1234567890abcdef1234567890abcdef1234567890abcdef12345678",
|
||||
...
|
||||
]
|
||||
"snapshot_id": "17f97bcde958748af076b926af59823943db59e80ce7170b40f124dfa28f64aa",
|
||||
"timestamp": "2025-06-01T12:04:31Z",
|
||||
"blob_count": 2,
|
||||
"total_compressed_size": 57344,
|
||||
"blobs": [
|
||||
{
|
||||
"hash": "cafebabe1234567890abcdef1234567890abcdef1234567890abcdef12345678",
|
||||
"compressed_size": 24576
|
||||
},
|
||||
{
|
||||
"hash": "deadbeef1234567890abcdef1234567890abcdef1234567890abcdef12345678",
|
||||
"compressed_size": 32768
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Why Manifest is Unencrypted
|
||||
The manifest must be readable without the private key to enable:
|
||||
1. **Pruning operations** - Identifying unreferenced blobs for deletion
|
||||
2. **Storage analysis** - Understanding space usage without decryption
|
||||
3. **Verification** - Checking blob existence without decryption
|
||||
4. **Cross-snapshot deduplication analysis** - Finding shared blobs between snapshots
|
||||
- `snapshot_id`: the remote key, not the snapshot ID.
|
||||
- `timestamp`: when the manifest was written, at the end of the backup.
|
||||
- `blob_count`: the number of blobs.
|
||||
- `total_compressed_size`: the total size of those blobs, in bytes.
|
||||
- `blobs`: the hash and the size of each blob.
|
||||
|
||||
The manifest only contains blob hashes, not file names or any other sensitive information.
|
||||
The manifest holds no file name and no path.
|
||||
|
||||
## Security Considerations
|
||||
The manifest is not encrypted so that these commands work without the private
|
||||
key:
|
||||
|
||||
### What's Encrypted
|
||||
- **All file content** (in blobs)
|
||||
- **All file metadata** (paths, permissions, timestamps, ownership in db.zst.age)
|
||||
- **File-to-chunk mappings** (in db.zst.age)
|
||||
- `prune`, which must know which blobs are still in use
|
||||
- `snapshot verify`, which checks that each blob is present
|
||||
- `snapshot list` and `remote info`, which report what the destination holds
|
||||
|
||||
### What's Not Encrypted
|
||||
- **Blob hashes** (in manifest.json.zst)
|
||||
- **Snapshot IDs** (directory names)
|
||||
- **Blob count per snapshot** (in manifest.json.zst)
|
||||
## What is protected
|
||||
|
||||
### Privacy Implications
|
||||
From the unencrypted data, an observer can determine:
|
||||
- When backups were taken (from snapshot IDs)
|
||||
- Which hostname created backups (from snapshot IDs)
|
||||
- How many blobs each snapshot references
|
||||
- Which blobs are shared between snapshots (deduplication patterns)
|
||||
- The size of each encrypted blob
|
||||
These are encrypted:
|
||||
|
||||
An observer cannot determine:
|
||||
- File names or paths
|
||||
- File contents
|
||||
- File permissions or ownership
|
||||
- Directory structure
|
||||
- Which chunks belong to which files
|
||||
- file contents, in the blobs
|
||||
- in `db.zst.age`:
|
||||
- file names and paths
|
||||
- permissions and owners
|
||||
- modification times
|
||||
- which chunks make up which file
|
||||
|
||||
## Consistency Guarantees
|
||||
These are not encrypted:
|
||||
|
||||
1. **Blobs are immutable** - Once written, a blob is never modified
|
||||
2. **Blobs are written before metadata** - A snapshot's metadata is only written after all its blobs are successfully uploaded
|
||||
3. **Metadata is written atomically** - Both db.zst.age and manifest.json.zst are written as complete files
|
||||
4. **Snapshots are marked complete in local DB only after metadata upload** - Ensures consistency between local and remote state
|
||||
- the remote key of each snapshot
|
||||
- the time each backup finished, in the manifest
|
||||
- the hash and the compressed size of each blob, in the manifest
|
||||
- the number of blobs and their total size for each snapshot, in the manifest
|
||||
|
||||
## Pruning Safety
|
||||
Someone who can read the destination therefore learns these:
|
||||
|
||||
The prune operation is safe because:
|
||||
1. It only deletes blobs not referenced in any manifest
|
||||
2. Manifests are unencrypted and can be read without keys
|
||||
3. The operation compares the latest local DB snapshot with the latest S3 snapshot to ensure consistency
|
||||
4. Pruning will fail if these don't match, preventing accidental deletion of needed blobs
|
||||
- when each backup finished
|
||||
- how many blobs each snapshot uses, and how large they are
|
||||
- which blobs two snapshots share
|
||||
- whether a snapshot ID they guess exists
|
||||
|
||||
## Restoration Requirements
|
||||
They do not learn these:
|
||||
|
||||
To restore from a backup, you need:
|
||||
1. **The Age private key** - To decrypt blobs and database
|
||||
2. **The snapshot metadata** - Both files from the snapshot's metadata directory
|
||||
3. **All referenced blobs** - As listed in the manifest
|
||||
- any hostname or snapshot name, unless they guess it
|
||||
- file names or paths
|
||||
- file contents
|
||||
- permissions or owners
|
||||
- the directory structure
|
||||
- which chunks belong to which file
|
||||
|
||||
The restoration process:
|
||||
1. Download and decrypt the database dump to understand file structure
|
||||
2. Download and decrypt the required blobs
|
||||
3. Reconstruct files from their chunks
|
||||
4. Restore file metadata (permissions, timestamps, etc.)
|
||||
The times and sizes could not be hidden by encrypting the manifest. The
|
||||
destination itself shows when each object was written and how large it is.
|
||||
|
||||
## Accepted risks
|
||||
|
||||
These are known properties of the format. They are written down so that you can
|
||||
weigh them.
|
||||
|
||||
### No proof of who wrote the data
|
||||
|
||||
A restore and `snapshot verify --deep` prove two things. The data decrypts with
|
||||
your private key, and it matches its hashes. They do not prove who wrote it.
|
||||
|
||||
The hashes use no secret. Someone who knows a recipient's public key and can
|
||||
replace objects at the destination can put a snapshot of their own there. The
|
||||
public key is not stored at the destination, but a backed-up machine that is
|
||||
broken into has it.
|
||||
|
||||
The defences are on the destination's side:
|
||||
|
||||
- object versioning or object lock on the bucket
|
||||
- credentials for the backed-up machine that cannot delete or overwrite
|
||||
- running `prune` from a trusted machine
|
||||
|
||||
An S3 `PutObject` replaces an existing object. Permission to write alone
|
||||
therefore does not stop overwrites.
|
||||
|
||||
### Sizes are visible
|
||||
|
||||
An object's size follows exactly from the size of its compressed contents.
|
||||
|
||||
All the new chunks in one blob share one zstd stream. The stream looks back 8
|
||||
MiB, or 4 MiB at compression levels 1 and 2. A blob is closed at
|
||||
`blob_size_limit` and at the end of each configured path. A chunk that is
|
||||
already stored is never packed again.
|
||||
|
||||
Data that an outsider writes compresses smaller when it matches a secret in the
|
||||
same chunk, so the blob gets smaller. The blob's size therefore shows the
|
||||
outsider whether their data matched the secret.
|
||||
|
||||
This works only when all of these hold:
|
||||
|
||||
- The outsider can write into a backed-up file.
|
||||
- The outsider can see the blob sizes at the destination.
|
||||
- The outsider's data and the secret land in the same chunk.
|
||||
- The file keeps changing.
|
||||
|
||||
Back up a directory that outsiders can write to as a snapshot of its own.
|
||||
|
||||
### Chunk boundaries use no secret
|
||||
|
||||
The FastCDC parameters are fixed and public. `chunk_size` sets the average chunk
|
||||
size. No chunk is larger than four times this size. Only a file's last chunk can
|
||||
be smaller than a quarter of it. With the default of 10 MB, those limits are 40
|
||||
MB and 2.5 MB. A file of 2.5 MB or less is one chunk.
|
||||
|
||||
At the default `blob_size_limit` of 10 GB, a blob holds hundreds of chunks. The
|
||||
size of the blob then does not show the size of any one chunk. A limit close to
|
||||
`chunk_size` begins to show chunk sizes.
|
||||
|
||||
### Decrypted data on the local disk
|
||||
|
||||
These commands write unencrypted data to the temporary directory:
|
||||
|
||||
- `snapshot restore` writes decrypted blobs under `vaultik-blobcache-*/`. There
|
||||
is no limit on their total size.
|
||||
- `snapshot restore` writes the decrypted database to
|
||||
`vaultik-restore-*/snapshot.db`.
|
||||
- `snapshot verify --deep` writes the decrypted database to
|
||||
`vaultik-verify-*/snapshot.db`.
|
||||
- `snapshot create` writes a copy of the local index to
|
||||
`vaultik-snapshot-*/snapshot.db`.
|
||||
|
||||
These files are readable only by their owner, and are removed when the command
|
||||
ends. A `kill -9` or a power loss leaves them behind. Delete any leftover
|
||||
`vaultik-*` entry in the temporary directory yourself.
|
||||
|
||||
Trust the temporary directory as much as the directory you restore into.
|
||||
|
||||
### Commands need different permissions
|
||||
|
||||
`snapshot create` needs only permission to write objects. Other commands need
|
||||
more:
|
||||
|
||||
- `snapshot list`, `snapshot verify`, `snapshot restore` and `remote info` list
|
||||
and read.
|
||||
- `snapshot purge`, `snapshot remove` and `remote nuke` list and delete.
|
||||
- `prune` lists, reads and deletes.
|
||||
|
||||
`snapshot create --prune` runs a prune on the backed-up machine. That machine
|
||||
then holds credentials that can delete its own backups. To keep it to
|
||||
write-only credentials, run `prune` from a separate, trusted machine.
|
||||
|
||||
### New recipients cannot read old data
|
||||
|
||||
Changing `age_recipients` does not re-encrypt what is already stored. A chunk
|
||||
that is already stored is not written again, so it stays encrypted to the
|
||||
earlier recipients.
|
||||
|
||||
A new snapshot that uses such chunks cannot be restored with a new recipient's
|
||||
key alone. To make everything readable by a new key, run
|
||||
`vaultik database delete` and take a full backup to a new destination.
|
||||
|
||||
### X25519 recipients only
|
||||
|
||||
`vaultik` accepts age X25519 public keys only. It rejects age SSH and plugin
|
||||
recipients.
|
||||
|
||||
The encrypted data is held by a third party for a long time. There is no second
|
||||
kind of recipient to fall back on if X25519 is broken, and no post-quantum
|
||||
option.
|
||||
|
||||
## Order of writes
|
||||
|
||||
A backup writes in a fixed order:
|
||||
|
||||
- the marker file
|
||||
- the blobs
|
||||
- `db.zst.age`
|
||||
- `manifest.json.zst`
|
||||
|
||||
A snapshot's metadata is written only after all its blobs are uploaded. The
|
||||
manifest is written last. A directory under `metadata/` that has no manifest
|
||||
therefore belongs to a backup that did not finish.
|
||||
|
||||
The local index marks a snapshot complete only after both metadata files are
|
||||
uploaded. A backup that stops during that upload leaves the snapshot
|
||||
unfinished. The next backup removes it from the local index and stores its data
|
||||
again.
|
||||
|
||||
One short gap remains. A backup can stop after the upload and before the
|
||||
snapshot is marked complete. The destination then holds a snapshot that can be
|
||||
restored, and the local index does not list it. `snapshot list` shows it as
|
||||
remote only.
|
||||
|
||||
An upload that is cut off leaves no partial object under the final name on S3
|
||||
or on a local directory. The same holds for an rclone remote that can move
|
||||
objects. An rclone remote that cannot move objects can keep a partial object.
|
||||
|
||||
## Pruning
|
||||
|
||||
`prune` deletes two kinds of object at the destination:
|
||||
|
||||
- a blob that no manifest lists
|
||||
- a directory under `metadata/` that has no manifest
|
||||
|
||||
It reads the manifests, which are not encrypted, so it needs no private key.
|
||||
|
||||
These rules keep it from deleting data that is in use:
|
||||
|
||||
- It keeps every blob that any manifest lists.
|
||||
- It deletes nothing when any manifest cannot be downloaded or decoded.
|
||||
- It deletes a blob only once the blob is older than `prune_grace_period`.
|
||||
- It deletes a directory only once every object in it is that old.
|
||||
- It keeps a directory whose snapshot is in the local index.
|
||||
- It leaves alone a directory whose name is not a remote key.
|
||||
|
||||
`prune_grace_period` defaults to `24h`. It protects a backup that is still
|
||||
running. Such a backup has uploaded blobs and perhaps `db.zst.age`, but not yet
|
||||
its manifest. A prune from this machine or another keeps those objects, as long
|
||||
as no backup takes longer than the grace period.
|
||||
|
||||
The grace period does not protect one case. A backup does not upload a blob
|
||||
that is already at the destination, so that blob keeps the time of its first
|
||||
upload. The blob can be left over from an earlier backup that stopped before
|
||||
its manifest. A prune that runs before the new manifest is written can then
|
||||
delete that blob. The new snapshot cannot be restored afterwards.
|
||||
|
||||
To avoid this case, give each machine and each user its own destination.
|
||||
|
||||
`remote nuke` applies none of these rules.
|
||||
|
||||
## What a restore needs
|
||||
|
||||
- the age private key
|
||||
- the snapshot's `db.zst.age`
|
||||
- the blobs that hold the files to restore
|
||||
|
||||
A restore does not read the manifest and does not need the local index.
|
||||
|
||||
It works in these steps:
|
||||
|
||||
- It downloads and decrypts `db.zst.age`.
|
||||
- It checks that the database is the one for the snapshot that was asked for.
|
||||
- It downloads and decrypts the blobs it needs. It checks each blob against its
|
||||
name.
|
||||
- It writes each file from its chunks.
|
||||
- It sets each file's permissions and times, and its owner when it runs as
|
||||
root.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module git.eeqj.de/sneak/vaultik
|
||||
module sneak.berlin/go/vaultik
|
||||
|
||||
go 1.24.4
|
||||
go 1.26.1
|
||||
|
||||
require (
|
||||
filippo.io/age v1.2.1
|
||||
@@ -17,13 +17,15 @@ require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/johannesboyne/gofakes3 v0.0.0-20250603205740-ed9094be7668
|
||||
github.com/klauspost/compress v1.18.1
|
||||
github.com/mattn/go-sqlite3 v1.14.29
|
||||
github.com/rclone/rclone v1.72.1
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/spf13/afero v1.15.0
|
||||
github.com/spf13/cobra v1.10.1
|
||||
github.com/spf13/pflag v1.0.10
|
||||
github.com/stretchr/testify v1.11.1
|
||||
go.uber.org/dig v1.19.0
|
||||
go.uber.org/fx v1.24.0
|
||||
golang.org/x/sync v0.18.0
|
||||
golang.org/x/sys v0.38.0
|
||||
golang.org/x/term v0.37.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
modernc.org/sqlite v1.38.0
|
||||
@@ -186,7 +188,6 @@ require (
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
@@ -217,7 +218,6 @@ require (
|
||||
github.com/relvacode/iso8601 v1.7.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rfjakob/eme v1.1.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
|
||||
@@ -228,7 +228,6 @@ require (
|
||||
github.com/smarty/assertions v1.16.0 // indirect
|
||||
github.com/sony/gobreaker v1.0.0 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.25-0.20251022131615-eb24eb109368 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 // indirect
|
||||
github.com/tidwall/gjson v1.18.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
@@ -258,7 +257,6 @@ require (
|
||||
go.opentelemetry.io/otel/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.38.0 // indirect
|
||||
go.shabbyrobe.org/gocovmerge v0.0.0-20230507111327-fa4f82cfbf4d // indirect
|
||||
go.uber.org/dig v1.19.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
@@ -266,8 +264,6 @@ require (
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/oauth2 v0.33.0 // indirect
|
||||
golang.org/x/sync v0.18.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
golang.org/x/tools v0.38.0 // indirect
|
||||
|
||||
@@ -202,8 +202,6 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc=
|
||||
github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts=
|
||||
github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
|
||||
github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
|
||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 h1:z0uK8UQqjMVYzvk4tiiu3obv2B44+XBsvgEJREQfnO8=
|
||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9/go.mod h1:Jl2neWsQaDanWORdqZ4emBl50J4/aRBBS4FyyG9/PFo=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
@@ -593,16 +591,12 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
|
||||
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
||||
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/mattn/go-sqlite3 v1.14.29 h1:1O6nRLJKvsi1H2Sj0Hzdfojwt8GiGKm+LOfLaBFaouQ=
|
||||
github.com/mattn/go-sqlite3 v1.14.29/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
|
||||
github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
|
||||
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
|
||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||
github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
|
||||
@@ -707,8 +701,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4=
|
||||
github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
@@ -723,8 +715,6 @@ github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDj
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
|
||||
github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw=
|
||||
github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
|
||||
github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc=
|
||||
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
|
||||
+14
-1
@@ -2,5 +2,18 @@ package blob
|
||||
|
||||
import "errors"
|
||||
|
||||
// ErrBlobSizeLimitExceeded is returned when adding a chunk would exceed the blob size limit
|
||||
// ErrBlobSizeLimitExceeded is returned when adding a chunk would exceed
|
||||
// the blob size limit.
|
||||
var ErrBlobSizeLimitExceeded = errors.New("adding chunk would exceed blob size limit")
|
||||
|
||||
// ErrNoRecipients is returned when a Packer is created without any age
|
||||
// recipients; blobs must always be encrypted.
|
||||
var ErrNoRecipients = errors.New("recipients are required - blobs must be encrypted")
|
||||
|
||||
// ErrInvalidMaxBlobSize is returned when the configured maximum blob size
|
||||
// is zero or negative.
|
||||
var ErrInvalidMaxBlobSize = errors.New("max blob size must be positive")
|
||||
|
||||
// ErrNoFilesystem is returned when a Packer is created without a filesystem
|
||||
// for temporary files.
|
||||
var ErrNoFilesystem = errors.New("filesystem is required")
|
||||
|
||||
+286
-186
@@ -1,14 +1,16 @@
|
||||
// Package blob handles the creation of blobs - the final storage units for Vaultik.
|
||||
// A blob is a large file (up to 10GB) containing many compressed and encrypted chunks
|
||||
// from multiple source files. Blobs are content-addressed, meaning their filename
|
||||
// is derived from the SHA256 hash of their compressed and encrypted content.
|
||||
// from multiple source files. Blobs are content-addressed: a blob's filename is
|
||||
// hex(SHA256(SHA256(uncompressed blob contents))), computed from the concatenated
|
||||
// chunk data before compression and encryption, not from the stored bytes. See
|
||||
// blobgen.DoubleSHA256 and docs/REPOSTRUCTURE.md.
|
||||
//
|
||||
// The blob creation process:
|
||||
// 1. Chunks are accumulated from multiple files
|
||||
// 2. The collection is compressed using zstd
|
||||
// 3. The compressed data is encrypted using age
|
||||
// 4. The encrypted blob is hashed to create its content-addressed name
|
||||
// 5. The blob is uploaded to S3 using the hash as the filename
|
||||
// 1. Chunks are accumulated from multiple files
|
||||
// 2. Each chunk's uncompressed bytes are fed to a running SHA-256 and, in the same
|
||||
// pass, compressed with zstd and encrypted with age into the temp file
|
||||
// 3. On finalize, the name is the double SHA-256 of that uncompressed content
|
||||
// 4. The blob is uploaded to S3 using the name as the filename
|
||||
//
|
||||
// This design optimizes storage efficiency by batching many small chunks into
|
||||
// larger blobs, reducing the number of S3 operations and associated costs.
|
||||
@@ -18,34 +20,42 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/blobgen"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"github.com/google/uuid"
|
||||
"github.com/spf13/afero"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// BlobHandler is a callback function invoked when a blob is finalized and ready for upload.
|
||||
// The handler receives a BlobWithReader containing the blob metadata and a reader for
|
||||
// the compressed and encrypted blob content. The handler is responsible for uploading
|
||||
// the blob to storage and cleaning up any temporary files.
|
||||
type BlobHandler func(blob *BlobWithReader) error
|
||||
// Handler is a callback function invoked when a blob is finalized and
|
||||
// ready for upload. The handler receives a WithReader containing the
|
||||
// blob metadata and a reader for the compressed and encrypted blob content.
|
||||
// The handler is responsible for uploading the blob to storage and cleaning
|
||||
// up any temporary files.
|
||||
type Handler func(blob *WithReader) error
|
||||
|
||||
// PackerConfig holds configuration for creating a Packer.
|
||||
// All fields except BlobHandler are required.
|
||||
type PackerConfig struct {
|
||||
MaxBlobSize int64 // Maximum size of a blob before forcing finalization
|
||||
CompressionLevel int // Zstd compression level (1-19, higher = better compression)
|
||||
Recipients []string // Age recipients for encryption
|
||||
Repositories *database.Repositories // Database repositories for tracking blob metadata
|
||||
BlobHandler BlobHandler // Optional callback when blob is ready for upload
|
||||
Fs afero.Fs // Filesystem for temporary files
|
||||
// MaxBlobSize is the maximum size of a blob before forcing finalization.
|
||||
MaxBlobSize int64
|
||||
// CompressionLevel is the zstd level (1-19, higher = better compression).
|
||||
CompressionLevel int
|
||||
// Recipients holds the age recipients for encryption.
|
||||
Recipients []string
|
||||
// Repositories provides database access for tracking blob metadata.
|
||||
Repositories *database.Repositories
|
||||
// BlobHandler is an optional callback when a blob is ready for upload.
|
||||
BlobHandler Handler
|
||||
// Fs is the filesystem used for temporary files.
|
||||
Fs afero.Fs
|
||||
}
|
||||
|
||||
// PendingChunk represents a chunk waiting to be inserted into the database.
|
||||
@@ -61,7 +71,7 @@ type Packer struct {
|
||||
maxBlobSize int64
|
||||
compressionLevel int
|
||||
recipients []string // Age recipients for encryption
|
||||
blobHandler BlobHandler // Called when blob is ready
|
||||
blobHandler Handler // Called when blob is ready
|
||||
repos *database.Repositories // For creating blob records
|
||||
fs afero.Fs // Filesystem for temporary files
|
||||
|
||||
@@ -108,22 +118,23 @@ type FinishedBlob struct {
|
||||
ID string
|
||||
Hash string
|
||||
Data []byte // Compressed data
|
||||
Chunks []*BlobChunkRef
|
||||
Chunks []*ChunkPosition
|
||||
CreatedTS time.Time
|
||||
Uncompressed int64
|
||||
Compressed int64
|
||||
}
|
||||
|
||||
// BlobChunkRef represents a chunk's position within a blob
|
||||
type BlobChunkRef struct {
|
||||
// ChunkPosition represents a chunk's position within a blob
|
||||
type ChunkPosition struct {
|
||||
ChunkHash string
|
||||
Offset int64
|
||||
Length int64
|
||||
}
|
||||
|
||||
// BlobWithReader wraps a FinishedBlob with its data reader
|
||||
type BlobWithReader struct {
|
||||
// WithReader wraps a FinishedBlob with its data reader
|
||||
type WithReader struct {
|
||||
*FinishedBlob
|
||||
|
||||
Reader io.ReadSeeker
|
||||
TempFile afero.File // Optional, only set for disk-based blobs
|
||||
InsertedChunkHashes []string // Chunk hashes that were inserted to DB with this blob
|
||||
@@ -134,14 +145,17 @@ type BlobWithReader struct {
|
||||
// Returns an error if required configuration fields are missing or invalid.
|
||||
func NewPacker(cfg PackerConfig) (*Packer, error) {
|
||||
if len(cfg.Recipients) == 0 {
|
||||
return nil, fmt.Errorf("recipients are required - blobs must be encrypted")
|
||||
return nil, ErrNoRecipients
|
||||
}
|
||||
|
||||
if cfg.MaxBlobSize <= 0 {
|
||||
return nil, fmt.Errorf("max blob size must be positive")
|
||||
return nil, ErrInvalidMaxBlobSize
|
||||
}
|
||||
|
||||
if cfg.Fs == nil {
|
||||
return nil, fmt.Errorf("filesystem is required")
|
||||
return nil, ErrNoFilesystem
|
||||
}
|
||||
|
||||
return &Packer{
|
||||
maxBlobSize: cfg.MaxBlobSize,
|
||||
compressionLevel: cfg.CompressionLevel,
|
||||
@@ -157,9 +171,10 @@ func NewPacker(cfg PackerConfig) (*Packer, error) {
|
||||
// The handler is responsible for uploading the blob to storage.
|
||||
// If no handler is set, finalized blobs are stored in memory and can be
|
||||
// retrieved with GetFinishedBlobs().
|
||||
func (p *Packer) SetBlobHandler(handler BlobHandler) {
|
||||
func (p *Packer) SetBlobHandler(handler Handler) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.blobHandler = handler
|
||||
}
|
||||
|
||||
@@ -169,6 +184,7 @@ func (p *Packer) SetBlobHandler(handler BlobHandler) {
|
||||
func (p *Packer) AddPendingChunk(hash string, size int64) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.pendingChunks = append(p.pendingChunks, PendingChunk{Hash: hash, Size: size})
|
||||
}
|
||||
|
||||
@@ -177,13 +193,14 @@ func (p *Packer) AddPendingChunk(hash string, size int64) {
|
||||
// In this case, the caller should finalize the current blob and retry.
|
||||
// The chunk data is written immediately and can be garbage collected after this call.
|
||||
// Thread-safe.
|
||||
func (p *Packer) AddChunk(chunk *ChunkRef) error {
|
||||
func (p *Packer) AddChunk(ctx context.Context, chunk *ChunkRef) error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
// Initialize new blob if needed
|
||||
if p.currentBlob == nil {
|
||||
if err := p.startNewBlob(); err != nil {
|
||||
err := p.startNewBlob(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("starting new blob: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -202,7 +219,8 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error {
|
||||
}
|
||||
|
||||
// Add chunk to current blob
|
||||
if err := p.addChunkToCurrentBlob(chunk); err != nil {
|
||||
err := p.addChunkToCurrentBlob(chunk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -213,12 +231,13 @@ func (p *Packer) AddChunk(chunk *ChunkRef) error {
|
||||
// This should be called after all chunks have been added to ensure no data is lost.
|
||||
// If a BlobHandler is set, it will be called with the finalized blob.
|
||||
// Thread-safe.
|
||||
func (p *Packer) Flush() error {
|
||||
func (p *Packer) Flush(ctx context.Context) error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
if p.currentBlob != nil && len(p.currentBlob.chunks) > 0 {
|
||||
if err := p.finalizeCurrentBlob(); err != nil {
|
||||
err := p.finalizeCurrentBlob(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("finalizing blob: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -226,13 +245,24 @@ func (p *Packer) Flush() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DiscardCurrentBlob closes and removes the temp file of the blob being
|
||||
// packed, if any, and drops that blob. A caller that stops before Flush
|
||||
// must call it, or the temp file is left behind. Thread-safe.
|
||||
func (p *Packer) DiscardCurrentBlob() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
p.cleanupTempFile()
|
||||
p.currentBlob = nil
|
||||
}
|
||||
|
||||
// FinalizeBlob finalizes the current blob being assembled.
|
||||
// This compresses the accumulated chunks, encrypts the result, and computes
|
||||
// the content-addressed hash. The finalized blob is either passed to the
|
||||
// BlobHandler (if set) or stored internally.
|
||||
// Caller must handle retrying any chunk that triggered size limit exceeded.
|
||||
// Not thread-safe - caller must hold the lock.
|
||||
func (p *Packer) FinalizeBlob() error {
|
||||
func (p *Packer) FinalizeBlob(ctx context.Context) error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
@@ -240,7 +270,7 @@ func (p *Packer) FinalizeBlob() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
return p.finalizeCurrentBlob()
|
||||
return p.finalizeCurrentBlob(ctx)
|
||||
}
|
||||
|
||||
// GetFinishedBlobs returns all completed blobs and clears the internal list.
|
||||
@@ -253,11 +283,37 @@ func (p *Packer) GetFinishedBlobs() []*FinishedBlob {
|
||||
|
||||
blobs := p.finishedBlobs
|
||||
p.finishedBlobs = make([]*FinishedBlob, 0)
|
||||
|
||||
return blobs
|
||||
}
|
||||
|
||||
// PackChunks is a convenience method to pack multiple chunks at once.
|
||||
func (p *Packer) PackChunks(ctx context.Context, chunks []*ChunkRef) error {
|
||||
for _, chunk := range chunks {
|
||||
err := p.AddChunk(ctx, chunk)
|
||||
if errors.Is(err, ErrBlobSizeLimitExceeded) {
|
||||
// Finalize current blob and retry
|
||||
err = p.FinalizeBlob(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("finalizing blob before retry: %w", err)
|
||||
}
|
||||
|
||||
// Retry the chunk
|
||||
err = p.AddChunk(ctx, chunk)
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"adding chunk %s after finalize: %w", chunk.Hash, err)
|
||||
}
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("adding chunk %s: %w", chunk.Hash, err)
|
||||
}
|
||||
}
|
||||
|
||||
return p.Flush(ctx)
|
||||
}
|
||||
|
||||
// startNewBlob initializes a new blob (must be called with lock held)
|
||||
func (p *Packer) startNewBlob() error {
|
||||
func (p *Packer) startNewBlob(ctx context.Context) error {
|
||||
// Generate UUID for the blob
|
||||
blobID := uuid.New().String()
|
||||
|
||||
@@ -267,18 +323,24 @@ func (p *Packer) startNewBlob() error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing blob ID: %w", err)
|
||||
}
|
||||
|
||||
blob := &database.Blob{
|
||||
ID: blobIDTyped,
|
||||
Hash: types.BlobHash("temp-placeholder-" + blobID), // Temporary placeholder until finalized
|
||||
ID: blobIDTyped,
|
||||
// Temporary placeholder hash until finalized.
|
||||
Hash: types.BlobHash("temp-placeholder-" + blobID),
|
||||
CreatedTS: time.Now().UTC(),
|
||||
FinishedTS: nil,
|
||||
UncompressedSize: 0,
|
||||
CompressedSize: 0,
|
||||
UploadedTS: nil,
|
||||
}
|
||||
if err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return p.repos.Blobs.Create(ctx, tx, blob)
|
||||
}); err != nil {
|
||||
|
||||
err = p.repos.WithTx(
|
||||
ctx,
|
||||
func(txCtx context.Context, tx *sql.Tx) error {
|
||||
return p.repos.Blobs.Create(txCtx, tx, blob)
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating blob record: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -294,6 +356,7 @@ func (p *Packer) startNewBlob() error {
|
||||
if err != nil {
|
||||
_ = tempFile.Close()
|
||||
_ = p.fs.Remove(tempFile.Name())
|
||||
|
||||
return fmt.Errorf("creating blobgen writer: %w", err)
|
||||
}
|
||||
|
||||
@@ -307,15 +370,20 @@ func (p *Packer) startNewBlob() error {
|
||||
size: 0,
|
||||
}
|
||||
|
||||
log.Debug("Created new blob container", "blob_id", blobID, "temp_file", tempFile.Name())
|
||||
log.Debug("Created new blob container",
|
||||
"blob_id", blobID, "temp_file", tempFile.Name())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// addChunkToCurrentBlob adds a chunk to the current blob (must be called with lock held)
|
||||
// addChunkToCurrentBlob adds a chunk to the current blob (must be called
|
||||
// with lock held).
|
||||
func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error {
|
||||
// Skip if chunk already in current blob
|
||||
if p.currentBlob.chunkSet[chunk.Hash] {
|
||||
log.Debug("Skipping duplicate chunk already in current blob", "chunk_hash", chunk.Hash)
|
||||
log.Debug("Skipping duplicate chunk already in current blob",
|
||||
"chunk_hash", chunk.Hash)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -323,7 +391,8 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error {
|
||||
offset := p.currentBlob.size
|
||||
|
||||
// Write to the blobgen writer (compression -> encryption -> disk)
|
||||
if _, err := p.currentBlob.writer.Write(chunk.Data); err != nil {
|
||||
_, err := p.currentBlob.writer.Write(chunk.Data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing to blob stream: %w", err)
|
||||
}
|
||||
|
||||
@@ -356,106 +425,29 @@ func (p *Packer) addChunkToCurrentBlob(chunk *ChunkRef) error {
|
||||
}
|
||||
|
||||
// finalizeCurrentBlob completes the current blob (must be called with lock held)
|
||||
func (p *Packer) finalizeCurrentBlob() error {
|
||||
func (p *Packer) finalizeCurrentBlob(ctx context.Context) error {
|
||||
if p.currentBlob == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close blobgen writer to flush all data
|
||||
if err := p.currentBlob.writer.Close(); err != nil {
|
||||
p.cleanupTempFile()
|
||||
return fmt.Errorf("closing blobgen writer: %w", err)
|
||||
}
|
||||
|
||||
// Sync file to ensure all data is written
|
||||
if err := p.currentBlob.tempFile.Sync(); err != nil {
|
||||
p.cleanupTempFile()
|
||||
return fmt.Errorf("syncing temp file: %w", err)
|
||||
}
|
||||
|
||||
// Get the final size (encrypted if applicable)
|
||||
finalSize, err := p.currentBlob.tempFile.Seek(0, io.SeekCurrent)
|
||||
blobHash, finalSize, err := p.closeBlobWriter()
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
return fmt.Errorf("getting file size: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Reset to beginning for reading
|
||||
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
|
||||
p.cleanupTempFile()
|
||||
return fmt.Errorf("seeking to start: %w", err)
|
||||
}
|
||||
chunkRefs := p.buildChunkRefs()
|
||||
|
||||
// Get hash from blobgen writer (of final encrypted data)
|
||||
finalHash := p.currentBlob.writer.Sum256()
|
||||
blobHash := hex.EncodeToString(finalHash)
|
||||
|
||||
// Create chunk references with offsets
|
||||
chunkRefs := make([]*BlobChunkRef, 0, len(p.currentBlob.chunks))
|
||||
|
||||
for _, chunk := range p.currentBlob.chunks {
|
||||
chunkRefs = append(chunkRefs, &BlobChunkRef{
|
||||
ChunkHash: chunk.Hash,
|
||||
Offset: chunk.Offset,
|
||||
Length: chunk.Size,
|
||||
})
|
||||
}
|
||||
|
||||
// Get pending chunks (will be inserted to DB and reported to handler)
|
||||
chunksToInsert := p.pendingChunks
|
||||
p.pendingChunks = nil // Clear pending list
|
||||
p.pendingChunks = nil
|
||||
|
||||
// Insert pending chunks, blob_chunks, and update blob in a single transaction
|
||||
if p.repos != nil {
|
||||
blobIDTyped, parseErr := types.ParseBlobID(p.currentBlob.id)
|
||||
if parseErr != nil {
|
||||
p.cleanupTempFile()
|
||||
return fmt.Errorf("parsing blob ID: %w", parseErr)
|
||||
}
|
||||
err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
// First insert all pending chunks (required for blob_chunks FK)
|
||||
for _, chunk := range chunksToInsert {
|
||||
dbChunk := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash(chunk.Hash),
|
||||
Size: chunk.Size,
|
||||
}
|
||||
if err := p.repos.Chunks.Create(ctx, tx, dbChunk); err != nil {
|
||||
return fmt.Errorf("creating chunk: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Insert all blob_chunk records in batch
|
||||
for _, chunk := range p.currentBlob.chunks {
|
||||
blobChunk := &database.BlobChunk{
|
||||
BlobID: blobIDTyped,
|
||||
ChunkHash: types.ChunkHash(chunk.Hash),
|
||||
Offset: chunk.Offset,
|
||||
Length: chunk.Size,
|
||||
}
|
||||
if err := p.repos.BlobChunks.Create(ctx, tx, blobChunk); err != nil {
|
||||
return fmt.Errorf("creating blob_chunk: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Update blob record with final hash and sizes
|
||||
return p.repos.Blobs.UpdateFinished(ctx, tx, p.currentBlob.id, blobHash,
|
||||
p.currentBlob.size, finalSize)
|
||||
})
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
return fmt.Errorf("finalizing blob transaction: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Committed blob transaction",
|
||||
"chunks_inserted", len(chunksToInsert),
|
||||
"blob_chunks_inserted", len(p.currentBlob.chunks))
|
||||
err = p.commitBlobToDatabase(ctx, blobHash, finalSize, chunksToInsert)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create finished blob
|
||||
finished := &FinishedBlob{
|
||||
ID: p.currentBlob.id,
|
||||
Hash: blobHash,
|
||||
Data: nil, // We don't load data into memory anymore
|
||||
Chunks: chunkRefs,
|
||||
CreatedTS: p.currentBlob.startTime,
|
||||
Uncompressed: p.currentBlob.size,
|
||||
@@ -464,63 +456,192 @@ func (p *Packer) finalizeCurrentBlob() error {
|
||||
|
||||
compressionRatio := float64(finished.Compressed) / float64(finished.Uncompressed)
|
||||
log.Info("Finalized blob (compressed and encrypted)",
|
||||
"hash", blobHash,
|
||||
"chunks", len(chunkRefs),
|
||||
"uncompressed", finished.Uncompressed,
|
||||
"compressed", finished.Compressed,
|
||||
"hash", blobHash, "chunks", len(chunkRefs),
|
||||
"uncompressed", finished.Uncompressed, "compressed", finished.Compressed,
|
||||
"ratio", fmt.Sprintf("%.2f", compressionRatio),
|
||||
"duration", time.Since(p.currentBlob.startTime))
|
||||
|
||||
// Collect inserted chunk hashes for the scanner to track
|
||||
var insertedChunkHashes []string
|
||||
for _, chunk := range chunksToInsert {
|
||||
insertedChunkHashes = append(insertedChunkHashes, chunk.Hash)
|
||||
}
|
||||
|
||||
// Call blob handler if set
|
||||
return p.deliverFinishedBlob(finished, insertedChunkHashes)
|
||||
}
|
||||
|
||||
// closeBlobWriter closes the writer, syncs to disk, and returns the blob
|
||||
// hash and final size.
|
||||
func (p *Packer) closeBlobWriter() (string, int64, error) {
|
||||
err := p.currentBlob.writer.Close()
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return "", 0, fmt.Errorf("closing blobgen writer: %w", err)
|
||||
}
|
||||
|
||||
err = p.currentBlob.tempFile.Sync()
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return "", 0, fmt.Errorf("syncing temp file: %w", err)
|
||||
}
|
||||
|
||||
finalSize, err := p.currentBlob.tempFile.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return "", 0, fmt.Errorf("getting file size: %w", err)
|
||||
}
|
||||
|
||||
_, err = p.currentBlob.tempFile.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return "", 0, fmt.Errorf("seeking to start: %w", err)
|
||||
}
|
||||
|
||||
finalHash := p.currentBlob.writer.ContentID()
|
||||
|
||||
return hex.EncodeToString(finalHash), finalSize, nil
|
||||
}
|
||||
|
||||
// buildChunkRefs creates ChunkPosition entries from the current blob's chunks
|
||||
func (p *Packer) buildChunkRefs() []*ChunkPosition {
|
||||
refs := make([]*ChunkPosition, 0, len(p.currentBlob.chunks))
|
||||
for _, chunk := range p.currentBlob.chunks {
|
||||
refs = append(refs, &ChunkPosition{
|
||||
ChunkHash: chunk.Hash, Offset: chunk.Offset, Length: chunk.Size,
|
||||
})
|
||||
}
|
||||
|
||||
return refs
|
||||
}
|
||||
|
||||
// commitBlobToDatabase inserts pending chunks, blob_chunks, and updates the blob record
|
||||
func (p *Packer) commitBlobToDatabase(
|
||||
ctx context.Context,
|
||||
blobHash string, finalSize int64, chunksToInsert []PendingChunk,
|
||||
) error {
|
||||
if p.repos == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
blobIDTyped, parseErr := types.ParseBlobID(p.currentBlob.id)
|
||||
if parseErr != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("parsing blob ID: %w", parseErr)
|
||||
}
|
||||
|
||||
err := p.repos.WithTx(
|
||||
ctx,
|
||||
func(txCtx context.Context, tx *sql.Tx) error {
|
||||
return p.insertBlobRecords(txCtx, tx, blobIDTyped, blobHash,
|
||||
finalSize, chunksToInsert)
|
||||
})
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("finalizing blob transaction: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Committed blob transaction",
|
||||
"chunks_inserted", len(chunksToInsert),
|
||||
"blob_chunks_inserted", len(p.currentBlob.chunks))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// insertBlobRecords inserts pending chunks and blob_chunk rows, then marks
|
||||
// the blob finished, all within the supplied transaction.
|
||||
func (p *Packer) insertBlobRecords(
|
||||
ctx context.Context,
|
||||
tx *sql.Tx,
|
||||
blobIDTyped types.BlobID,
|
||||
blobHash string,
|
||||
finalSize int64,
|
||||
chunksToInsert []PendingChunk,
|
||||
) error {
|
||||
for _, chunk := range chunksToInsert {
|
||||
dbChunk := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size,
|
||||
}
|
||||
|
||||
err := p.repos.Chunks.Create(ctx, tx, dbChunk)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating chunk: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, chunk := range p.currentBlob.chunks {
|
||||
blobChunk := &database.BlobChunk{
|
||||
BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash),
|
||||
Offset: chunk.Offset, Length: chunk.Size,
|
||||
}
|
||||
|
||||
err := p.repos.BlobChunks.Create(ctx, tx, blobChunk)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating blob_chunk: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return p.repos.Blobs.UpdateFinished(ctx, tx, p.currentBlob.id, blobHash,
|
||||
p.currentBlob.size, finalSize)
|
||||
}
|
||||
|
||||
// deliverFinishedBlob passes the blob to the handler or stores it internally
|
||||
func (p *Packer) deliverFinishedBlob(
|
||||
finished *FinishedBlob, insertedChunkHashes []string,
|
||||
) error {
|
||||
if p.blobHandler != nil {
|
||||
// Reset file position for handler
|
||||
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
|
||||
_, err := p.currentBlob.tempFile.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("seeking for handler: %w", err)
|
||||
}
|
||||
|
||||
// Create a blob reader that includes the data stream
|
||||
blobWithReader := &BlobWithReader{
|
||||
blobWithReader := &WithReader{
|
||||
FinishedBlob: finished,
|
||||
Reader: p.currentBlob.tempFile,
|
||||
TempFile: p.currentBlob.tempFile,
|
||||
InsertedChunkHashes: insertedChunkHashes,
|
||||
}
|
||||
|
||||
if err := p.blobHandler(blobWithReader); err != nil {
|
||||
p.cleanupTempFile()
|
||||
return fmt.Errorf("blob handler failed: %w", err)
|
||||
}
|
||||
// Note: blob handler is responsible for closing/cleaning up temp file
|
||||
p.currentBlob = nil
|
||||
} else {
|
||||
log.Debug("No blob handler callback configured", "blob_hash", blobHash[:8]+"...")
|
||||
// No handler, need to read data for legacy behavior
|
||||
if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil {
|
||||
p.cleanupTempFile()
|
||||
return fmt.Errorf("seeking to read data: %w", err)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(p.currentBlob.tempFile)
|
||||
err = p.blobHandler(blobWithReader)
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
return fmt.Errorf("reading blob data: %w", err)
|
||||
|
||||
return fmt.Errorf("blob handler failed: %w", err)
|
||||
}
|
||||
finished.Data = data
|
||||
|
||||
p.finishedBlobs = append(p.finishedBlobs, finished)
|
||||
|
||||
// Cleanup
|
||||
p.cleanupTempFile()
|
||||
p.currentBlob = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// No handler - read data for legacy behavior
|
||||
log.Debug("No blob handler callback configured", "blob_hash", finished.Hash[:8]+"...")
|
||||
|
||||
_, err := p.currentBlob.tempFile.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("seeking to read data: %w", err)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(p.currentBlob.tempFile)
|
||||
if err != nil {
|
||||
p.cleanupTempFile()
|
||||
|
||||
return fmt.Errorf("reading blob data: %w", err)
|
||||
}
|
||||
|
||||
finished.Data = data
|
||||
p.finishedBlobs = append(p.finishedBlobs, finished)
|
||||
p.cleanupTempFile()
|
||||
p.currentBlob = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -532,24 +653,3 @@ func (p *Packer) cleanupTempFile() {
|
||||
_ = p.fs.Remove(name)
|
||||
}
|
||||
}
|
||||
|
||||
// PackChunks is a convenience method to pack multiple chunks at once
|
||||
func (p *Packer) PackChunks(chunks []*ChunkRef) error {
|
||||
for _, chunk := range chunks {
|
||||
err := p.AddChunk(chunk)
|
||||
if err == ErrBlobSizeLimitExceeded {
|
||||
// Finalize current blob and retry
|
||||
if err := p.FinalizeBlob(); err != nil {
|
||||
return fmt.Errorf("finalizing blob before retry: %w", err)
|
||||
}
|
||||
// Retry the chunk
|
||||
if err := p.AddChunk(chunk); err != nil {
|
||||
return fmt.Errorf("adding chunk %s after finalize: %w", chunk.Hash, err)
|
||||
}
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("adding chunk %s: %w", chunk.Hash, err)
|
||||
}
|
||||
}
|
||||
|
||||
return p.Flush()
|
||||
}
|
||||
|
||||
+254
-321
@@ -1,4 +1,4 @@
|
||||
package blob
|
||||
package blob_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -6,380 +6,313 @@ import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/spf13/afero"
|
||||
"sneak.berlin/go/vaultik/internal/blob"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
const (
|
||||
// Test key from test/insecure-integration-test.key
|
||||
testPrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5"
|
||||
testPublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"
|
||||
testPrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7A" +
|
||||
"PHXA2QS2NJA5"
|
||||
testPublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"
|
||||
|
||||
defaultMaxBlobSize = 10 * 1024 * 1024 // 10MB
|
||||
testChunkSize = 1000
|
||||
testChunkCount = 10
|
||||
)
|
||||
|
||||
func TestPacker(t *testing.T) {
|
||||
// Initialize logger for tests
|
||||
log.Initialize(log.Config{})
|
||||
// parseTestIdentity parses the fixed test age identity.
|
||||
func parseTestIdentity(t *testing.T) *age.X25519Identity {
|
||||
t.Helper()
|
||||
|
||||
// Parse test identity
|
||||
identity, err := age.ParseX25519Identity(testPrivateKey)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse test identity: %v", err)
|
||||
}
|
||||
|
||||
t.Run("single chunk creates single blob", func(t *testing.T) {
|
||||
// Create test database
|
||||
db, err := database.NewTestDB()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test db: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
cfg := PackerConfig{
|
||||
MaxBlobSize: 10 * 1024 * 1024, // 10MB
|
||||
CompressionLevel: 3,
|
||||
Recipients: []string{testPublicKey},
|
||||
Repositories: repos,
|
||||
Fs: afero.NewMemMapFs(),
|
||||
}
|
||||
packer, err := NewPacker(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create packer: %v", err)
|
||||
}
|
||||
|
||||
// Create a chunk
|
||||
data := []byte("Hello, World!")
|
||||
hash := sha256.Sum256(data)
|
||||
hashStr := hex.EncodeToString(hash[:])
|
||||
|
||||
// Create chunk in database first
|
||||
dbChunk := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash(hashStr),
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return identity
|
||||
}
|
||||
|
||||
// newTestPacker creates a test database and a Packer backed by it.
|
||||
func newTestPacker(
|
||||
t *testing.T, maxBlobSize int64,
|
||||
) (*database.Repositories, *blob.Packer) {
|
||||
t.Helper()
|
||||
|
||||
db, err := database.NewTestDB()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test db: %v", err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
packer, err := blob.NewPacker(blob.PackerConfig{
|
||||
MaxBlobSize: maxBlobSize,
|
||||
CompressionLevel: 3,
|
||||
Recipients: []string{testPublicKey},
|
||||
Repositories: repos,
|
||||
Fs: afero.NewMemMapFs(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create packer: %v", err)
|
||||
}
|
||||
|
||||
return repos, packer
|
||||
}
|
||||
|
||||
// makeChunk creates a ChunkRef for data and registers the chunk in the
|
||||
// database.
|
||||
func makeChunk(
|
||||
t *testing.T, repos *database.Repositories, data []byte,
|
||||
) *blob.ChunkRef {
|
||||
t.Helper()
|
||||
|
||||
hash := sha256.Sum256(data)
|
||||
hashStr := hex.EncodeToString(hash[:])
|
||||
|
||||
dbChunk := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash(hashStr),
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
|
||||
err := repos.WithTx(
|
||||
context.Background(),
|
||||
func(ctx context.Context, tx *sql.Tx) error {
|
||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk in db: %v", err)
|
||||
}
|
||||
|
||||
return &blob.ChunkRef{
|
||||
Hash: hashStr,
|
||||
Data: data,
|
||||
}
|
||||
}
|
||||
|
||||
// decryptAndDecompress reverses the blob pipeline: age decrypt, then zstd
|
||||
// decompress.
|
||||
func decryptAndDecompress(
|
||||
t *testing.T, blobData []byte, identity *age.X25519Identity,
|
||||
) []byte {
|
||||
t.Helper()
|
||||
|
||||
decrypted, err := age.Decrypt(bytes.NewReader(blobData), identity)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decrypt blob: %v", err)
|
||||
}
|
||||
|
||||
reader, err := zstd.NewReader(decrypted)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create decompressor: %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
var decompressed bytes.Buffer
|
||||
|
||||
_, err = io.Copy(&decompressed, reader)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decompress: %v", err)
|
||||
}
|
||||
|
||||
return decompressed.Bytes()
|
||||
}
|
||||
|
||||
func TestPackerSingleChunk(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
identity := parseTestIdentity(t)
|
||||
repos, packer := newTestPacker(t, defaultMaxBlobSize)
|
||||
ctx := context.Background()
|
||||
|
||||
data := []byte("Hello, World!")
|
||||
chunk := makeChunk(t, repos, data)
|
||||
|
||||
err := packer.AddChunk(ctx, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to add chunk: %v", err)
|
||||
}
|
||||
|
||||
err = packer.Flush(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to flush: %v", err)
|
||||
}
|
||||
|
||||
blobs := packer.GetFinishedBlobs()
|
||||
if len(blobs) != 1 {
|
||||
t.Fatalf("expected 1 blob, got %d", len(blobs))
|
||||
}
|
||||
|
||||
finished := blobs[0]
|
||||
if len(finished.Chunks) != 1 {
|
||||
t.Errorf("expected 1 chunk in blob, got %d", len(finished.Chunks))
|
||||
}
|
||||
|
||||
// Note: Very small data may not compress well
|
||||
t.Logf("Compression: %d -> %d bytes",
|
||||
finished.Uncompressed, finished.Compressed)
|
||||
|
||||
decompressed := decryptAndDecompress(t, finished.Data, identity)
|
||||
if !bytes.Equal(decompressed, data) {
|
||||
t.Error("decompressed data doesn't match original")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPackerMultipleChunks(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
repos, packer := newTestPacker(t, defaultMaxBlobSize)
|
||||
ctx := context.Background()
|
||||
|
||||
chunks := make([]*blob.ChunkRef, testChunkCount)
|
||||
for i := range testChunkCount {
|
||||
data := bytes.Repeat([]byte{byte(i)}, testChunkSize)
|
||||
chunks[i] = makeChunk(t, repos, data)
|
||||
}
|
||||
|
||||
for _, chunk := range chunks {
|
||||
err := packer.AddChunk(ctx, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk in db: %v", err)
|
||||
t.Fatalf("failed to add chunk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
chunk := &ChunkRef{
|
||||
Hash: hashStr,
|
||||
Data: data,
|
||||
}
|
||||
err := packer.Flush(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to flush: %v", err)
|
||||
}
|
||||
|
||||
// Add chunk
|
||||
if err := packer.AddChunk(chunk); err != nil {
|
||||
t.Fatalf("failed to add chunk: %v", err)
|
||||
}
|
||||
blobs := packer.GetFinishedBlobs()
|
||||
if len(blobs) != 1 {
|
||||
t.Fatalf("expected 1 blob, got %d", len(blobs))
|
||||
}
|
||||
|
||||
// Flush
|
||||
if err := packer.Flush(); err != nil {
|
||||
t.Fatalf("failed to flush: %v", err)
|
||||
}
|
||||
if len(blobs[0].Chunks) != testChunkCount {
|
||||
t.Errorf("expected %d chunks in blob, got %d",
|
||||
testChunkCount, len(blobs[0].Chunks))
|
||||
}
|
||||
|
||||
// Get finished blobs
|
||||
blobs := packer.GetFinishedBlobs()
|
||||
if len(blobs) != 1 {
|
||||
t.Fatalf("expected 1 blob, got %d", len(blobs))
|
||||
}
|
||||
// Verify offsets are correct
|
||||
expectedOffset := int64(0)
|
||||
|
||||
blob := blobs[0]
|
||||
if len(blob.Chunks) != 1 {
|
||||
t.Errorf("expected 1 chunk in blob, got %d", len(blob.Chunks))
|
||||
for i, chunkRef := range blobs[0].Chunks {
|
||||
if chunkRef.Offset != expectedOffset {
|
||||
t.Errorf("chunk %d: expected offset %d, got %d",
|
||||
i, expectedOffset, chunkRef.Offset)
|
||||
}
|
||||
|
||||
// Note: Very small data may not compress well
|
||||
t.Logf("Compression: %d -> %d bytes", blob.Uncompressed, blob.Compressed)
|
||||
|
||||
// Decrypt the blob data
|
||||
decrypted, err := age.Decrypt(bytes.NewReader(blob.Data), identity)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decrypt blob: %v", err)
|
||||
if chunkRef.Length != testChunkSize {
|
||||
t.Errorf("chunk %d: expected length %d, got %d",
|
||||
i, testChunkSize, chunkRef.Length)
|
||||
}
|
||||
|
||||
// Decompress the decrypted data
|
||||
reader, err := zstd.NewReader(decrypted)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create decompressor: %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
expectedOffset += chunkRef.Length
|
||||
}
|
||||
}
|
||||
|
||||
var decompressed bytes.Buffer
|
||||
if _, err := io.Copy(&decompressed, reader); err != nil {
|
||||
t.Fatalf("failed to decompress: %v", err)
|
||||
}
|
||||
func TestPackerSizeLimit(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
if !bytes.Equal(decompressed.Bytes(), data) {
|
||||
t.Error("decompressed data doesn't match original")
|
||||
}
|
||||
})
|
||||
const (
|
||||
maxBlobSize = 5000 // 5KB max, forces multiple blobs
|
||||
maxBlobawoOverhead = 6000 // allow some overhead over the limit
|
||||
)
|
||||
|
||||
t.Run("multiple chunks packed together", func(t *testing.T) {
|
||||
// Create test database
|
||||
db, err := database.NewTestDB()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test db: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
cfg := PackerConfig{
|
||||
MaxBlobSize: 10 * 1024 * 1024, // 10MB
|
||||
CompressionLevel: 3,
|
||||
Recipients: []string{testPublicKey},
|
||||
Repositories: repos,
|
||||
Fs: afero.NewMemMapFs(),
|
||||
}
|
||||
packer, err := NewPacker(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create packer: %v", err)
|
||||
}
|
||||
repos, packer := newTestPacker(t, maxBlobSize)
|
||||
ctx := context.Background()
|
||||
|
||||
// Create multiple small chunks
|
||||
chunks := make([]*ChunkRef, 10)
|
||||
for i := 0; i < 10; i++ {
|
||||
data := bytes.Repeat([]byte{byte(i)}, 1000)
|
||||
hash := sha256.Sum256(data)
|
||||
hashStr := hex.EncodeToString(hash[:])
|
||||
|
||||
// Create chunk in database first
|
||||
dbChunk := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash(hashStr),
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk in db: %v", err)
|
||||
}
|
||||
chunks := make([]*blob.ChunkRef, testChunkCount)
|
||||
for i := range testChunkCount {
|
||||
data := bytes.Repeat([]byte{byte(i)}, testChunkSize) // 1KB each
|
||||
chunks[i] = makeChunk(t, repos, data)
|
||||
}
|
||||
|
||||
chunks[i] = &ChunkRef{
|
||||
Hash: hashStr,
|
||||
Data: data,
|
||||
}
|
||||
}
|
||||
blobCount := 0
|
||||
|
||||
// Add all chunks
|
||||
for _, chunk := range chunks {
|
||||
err := packer.AddChunk(chunk)
|
||||
// Add chunks and handle size limit errors
|
||||
for _, chunk := range chunks {
|
||||
err := packer.AddChunk(ctx, chunk)
|
||||
if errors.Is(err, blob.ErrBlobSizeLimitExceeded) {
|
||||
// Finalize current blob
|
||||
err = packer.FinalizeBlob(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to add chunk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Flush
|
||||
if err := packer.Flush(); err != nil {
|
||||
t.Fatalf("failed to flush: %v", err)
|
||||
}
|
||||
|
||||
// Should have one blob with all chunks
|
||||
blobs := packer.GetFinishedBlobs()
|
||||
if len(blobs) != 1 {
|
||||
t.Fatalf("expected 1 blob, got %d", len(blobs))
|
||||
}
|
||||
|
||||
if len(blobs[0].Chunks) != 10 {
|
||||
t.Errorf("expected 10 chunks in blob, got %d", len(blobs[0].Chunks))
|
||||
}
|
||||
|
||||
// Verify offsets are correct
|
||||
expectedOffset := int64(0)
|
||||
for i, chunkRef := range blobs[0].Chunks {
|
||||
if chunkRef.Offset != expectedOffset {
|
||||
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunkRef.Offset)
|
||||
}
|
||||
if chunkRef.Length != 1000 {
|
||||
t.Errorf("chunk %d: expected length 1000, got %d", i, chunkRef.Length)
|
||||
t.Fatalf("failed to finalize blob: %v", err)
|
||||
}
|
||||
expectedOffset += chunkRef.Length
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("blob size limit enforced", func(t *testing.T) {
|
||||
// Create test database
|
||||
db, err := database.NewTestDB()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test db: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Small blob size limit to force multiple blobs
|
||||
cfg := PackerConfig{
|
||||
MaxBlobSize: 5000, // 5KB max
|
||||
CompressionLevel: 3,
|
||||
Recipients: []string{testPublicKey},
|
||||
Repositories: repos,
|
||||
Fs: afero.NewMemMapFs(),
|
||||
}
|
||||
packer, err := NewPacker(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create packer: %v", err)
|
||||
}
|
||||
blobCount++
|
||||
|
||||
// Create chunks that will exceed the limit
|
||||
chunks := make([]*ChunkRef, 10)
|
||||
for i := 0; i < 10; i++ {
|
||||
data := bytes.Repeat([]byte{byte(i)}, 1000) // 1KB each
|
||||
hash := sha256.Sum256(data)
|
||||
hashStr := hex.EncodeToString(hash[:])
|
||||
|
||||
// Create chunk in database first
|
||||
dbChunk := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash(hashStr),
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||
})
|
||||
// Retry adding the chunk
|
||||
err = packer.AddChunk(ctx, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk in db: %v", err)
|
||||
t.Fatalf("failed to add chunk after finalize: %v", err)
|
||||
}
|
||||
|
||||
chunks[i] = &ChunkRef{
|
||||
Hash: hashStr,
|
||||
Data: data,
|
||||
}
|
||||
}
|
||||
|
||||
blobCount := 0
|
||||
|
||||
// Add chunks and handle size limit errors
|
||||
for _, chunk := range chunks {
|
||||
err := packer.AddChunk(chunk)
|
||||
if err == ErrBlobSizeLimitExceeded {
|
||||
// Finalize current blob
|
||||
if err := packer.FinalizeBlob(); err != nil {
|
||||
t.Fatalf("failed to finalize blob: %v", err)
|
||||
}
|
||||
blobCount++
|
||||
// Retry adding the chunk
|
||||
if err := packer.AddChunk(chunk); err != nil {
|
||||
t.Fatalf("failed to add chunk after finalize: %v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatalf("failed to add chunk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Flush remaining
|
||||
if err := packer.Flush(); err != nil {
|
||||
t.Fatalf("failed to flush: %v", err)
|
||||
}
|
||||
|
||||
// Get all blobs
|
||||
blobs := packer.GetFinishedBlobs()
|
||||
totalBlobs := blobCount + len(blobs)
|
||||
|
||||
// Should have multiple blobs due to size limit
|
||||
if totalBlobs < 2 {
|
||||
t.Errorf("expected multiple blobs due to size limit, got %d", totalBlobs)
|
||||
}
|
||||
|
||||
// Verify each blob respects size limit (approximately)
|
||||
for _, blob := range blobs {
|
||||
if blob.Compressed > 6000 { // Allow some overhead
|
||||
t.Errorf("blob size %d exceeds limit", blob.Compressed)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("with encryption", func(t *testing.T) {
|
||||
// Create test database
|
||||
db, err := database.NewTestDB()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create test db: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Generate test identity (using the one from parent test)
|
||||
cfg := PackerConfig{
|
||||
MaxBlobSize: 10 * 1024 * 1024, // 10MB
|
||||
CompressionLevel: 3,
|
||||
Recipients: []string{testPublicKey},
|
||||
Repositories: repos,
|
||||
Fs: afero.NewMemMapFs(),
|
||||
} else if err != nil {
|
||||
t.Fatalf("failed to add chunk: %v", err)
|
||||
}
|
||||
packer, err := NewPacker(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create packer: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create test data
|
||||
data := bytes.Repeat([]byte("Test data for encryption!"), 100)
|
||||
hash := sha256.Sum256(data)
|
||||
hashStr := hex.EncodeToString(hash[:])
|
||||
err := packer.Flush(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to flush: %v", err)
|
||||
}
|
||||
|
||||
// Create chunk in database first
|
||||
dbChunk := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash(hashStr),
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
err = repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error {
|
||||
return repos.Chunks.Create(ctx, tx, dbChunk)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk in db: %v", err)
|
||||
}
|
||||
blobs := packer.GetFinishedBlobs()
|
||||
|
||||
chunk := &ChunkRef{
|
||||
Hash: hashStr,
|
||||
Data: data,
|
||||
}
|
||||
totalBlobs := blobCount + len(blobs)
|
||||
if totalBlobs < 2 {
|
||||
t.Errorf("expected multiple blobs due to size limit, got %d", totalBlobs)
|
||||
}
|
||||
|
||||
// Add chunk and flush
|
||||
if err := packer.AddChunk(chunk); err != nil {
|
||||
t.Fatalf("failed to add chunk: %v", err)
|
||||
}
|
||||
if err := packer.Flush(); err != nil {
|
||||
t.Fatalf("failed to flush: %v", err)
|
||||
// Verify each blob respects size limit (approximately)
|
||||
for _, finished := range blobs {
|
||||
if finished.Compressed > maxBlobawoOverhead {
|
||||
t.Errorf("blob size %d exceeds limit", finished.Compressed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get blob
|
||||
blobs := packer.GetFinishedBlobs()
|
||||
if len(blobs) != 1 {
|
||||
t.Fatalf("expected 1 blob, got %d", len(blobs))
|
||||
}
|
||||
func TestPackerEncryption(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
blob := blobs[0]
|
||||
identity := parseTestIdentity(t)
|
||||
repos, packer := newTestPacker(t, defaultMaxBlobSize)
|
||||
ctx := context.Background()
|
||||
|
||||
// Decrypt the blob
|
||||
decrypted, err := age.Decrypt(bytes.NewReader(blob.Data), identity)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decrypt blob: %v", err)
|
||||
}
|
||||
data := bytes.Repeat([]byte("Test data for encryption!"), 100)
|
||||
chunk := makeChunk(t, repos, data)
|
||||
|
||||
var decryptedData bytes.Buffer
|
||||
if _, err := decryptedData.ReadFrom(decrypted); err != nil {
|
||||
t.Fatalf("failed to read decrypted data: %v", err)
|
||||
}
|
||||
err := packer.AddChunk(ctx, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to add chunk: %v", err)
|
||||
}
|
||||
|
||||
// Decompress
|
||||
reader, err := zstd.NewReader(&decryptedData)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create decompressor: %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
err = packer.Flush(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to flush: %v", err)
|
||||
}
|
||||
|
||||
var decompressed bytes.Buffer
|
||||
if _, err := decompressed.ReadFrom(reader); err != nil {
|
||||
t.Fatalf("failed to decompress: %v", err)
|
||||
}
|
||||
blobs := packer.GetFinishedBlobs()
|
||||
if len(blobs) != 1 {
|
||||
t.Fatalf("expected 1 blob, got %d", len(blobs))
|
||||
}
|
||||
|
||||
// Verify data
|
||||
if !bytes.Equal(decompressed.Bytes(), data) {
|
||||
t.Error("decrypted and decompressed data doesn't match original")
|
||||
}
|
||||
})
|
||||
decompressed := decryptAndDecompress(t, blobs[0].Data, identity)
|
||||
if !bytes.Equal(decompressed, data) {
|
||||
t.Error("decrypted and decompressed data doesn't match original")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
package blobgen
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// CompressResult contains the results of compression
|
||||
type CompressResult struct {
|
||||
Data []byte
|
||||
UncompressedSize int64
|
||||
CompressedSize int64
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
// CompressData compresses and encrypts data, returning the result with hash
|
||||
func CompressData(data []byte, compressionLevel int, recipients []string) (*CompressResult, error) {
|
||||
var buf bytes.Buffer
|
||||
|
||||
// Create writer
|
||||
w, err := NewWriter(&buf, compressionLevel, recipients)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating writer: %w", err)
|
||||
}
|
||||
|
||||
// Write data
|
||||
if _, err := w.Write(data); err != nil {
|
||||
_ = w.Close()
|
||||
return nil, fmt.Errorf("writing data: %w", err)
|
||||
}
|
||||
|
||||
// Close to flush
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, fmt.Errorf("closing writer: %w", err)
|
||||
}
|
||||
|
||||
return &CompressResult{
|
||||
Data: buf.Bytes(),
|
||||
UncompressedSize: int64(len(data)),
|
||||
CompressedSize: int64(buf.Len()),
|
||||
SHA256: hex.EncodeToString(w.Sum256()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CompressStream compresses and encrypts from reader to writer, returning hash
|
||||
func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipients []string) (written int64, hash string, err error) {
|
||||
// Create writer
|
||||
w, err := NewWriter(dst, compressionLevel, recipients)
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("creating writer: %w", err)
|
||||
}
|
||||
|
||||
closed := false
|
||||
defer func() {
|
||||
if !closed {
|
||||
_ = w.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
// Copy data
|
||||
if _, err := io.Copy(w, src); err != nil {
|
||||
return 0, "", fmt.Errorf("copying data: %w", err)
|
||||
}
|
||||
|
||||
// Close to flush
|
||||
if err := w.Close(); err != nil {
|
||||
return 0, "", fmt.Errorf("closing writer: %w", err)
|
||||
}
|
||||
closed = true
|
||||
|
||||
return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
package blobgen
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// testRecipient is a static age recipient for tests.
|
||||
const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s89gmvv"
|
||||
|
||||
// TestCompressStreamNoDoubleClose is a regression test for issue #28.
|
||||
// It verifies that CompressStream does not panic or return an error due to
|
||||
// double-closing the underlying blobgen.Writer. Before the fix in PR #33,
|
||||
// the explicit Close() on the happy path combined with defer Close() would
|
||||
// cause a double close.
|
||||
func TestCompressStreamNoDoubleClose(t *testing.T) {
|
||||
input := []byte("regression test data for issue #28 double-close fix")
|
||||
var buf bytes.Buffer
|
||||
|
||||
written, hash, err := CompressStream(&buf, bytes.NewReader(input), 3, []string{testRecipient})
|
||||
require.NoError(t, err, "CompressStream should not return an error")
|
||||
assert.True(t, written > 0, "expected bytes written > 0")
|
||||
assert.NotEmpty(t, hash, "expected non-empty hash")
|
||||
assert.True(t, buf.Len() > 0, "expected non-empty output")
|
||||
}
|
||||
|
||||
// TestCompressStreamLargeInput exercises CompressStream with a larger payload
|
||||
// to ensure no double-close issues surface under heavier I/O.
|
||||
func TestCompressStreamLargeInput(t *testing.T) {
|
||||
data := make([]byte, 512*1024) // 512 KB
|
||||
_, err := rand.Read(data)
|
||||
require.NoError(t, err)
|
||||
|
||||
var buf bytes.Buffer
|
||||
written, hash, err := CompressStream(&buf, bytes.NewReader(data), 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, written > 0)
|
||||
assert.NotEmpty(t, hash)
|
||||
}
|
||||
|
||||
// TestCompressStreamEmptyInput verifies CompressStream handles empty input
|
||||
// without double-close issues.
|
||||
func TestCompressStreamEmptyInput(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
_, hash, err := CompressStream(&buf, strings.NewReader(""), 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, hash)
|
||||
}
|
||||
|
||||
// TestCompressDataNoDoubleClose mirrors the stream test for CompressData,
|
||||
// ensuring the explicit Close + error-path Close pattern is also safe.
|
||||
func TestCompressDataNoDoubleClose(t *testing.T) {
|
||||
input := []byte("CompressData regression test for double-close")
|
||||
result, err := CompressData(input, 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.CompressedSize > 0)
|
||||
assert.True(t, result.UncompressedSize == int64(len(input)))
|
||||
assert.NotEmpty(t, result.SHA256)
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package blobgen_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// ageChunkSize is age's STREAM plaintext chunk size (64 KiB); each encrypted
|
||||
// chunk adds a 16-byte ChaCha20-Poly1305 tag.
|
||||
const (
|
||||
ageChunkSize = 64 * 1024
|
||||
ageChunkTagSize = 16
|
||||
ageSegmentSize = ageChunkSize + ageChunkTagSize
|
||||
ageNonceSize = 16
|
||||
)
|
||||
|
||||
// makeIdentity returns a fresh X25519 identity and its recipient string.
|
||||
func makeIdentity(t *testing.T) (*age.X25519Identity, string) {
|
||||
t.Helper()
|
||||
|
||||
id, err := age.GenerateX25519Identity()
|
||||
require.NoError(t, err)
|
||||
|
||||
return id, id.Recipient().String()
|
||||
}
|
||||
|
||||
// randomBytes returns n cryptographically random bytes, which do not compress
|
||||
// so the encrypted payload spans multiple age segments.
|
||||
func randomBytes(t *testing.T, n int) []byte {
|
||||
t.Helper()
|
||||
|
||||
b := make([]byte, n)
|
||||
_, err := rand.Read(b)
|
||||
require.NoError(t, err)
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
// compressibleBytes returns n bytes of a repeating pattern, which zstd packs
|
||||
// down to a small payload.
|
||||
func compressibleBytes(n int) []byte {
|
||||
pattern := bytes.Repeat([]byte("compressible-"), n/13+1)
|
||||
|
||||
return pattern[:n]
|
||||
}
|
||||
|
||||
// encryptBlob compresses, encrypts and returns a blob for plaintext at
|
||||
// compression level 1.
|
||||
func encryptBlob(t *testing.T, plaintext []byte, recipients ...string) []byte {
|
||||
t.Helper()
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
w, err := blobgen.NewWriter(&buf, 1, recipients)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = w.Write(plaintext)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, w.Close())
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// ageHeaderLen returns the byte length of blob's age header, i.e. the offset
|
||||
// of the 16-byte payload nonce that follows it. The header ends with a MAC
|
||||
// line "--- <mac>\n"; the nonce begins right after that newline.
|
||||
func ageHeaderLen(t *testing.T, blob []byte) int {
|
||||
t.Helper()
|
||||
|
||||
i := bytes.Index(blob, []byte("\n--- "))
|
||||
require.GreaterOrEqual(t, i, 0, "age MAC footer line not found")
|
||||
|
||||
nl := bytes.IndexByte(blob[i+1:], '\n')
|
||||
require.GreaterOrEqual(t, nl, 0, "newline ending MAC line not found")
|
||||
|
||||
return i + 1 + nl + 1
|
||||
}
|
||||
|
||||
// requireBlobUnreadable asserts that data never decrypts to a plaintext with a
|
||||
// nil error: either NewReader fails, or reading it does.
|
||||
func requireBlobUnreadable(t *testing.T, data []byte, id age.Identity) {
|
||||
t.Helper()
|
||||
|
||||
r, err := blobgen.NewReader(bytes.NewReader(data), id)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
_, err = io.ReadAll(r)
|
||||
_ = r.Close()
|
||||
|
||||
require.Error(t, err, "reading a damaged blob must fail")
|
||||
}
|
||||
|
||||
// errFailWriter is returned by failAfterWriter once its byte limit is passed.
|
||||
var errFailWriter = errors.New("destination write failed")
|
||||
|
||||
// failAfterWriter accepts writes until more than limit bytes have been sent,
|
||||
// then fails every write. It models a destination that dies mid-blob.
|
||||
type failAfterWriter struct {
|
||||
limit int
|
||||
written int
|
||||
}
|
||||
|
||||
func (f *failAfterWriter) Write(p []byte) (int, error) {
|
||||
f.written += len(p)
|
||||
if f.written > f.limit {
|
||||
return 0, errFailWriter
|
||||
}
|
||||
|
||||
return len(p), nil
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package blobgen
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// ErrOutputTooLarge is returned by a reader from LimitReader once it has
|
||||
// been asked for more than its limit. It bounds how far an untrusted
|
||||
// compressed stream may expand, so a small, highly compressible object
|
||||
// from the store cannot decompress without limit.
|
||||
var ErrOutputTooLarge = errors.New("output exceeds size limit")
|
||||
|
||||
// LimitReader returns a reader that yields at most limit bytes from r and
|
||||
// then fails with ErrOutputTooLarge. Unlike io.LimitReader, which reports
|
||||
// a silent io.EOF at the limit (indistinguishable from a stream that
|
||||
// simply ended), this fails, so a caller decoding or copying the stream
|
||||
// sees an error rather than a truncated value. A stream of exactly limit
|
||||
// bytes reads back cleanly to EOF; the first byte beyond it is the error.
|
||||
func LimitReader(r io.Reader, limit int64) io.Reader {
|
||||
// remaining counts down from limit+1: the extra byte is the one that,
|
||||
// if it ever arrives, proves the stream is longer than the limit.
|
||||
return &limitReader{r: r, remaining: limit + 1}
|
||||
}
|
||||
|
||||
type limitReader struct {
|
||||
r io.Reader
|
||||
remaining int64
|
||||
}
|
||||
|
||||
func (l *limitReader) Read(p []byte) (int, error) {
|
||||
if l.remaining <= 0 {
|
||||
return 0, ErrOutputTooLarge
|
||||
}
|
||||
|
||||
if int64(len(p)) > l.remaining {
|
||||
p = p[:l.remaining]
|
||||
}
|
||||
|
||||
n, err := l.r.Read(p)
|
||||
l.remaining -= int64(n)
|
||||
|
||||
if l.remaining <= 0 {
|
||||
// The (limit+1)th byte was just read: the stream is too long.
|
||||
return n, ErrOutputTooLarge
|
||||
}
|
||||
|
||||
return n, err
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package blobgen_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// TestLimitReaderPassesExactSize checks that a stream of exactly the limit
|
||||
// reads back cleanly to EOF: the bound must not reject a legitimate blob
|
||||
// whose plaintext equals its recorded size.
|
||||
func TestLimitReaderPassesExactSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const n = 1000
|
||||
|
||||
r := blobgen.LimitReader(bytes.NewReader(bytes.Repeat([]byte("a"), n)), n)
|
||||
|
||||
got, err := io.ReadAll(r)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, n)
|
||||
}
|
||||
|
||||
// TestLimitReaderFailsPastLimit feeds a large, highly compressible run of
|
||||
// zeros — the decompressed output a zip bomb would produce — through a
|
||||
// small limit and checks it fails within the bound rather than passing
|
||||
// the whole stream through.
|
||||
func TestLimitReaderFailsPastLimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const limit = 1000
|
||||
|
||||
r := blobgen.LimitReader(
|
||||
bytes.NewReader(bytes.Repeat([]byte{0}, limit*1000)), limit)
|
||||
|
||||
n, err := io.Copy(io.Discard, r)
|
||||
require.ErrorIs(t, err, blobgen.ErrOutputTooLarge)
|
||||
require.LessOrEqual(t, n, int64(limit)+1,
|
||||
"reader must stop within one byte of the limit")
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package blobgen_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// TestNewReaderWrongIdentity covers issue case 4: opening a blob with an
|
||||
// identity other than the recipient reports no matching identity.
|
||||
func TestNewReaderWrongIdentity(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, recipient := makeIdentity(t)
|
||||
other, _ := makeIdentity(t)
|
||||
|
||||
blob := encryptBlob(t, []byte("secret payload"), recipient)
|
||||
|
||||
_, err := blobgen.NewReader(bytes.NewReader(blob), other)
|
||||
require.Error(t, err)
|
||||
|
||||
var noMatch *age.NoIdentityMatchError
|
||||
assert.ErrorAs(t, err, &noMatch)
|
||||
}
|
||||
|
||||
// TestNewReaderTruncated covers issue case 6: a multi-segment blob cut at
|
||||
// several points must never read back as valid data. The point immediately
|
||||
// after the header and nonce is intentionally excluded: it reads as a valid
|
||||
// empty blob today and is the regression case for
|
||||
// https://git.eeqj.de/sneak/vaultik/issues/152.
|
||||
func TestNewReaderTruncated(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
id, recipient := makeIdentity(t)
|
||||
blob := encryptBlob(t, randomBytes(t, 4*65536+123), recipient)
|
||||
h := ageHeaderLen(t, blob)
|
||||
|
||||
require.Greater(t, len(blob), h+ageNonceSize+ageSegmentSize,
|
||||
"test needs a blob of at least two age segments")
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
size int
|
||||
}{
|
||||
{"inside header", h / 2},
|
||||
{"inside nonce", h + 8},
|
||||
{"inside first segment", h + ageNonceSize + 100},
|
||||
{"end of first full segment", h + ageNonceSize + ageSegmentSize},
|
||||
{"last byte removed", len(blob) - 1},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
requireBlobUnreadable(t, blob[:tc.size], id)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewReaderCorrupted covers issue case 7: one flipped byte in each region
|
||||
// of a multi-segment blob makes it unreadable.
|
||||
func TestNewReaderCorrupted(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
id, recipient := makeIdentity(t)
|
||||
blob := encryptBlob(t, randomBytes(t, 4*65536+123), recipient)
|
||||
h := ageHeaderLen(t, blob)
|
||||
|
||||
firstNL := bytes.IndexByte(blob, '\n')
|
||||
require.Positive(t, firstNL, "header must have a version line")
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
pos int
|
||||
}{
|
||||
{"header stanza", firstNL + 5},
|
||||
{"header MAC line", h - 2},
|
||||
{"nonce", h + 4},
|
||||
{"body segment", h + ageNonceSize + 50},
|
||||
{"final tag", len(blob) - 1},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
corrupt := append([]byte(nil), blob...)
|
||||
corrupt[tc.pos] ^= 0xff
|
||||
requireBlobUnreadable(t, corrupt, id)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewReaderTrailingAndGarbage covers issue case 8: bytes appended after a
|
||||
// valid blob, empty input, and random garbage each fail to read.
|
||||
func TestNewReaderTrailingAndGarbage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
id, recipient := makeIdentity(t)
|
||||
|
||||
valid := encryptBlob(t, []byte("small payload"), recipient)
|
||||
appended := append(append([]byte(nil), valid...), []byte("trailing junk")...)
|
||||
|
||||
t.Run("appended bytes", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
requireBlobUnreadable(t, appended, id)
|
||||
})
|
||||
t.Run("empty input", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
requireBlobUnreadable(t, []byte{}, id)
|
||||
})
|
||||
t.Run("random garbage", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
requireBlobUnreadable(t, randomBytes(t, 512), id)
|
||||
})
|
||||
}
|
||||
|
||||
// TestNewWriterInvalidLevel covers the rejected end of issue case 9: an
|
||||
// out-of-range compression level errors and writes nothing to the destination.
|
||||
func TestNewWriterInvalidLevel(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, recipient := makeIdentity(t)
|
||||
|
||||
for _, level := range []int{0, -1, 20} {
|
||||
t.Run(fmt.Sprintf("level%d", level), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
w, err := blobgen.NewWriter(&buf, level, []string{recipient})
|
||||
require.ErrorIs(t, err, blobgen.ErrInvalidCompressionLevel)
|
||||
assert.Nil(t, w)
|
||||
assert.Zero(t, buf.Len(), "nothing written on an invalid level")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewWriterInvalidRecipients covers issue case 10: nil and empty recipient
|
||||
// lists and an unparsable recipient string each error.
|
||||
func TestNewWriterInvalidRecipients(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
recipients []string
|
||||
}{
|
||||
{"nil list", nil},
|
||||
{"empty list", []string{}},
|
||||
{"invalid recipient string", []string{"not-a-recipient"}},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
w, err := blobgen.NewWriter(&buf, 1, tc.recipients)
|
||||
require.Error(t, err)
|
||||
assert.Nil(t, w)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewWriterFailingDestination covers issue case 11: a destination that
|
||||
// fails mid-blob surfaces its error from Write or Close.
|
||||
func TestNewWriterFailingDestination(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, recipient := makeIdentity(t)
|
||||
|
||||
// The limit clears the age header and nonce so NewWriter succeeds, then
|
||||
// trips once the compressed body starts flowing.
|
||||
dst := &failAfterWriter{limit: 512}
|
||||
|
||||
w, err := blobgen.NewWriter(dst, 1, []string{recipient})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, writeErr := w.Write(randomBytes(t, 256*1024))
|
||||
closeErr := w.Close()
|
||||
|
||||
assert.True(t, writeErr != nil || closeErr != nil,
|
||||
"destination failure must surface from Write or Close")
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package blobgen
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
@@ -20,10 +21,12 @@ type Reader struct {
|
||||
bytesRead int64
|
||||
}
|
||||
|
||||
// NewReader creates a new Reader that decrypts, decompresses, and verifies data
|
||||
func NewReader(r io.Reader, identity age.Identity) (*Reader, error) {
|
||||
// NewReader creates a new Reader that decrypts, decompresses, and verifies
|
||||
// data. Every supplied identity is offered to age.Decrypt, so a blob
|
||||
// encrypted to any one of them can be read.
|
||||
func NewReader(r io.Reader, identities ...age.Identity) (*Reader, error) {
|
||||
// Create decryption reader
|
||||
decReader, err := age.Decrypt(r, identity)
|
||||
decReader, err := age.Decrypt(r, identities...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating decryption reader: %w", err)
|
||||
}
|
||||
@@ -50,19 +53,39 @@ func NewReader(r io.Reader, identity age.Identity) (*Reader, error) {
|
||||
}
|
||||
|
||||
// Read implements io.Reader
|
||||
func (r *Reader) Read(p []byte) (n int, err error) {
|
||||
n, err = r.teeReader.Read(p)
|
||||
func (r *Reader) Read(p []byte) (int, error) {
|
||||
n, err := r.teeReader.Read(p)
|
||||
r.bytesRead += int64(n)
|
||||
|
||||
// When the ciphertext is cut right after the age header plus its
|
||||
// 16-byte nonce, the age reader's first read fails with
|
||||
// io.ErrUnexpectedEOF, and the zstd decoder maps that to a clean
|
||||
// io.EOF at frame start. That makes a truncated stream look like a
|
||||
// valid empty one. Distinguish the two: on EOF, read once more from
|
||||
// the age reader. A genuine end leaves it at (0, io.EOF); a truncated
|
||||
// stream leaves its stored io.ErrUnexpectedEOF, which we surface.
|
||||
if errors.Is(err, io.EOF) {
|
||||
var probe [1]byte
|
||||
|
||||
m, ageErr := r.decryptor.Read(probe[:])
|
||||
if m != 0 || !errors.Is(ageErr, io.EOF) {
|
||||
return n, io.ErrUnexpectedEOF
|
||||
}
|
||||
}
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close closes the decompressor
|
||||
func (r *Reader) Close() error {
|
||||
r.decompressor.Close()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sum256 returns the SHA256 hash of all data read
|
||||
// Sum256 returns the single SHA-256 of the plaintext read so far. This is the
|
||||
// first hash only; the stored object name is its double hash, which callers
|
||||
// obtain by passing this digest to DoubleSHA256.
|
||||
func (r *Reader) Sum256() []byte {
|
||||
return r.hasher.Sum(nil)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package blobgen_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// TestMultipleRecipients verifies that data written for several recipients can
|
||||
// be read back by each recipient's identity. Moved from internal/crypto, which
|
||||
// held the only multi-recipient test; blobgen is now the sole encryption path.
|
||||
func TestMultipleRecipients(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
identities := make([]*age.X25519Identity, 3)
|
||||
recipients := make([]string, 3)
|
||||
|
||||
for i := range identities {
|
||||
identity, err := age.GenerateX25519Identity()
|
||||
require.NoError(t, err)
|
||||
|
||||
identities[i] = identity
|
||||
recipients[i] = identity.Recipient().String()
|
||||
}
|
||||
|
||||
plaintext := []byte("Secret message for multiple recipients")
|
||||
|
||||
var encrypted bytes.Buffer
|
||||
|
||||
writer, err := blobgen.NewWriter(&encrypted, 3, recipients)
|
||||
require.NoError(t, err)
|
||||
_, err = writer.Write(plaintext)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
// Every recipient's identity must recover the original plaintext.
|
||||
for i, identity := range identities {
|
||||
reader, err := blobgen.NewReader(
|
||||
bytes.NewReader(encrypted.Bytes()), identity)
|
||||
require.NoError(t, err, "recipient %d should open the reader", i+1)
|
||||
|
||||
got, err := io.ReadAll(reader)
|
||||
require.NoError(t, err, "recipient %d should read the plaintext", i+1)
|
||||
require.NoError(t, reader.Close())
|
||||
|
||||
assert.Equal(t, plaintext, got,
|
||||
"recipient %d should recover the original plaintext", i+1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package blobgen_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// checkRoundTrip writes input through a Writer, reads it back through a Reader,
|
||||
// and verifies the plaintext, the byte counts, and the content hashes.
|
||||
func checkRoundTrip(
|
||||
t *testing.T, id *age.X25519Identity, recipient string,
|
||||
level int, input []byte,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
w, err := blobgen.NewWriter(&buf, level, []string{recipient})
|
||||
require.NoError(t, err)
|
||||
|
||||
n, err := w.Write(input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, len(input), n)
|
||||
require.NoError(t, w.Close())
|
||||
require.Equal(t, int64(len(input)), w.BytesWritten())
|
||||
|
||||
r, err := blobgen.NewReader(bytes.NewReader(buf.Bytes()), id)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := io.ReadAll(r)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, r.Close())
|
||||
|
||||
assert.Equal(t, input, got, "decrypted output must equal input")
|
||||
require.Equal(t, int64(len(input)), r.BytesRead())
|
||||
|
||||
// The hash values are checked by decrypting: the reader's single SHA-256
|
||||
// is the hash of the plaintext, and hashing it once more (DoubleSHA256)
|
||||
// gives the writer's ContentID.
|
||||
single := sha256.Sum256(got)
|
||||
assert.Equal(t, single[:], r.Sum256())
|
||||
assert.Equal(t, blobgen.DoubleSHA256(r.Sum256()), w.ContentID())
|
||||
}
|
||||
|
||||
// TestWriterReaderRoundTrip covers issue cases 1 and 2: every size round trips
|
||||
// for both random and compressible data, and the reader hash, its double hash
|
||||
// and the byte counts all agree.
|
||||
func TestWriterReaderRoundTrip(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
id, recipient := makeIdentity(t)
|
||||
|
||||
// Sizes exercise the age segment boundary (64 KiB) from just below to a
|
||||
// few segments above it, plus the empty and single-byte edges.
|
||||
sizes := []int{0, 1, 65535, 65536, 65537, 4*65536 + 123}
|
||||
|
||||
kinds := []struct {
|
||||
name string
|
||||
fill func(*testing.T, int) []byte
|
||||
}{
|
||||
{"random", randomBytes},
|
||||
{"compressible", func(_ *testing.T, n int) []byte {
|
||||
return compressibleBytes(n)
|
||||
}},
|
||||
}
|
||||
|
||||
for _, k := range kinds {
|
||||
for _, size := range sizes {
|
||||
name := fmt.Sprintf("%s/%d", k.name, size)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
checkRoundTrip(t, id, recipient, 1, k.fill(t, size))
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestZeroLengthNoWrite covers issue case 3: a Writer closed with no Write at
|
||||
// all produces the double hash of the empty input, and the blob reads back as
|
||||
// empty with no error.
|
||||
func TestZeroLengthNoWrite(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
id, recipient := makeIdentity(t)
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
w, err := blobgen.NewWriter(&buf, 1, []string{recipient})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, w.Close())
|
||||
assert.Equal(t, int64(0), w.BytesWritten())
|
||||
|
||||
empty := sha256.Sum256(nil)
|
||||
doubled := sha256.Sum256(empty[:])
|
||||
assert.Equal(t, doubled[:], w.ContentID(),
|
||||
"ContentID of empty input is SHA256(SHA256(\"\"))")
|
||||
|
||||
r, err := blobgen.NewReader(bytes.NewReader(buf.Bytes()), id)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := io.ReadAll(r)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, r.Close())
|
||||
|
||||
assert.Empty(t, got, "empty blob decrypts to empty output")
|
||||
assert.Equal(t, int64(0), r.BytesRead())
|
||||
assert.Equal(t, empty[:], r.Sum256())
|
||||
}
|
||||
|
||||
// TestNewWriterValidLevelsRoundTrip covers the accepted end of issue case 9:
|
||||
// the boundary compression levels 1 and 19 both round trip.
|
||||
func TestNewWriterValidLevelsRoundTrip(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
id, recipient := makeIdentity(t)
|
||||
input := randomBytes(t, 4096)
|
||||
|
||||
for _, level := range []int{1, 19} {
|
||||
t.Run(fmt.Sprintf("level%d", level), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
checkRoundTrip(t, id, recipient, level, input)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package blobgen_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// TestReaderRejectsHeaderNonceTruncation guards against a stream cut right
|
||||
// after the age header plus its 16-byte nonce. age.Decrypt still succeeds on
|
||||
// such an object, and the zstd decoder maps the age reader's
|
||||
// io.ErrUnexpectedEOF to a clean io.EOF at frame start, so without the extra
|
||||
// check the truncated stream would read as a valid empty one. Reading it must
|
||||
// now fail.
|
||||
func TestReaderRejectsHeaderNonceTruncation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
identity, err := age.GenerateX25519Identity()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Encrypting empty plaintext yields header + nonce(16) + a single
|
||||
// 16-byte final chunk tag. Dropping the trailing tag leaves exactly the
|
||||
// age header plus its nonce — the truncation point that triggers the bug.
|
||||
var full bytes.Buffer
|
||||
|
||||
w, err := age.Encrypt(&full, identity.Recipient())
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, w.Close())
|
||||
|
||||
truncated := full.Bytes()[:full.Len()-16]
|
||||
|
||||
reader, err := blobgen.NewReader(bytes.NewReader(truncated), identity)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
_, err = io.ReadAll(reader)
|
||||
require.Error(t, err)
|
||||
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
||||
}
|
||||
|
||||
// TestReaderReadsGenuinelyEmptyBlob confirms the truncation check does not
|
||||
// reject a legitimately empty payload: a blob written with no data must round
|
||||
// trip back to zero bytes with no error.
|
||||
func TestReaderReadsGenuinelyEmptyBlob(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
identity, err := age.GenerateX25519Identity()
|
||||
require.NoError(t, err)
|
||||
|
||||
var encrypted bytes.Buffer
|
||||
|
||||
writer, err := blobgen.NewWriter(
|
||||
&encrypted, 3, []string{identity.Recipient().String()})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
reader, err := blobgen.NewReader(bytes.NewReader(encrypted.Bytes()), identity)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, data)
|
||||
}
|
||||
+76
-27
@@ -1,7 +1,11 @@
|
||||
// Package blobgen implements the blob data pipeline: streaming zstd
|
||||
// compression, age encryption, and SHA256 content hashing for blob
|
||||
// creation, plus the matching decrypt/decompress/verify reader.
|
||||
package blobgen
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
@@ -11,6 +15,44 @@ import (
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// DoubleSHA256 returns the double SHA-256 of content whose single SHA-256
|
||||
// digest is sum: it hashes that digest once more. Stored objects — a blob, and
|
||||
// the metadata database export — are named by this second hash.
|
||||
//
|
||||
// The second hash does not hide whether known content is stored: an attacker
|
||||
// who can reproduce an object's entire plaintext computes the same name simply
|
||||
// by hashing twice, exactly as this code does. What limits that is blob
|
||||
// packing, not the double hash — a blob's name covers all of its concatenated
|
||||
// chunk plaintext, so a name can be confirmed only by someone who can
|
||||
// reproduce the whole blob (a snapshot made entirely of known content, or a
|
||||
// known file large enough to fill blobs on its own). An ordinary file that
|
||||
// shares a blob with other, unknown data cannot be confirmed this way.
|
||||
func DoubleSHA256(sum []byte) []byte {
|
||||
h := sha256.Sum256(sum)
|
||||
|
||||
return h[:]
|
||||
}
|
||||
|
||||
// Zstd compression level bounds accepted by NewWriter.
|
||||
const (
|
||||
minCompressionLevel = 1
|
||||
maxCompressionLevel = 19
|
||||
)
|
||||
|
||||
// reservedCompressionCPUs is how many CPUs are left free of zstd
|
||||
// compression work for I/O and hashing.
|
||||
const reservedCompressionCPUs = 2
|
||||
|
||||
// ErrInvalidCompressionLevel is returned when the zstd compression level
|
||||
// is outside the accepted 1-19 range.
|
||||
var ErrInvalidCompressionLevel = errors.New(
|
||||
"invalid compression level: must be between 1 and 19")
|
||||
|
||||
// errInvalidRecipient is returned when a recipient string does not parse as
|
||||
// an X25519 age1... public key. It omits the value, which can be sensitive.
|
||||
var errInvalidRecipient = errors.New(
|
||||
"not a valid X25519 age1... recipient")
|
||||
|
||||
// Writer wraps compression and encryption with SHA256 hashing.
|
||||
// Data flows: input -> tee(hasher, compressor -> encryptor -> destination)
|
||||
// The hash is computed on the uncompressed input for deterministic content-addressing.
|
||||
@@ -23,11 +65,15 @@ type Writer struct {
|
||||
bytesWritten int64
|
||||
}
|
||||
|
||||
// NewWriter creates a new Writer that compresses, encrypts, and hashes data.
|
||||
// The hash is computed on the uncompressed input for deterministic content-addressing.
|
||||
func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer, error) {
|
||||
// NewWriter creates a new Writer that compresses, encrypts, and hashes
|
||||
// data. The hash is computed on the uncompressed input for deterministic
|
||||
// content-addressing.
|
||||
func NewWriter(
|
||||
w io.Writer, compressionLevel int, recipients []string,
|
||||
) (*Writer, error) {
|
||||
// Validate compression level
|
||||
if err := validateCompressionLevel(compressionLevel); err != nil {
|
||||
err := validateCompressionLevel(compressionLevel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -36,11 +82,15 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
||||
|
||||
// Parse recipients
|
||||
var ageRecipients []age.Recipient
|
||||
for _, recipient := range recipients {
|
||||
|
||||
for i, recipient := range recipients {
|
||||
// The recipient string can be sensitive (e.g. a secret key pasted by
|
||||
// mistake), so the error names its position, never its value.
|
||||
r, err := age.ParseX25519Recipient(recipient)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing recipient %s: %w", recipient, err)
|
||||
return nil, fmt.Errorf("%w: recipient %d", errInvalidRecipient, i)
|
||||
}
|
||||
|
||||
ageRecipients = append(ageRecipients, r)
|
||||
}
|
||||
|
||||
@@ -51,10 +101,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
||||
}
|
||||
|
||||
// Calculate compression concurrency: CPUs - 2, minimum 1
|
||||
concurrency := runtime.NumCPU() - 2
|
||||
if concurrency < 1 {
|
||||
concurrency = 1
|
||||
}
|
||||
concurrency := max(runtime.NumCPU()-reservedCompressionCPUs, 1)
|
||||
|
||||
// Create compression writer with encryption as destination
|
||||
compressor, err := zstd.NewWriter(encWriter,
|
||||
@@ -63,6 +110,7 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
||||
)
|
||||
if err != nil {
|
||||
_ = encWriter.Close()
|
||||
|
||||
return nil, fmt.Errorf("creating compression writer: %w", err)
|
||||
}
|
||||
|
||||
@@ -79,37 +127,36 @@ func NewWriter(w io.Writer, compressionLevel int, recipients []string) (*Writer,
|
||||
}
|
||||
|
||||
// Write implements io.Writer
|
||||
func (w *Writer) Write(p []byte) (n int, err error) {
|
||||
n, err = w.teeWriter.Write(p)
|
||||
func (w *Writer) Write(p []byte) (int, error) {
|
||||
n, err := w.teeWriter.Write(p)
|
||||
w.bytesWritten += int64(n)
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Close closes all layers and returns any errors
|
||||
func (w *Writer) Close() error {
|
||||
// Close compressor first
|
||||
if err := w.compressor.Close(); err != nil {
|
||||
err := w.compressor.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("closing compressor: %w", err)
|
||||
}
|
||||
|
||||
// Then close encryptor
|
||||
if err := w.encryptor.Close(); err != nil {
|
||||
err = w.encryptor.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("closing encryptor: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sum256 returns the double SHA256 hash of the uncompressed input data.
|
||||
// Double hashing (SHA256(SHA256(data))) prevents information leakage about
|
||||
// the plaintext - an attacker cannot confirm existence of known content
|
||||
// by computing its hash and checking for a matching blob filename.
|
||||
func (w *Writer) Sum256() []byte {
|
||||
// First hash: SHA256(plaintext)
|
||||
firstHash := w.hasher.Sum(nil)
|
||||
// Second hash: SHA256(firstHash) - this is the blob ID
|
||||
secondHash := sha256.Sum256(firstHash)
|
||||
return secondHash[:]
|
||||
// ContentID returns the double SHA-256 of the uncompressed input data: the
|
||||
// name under which this content is stored. It is the second hash of the
|
||||
// running SHA-256, via DoubleSHA256; see that function for what naming content
|
||||
// this way does and does not hide.
|
||||
func (w *Writer) ContentID() []byte {
|
||||
return DoubleSHA256(w.hasher.Sum(nil))
|
||||
}
|
||||
|
||||
// BytesWritten returns the number of uncompressed bytes written
|
||||
@@ -119,9 +166,11 @@ func (w *Writer) BytesWritten() int64 {
|
||||
|
||||
func validateCompressionLevel(level int) error {
|
||||
// Zstd compression levels: 1-19 (default is 3)
|
||||
// SpeedFastest = 1, SpeedDefault = 3, SpeedBetterCompression = 7, SpeedBestCompression = 11
|
||||
if level < 1 || level > 19 {
|
||||
return fmt.Errorf("invalid compression level %d: must be between 1 and 19", level)
|
||||
// SpeedFastest = 1, SpeedDefault = 3, SpeedBetterCompression = 7,
|
||||
// SpeedBestCompression = 11
|
||||
if level < minCompressionLevel || level > maxCompressionLevel {
|
||||
return fmt.Errorf("%w: got %d", ErrInvalidCompressionLevel, level)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package blobgen
|
||||
package blobgen_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -9,12 +9,16 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// TestWriterHashIsDoubleHash verifies that Writer.Sum256() returns
|
||||
// the double hash SHA256(SHA256(plaintext)) for security.
|
||||
// Double hashing prevents attackers from confirming existence of known content.
|
||||
// TestWriterHashIsDoubleHash verifies that Writer.ContentID() returns
|
||||
// SHA256(SHA256(plaintext)). Stored objects are named by this second hash so a
|
||||
// name is not the plaintext's own SHA-256; this does not stop someone who
|
||||
// already holds the plaintext from confirming it.
|
||||
func TestWriterHashIsDoubleHash(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Test data - random data that doesn't compress well
|
||||
testData := make([]byte, 1024*1024) // 1MB
|
||||
_, err := rand.Read(testData)
|
||||
@@ -27,7 +31,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) {
|
||||
var encryptedBuf bytes.Buffer
|
||||
|
||||
// Create blobgen writer
|
||||
writer, err := NewWriter(&encryptedBuf, 3, []string{testRecipient})
|
||||
writer, err := blobgen.NewWriter(&encryptedBuf, 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Write test data
|
||||
@@ -40,7 +44,7 @@ func TestWriterHashIsDoubleHash(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get the hash from the writer
|
||||
writerHash := hex.EncodeToString(writer.Sum256())
|
||||
writerHash := hex.EncodeToString(writer.ContentID())
|
||||
|
||||
// Calculate the expected double hash: SHA256(SHA256(plaintext))
|
||||
firstHash := sha256.Sum256(testData)
|
||||
@@ -57,16 +61,18 @@ func TestWriterHashIsDoubleHash(t *testing.T) {
|
||||
|
||||
// The writer hash should match the double hash
|
||||
assert.Equal(t, expectedDoubleHash, writerHash,
|
||||
"Writer.Sum256() should return SHA256(SHA256(plaintext)) for security")
|
||||
"Writer.ContentID() must be SHA256(SHA256(plaintext))")
|
||||
|
||||
// Verify it's NOT the single hash (would leak information)
|
||||
// It must be the second hash, not the plaintext's own SHA-256.
|
||||
assert.NotEqual(t, singleHashStr, writerHash,
|
||||
"Writer hash should not be single hash (would allow content confirmation attacks)")
|
||||
"Writer hash must be the double hash, not the single SHA-256")
|
||||
}
|
||||
|
||||
// TestWriterDeterministicHash verifies that the same input always produces
|
||||
// the same hash, even with non-deterministic encryption.
|
||||
func TestWriterDeterministicHash(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Test data
|
||||
testData := []byte("Hello, World! This is test data for deterministic hashing.")
|
||||
|
||||
@@ -76,20 +82,20 @@ func TestWriterDeterministicHash(t *testing.T) {
|
||||
// Create two writers and verify they produce the same hash
|
||||
var buf1, buf2 bytes.Buffer
|
||||
|
||||
writer1, err := NewWriter(&buf1, 3, []string{testRecipient})
|
||||
writer1, err := blobgen.NewWriter(&buf1, 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
_, err = writer1.Write(testData)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer1.Close())
|
||||
|
||||
writer2, err := NewWriter(&buf2, 3, []string{testRecipient})
|
||||
writer2, err := blobgen.NewWriter(&buf2, 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
_, err = writer2.Write(testData)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer2.Close())
|
||||
|
||||
hash1 := hex.EncodeToString(writer1.Sum256())
|
||||
hash2 := hex.EncodeToString(writer2.Sum256())
|
||||
hash1 := hex.EncodeToString(writer1.ContentID())
|
||||
hash2 := hex.EncodeToString(writer2.ContentID())
|
||||
|
||||
// Hashes should be identical (deterministic)
|
||||
assert.Equal(t, hash1, hash2, "Same input should produce same hash")
|
||||
@@ -103,3 +109,20 @@ func TestWriterDeterministicHash(t *testing.T) {
|
||||
t.Logf("Encrypted size 1: %d bytes", buf1.Len())
|
||||
t.Logf("Encrypted size 2: %d bytes", buf2.Len())
|
||||
}
|
||||
|
||||
// TestNewWriterSecretKeyNotEchoed verifies that a secret key mistakenly passed
|
||||
// as a recipient does not appear in the returned error. A recipient string can
|
||||
// be sensitive, so the error must name only the position, not the value.
|
||||
func TestNewWriterSecretKeyNotEchoed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
secretKey := "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GX" +
|
||||
"VEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5"
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
_, err := blobgen.NewWriter(&buf, 3, []string{secretKey})
|
||||
require.Error(t, err)
|
||||
assert.NotContains(t, err.Error(), secretKey,
|
||||
"error must not echo the recipient value")
|
||||
}
|
||||
|
||||
+51
-27
@@ -1,16 +1,21 @@
|
||||
// Package chunker splits input data into content-defined chunks using the
|
||||
// FastCDC algorithm so that identical data sequences produce identical
|
||||
// chunks regardless of their position in the file.
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Chunk represents a single chunk of data produced by the content-defined chunking algorithm.
|
||||
// Each chunk is identified by its SHA256 hash and contains the raw data along with
|
||||
// its position and size information from the original file.
|
||||
// Chunk represents a single chunk of data produced by the content-defined
|
||||
// chunking algorithm. Each chunk is identified by its SHA256 hash and
|
||||
// contains the raw data along with its position and size information from
|
||||
// the original file.
|
||||
type Chunk struct {
|
||||
Hash string // Content hash of the chunk
|
||||
Data []byte // Chunk data
|
||||
@@ -28,6 +33,11 @@ type Chunker struct {
|
||||
maxChunkSize int
|
||||
}
|
||||
|
||||
// ChunkSizeSpread is the FastCDC-recommended factor between the average
|
||||
// chunk size and the minimum (avg/spread) and maximum (avg*spread) sizes.
|
||||
// The largest chunk the chunker can emit is therefore avg*ChunkSizeSpread.
|
||||
const ChunkSizeSpread = 4
|
||||
|
||||
// NewChunker creates a new chunker with the specified average chunk size.
|
||||
// The actual chunk sizes will vary between avgChunkSize/4 and avgChunkSize*4
|
||||
// as recommended by the FastCDC algorithm. Typical values for avgChunkSize
|
||||
@@ -36,27 +46,31 @@ func NewChunker(avgChunkSize int64) *Chunker {
|
||||
// FastCDC recommends min = avg/4 and max = avg*4
|
||||
return &Chunker{
|
||||
avgChunkSize: int(avgChunkSize),
|
||||
minChunkSize: int(avgChunkSize / 4),
|
||||
maxChunkSize: int(avgChunkSize * 4),
|
||||
minChunkSize: int(avgChunkSize / ChunkSizeSpread),
|
||||
maxChunkSize: int(avgChunkSize * ChunkSizeSpread),
|
||||
}
|
||||
}
|
||||
|
||||
// ChunkReader splits the reader into content-defined chunks and returns all chunks at once.
|
||||
// This method loads all chunk data into memory, so it should only be used for
|
||||
// reasonably sized inputs. For large files or streams, use ChunkReaderStreaming instead.
|
||||
// ChunkReader splits the reader into content-defined chunks and returns all
|
||||
// chunks at once. This method loads all chunk data into memory, so it should
|
||||
// only be used for reasonably sized inputs. For large files or streams, use
|
||||
// ChunkReaderStreaming instead.
|
||||
// Returns an error if chunking fails or if reading from the input fails.
|
||||
func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) {
|
||||
chunker := AcquireReusableChunker(r, c.minChunkSize, c.avgChunkSize, c.maxChunkSize)
|
||||
chunker := AcquireReusableChunker(
|
||||
r, c.minChunkSize, c.avgChunkSize, c.maxChunkSize)
|
||||
defer chunker.Release()
|
||||
|
||||
var chunks []Chunk
|
||||
|
||||
offset := int64(0)
|
||||
|
||||
for {
|
||||
chunk, err := chunker.Next()
|
||||
if err == io.EOF {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading chunk: %w", err)
|
||||
}
|
||||
@@ -83,30 +97,36 @@ func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) {
|
||||
|
||||
// ChunkCallback is a function called for each chunk as it's processed.
|
||||
// The callback receives a Chunk containing the hash, data, offset, and size.
|
||||
// If the callback returns an error, chunk processing stops and the error is propagated.
|
||||
// If the callback returns an error, chunk processing stops and the error is
|
||||
// propagated.
|
||||
type ChunkCallback func(chunk Chunk) error
|
||||
|
||||
// ChunkReaderStreaming splits the reader into chunks and calls the callback for each chunk.
|
||||
// This is the preferred method for processing large files or streams as it doesn't
|
||||
// accumulate all chunks in memory. The callback is invoked for each chunk as it's
|
||||
// produced, allowing for streaming processing and immediate storage or transmission.
|
||||
// Returns the SHA256 hash of the entire file content and an error if chunking fails,
|
||||
// reading fails, or if the callback returns an error.
|
||||
func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (string, error) {
|
||||
// ChunkReaderStreaming splits the reader into chunks and calls the callback
|
||||
// for each chunk. This is the preferred method for processing large files or
|
||||
// streams as it doesn't accumulate all chunks in memory. The callback is
|
||||
// invoked for each chunk as it's produced, allowing for streaming processing
|
||||
// and immediate storage or transmission.
|
||||
// Returns the SHA256 hash of the entire file content and an error if
|
||||
// chunking fails, reading fails, or if the callback returns an error.
|
||||
func (c *Chunker) ChunkReaderStreaming(
|
||||
r io.Reader, callback ChunkCallback,
|
||||
) (string, error) {
|
||||
// Create a tee reader to calculate full file hash while chunking
|
||||
fileHasher := sha256.New()
|
||||
teeReader := io.TeeReader(r, fileHasher)
|
||||
|
||||
chunker := AcquireReusableChunker(teeReader, c.minChunkSize, c.avgChunkSize, c.maxChunkSize)
|
||||
chunker := AcquireReusableChunker(
|
||||
teeReader, c.minChunkSize, c.avgChunkSize, c.maxChunkSize)
|
||||
defer chunker.Release()
|
||||
|
||||
offset := int64(0)
|
||||
|
||||
for {
|
||||
chunk, err := chunker.Next()
|
||||
if err == io.EOF {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("reading chunk: %w", err)
|
||||
}
|
||||
@@ -114,15 +134,17 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str
|
||||
// Calculate chunk hash
|
||||
hash := sha256.Sum256(chunk.Data)
|
||||
|
||||
// Pass the data directly - caller must process it before we call Next() again
|
||||
// (chunker reuses its internal buffer, but since we process synchronously
|
||||
// and completely before continuing, no copy is needed)
|
||||
if err := callback(Chunk{
|
||||
// Pass the data directly - caller must process it before we call
|
||||
// Next() again (chunker reuses its internal buffer, but since we
|
||||
// process synchronously and completely before continuing, no copy
|
||||
// is needed)
|
||||
err = callback(Chunk{
|
||||
Hash: hex.EncodeToString(hash[:]),
|
||||
Data: chunk.Data,
|
||||
Offset: offset,
|
||||
Size: int64(len(chunk.Data)),
|
||||
}); err != nil {
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("callback error: %w", err)
|
||||
}
|
||||
|
||||
@@ -138,12 +160,14 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str
|
||||
// For large files, consider using ChunkReaderStreaming with a file handle instead.
|
||||
// Returns an error if the file cannot be opened or if chunking fails.
|
||||
func (c *Chunker) ChunkFile(path string) ([]Chunk, error) {
|
||||
file, err := os.Open(path)
|
||||
file, err := os.Open(path) //nolint:gosec // G304: path is caller-supplied by design
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening file: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := file.Close(); err != nil && err.Error() != "invalid argument" {
|
||||
err := file.Close()
|
||||
if err != nil && err.Error() != "invalid argument" {
|
||||
// Log error or handle as needed
|
||||
_ = err
|
||||
}
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
package chunker
|
||||
package chunker_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/chunker"
|
||||
)
|
||||
|
||||
func TestChunkerExpectedChunkCount(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
fileSize int
|
||||
@@ -38,16 +42,19 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
chunker := NewChunker(tt.avgChunkSize)
|
||||
t.Parallel()
|
||||
|
||||
c := chunker.NewChunker(tt.avgChunkSize)
|
||||
|
||||
// Create data with some variation to trigger chunk boundaries
|
||||
data := make([]byte, tt.fileSize)
|
||||
for i := 0; i < len(data); i++ {
|
||||
for i := range data {
|
||||
// Use a pattern that should create boundaries
|
||||
//nolint:gosec // G115: intentional byte truncation
|
||||
data[i] = byte((i * 17) ^ (i >> 5))
|
||||
}
|
||||
|
||||
chunks, err := chunker.ChunkReader(bytes.NewReader(data))
|
||||
chunks, err := c.ChunkReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("chunking failed: %v", err)
|
||||
}
|
||||
@@ -59,6 +66,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
|
||||
t.Errorf("too few chunks: got %d, expected at least %d",
|
||||
len(chunks), tt.minExpected)
|
||||
}
|
||||
|
||||
if len(chunks) > tt.maxExpected {
|
||||
t.Errorf("too many chunks: got %d, expected at most %d",
|
||||
len(chunks), tt.maxExpected)
|
||||
@@ -69,6 +77,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
|
||||
for _, chunk := range chunks {
|
||||
reconstructed = append(reconstructed, chunk.Data...)
|
||||
}
|
||||
|
||||
if !bytes.Equal(data, reconstructed) {
|
||||
t.Error("reconstructed data doesn't match original")
|
||||
}
|
||||
|
||||
@@ -1,104 +1,120 @@
|
||||
package chunker
|
||||
package chunker_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/chunker"
|
||||
)
|
||||
|
||||
func TestChunker(t *testing.T) {
|
||||
t.Run("small file produces single chunk", func(t *testing.T) {
|
||||
chunker := NewChunker(1024 * 1024) // 1MB average
|
||||
data := bytes.Repeat([]byte("hello"), 100) // 500 bytes
|
||||
func TestChunkerSmallFileSingleChunk(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
chunks, err := chunker.ChunkReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("chunking failed: %v", err)
|
||||
c := chunker.NewChunker(1024 * 1024) // 1MB average
|
||||
data := bytes.Repeat([]byte("hello"), 100) // 500 bytes
|
||||
|
||||
chunks, err := c.ChunkReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("chunking failed: %v", err)
|
||||
}
|
||||
|
||||
if len(chunks) != 1 {
|
||||
t.Errorf("expected 1 chunk, got %d", len(chunks))
|
||||
}
|
||||
|
||||
if chunks[0].Size != int64(len(data)) {
|
||||
t.Errorf("expected chunk size %d, got %d", len(data), chunks[0].Size)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkerLargeFileMultipleChunks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c := chunker.NewChunker(256 * 1024) // 256KB average chunk size
|
||||
|
||||
// Generate 2MB of random data
|
||||
data := make([]byte, 2*1024*1024)
|
||||
|
||||
_, err := rand.Read(data)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate random data: %v", err)
|
||||
}
|
||||
|
||||
chunks, err := c.ChunkReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("chunking failed: %v", err)
|
||||
}
|
||||
|
||||
// Should produce multiple chunks - with FastCDC we expect around 8
|
||||
// chunks for 2MB with 256KB average
|
||||
if len(chunks) < 4 || len(chunks) > 16 {
|
||||
t.Errorf("expected 4-16 chunks, got %d", len(chunks))
|
||||
}
|
||||
|
||||
// Verify chunks reconstruct original data
|
||||
reconstructed := make([]byte, 0, len(data))
|
||||
for _, chunk := range chunks {
|
||||
reconstructed = append(reconstructed, chunk.Data...)
|
||||
}
|
||||
|
||||
if !bytes.Equal(data, reconstructed) {
|
||||
t.Error("reconstructed data doesn't match original")
|
||||
}
|
||||
|
||||
// Verify offsets
|
||||
var expectedOffset int64
|
||||
|
||||
for i, chunk := range chunks {
|
||||
if chunk.Offset != expectedOffset {
|
||||
t.Errorf("chunk %d: expected offset %d, got %d",
|
||||
i, expectedOffset, chunk.Offset)
|
||||
}
|
||||
|
||||
if len(chunks) != 1 {
|
||||
t.Errorf("expected 1 chunk, got %d", len(chunks))
|
||||
expectedOffset += chunk.Size
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkerDeterministic(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
chunker1 := chunker.NewChunker(256 * 1024)
|
||||
chunker2 := chunker.NewChunker(256 * 1024)
|
||||
|
||||
// Use deterministic data
|
||||
data := bytes.Repeat([]byte("abcdefghijklmnopqrstuvwxyz"), 20000) // ~520KB
|
||||
|
||||
chunks1, err := chunker1.ChunkReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("chunking failed: %v", err)
|
||||
}
|
||||
|
||||
chunks2, err := chunker2.ChunkReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("chunking failed: %v", err)
|
||||
}
|
||||
|
||||
// Should produce same chunks
|
||||
if len(chunks1) != len(chunks2) {
|
||||
t.Fatalf("different number of chunks: %d vs %d",
|
||||
len(chunks1), len(chunks2))
|
||||
}
|
||||
|
||||
for i := range chunks1 {
|
||||
if chunks1[i].Hash != chunks2[i].Hash {
|
||||
t.Errorf("chunk %d: different hashes", i)
|
||||
}
|
||||
|
||||
if chunks[0].Size != int64(len(data)) {
|
||||
t.Errorf("expected chunk size %d, got %d", len(data), chunks[0].Size)
|
||||
if chunks1[i].Size != chunks2[i].Size {
|
||||
t.Errorf("chunk %d: different sizes", i)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("large file produces multiple chunks", func(t *testing.T) {
|
||||
chunker := NewChunker(256 * 1024) // 256KB average chunk size
|
||||
|
||||
// Generate 2MB of random data
|
||||
data := make([]byte, 2*1024*1024)
|
||||
if _, err := rand.Read(data); err != nil {
|
||||
t.Fatalf("failed to generate random data: %v", err)
|
||||
}
|
||||
|
||||
chunks, err := chunker.ChunkReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("chunking failed: %v", err)
|
||||
}
|
||||
|
||||
// Should produce multiple chunks - with FastCDC we expect around 8 chunks for 2MB with 256KB average
|
||||
if len(chunks) < 4 || len(chunks) > 16 {
|
||||
t.Errorf("expected 4-16 chunks, got %d", len(chunks))
|
||||
}
|
||||
|
||||
// Verify chunks reconstruct original data
|
||||
var reconstructed []byte
|
||||
for _, chunk := range chunks {
|
||||
reconstructed = append(reconstructed, chunk.Data...)
|
||||
}
|
||||
|
||||
if !bytes.Equal(data, reconstructed) {
|
||||
t.Error("reconstructed data doesn't match original")
|
||||
}
|
||||
|
||||
// Verify offsets
|
||||
var expectedOffset int64
|
||||
for i, chunk := range chunks {
|
||||
if chunk.Offset != expectedOffset {
|
||||
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset)
|
||||
}
|
||||
expectedOffset += chunk.Size
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("deterministic chunking", func(t *testing.T) {
|
||||
chunker1 := NewChunker(256 * 1024)
|
||||
chunker2 := NewChunker(256 * 1024)
|
||||
|
||||
// Use deterministic data
|
||||
data := bytes.Repeat([]byte("abcdefghijklmnopqrstuvwxyz"), 20000) // ~520KB
|
||||
|
||||
chunks1, err := chunker1.ChunkReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("chunking failed: %v", err)
|
||||
}
|
||||
|
||||
chunks2, err := chunker2.ChunkReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("chunking failed: %v", err)
|
||||
}
|
||||
|
||||
// Should produce same chunks
|
||||
if len(chunks1) != len(chunks2) {
|
||||
t.Fatalf("different number of chunks: %d vs %d", len(chunks1), len(chunks2))
|
||||
}
|
||||
|
||||
for i := range chunks1 {
|
||||
if chunks1[i].Hash != chunks2[i].Hash {
|
||||
t.Errorf("chunk %d: different hashes", i)
|
||||
}
|
||||
if chunks1[i].Size != chunks2[i].Size {
|
||||
t.Errorf("chunk %d: different sizes", i)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkBoundaries(t *testing.T) {
|
||||
chunker := NewChunker(256 * 1024) // 256KB average
|
||||
t.Parallel()
|
||||
|
||||
c := chunker.NewChunker(256 * 1024) // 256KB average
|
||||
|
||||
// FastCDC uses avg/4 for min and avg*4 for max
|
||||
avgSize := int64(256 * 1024)
|
||||
@@ -107,11 +123,13 @@ func TestChunkBoundaries(t *testing.T) {
|
||||
|
||||
// Test that minimum chunk size is respected
|
||||
data := make([]byte, minSize+1024)
|
||||
if _, err := rand.Read(data); err != nil {
|
||||
|
||||
_, err := rand.Read(data)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate random data: %v", err)
|
||||
}
|
||||
|
||||
chunks, err := chunker.ChunkReader(bytes.NewReader(data))
|
||||
chunks, err := c.ChunkReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("chunking failed: %v", err)
|
||||
}
|
||||
@@ -119,10 +137,13 @@ func TestChunkBoundaries(t *testing.T) {
|
||||
for i, chunk := range chunks {
|
||||
// Last chunk can be smaller than minimum
|
||||
if i < len(chunks)-1 && chunk.Size < minSize {
|
||||
t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize)
|
||||
t.Errorf("chunk %d size %d is below minimum %d",
|
||||
i, chunk.Size, minSize)
|
||||
}
|
||||
|
||||
if chunk.Size > maxSize {
|
||||
t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize)
|
||||
t.Errorf("chunk %d size %d exceeds maximum %d",
|
||||
i, chunk.Size, maxSize)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+77
-46
@@ -1,6 +1,7 @@
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"sync"
|
||||
@@ -27,32 +28,52 @@ type ReusableChunker struct {
|
||||
}
|
||||
|
||||
// reusableChunkerPool pools ReusableChunker instances to avoid allocations.
|
||||
//
|
||||
//nolint:gochecknoglobals // process-wide object pool by design
|
||||
var reusableChunkerPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
New: func() any {
|
||||
return &ReusableChunker{}
|
||||
},
|
||||
}
|
||||
|
||||
// bufferPools contains pools for different buffer sizes.
|
||||
// Key is the buffer size.
|
||||
//
|
||||
//nolint:gochecknoglobals // process-wide buffer pools by design
|
||||
var bufferPools = sync.Map{}
|
||||
|
||||
func getBuffer(size int) []byte {
|
||||
poolI, _ := bufferPools.LoadOrStore(size, &sync.Pool{
|
||||
New: func() interface{} {
|
||||
New: func() any {
|
||||
buf := make([]byte, size)
|
||||
|
||||
return &buf
|
||||
},
|
||||
})
|
||||
pool := poolI.(*sync.Pool)
|
||||
return *pool.Get().(*[]byte)
|
||||
|
||||
pool, ok := poolI.(*sync.Pool)
|
||||
if !ok {
|
||||
panic("bufferPools holds a non-pool value")
|
||||
}
|
||||
|
||||
buf, ok := pool.Get().(*[]byte)
|
||||
if !ok {
|
||||
panic("buffer pool holds a non-buffer value")
|
||||
}
|
||||
|
||||
return *buf
|
||||
}
|
||||
|
||||
func putBuffer(buf []byte) {
|
||||
size := cap(buf)
|
||||
|
||||
poolI, ok := bufferPools.Load(size)
|
||||
if ok {
|
||||
pool := poolI.(*sync.Pool)
|
||||
pool, isPool := poolI.(*sync.Pool)
|
||||
if !isPool {
|
||||
panic("bufferPools holds a non-pool value")
|
||||
}
|
||||
|
||||
b := buf[:size]
|
||||
pool.Put(&b)
|
||||
}
|
||||
@@ -66,17 +87,28 @@ type FastCDCChunk struct {
|
||||
Fingerprint uint64
|
||||
}
|
||||
|
||||
// AcquireReusableChunker gets a chunker from the pool and initializes it for the given reader.
|
||||
func AcquireReusableChunker(rd io.Reader, minSize, avgSize, maxSize int) *ReusableChunker {
|
||||
c := reusableChunkerPool.Get().(*ReusableChunker)
|
||||
// bufSizeFactor sizes the internal read buffer relative to the maximum
|
||||
// chunk size so a full chunk plus read-ahead always fits.
|
||||
const bufSizeFactor = 2
|
||||
|
||||
bufSize := maxSize * 2
|
||||
// AcquireReusableChunker gets a chunker from the pool and initializes it
|
||||
// for the given reader.
|
||||
func AcquireReusableChunker(
|
||||
rd io.Reader, minSize, avgSize, maxSize int,
|
||||
) *ReusableChunker {
|
||||
c, ok := reusableChunkerPool.Get().(*ReusableChunker)
|
||||
if !ok {
|
||||
panic("reusableChunkerPool holds a non-chunker value")
|
||||
}
|
||||
|
||||
bufSize := maxSize * bufSizeFactor
|
||||
|
||||
// Reuse buffer if it's the right size, otherwise get a new one
|
||||
if c.buf == nil || cap(c.buf) != bufSize {
|
||||
if c.buf != nil {
|
||||
putBuffer(c.buf)
|
||||
}
|
||||
|
||||
c.buf = getBuffer(bufSize)
|
||||
} else {
|
||||
// Restore buffer to full capacity (may have been truncated by previous EOF)
|
||||
@@ -108,41 +140,14 @@ func (c *ReusableChunker) Release() {
|
||||
reusableChunkerPool.Put(c)
|
||||
}
|
||||
|
||||
func (c *ReusableChunker) fillBuffer() error {
|
||||
n := len(c.buf) - c.cursor
|
||||
if n >= c.maxSize {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Move all data after the cursor to the start of the buffer
|
||||
copy(c.buf[:n], c.buf[c.cursor:])
|
||||
c.cursor = 0
|
||||
|
||||
if c.eof {
|
||||
c.buf = c.buf[:n]
|
||||
return nil
|
||||
}
|
||||
|
||||
// Restore buffer to full capacity for reading
|
||||
c.buf = c.buf[:c.bufSize]
|
||||
|
||||
// Fill the rest of the buffer
|
||||
m, err := io.ReadFull(c.rd, c.buf[n:])
|
||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
c.buf = c.buf[:n+m]
|
||||
c.eof = true
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Next returns the next chunk or io.EOF when done.
|
||||
// The returned Data slice is only valid until the next call to Next.
|
||||
func (c *ReusableChunker) Next() (FastCDCChunk, error) {
|
||||
if err := c.fillBuffer(); err != nil {
|
||||
err := c.fillBuffer()
|
||||
if err != nil {
|
||||
return FastCDCChunk{}, err
|
||||
}
|
||||
|
||||
if len(c.buf) == 0 {
|
||||
return FastCDCChunk{}, io.EOF
|
||||
}
|
||||
@@ -162,6 +167,37 @@ func (c *ReusableChunker) Next() (FastCDCChunk, error) {
|
||||
return chunk, nil
|
||||
}
|
||||
|
||||
func (c *ReusableChunker) fillBuffer() error {
|
||||
n := len(c.buf) - c.cursor
|
||||
if n >= c.maxSize {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Move all data after the cursor to the start of the buffer
|
||||
copy(c.buf[:n], c.buf[c.cursor:])
|
||||
c.cursor = 0
|
||||
|
||||
if c.eof {
|
||||
c.buf = c.buf[:n]
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Restore buffer to full capacity for reading
|
||||
c.buf = c.buf[:c.bufSize]
|
||||
|
||||
// Fill the rest of the buffer
|
||||
m, err := io.ReadFull(c.rd, c.buf[n:])
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
c.buf = c.buf[:n+m]
|
||||
c.eof = true
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) {
|
||||
fp := uint64(0)
|
||||
i := c.minSize
|
||||
@@ -189,14 +225,9 @@ func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) {
|
||||
return i, fp
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// 256 random uint64s for the rolling hash function (from FastCDC paper)
|
||||
//
|
||||
//nolint:gochecknoglobals // immutable FastCDC gear lookup table
|
||||
var table = [256]uint64{
|
||||
0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c,
|
||||
0xeac874b86f0f57b9, 0xa5ccedec95ec79c7, 0xe15a3320ad42ac0a, 0x5ed3583fa63cec15,
|
||||
|
||||
+318
-70
@@ -1,54 +1,110 @@
|
||||
// Package cli implements the vaultik command-line interface: cobra
|
||||
// commands, fx application wiring, and process-level concerns such as
|
||||
// signal handling and the PID lock.
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/pidlock"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/adrg/xdg"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/dig"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/globals"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/pidlock"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// AppOptions contains common options for creating the fx application.
|
||||
// It includes the configuration file path, logging options, and additional
|
||||
// fx modules and invocations that should be included in the application.
|
||||
// shutdownTimeout bounds how long a signal-triggered graceful shutdown
|
||||
// may take before we give up.
|
||||
const shutdownTimeout = 30 * time.Second
|
||||
|
||||
// lockMode says whether a command mutates persistent state — the local
|
||||
// index database or the remote store — and so must hold the process-wide
|
||||
// PID lock, or only reads that state and may run alongside a mutator.
|
||||
type lockMode int
|
||||
|
||||
const (
|
||||
// mutating commands (snapshot create, snapshot purge, snapshot remove,
|
||||
// prune, remote nuke) write the local index or the remote store. They
|
||||
// hold the PID lock so that at most one runs at a time.
|
||||
mutating lockMode = iota
|
||||
// readOnly commands (info, snapshot list, snapshot verify, remote info,
|
||||
// snapshot restore) do not write the local index or the remote store,
|
||||
// so they run without the lock and are never blocked by a running
|
||||
// mutator. restore writes only to the target directory it is given.
|
||||
readOnly
|
||||
)
|
||||
|
||||
// AppOptions contains common options for creating and running the fx
|
||||
// application: the configuration file path, logging options, additional fx
|
||||
// modules and invocations, and whether the command mutates persistent
|
||||
// state (which decides whether it takes the PID lock).
|
||||
type AppOptions struct {
|
||||
ConfigPath string
|
||||
LogOptions log.LogOptions
|
||||
LogOptions log.Options
|
||||
Modules []fx.Option
|
||||
Invokes []fx.Option
|
||||
Mode lockMode
|
||||
}
|
||||
|
||||
// setupGlobals sets up the globals with application startup time
|
||||
func setupGlobals(lc fx.Lifecycle, g *globals.Globals) {
|
||||
// setupGlobals records the startup time and, when an output-suppression
|
||||
// flag is active, marks the UI writer quiet so that Begin/Complete/
|
||||
// Info/Notice/Detail/Progress are silenced. Warning and Error are NOT
|
||||
// silenced — per the documented convention that --quiet suppresses
|
||||
// non-error output only. The startup banner is printed by Entry
|
||||
// before cobra parses arguments, gated by the same arg-level check.
|
||||
//
|
||||
// --json quiets the UI here too, because stdout then carries a JSON
|
||||
// document and human narration would corrupt it. Unlike Quiet it does
|
||||
// not lower the stderr log level (issue #112), so --verbose/--debug
|
||||
// still surface diagnostics alongside the document.
|
||||
func setupGlobals(
|
||||
lc fx.Lifecycle, g *globals.Globals, v *vaultik.Vaultik, opts log.Options,
|
||||
) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
OnStart: func(_ context.Context) error {
|
||||
g.StartTime = time.Now().UTC()
|
||||
|
||||
if opts.Cron || opts.Quiet || opts.JSON {
|
||||
v.UI.SetQuiet(true)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// writeStartupBanner prints the two-line application banner followed by a
|
||||
// blank line. Used both from the fx hook (for subcommand invocations) and
|
||||
// from the root cobra Run handler (for `vaultik` with no subcommand).
|
||||
func writeStartupBanner(w *ui.Writer, startTime time.Time, shortCommit string) {
|
||||
w.Bannerf("%s %s by %s (commit %s, built on %s) starting up at %s.",
|
||||
globals.Appname, globals.Version, globals.Author,
|
||||
shortCommit, globals.CommitDate,
|
||||
startTime.Format(time.RFC3339))
|
||||
w.Bannerf("%s", globals.Homepage)
|
||||
w.Bannerf("")
|
||||
}
|
||||
|
||||
// NewApp creates a new fx application with common modules.
|
||||
// It sets up the base modules (config, database, logging, globals) and
|
||||
// combines them with any additional modules specified in the options.
|
||||
// The returned fx.App is ready to be started with RunApp.
|
||||
func NewApp(opts AppOptions) *fx.App {
|
||||
baseModules := []fx.Option{
|
||||
fx.Supply(config.ConfigPath(opts.ConfigPath)),
|
||||
fx.Supply(config.Path(opts.ConfigPath)),
|
||||
fx.Supply(opts.LogOptions),
|
||||
fx.Provide(globals.New),
|
||||
fx.Provide(log.New),
|
||||
@@ -62,83 +118,275 @@ func NewApp(opts AppOptions) *fx.App {
|
||||
fx.NopLogger,
|
||||
}
|
||||
|
||||
allOptions := append(baseModules, opts.Modules...)
|
||||
capacity := len(baseModules) + len(opts.Modules) + len(opts.Invokes)
|
||||
allOptions := make([]fx.Option, 0, capacity)
|
||||
allOptions = append(allOptions, baseModules...)
|
||||
allOptions = append(allOptions, opts.Modules...)
|
||||
allOptions = append(allOptions, opts.Invokes...)
|
||||
|
||||
return fx.New(allOptions...)
|
||||
}
|
||||
|
||||
// RunApp starts and stops the fx application within the given context.
|
||||
// It handles graceful shutdown on interrupt signals (SIGINT, SIGTERM) and
|
||||
// ensures the application stops cleanly. The function blocks until the
|
||||
// application completes or is interrupted. Returns an error if startup fails.
|
||||
func RunApp(ctx context.Context, app *fx.App) error {
|
||||
// Set up signal handling for graceful shutdown
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
|
||||
// startupError carries a startup failure message that has been cleaned
|
||||
// of fx dependency-injection noise. A distinct type (rather than
|
||||
// errors.New) keeps the dynamic message out of err113's sight while
|
||||
// preserving the exact user-facing text.
|
||||
type startupError struct {
|
||||
msg string
|
||||
}
|
||||
|
||||
// Create a context that will be cancelled on signal
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
func (e *startupError) Error() string {
|
||||
return e.msg
|
||||
}
|
||||
|
||||
// cleanStartupError strips fx's dependency-injection call-chain noise from
|
||||
// startup errors. fx wraps the underlying error with messages like
|
||||
//
|
||||
// could not build arguments for function "X" (file:line): failed to build T:
|
||||
// could not build arguments for function "Y" (file:line): failed to build U:
|
||||
// received non-nil error from function "Z" (file:line): <real error>
|
||||
//
|
||||
// Users care about the real error, not the DI plumbing. dig.RootCause,
|
||||
// from the library fx is built on, unwraps those layers and returns the
|
||||
// real error whole. Cutting the text at "): " instead would also cut a
|
||||
// real error that contains it, such as the blob_size_limit one.
|
||||
func cleanStartupError(err error) error {
|
||||
return &startupError{msg: dig.RootCause(err).Error()}
|
||||
}
|
||||
|
||||
// RunApp starts the fx application, blocks until it is asked to stop, and
|
||||
// then stops it. The app is asked to stop either by an OS interrupt
|
||||
// (SIGINT/SIGTERM — fx installs its own handler when app.Wait is called) or,
|
||||
// on normal completion, by the finished operation calling
|
||||
// Shutdowner.Shutdown(); both arrive on the app.Wait channel.
|
||||
//
|
||||
// Stopping runs the fx OnStop hooks, and RunApp does not return until Stop
|
||||
// returns. On an interrupt the operation's OnStop hook cancels the running
|
||||
// command and waits for it to unwind — removing its decrypted scratch files —
|
||||
// so the process cannot proceed to exit mid-cleanup (issue #159). Waiting for
|
||||
// Stop before returning is what makes that hook effective: routing the
|
||||
// interrupt through app.Stop and not returning until it completes is required,
|
||||
// because fx also fires the app.Wait channel on the signal, and an earlier
|
||||
// version returned on that alone — unwinding to os.Exit while the concurrent
|
||||
// cleanup still ran. The stop is bounded by shutdownTimeout. Returns an error
|
||||
// if startup fails.
|
||||
func RunApp(ctx context.Context, app *fx.App) error {
|
||||
err := app.Start(ctx)
|
||||
if err != nil {
|
||||
return cleanStartupError(err)
|
||||
}
|
||||
|
||||
// Block until an interrupt or the finished operation's
|
||||
// Shutdowner.Shutdown() arrives, then stop the app in this goroutine so we
|
||||
// return only after its OnStop hooks — including the operation's cleanup
|
||||
// wait — have run. Detach the stop from ctx's cancellation but keep its
|
||||
// values, and bound it by shutdownTimeout.
|
||||
<-app.Wait()
|
||||
|
||||
shutdownCtx, cancel := context.WithTimeout(
|
||||
context.WithoutCancel(ctx), shutdownTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Start the app
|
||||
if err := app.Start(ctx); err != nil {
|
||||
return fmt.Errorf("failed to start app: %w", err)
|
||||
err = app.Stop(shutdownCtx)
|
||||
if err != nil {
|
||||
log.Error("Error during shutdown", "error", err)
|
||||
}
|
||||
|
||||
// Handle shutdown
|
||||
shutdownComplete := make(chan struct{})
|
||||
go func() {
|
||||
defer close(shutdownComplete)
|
||||
<-sigChan
|
||||
log.Notice("Received interrupt signal, shutting down gracefully...")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create a timeout context for shutdown
|
||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer shutdownCancel()
|
||||
// errReported marks a failure the operation has already shown the user
|
||||
// (or, under `snapshot verify --json`, put in its document). Entry
|
||||
// turns it into a non-zero exit status without printing anything
|
||||
// further, so the error line is not doubled. It flows up from
|
||||
// RunOperation through cobra to Entry.
|
||||
var errReported = errors.New("operation failed")
|
||||
|
||||
if err := app.Stop(shutdownCtx); err != nil {
|
||||
log.Error("Error during shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
// errInterrupted marks an operation that SIGINT or SIGTERM stopped
|
||||
// before it finished. Entry shows it and returns exitCodeInterrupted.
|
||||
var errInterrupted = errors.New("interrupted before the command finished")
|
||||
|
||||
// Wait for either the signal handler to complete shutdown or the app to request shutdown
|
||||
select {
|
||||
case <-shutdownComplete:
|
||||
// Shutdown completed via signal
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
// Context cancelled (shouldn't happen in normal operation)
|
||||
if err := app.Stop(context.Background()); err != nil {
|
||||
log.Error("Error stopping app", "error", err)
|
||||
}
|
||||
return ctx.Err()
|
||||
case <-app.Done():
|
||||
// App finished running (e.g., backup completed)
|
||||
// RunOperation runs op against the Vaultik instance inside the fx app
|
||||
// and turns a failure into a returned error rather than an os.Exit from
|
||||
// within the goroutine. An os.Exit there skipped main's deferred
|
||||
// profile writers -- so profiling a failing command yielded a truncated
|
||||
// profile (issue #75) -- and RunWithApp's PID-lock release, and denied
|
||||
// the app any graceful shutdown; returning the error to the top runs
|
||||
// all three.
|
||||
//
|
||||
// op runs in a goroutine so OnStart returns promptly and an interrupt
|
||||
// can still cancel through OnStop; when it finishes, success or failure,
|
||||
// it triggers shutdown, which is what lets RunWithApp return. On an
|
||||
// interrupt OnStop cancels op and waits for the goroutine to return, so
|
||||
// op's cleanup (removing decrypted scratch files) runs before the
|
||||
// process exits; the wait is bounded by shutdownTimeout. report is
|
||||
// called with a failure so the caller can show it to the user before
|
||||
// it becomes errReported.
|
||||
//
|
||||
// The run counts as interrupted unless op returned, without an
|
||||
// interrupt having cancelled it, before RunWithApp returned. An
|
||||
// interrupted op is not reported, whatever it returned; RunOperation
|
||||
// returns errInterrupted instead.
|
||||
func RunOperation(
|
||||
ctx context.Context, opts AppOptions,
|
||||
op func(v *vaultik.Vaultik) error, report func(err error),
|
||||
) error {
|
||||
var (
|
||||
mu sync.Mutex
|
||||
finished bool // op returned before any interrupt cancelled it
|
||||
failed bool // op finished with an error
|
||||
)
|
||||
|
||||
opts.Invokes = append(opts.Invokes,
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
var stop func(context.Context) bool
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(_ context.Context) error {
|
||||
stop = v.StartOperation(func() {
|
||||
err := op(v)
|
||||
|
||||
// Only stop, called from OnStop below, cancels the
|
||||
// Vaultik context, so a live context means no
|
||||
// interrupt cancelled op. Check the context, not
|
||||
// err: an interrupted op need not return
|
||||
// context.Canceled (`snapshot verify --json`
|
||||
// returns a verification failure).
|
||||
if v.Context().Err() == nil {
|
||||
if err != nil {
|
||||
report(err)
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
finished = true
|
||||
failed = err != nil
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
stopErr := v.Shutdowner.Shutdown()
|
||||
if stopErr != nil {
|
||||
log.Error("Failed to shutdown", "error", stopErr)
|
||||
}
|
||||
})
|
||||
|
||||
return nil
|
||||
},
|
||||
// On an interrupt, cancel the operation and wait for it to
|
||||
// unwind so its cleanup defers (which remove decrypted
|
||||
// scratch files from the temp directory) run before the
|
||||
// process exits. The wait is bounded by ctx, the existing
|
||||
// shutdownTimeout.
|
||||
OnStop: func(ctx context.Context) error {
|
||||
if !stop(ctx) {
|
||||
log.Warn("Shutdown timed out before the operation " +
|
||||
"finished; decrypted temporary files may remain")
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}))
|
||||
|
||||
err := RunWithApp(ctx, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// RunWithApp returns only after the app was asked to stop, either by
|
||||
// an interrupt or by the goroutine's Shutdown call. When op finished
|
||||
// without being cancelled, the goroutine set finished before that
|
||||
// call. So if finished is unset here, an interrupt stopped the app,
|
||||
// and op either returned after it was cancelled or is still running
|
||||
// because the shutdown timed out.
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
switch {
|
||||
case !finished:
|
||||
return errInterrupted
|
||||
case failed:
|
||||
return errReported
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// runVaultikApp runs the standard single-operation command lifecycle
|
||||
// shared by the snapshot list/purge/remove and remote nuke subcommands:
|
||||
// resolve the config, then run op against the Vaultik instance through
|
||||
// RunOperation, reporting a failure prefixed with failMsg on stderr. mode
|
||||
// says whether the command takes the PID lock. jsonOutput marks a command
|
||||
// whose stdout is a JSON document: it quiets the UI but, unlike Quiet,
|
||||
// leaves the stderr log level alone.
|
||||
func runVaultikApp(
|
||||
cmd *cobra.Command, mode lockMode, jsonOutput bool,
|
||||
failMsg string, op func(v *vaultik.Vaultik) error,
|
||||
) error {
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunOperation(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
JSON: jsonOutput,
|
||||
},
|
||||
Mode: mode,
|
||||
}, op, func(err error) {
|
||||
log.Error(failMsg, "error", err)
|
||||
ReportErrorf("%s: %v", failMsg, err)
|
||||
})
|
||||
}
|
||||
|
||||
// RunWithApp is a helper that creates and runs an fx app with the given options.
|
||||
// It combines NewApp and RunApp into a single convenient function. This is the
|
||||
// preferred way to run CLI commands that need the full application context.
|
||||
// It acquires a PID lock before starting to prevent concurrent instances.
|
||||
// A mutating command takes the process-wide PID lock before starting so that
|
||||
// only one runs at a time; a read-only command runs without it and is not
|
||||
// blocked while a mutator holds the lock (opts.Mode).
|
||||
func RunWithApp(ctx context.Context, opts AppOptions) error {
|
||||
// Acquire PID lock to prevent concurrent instances
|
||||
lockDir := filepath.Join(xdg.DataHome, "berlin.sneak.app.vaultik")
|
||||
release, err := acquireLockIfMutating(opts.Mode,
|
||||
filepath.Join(xdg.DataHome, "vaultik"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer release()
|
||||
|
||||
app := NewApp(opts)
|
||||
|
||||
return RunApp(ctx, app)
|
||||
}
|
||||
|
||||
// acquireLockIfMutating takes the process-wide PID lock in lockDir for a
|
||||
// mutating command and returns a function that releases it. A read-only
|
||||
// command takes no lock, so it returns a no-op release and is never blocked
|
||||
// while a mutator holds the lock. ErrAlreadyRunning (another mutator holds
|
||||
// the lock) is surfaced as a "cannot start" error.
|
||||
func acquireLockIfMutating(mode lockMode, lockDir string) (func(), error) {
|
||||
if mode != mutating {
|
||||
return func() {}, nil
|
||||
}
|
||||
|
||||
lock, err := pidlock.Acquire(lockDir)
|
||||
if err != nil {
|
||||
if errors.Is(err, pidlock.ErrAlreadyRunning) {
|
||||
return fmt.Errorf("cannot start: %w", err)
|
||||
return nil, fmt.Errorf("cannot start: %w", err)
|
||||
}
|
||||
return fmt.Errorf("failed to acquire lock: %w", err)
|
||||
|
||||
return nil, fmt.Errorf("failed to acquire lock: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := lock.Release(); err != nil {
|
||||
|
||||
return func() {
|
||||
err := lock.Release()
|
||||
if err != nil {
|
||||
log.Warn("Failed to release PID lock", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
app := NewApp(opts)
|
||||
return RunApp(ctx, app)
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
package cli //nolint:testpackage // needs access to unexported cleanStartupError
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/pidlock"
|
||||
)
|
||||
|
||||
// TestCleanStartupError starts a real fx app whose config has a
|
||||
// blob_size_limit below the minimum. fx puts annotations naming the
|
||||
// functions it was building in front of the config error; only those
|
||||
// may be removed, although the config error itself contains "): ".
|
||||
func TestCleanStartupError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
|
||||
err := os.WriteFile(path, []byte("snapshots:\n"+
|
||||
" test:\n"+
|
||||
" paths: [/tmp/vaultik-test-source]\n"+
|
||||
"storage_url: file:///tmp/vaultik-test-storage\n"+
|
||||
"chunk_size: 10MB\n"+
|
||||
"blob_size_limit: 30MB\n"), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
app := fx.New(
|
||||
fx.NopLogger,
|
||||
fx.Supply(config.Path(path)),
|
||||
config.Module,
|
||||
fx.Invoke(func(*config.Config) {}),
|
||||
)
|
||||
|
||||
err = app.Start(t.Context())
|
||||
if err == nil {
|
||||
t.Fatal("app started; want a blob_size_limit error")
|
||||
}
|
||||
|
||||
want := "failed to load config: invalid config: blob_size_limit must " +
|
||||
"be at least the largest chunk the chunker can emit (chunk_size " +
|
||||
"times the FastCDC size spread): need at least 40000000 bytes"
|
||||
|
||||
got := cleanStartupError(err).Error()
|
||||
if got != want {
|
||||
t.Errorf("got %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCleanStartupErrorNotFromFx checks that an error fx did not wrap is
|
||||
// shown unchanged.
|
||||
func TestCleanStartupErrorNotFromFx(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
//nolint:err113 // a one-off error standing in for a startup failure
|
||||
got := cleanStartupError(errors.New("plain error")).Error()
|
||||
if got != "plain error" {
|
||||
t.Errorf("got %q, want %q", got, "plain error")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLockScopedToMutatingCommands proves the partition the PID lock now
|
||||
// enforces: a read-only command runs while a mutator holds the lock, and
|
||||
// two mutating commands still mutually exclude.
|
||||
func TestLockScopedToMutatingCommands(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
lockDir := filepath.Join(t.TempDir(), "vaultik")
|
||||
|
||||
// A mutating command takes the process-wide lock.
|
||||
releaseMutator, err := acquireLockIfMutating(mutating, lockDir)
|
||||
if err != nil {
|
||||
t.Fatalf("mutating command could not acquire lock: %v", err)
|
||||
}
|
||||
|
||||
// A read-only command runs to completion even while the lock is held.
|
||||
releaseReader, err := acquireLockIfMutating(readOnly, lockDir)
|
||||
if err != nil {
|
||||
t.Fatalf("read-only command was blocked by held lock: %v", err)
|
||||
}
|
||||
|
||||
releaseReader()
|
||||
|
||||
// A second mutating command is refused while the first holds the lock.
|
||||
_, err = acquireLockIfMutating(mutating, lockDir)
|
||||
if !errors.Is(err, pidlock.ErrAlreadyRunning) {
|
||||
t.Fatalf("second mutating command was not excluded, got: %v", err)
|
||||
}
|
||||
|
||||
// Once the first mutator releases, another mutating command may run.
|
||||
releaseMutator()
|
||||
|
||||
release, err := acquireLockIfMutating(mutating, lockDir)
|
||||
if err != nil {
|
||||
t.Fatalf("mutating command could not acquire released lock: %v", err)
|
||||
}
|
||||
|
||||
release()
|
||||
}
|
||||
@@ -0,0 +1,849 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"gopkg.in/yaml.v3"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
)
|
||||
|
||||
// configFileMode is the permission set for freshly written config files;
|
||||
// configs may hold S3 credentials, so keep them owner-only.
|
||||
const configFileMode = 0o600
|
||||
|
||||
// configSetArgs is the argument count of `config set <key> <value>`.
|
||||
const configSetArgs = 2
|
||||
|
||||
// configDirMode is the permission set for created config directories;
|
||||
// parent config dirs (e.g. ~/.config) are conventionally traversable.
|
||||
const configDirMode = 0o755
|
||||
|
||||
// configYAMLIndent matches the 2-space indentation of defaultConfigTemplate,
|
||||
// so `config set` writes the file back with the same indentation rather than
|
||||
// yaml.Marshal's 4-space default.
|
||||
const configYAMLIndent = 2
|
||||
|
||||
// yamlStringTag is YAML's tag for a string scalar.
|
||||
const yamlStringTag = "!!str"
|
||||
|
||||
var (
|
||||
errConfigExists = errors.New("config file already exists")
|
||||
errEmptyConfig = errors.New("empty config file")
|
||||
errKeyNotFound = errors.New("key not found")
|
||||
errNeedNumericIndex = errors.New("key is a list; use a numeric index")
|
||||
errIndexOutOfRange = errors.New("index out of range")
|
||||
errNotMapOrList = errors.New("key is not a map or list")
|
||||
errWouldNotLoad = errors.New(
|
||||
"vaultik would not load the config after this change, " +
|
||||
"so the file was not changed")
|
||||
)
|
||||
|
||||
const defaultConfigTemplate = `# vaultik configuration
|
||||
# Documentation: https://sneak.berlin/go/vaultik
|
||||
|
||||
# ─── REQUIRED ────────────────────────────────────────────────────────────────
|
||||
|
||||
# Age recipient public keys for encryption. snapshot create needs at least
|
||||
# one; listing, verifying and restoring do not, so a machine that only
|
||||
# restores can leave this empty.
|
||||
# Backups are encrypted to ALL listed recipients; any one of the corresponding
|
||||
# private keys can decrypt. Adding a recipient later does not re-encrypt data
|
||||
# already stored: deduplicated chunks and existing blobs stay encrypted to the
|
||||
# earlier recipients, so a newly added key cannot restore them on its own (see
|
||||
# docs/REPOSTRUCTURE.md, Accepted Risks). Generate a keypair and add its
|
||||
# public key with:
|
||||
# age-keygen -o vaultik_backup_private_key.txt
|
||||
# grep 'public key' vaultik_backup_private_key.txt
|
||||
# vaultik config set age_recipients.0 age1...
|
||||
age_recipients: []
|
||||
|
||||
# Named snapshots. Each snapshot backs up one or more paths and can have its
|
||||
# own exclude patterns in addition to the global excludes below.
|
||||
#
|
||||
# Exclude pattern semantics:
|
||||
# - Patterns starting with / are anchored to the snapshot path root
|
||||
# (e.g. "/Library/Caches" matches only ~/Library/Caches in a ~ snapshot)
|
||||
# - Patterns without a leading / match anywhere in the tree
|
||||
# (e.g. ".cache" matches any directory named .cache at any depth)
|
||||
# - Globs are supported: *, **, ?
|
||||
snapshots:
|
||||
home:
|
||||
paths:
|
||||
- "~"
|
||||
exclude:
|
||||
# Trash, temp, and filesystem metadata
|
||||
- "/.Trash"
|
||||
- "/.Trashes"
|
||||
- "/.fseventsd"
|
||||
- "/.Spotlight-V100"
|
||||
- "/.TemporaryItems"
|
||||
- "/tmp"
|
||||
- "/.rnd"
|
||||
- ".DS_Store"
|
||||
# Caches and package manager state (rebuildable)
|
||||
- ".cache"
|
||||
- ".bundle"
|
||||
- "/.cpan/build"
|
||||
- "/.cpan/sources"
|
||||
- "/.gradle/caches"
|
||||
- "/.dropbox"
|
||||
- "/.minikube/cache"
|
||||
- "/.local/share/containers/podman/machine"
|
||||
- "/.persepolis"
|
||||
- "/Library/Caches"
|
||||
- "/Library/Logs"
|
||||
- "/Library/Cookies"
|
||||
- "/Library/Metadata"
|
||||
- "/Library/Suggestions"
|
||||
- "/Library/PubSub"
|
||||
- "/Library/Homebrew"
|
||||
- "/Library/Developer"
|
||||
- "/Library/Google/GoogleSoftwareUpdate"
|
||||
- "/Library/Preferences/Macromedia/Flash Player"
|
||||
- "/Library/Preferences/SDMHelpData"
|
||||
- "/Library/VoiceTrigger/SAT"
|
||||
# Language/toolchain package caches (rebuildable from registries)
|
||||
- "/.npm"
|
||||
- "/.cargo/registry"
|
||||
- "/.cargo/git"
|
||||
- "/.rustup/toolchains"
|
||||
- "/go/pkg/mod"
|
||||
- "/.m2/repository"
|
||||
- "/.vagrant.d/boxes"
|
||||
- "node_modules"
|
||||
- "__pycache__"
|
||||
- ".venv"
|
||||
# Virtual machine disk images (huge; remove these lines to back them up)
|
||||
- "/Parallels"
|
||||
- "/Virtual Machines.localized"
|
||||
- "/VirtualBox VMs"
|
||||
- "/.orbstack"
|
||||
- "/Library/Containers/com.utmapp.UTM"
|
||||
# Downloaded LLM models (huge, re-downloadable)
|
||||
- "/.ollama/models"
|
||||
- "/.lmstudio/models"
|
||||
# Cloud-synced storage. These are synced to a provider already, and on
|
||||
# modern macOS may contain dataless placeholder files that the backup
|
||||
# would force-download in full.
|
||||
- "/Library/CloudStorage"
|
||||
- "/Library/Mobile Documents"
|
||||
# Android SDK and emulator images (re-downloadable)
|
||||
- "/Library/Android/sdk"
|
||||
- "/.android/avd"
|
||||
# Cloud-synced or restorable-from-server data
|
||||
- "/Library/Mail"
|
||||
- "/Library/Mail Downloads"
|
||||
- "/Library/Safari"
|
||||
- "/Library/Application Support/Evernote"
|
||||
- "/Library/Application Support/MobileSync"
|
||||
- "/Library/Application Support/SyncServices"
|
||||
- "/Library/Application Support/protonmail/bridge/cache"
|
||||
- "/Library/Application Support/Syncthing/index-*"
|
||||
- "/Library/Syncthing/folders"
|
||||
- "/Documents/Dropbox/.dropbox.cache"
|
||||
# Large rebuildable app data (games, media caches, device backups)
|
||||
- "/Applications/Fortnite"
|
||||
- "/Documents/Steam Content"
|
||||
- "/Library/Application Support/Ableton"
|
||||
- "/Library/Application Support/CrossOver Games"
|
||||
- "/Library/Application Support/SecondLife/cache"
|
||||
- "/Library/Application Support/Steam/SteamApps"
|
||||
- "/Library/Containers/com.docker.docker"
|
||||
- "/Library/Group Containers/group.com.apple.secure-control-center-preferences"
|
||||
- "/Library/iTunes/iPad Software Updates"
|
||||
- "/Library/iTunes/iPhone Software Updates"
|
||||
- "/Movies/CacheClip"
|
||||
- "/Movies/ProxyMedia"
|
||||
- "/Music/iTunes/Album Artwork"
|
||||
- "/Pictures/iPod Photo Cache"
|
||||
|
||||
# Third-party applications. OS-provided apps live in /System/Applications
|
||||
# on modern macOS and are never in /Applications, but Apple-installed
|
||||
# App Store apps (Safari, GarageBand, iWork, iMovie) are excluded since
|
||||
# they are re-downloadable.
|
||||
apps:
|
||||
paths:
|
||||
- /Applications
|
||||
exclude:
|
||||
- ".DS_Store"
|
||||
- "/Safari.app"
|
||||
- "/GarageBand.app"
|
||||
- "/iMovie.app"
|
||||
- "/Keynote.app"
|
||||
- "/Numbers.app"
|
||||
- "/Pages.app"
|
||||
- "/Xcode.app"
|
||||
- "/Spotify.app"
|
||||
- "/Steam.app"
|
||||
- "/VirtualBox.app"
|
||||
- "/Utilities/Adobe Installers"
|
||||
|
||||
# Storage backend (pick ONE of the three forms below).
|
||||
#
|
||||
# S3-compatible:
|
||||
# storage_url: "s3://mybucket/backups?endpoint=s3.example.com®ion=us-east-1"
|
||||
# (also set s3.access_key_id and s3.secret_access_key below)
|
||||
#
|
||||
# Local filesystem:
|
||||
# storage_url: "file:///mnt/backups/vaultik"
|
||||
#
|
||||
# Rclone (requires rclone configured separately):
|
||||
# storage_url: "rclone://myremote/path/to/backups"
|
||||
storage_url: ""
|
||||
|
||||
# ─── S3 CREDENTIALS (required for s3:// storage_url) ────────────────────────
|
||||
|
||||
# s3:
|
||||
# access_key_id: YOUR_ACCESS_KEY
|
||||
# secret_access_key: YOUR_SECRET_KEY
|
||||
# # region: us-east-1 # Default: us-east-1
|
||||
# # part_size: 5MiB # Upload part size, 5MiB to 5GiB. Default: 5MiB
|
||||
# # For the s3:// form, disable TLS with ?ssl=false in the URL, not use_ssl.
|
||||
|
||||
# ─── OPTIONAL ────────────────────────────────────────────────────────────────
|
||||
|
||||
# Global exclude patterns applied to ALL snapshots.
|
||||
# Snapshot-specific excludes are additive.
|
||||
# exclude:
|
||||
# - "*.log"
|
||||
# - "*.tmp"
|
||||
# - ".git"
|
||||
# - "node_modules"
|
||||
|
||||
# Average chunk size for content-defined chunking (FastCDC).
|
||||
# Smaller = better deduplication but more metadata overhead.
|
||||
# Minimum 1MB. Accepts: 1MB, 10M, 64MiB, etc.
|
||||
# Default: 10MB
|
||||
# chunk_size: 10MB
|
||||
|
||||
# Maximum blob size before splitting into a new blob.
|
||||
# Must be at least four times chunk_size (the largest chunk the chunker can
|
||||
# emit); a smaller limit would let a single-chunk blob exceed it.
|
||||
# Accepts: 1GB, 10G, 500MB, etc.
|
||||
# Default: 10GB
|
||||
# blob_size_limit: 10GB
|
||||
|
||||
# Zstd compression level (1-19). Higher = better ratio but slower.
|
||||
# Default: 3
|
||||
# compression_level: 3
|
||||
|
||||
# How old a blob that no snapshot lists must be before prune deletes it,
|
||||
# and how old every object in a directory under metadata/ with no manifest
|
||||
# must be before prune deletes that directory. A backup uploads its blobs
|
||||
# and its db.zst.age before its manifest, so on a destination shared
|
||||
# between hosts or users this must be longer than the longest backup run.
|
||||
# It does not protect a blob a backup reuses from an earlier run that
|
||||
# stopped before its manifest: that blob keeps the time of its first
|
||||
# upload, so a prune can delete it.
|
||||
# Accepts: 36h, 90m, etc. 0s deletes such blobs and directories at once.
|
||||
# Default: 24h
|
||||
# prune_grace_period: 24h
|
||||
|
||||
# Hostname used in snapshot IDs. Default: system hostname.
|
||||
# hostname: myserver
|
||||
|
||||
# Path to the local SQLite index database.
|
||||
# Default: the platform data directory, e.g.
|
||||
# macOS: ~/Library/Application Support/vaultik/index.sqlite
|
||||
# Linux: ~/.local/share/vaultik/index.sqlite
|
||||
# index_path: /path/to/index.sqlite
|
||||
`
|
||||
|
||||
// NewConfigCommand creates the config command group.
|
||||
func NewConfigCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "config",
|
||||
Short: "Manage the configuration file",
|
||||
Long: "Commands for creating, editing, and querying the vaultik config file.",
|
||||
// cobra checks Args only on a command that has a Run: without
|
||||
// one, a mistyped subcommand printed help and exited 0.
|
||||
Args: cobra.NoArgs,
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
_ = cmd.Help()
|
||||
},
|
||||
}
|
||||
|
||||
cmd.AddCommand(newConfigInitCommand())
|
||||
cmd.AddCommand(newConfigEditCommand())
|
||||
cmd.AddCommand(newConfigGetCommand())
|
||||
cmd.AddCommand(newConfigSetCommand())
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// newConfigInitCommand creates the 'config init' subcommand.
|
||||
func newConfigInitCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "init",
|
||||
Short: "Write a default config file",
|
||||
Long: `Creates a default configuration file with commented explanations
|
||||
for the settings it lists. If a config file already exists at the target
|
||||
path, the command refuses to overwrite it. The file is created with mode
|
||||
0600, because it can hold credentials.
|
||||
|
||||
The config is written to the path from --config, $VAULTIK_CONFIG, or
|
||||
config.yml in the platform default config directory
|
||||
(~/Library/Application Support/vaultik/ on macOS, ~/.config/vaultik/ on
|
||||
Linux, /etc/vaultik/ as root).`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
path := configPathForInit()
|
||||
|
||||
_, err := os.Stat(path)
|
||||
if err == nil {
|
||||
return fmt.Errorf("%w: %s", errConfigExists, path)
|
||||
}
|
||||
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
err = os.MkdirAll(dir, configDirMode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating config directory %s: %w", dir, err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing config file: %w", err)
|
||||
}
|
||||
|
||||
// A written-confirmation, not scriptable output: route it
|
||||
// through the UI so it is styled and --quiet silences it.
|
||||
out := commandUI(cmd)
|
||||
out.Infof("Config written to %s.", path)
|
||||
out.Infof(
|
||||
"Edit it to set your age_recipients, snapshots, and storage_url.")
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// newConfigEditCommand creates the 'config edit' subcommand.
|
||||
func newConfigEditCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "edit",
|
||||
Short: "Open the config file in $EDITOR (vi when it is not set)",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
path, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
editor := os.Getenv("EDITOR")
|
||||
if editor == "" {
|
||||
editor = "vi"
|
||||
}
|
||||
|
||||
// Run $EDITOR through the shell, as git does, so a value
|
||||
// with arguments such as "code --wait" works. The path
|
||||
// goes in as an argument ("$@"), not into the command
|
||||
// text, so a path with spaces or quotes arrives whole.
|
||||
// sh -c takes the word after the command as $0, which
|
||||
// is why editor appears twice.
|
||||
//nolint:gosec // G204: launching the operator's own $EDITOR is the point
|
||||
ed := exec.CommandContext(cmd.Context(),
|
||||
"sh", "-c", editor+` "$@"`, editor, path)
|
||||
ed.Stdin = os.Stdin
|
||||
ed.Stdout = os.Stdout
|
||||
ed.Stderr = os.Stderr
|
||||
|
||||
return ed.Run()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// newConfigGetCommand creates the 'config get' subcommand.
|
||||
func newConfigGetCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "get <key>",
|
||||
Short: "Print a config value by dotted path (e.g. storage_url, compression_level)",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
path, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
root, err := loadYAMLFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
node, err := yamlPathGet(root, strings.Split(args[0], "."))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The value is scriptable output: it must stay machine-plain
|
||||
// (no marker, no color) and is never silenced by --quiet, so it
|
||||
// is written straight to stdout rather than through the UI.
|
||||
w := cmd.OutOrStdout()
|
||||
|
||||
if node.Kind == yaml.ScalarNode {
|
||||
_, _ = fmt.Fprintln(w, node.Value)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
out, err := yaml.Marshal(node)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshaling value: %w", err)
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprint(w, string(out))
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// newConfigSetCommand creates the 'config set' subcommand.
|
||||
func newConfigSetCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "set <key> <value>",
|
||||
Short: "Set a config value by dotted path (e.g. compression_level 5)",
|
||||
Long: `Sets a scalar config value addressed by dotted YAML path and writes
|
||||
the file back, preserving comments and formatting. Missing keys along
|
||||
the path are created: as a list when the next path element is a
|
||||
number, otherwise as a map.
|
||||
|
||||
If the config loads, a change after which it would not, such as
|
||||
compression_level 99, is refused and the file is left unchanged. A
|
||||
config that does not load, such as the one config init writes before
|
||||
storage_url is set or one with a ${ENV:...} reference to a variable
|
||||
not set here, is checked one setting at a time: only a change after
|
||||
which the changed setting cannot be read, such as one giving it the
|
||||
wrong type, is refused.
|
||||
|
||||
Examples:
|
||||
vaultik config set storage_url "file:///mnt/backups"
|
||||
vaultik config set storage_url "s3://bucket/prefix?endpoint=host®ion=us-east-1"
|
||||
vaultik config set compression_level 9
|
||||
vaultik config set s3.bucket mybucket # legacy S3 fields still supported`,
|
||||
Args: cobra.ExactArgs(configSetArgs),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
path, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return writeConfigSet(commandUI(cmd), path, args[0], args[1])
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// writeConfigSet applies key=value to the config at path, writes it back
|
||||
// owner-only, and confirms the write by naming just the key through the
|
||||
// UI writer (styled, and silenced by --quiet). The value is never
|
||||
// echoed: it may be a secret such as s3.secret_access_key, and captured
|
||||
// stdout or a pasted terminal would then leak it.
|
||||
func writeConfigSet(out *ui.Writer, path, key, value string) error {
|
||||
root, err := loadYAMLFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
oldData, err := marshalConfigYAML(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshaling config: %w", err)
|
||||
}
|
||||
|
||||
keys := strings.Split(key, ".")
|
||||
|
||||
err = yamlPathSet(root, keys, value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
data, err := marshalConfigYAML(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshaling config: %w", err)
|
||||
}
|
||||
|
||||
setting, err := yamlPathOnly(root, keys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
settingData, err := marshalConfigYAML(setting)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshaling config: %w", err)
|
||||
}
|
||||
|
||||
err = checkConfigChange(oldData, data, settingData)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", errWouldNotLoad, err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(path, data, configFileMode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing config file: %w", err)
|
||||
}
|
||||
|
||||
// os.WriteFile does not change the mode of a file that already exists,
|
||||
// so a config that was group- or world-readable stays that way. As it
|
||||
// may hold S3 credentials, tighten it to owner-only after writing.
|
||||
info, statErr := os.Stat(path)
|
||||
if statErr == nil && info.Mode().Perm()&0o044 != 0 {
|
||||
err = os.Chmod(path, configFileMode)
|
||||
if err != nil {
|
||||
return fmt.Errorf("tightening config file permissions: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
out.Infof("Set %s.", key)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkConfigChange returns the reason vaultik would not load newData, the
|
||||
// config after a change, if the change brings that failure in. If oldData,
|
||||
// the config before it, loads, the new one must load too. A config that
|
||||
// does not load, such as one still being filled in or one with a ${...}
|
||||
// reference that cannot be filled in here, is checked one setting at a
|
||||
// time: settingData, the changed setting on its own, must decode into the
|
||||
// config type.
|
||||
func checkConfigChange(oldData, newData, settingData []byte) error {
|
||||
oldCfg, err := config.Parse(oldData)
|
||||
if err == nil && oldCfg.Validate() == nil {
|
||||
newCfg, err := config.Parse(newData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return newCfg.Validate()
|
||||
}
|
||||
|
||||
_, err = config.Parse(settingData)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// yamlPathOnly returns a document holding only the value at the dotted key
|
||||
// path in root, inside the maps and lists that lead to it there.
|
||||
func yamlPathOnly(root *yaml.Node, keys []string) (*yaml.Node, error) {
|
||||
node, err := yamlPathGet(root, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for i, key := range slices.Backward(keys) {
|
||||
parent, err := yamlPathGet(root, keys[:i])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if parent.Kind == yaml.SequenceNode {
|
||||
node = &yaml.Node{
|
||||
Kind: yaml.SequenceNode, Content: []*yaml.Node{node},
|
||||
}
|
||||
} else {
|
||||
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
|
||||
node = &yaml.Node{
|
||||
Kind: yaml.MappingNode, Content: []*yaml.Node{keyNode, node},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{node}}, nil
|
||||
}
|
||||
|
||||
// marshalConfigYAML renders a config document tree with 2-space indentation,
|
||||
// matching defaultConfigTemplate. yaml.Marshal defaults to 4 spaces, which
|
||||
// would reindent the whole file on the first `config set` despite the promise
|
||||
// to preserve formatting.
|
||||
func marshalConfigYAML(root *yaml.Node) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
|
||||
enc := yaml.NewEncoder(&buf)
|
||||
enc.SetIndent(configYAMLIndent)
|
||||
|
||||
err := enc.Encode(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = enc.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// loadYAMLFile parses a YAML file into a yaml.Node document tree,
|
||||
// which preserves comments and ordering for round-tripping.
|
||||
func loadYAMLFile(path string) (*yaml.Node, error) {
|
||||
data, err := os.ReadFile(path) //nolint:gosec // G304: config path is operator-supplied
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading config file: %w", err)
|
||||
}
|
||||
|
||||
var root yaml.Node
|
||||
|
||||
err = yaml.Unmarshal(data, &root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing config file: %w", err)
|
||||
}
|
||||
|
||||
// An empty file yields a zero node; normalize to an empty mapping document.
|
||||
if root.Kind == 0 {
|
||||
root = yaml.Node{
|
||||
Kind: yaml.DocumentNode,
|
||||
Content: []*yaml.Node{{Kind: yaml.MappingNode}},
|
||||
}
|
||||
}
|
||||
|
||||
return &root, nil
|
||||
}
|
||||
|
||||
// yamlPathGet navigates a dotted key path through mapping and sequence
|
||||
// nodes and returns the value node. Numeric path components index into
|
||||
// sequences (e.g. "age_recipients.0").
|
||||
func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
|
||||
node := root
|
||||
if node.Kind == yaml.DocumentNode {
|
||||
if len(node.Content) == 0 {
|
||||
return nil, errEmptyConfig
|
||||
}
|
||||
|
||||
node = node.Content[0]
|
||||
}
|
||||
|
||||
for i, key := range keys {
|
||||
switch node.Kind {
|
||||
case yaml.MappingNode:
|
||||
found := false
|
||||
|
||||
for j := 0; j+1 < len(node.Content); j += 2 {
|
||||
if node.Content[j].Value == key {
|
||||
node = node.Content[j+1]
|
||||
found = true
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%w: %s",
|
||||
errKeyNotFound, strings.Join(keys[:i+1], "."))
|
||||
}
|
||||
case yaml.SequenceNode:
|
||||
idx, err := strconv.Atoi(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s",
|
||||
errNeedNumericIndex, strings.Join(keys[:i], "."))
|
||||
}
|
||||
|
||||
if idx < 0 || idx >= len(node.Content) {
|
||||
return nil, fmt.Errorf("%w: index %d for %s (len %d)",
|
||||
errIndexOutOfRange, idx, strings.Join(keys[:i], "."),
|
||||
len(node.Content))
|
||||
}
|
||||
|
||||
node = node.Content[idx]
|
||||
case yaml.DocumentNode, yaml.ScalarNode, yaml.AliasNode:
|
||||
return nil, fmt.Errorf("%w: %s",
|
||||
errNotMapOrList, strings.Join(keys[:i], "."))
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %s",
|
||||
errNotMapOrList, strings.Join(keys[:i], "."))
|
||||
}
|
||||
}
|
||||
|
||||
return node, nil
|
||||
}
|
||||
|
||||
// yamlPathSet navigates a dotted key path, creating missing keys as needed,
|
||||
// and sets the final key to the given scalar value. Numeric path
|
||||
// components index into sequences; an index equal to the sequence length
|
||||
// appends a new element (e.g. "age_recipients.1" on a 1-element list).
|
||||
func yamlPathSet(root *yaml.Node, keys []string, value string) error {
|
||||
node := root
|
||||
if node.Kind == yaml.DocumentNode {
|
||||
if len(node.Content) == 0 {
|
||||
node.Content = []*yaml.Node{{Kind: yaml.MappingNode}}
|
||||
}
|
||||
|
||||
node = node.Content[0]
|
||||
}
|
||||
|
||||
for i := range keys {
|
||||
last := i == len(keys)-1
|
||||
|
||||
switch node.Kind {
|
||||
case yaml.MappingNode:
|
||||
node = yamlSetInMapping(node, keys, i, value, last)
|
||||
case yaml.SequenceNode:
|
||||
next, err := yamlSetInSequence(node, keys, i, value, last)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
node = next
|
||||
case yaml.DocumentNode, yaml.ScalarNode, yaml.AliasNode:
|
||||
return fmt.Errorf("%w: %s",
|
||||
errNotMapOrList, strings.Join(keys[:i], "."))
|
||||
default:
|
||||
return fmt.Errorf("%w: %s",
|
||||
errNotMapOrList, strings.Join(keys[:i], "."))
|
||||
}
|
||||
}
|
||||
|
||||
// config.Load reads the file through untyped YAML, which turns an
|
||||
// unquoted 00112233 into the number 38043 and 1e5 into 100000. Tagging
|
||||
// a string setting as a string makes the encoder quote such a value.
|
||||
// Other settings stay unquoted, so compression_level 9 is a number.
|
||||
// The encoder refuses to write a value that is not valid UTF-8 as a
|
||||
// string. Left untagged, such a value is written as base64 !!binary and
|
||||
// loads back unchanged.
|
||||
if configKeyIsString(keys) && utf8.ValidString(value) {
|
||||
node.Tag = yamlStringTag
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// configKeyIsString reports whether the dotted key names a string in
|
||||
// config.Config, following the fields' yaml tags, as s3.access_key_id and
|
||||
// snapshots.home.exclude.0 do.
|
||||
func configKeyIsString(keys []string) bool {
|
||||
typ := reflect.TypeFor[config.Config]()
|
||||
|
||||
for _, key := range keys {
|
||||
switch {
|
||||
case typ.Kind() == reflect.Map || typ.Kind() == reflect.Slice:
|
||||
// The key is a snapshot name or a list index.
|
||||
typ = typ.Elem()
|
||||
case typ.Kind() == reflect.Struct:
|
||||
field, ok := yamlField(typ, key)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
typ = field.Type
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return typ.Kind() == reflect.String
|
||||
}
|
||||
|
||||
// yamlField returns the field of struct type typ whose yaml tag names key.
|
||||
func yamlField(typ reflect.Type, key string) (reflect.StructField, bool) {
|
||||
for field := range typ.Fields() {
|
||||
name, _, _ := strings.Cut(field.Tag.Get("yaml"), ",")
|
||||
if name == key {
|
||||
return field, true
|
||||
}
|
||||
}
|
||||
|
||||
return reflect.StructField{}, false
|
||||
}
|
||||
|
||||
// yamlSetInMapping resolves (creating if needed) the value node for the
|
||||
// path element keys[i] within a mapping node, setting it to value when it
|
||||
// is the final path element, and returns the node to descend into.
|
||||
func yamlSetInMapping(
|
||||
node *yaml.Node, keys []string, i int, value string, last bool,
|
||||
) *yaml.Node {
|
||||
var valueNode *yaml.Node
|
||||
|
||||
for j := 0; j+1 < len(node.Content); j += 2 {
|
||||
if node.Content[j].Value == keys[i] {
|
||||
valueNode = node.Content[j+1]
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if valueNode == nil {
|
||||
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: keys[i]}
|
||||
valueNode = newPathNode(keys, i, value)
|
||||
node.Content = append(node.Content, keyNode, valueNode)
|
||||
} else if last {
|
||||
setScalar(valueNode, value)
|
||||
}
|
||||
|
||||
return valueNode
|
||||
}
|
||||
|
||||
// yamlSetInSequence indexes (or appends to) a sequence node using the
|
||||
// numeric path element keys[i], setting the element to value when it is
|
||||
// the final path element, and returns the node to descend into.
|
||||
func yamlSetInSequence(
|
||||
node *yaml.Node, keys []string, i int, value string, last bool,
|
||||
) (*yaml.Node, error) {
|
||||
idx, err := strconv.Atoi(keys[i])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s",
|
||||
errNeedNumericIndex, strings.Join(keys[:i], "."))
|
||||
}
|
||||
|
||||
if idx < 0 || idx > len(node.Content) {
|
||||
return nil, fmt.Errorf("%w: index %d for %s (len %d)",
|
||||
errIndexOutOfRange, idx, strings.Join(keys[:i], "."),
|
||||
len(node.Content))
|
||||
}
|
||||
|
||||
if idx == len(node.Content) {
|
||||
node.Content = append(node.Content, newPathNode(keys, i, value))
|
||||
} else if last {
|
||||
setScalar(node.Content[idx], value)
|
||||
}
|
||||
|
||||
return node.Content[idx], nil
|
||||
}
|
||||
|
||||
// newPathNode returns the node to create for the missing path element
|
||||
// keys[i]: the value when it is the final path element, otherwise an empty
|
||||
// list when the next path element is a number and an empty map when not.
|
||||
func newPathNode(keys []string, i int, value string) *yaml.Node {
|
||||
if i == len(keys)-1 {
|
||||
return &yaml.Node{Kind: yaml.ScalarNode, Value: value}
|
||||
}
|
||||
|
||||
_, err := strconv.Atoi(keys[i+1])
|
||||
if err == nil {
|
||||
return &yaml.Node{Kind: yaml.SequenceNode}
|
||||
}
|
||||
|
||||
return &yaml.Node{Kind: yaml.MappingNode}
|
||||
}
|
||||
|
||||
// setScalar overwrites a node in place with a plain scalar value.
|
||||
func setScalar(n *yaml.Node, value string) {
|
||||
n.Kind = yaml.ScalarNode
|
||||
n.Tag = ""
|
||||
n.Value = value
|
||||
n.Content = nil
|
||||
n.Style = 0
|
||||
}
|
||||
|
||||
// configPathForInit returns the config path to write, checking --config flag,
|
||||
// VAULTIK_CONFIG env, and the platform default.
|
||||
func configPathForInit() string {
|
||||
if rootFlags.ConfigPath != "" {
|
||||
return rootFlags.ConfigPath
|
||||
}
|
||||
|
||||
if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" {
|
||||
return envPath
|
||||
}
|
||||
|
||||
return DefaultConfigPath()
|
||||
}
|
||||
@@ -0,0 +1,680 @@
|
||||
package cli //nolint:testpackage // exercises unexported yamlPathGet/yamlPathSet
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
)
|
||||
|
||||
// TestDefaultConfigTemplateParses ensures the init template is valid YAML
|
||||
// that unmarshals into the Config struct with the expected snapshots.
|
||||
func TestDefaultConfigTemplateParses(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var cfg config.Config
|
||||
|
||||
err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("default config template is not valid YAML: %v", err)
|
||||
}
|
||||
|
||||
// A placeholder recipient would fail config.Load, so the template
|
||||
// leaves the list empty.
|
||||
if len(cfg.AgeRecipients) != 0 {
|
||||
t.Errorf("expected no age recipients, got %d", len(cfg.AgeRecipients))
|
||||
}
|
||||
|
||||
home, ok := cfg.Snapshots["home"]
|
||||
if !ok {
|
||||
t.Fatal("expected 'home' snapshot in default config")
|
||||
}
|
||||
|
||||
if len(home.Paths) == 0 {
|
||||
t.Error("home snapshot should have at least one path")
|
||||
}
|
||||
|
||||
if len(home.Exclude) == 0 {
|
||||
t.Error("home snapshot should have exclude patterns")
|
||||
}
|
||||
|
||||
apps, ok := cfg.Snapshots["apps"]
|
||||
if !ok {
|
||||
t.Fatal("expected 'apps' snapshot in default config")
|
||||
}
|
||||
|
||||
if len(apps.Paths) != 1 || apps.Paths[0] != "/Applications" {
|
||||
t.Errorf("apps snapshot should back up /Applications, got %v", apps.Paths)
|
||||
}
|
||||
|
||||
if len(apps.Exclude) == 0 {
|
||||
t.Error("apps snapshot should have exclude patterns")
|
||||
}
|
||||
}
|
||||
|
||||
// testRecipient is a valid age public key.
|
||||
const testRecipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
|
||||
|
||||
// TestConfigSetRecipientOnFreshConfig follows the README's getting started
|
||||
// steps: on the file `config init` writes, `config set age_recipients.0` and
|
||||
// `config set storage_url` give a config that loads with that recipient.
|
||||
// Neither is refused, although the config does not load until both are set;
|
||||
// once it loads, a recipient that is not a valid key is.
|
||||
func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
|
||||
err := os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode)
|
||||
if err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
out := ui.NewWithColor(&bytes.Buffer{}, false)
|
||||
|
||||
err = writeConfigSet(out, path, "age_recipients.0", testRecipient)
|
||||
if err != nil {
|
||||
t.Fatalf("config set age_recipients.0: %v", err)
|
||||
}
|
||||
|
||||
err = writeConfigSet(out, path, "storage_url", "file:///mnt/backups")
|
||||
if err != nil {
|
||||
t.Fatalf("config set storage_url: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := config.Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("config.Load: %v", err)
|
||||
}
|
||||
|
||||
if len(cfg.AgeRecipients) != 1 || cfg.AgeRecipients[0] != testRecipient {
|
||||
t.Errorf("age_recipients = %v, want [%s]", cfg.AgeRecipients, testRecipient)
|
||||
}
|
||||
|
||||
err = writeConfigSet(out, path, "age_recipients.0", "notakey")
|
||||
if err == nil || !strings.Contains(err.Error(), "not a valid recipient") {
|
||||
t.Errorf("config set age_recipients.0 notakey gives error %v, "+
|
||||
"want one saying it is not a valid recipient", err)
|
||||
}
|
||||
|
||||
_, err = config.Load(path)
|
||||
if err != nil {
|
||||
t.Errorf("config.Load after the refused change: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// configWithoutRecipients loads, but has no age_recipients key and no
|
||||
// snapshot named work.
|
||||
const configWithoutRecipients = `storage_url: file:///mnt/backups
|
||||
snapshots:
|
||||
home:
|
||||
paths:
|
||||
- "~"
|
||||
`
|
||||
|
||||
// configWithUnsetReference loads only where VAULTIK_TEST_UNSET_SECRET is
|
||||
// set, as a backup job's environment might set it. No test sets it.
|
||||
const configWithUnsetReference = configWithoutRecipients + `s3:
|
||||
secret_access_key: ${ENV:VAULTIK_TEST_UNSET_SECRET}
|
||||
`
|
||||
|
||||
// configWithLevelReference returns configWithoutRecipients with a ${FILE:...}
|
||||
// reference on compression_level to a file holding 9, so that the config
|
||||
// loads once the reference is filled in.
|
||||
func configWithLevelReference(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
levelPath := filepath.Join(t.TempDir(), "level")
|
||||
|
||||
err := os.WriteFile(levelPath, []byte("9"), configFileMode)
|
||||
if err != nil {
|
||||
t.Fatalf("write level: %v", err)
|
||||
}
|
||||
|
||||
return configWithoutRecipients +
|
||||
"compression_level: ${FILE:" + levelPath + "}\n"
|
||||
}
|
||||
|
||||
// TestConfigSetIndexUnderMissingKey checks that a list index under a key
|
||||
// the file does not have creates a list, and that the result loads.
|
||||
func TestConfigSetIndexUnderMissingKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
key string
|
||||
value string
|
||||
field func(cfg *config.Config) []string
|
||||
}{
|
||||
{"age_recipients.0", testRecipient,
|
||||
func(cfg *config.Config) []string { return cfg.AgeRecipients }},
|
||||
{"snapshots.work.paths.0", "/srv/work",
|
||||
func(cfg *config.Config) []string { return cfg.Snapshots["work"].Paths }},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.key, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
|
||||
err := os.WriteFile(path, []byte(configWithoutRecipients), configFileMode)
|
||||
if err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
|
||||
path, tt.key, tt.value)
|
||||
if err != nil {
|
||||
t.Fatalf("config set %s: %v", tt.key, err)
|
||||
}
|
||||
|
||||
cfg, err := config.Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("config.Load: %v", err)
|
||||
}
|
||||
|
||||
got := tt.field(cfg)
|
||||
if len(got) != 1 || got[0] != tt.value {
|
||||
t.Errorf("%s gives %v, want [%s]", tt.key, got, tt.value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigSetKeepsFileThatWouldNotLoad checks that config set refuses a
|
||||
// change after which the config would not load, with an error giving the
|
||||
// reason, and leaves the file as it was. On the file `config init` writes,
|
||||
// which does not load until storage_url is set, and on one that cannot be
|
||||
// read here, only a value of the wrong type is refused.
|
||||
func TestConfigSetKeepsFileThatWouldNotLoad(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
levelReference := configWithLevelReference(t)
|
||||
|
||||
tests := []struct {
|
||||
config string
|
||||
key string
|
||||
value string
|
||||
reason string
|
||||
}{
|
||||
// Index 1 of the empty list the missing key gives.
|
||||
{configWithoutRecipients, "age_recipients.1", testRecipient,
|
||||
"index out of range"},
|
||||
// A list set to a single value.
|
||||
{configWithoutRecipients, "snapshots.home.paths", "/srv",
|
||||
"into []string"},
|
||||
{configWithoutRecipients, "chunk_size", "fast",
|
||||
"invalid size format"},
|
||||
// Values of the right type that fail validation.
|
||||
{configWithoutRecipients, "compression_level", "99",
|
||||
"compression_level must be between 1 and 19"},
|
||||
{configWithoutRecipients, "chunk_size", "1KB",
|
||||
"chunk_size must be at least 1MB"},
|
||||
// With its reference filled in, the config loads.
|
||||
{levelReference, "blob_size_limit", "1MB",
|
||||
"blob_size_limit must be at least"},
|
||||
// The type check on a config that does not load yet.
|
||||
{defaultConfigTemplate, "snapshots.apps.paths", "/srv",
|
||||
"into []string"},
|
||||
// The type check on a config that cannot be read here.
|
||||
{configWithUnsetReference, "snapshots.home.paths", "/x",
|
||||
"cannot unmarshal !!str `/x`"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.key+" "+tt.value, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
|
||||
err := os.WriteFile(path, []byte(tt.config), configFileMode)
|
||||
if err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
|
||||
path, tt.key, tt.value)
|
||||
if err == nil || !strings.Contains(err.Error(), tt.reason) {
|
||||
t.Errorf("config set %s %s gives error %v, want one saying %q",
|
||||
tt.key, tt.value, err, tt.reason)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
|
||||
if string(data) != tt.config {
|
||||
t.Errorf("config set %s changed the file to:\n%s",
|
||||
tt.key, data)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigSetRefusesOnlyItsOwnFailure checks that setting another key
|
||||
// works on a config with a ${...} reference on compression_level, which
|
||||
// loads because loading fills the reference in, on one whose
|
||||
// compression_level already fails to load, and on one with a reference
|
||||
// that cannot be filled in here.
|
||||
func TestConfigSetRefusesOnlyItsOwnFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
configs := map[string]string{
|
||||
"reference": configWithLevelReference(t),
|
||||
"failing": configWithoutRecipients + "compression_level: fast\n",
|
||||
"unset reference": configWithUnsetReference,
|
||||
}
|
||||
|
||||
for name, configYAML := range configs {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
|
||||
err := os.WriteFile(path, []byte(configYAML), configFileMode)
|
||||
if err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
err = writeConfigSet(ui.NewWithColor(&bytes.Buffer{}, false),
|
||||
path, "hostname", "myhost")
|
||||
if err != nil {
|
||||
t.Fatalf("config set hostname: %v", err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
|
||||
if err != nil {
|
||||
t.Fatalf("read config: %v", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(string(data), "hostname: myhost") {
|
||||
t.Errorf("config set hostname wrote:\n%s", data)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigSetStringLooksLikeNumber sets string settings to values that
|
||||
// YAML reads as numbers or booleans when they are unquoted, and checks that
|
||||
// config.Load returns each one unchanged.
|
||||
func TestConfigSetStringLooksLikeNumber(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
key string
|
||||
value string
|
||||
field func(cfg *config.Config) string
|
||||
}{
|
||||
{"s3.access_key_id", "00112233",
|
||||
func(cfg *config.Config) string { return cfg.S3.AccessKeyID }},
|
||||
{"s3.secret_access_key", "12345678901234567890123456789012",
|
||||
func(cfg *config.Config) string { return cfg.S3.SecretAccessKey }},
|
||||
{"hostname", "007",
|
||||
func(cfg *config.Config) string { return cfg.Hostname }},
|
||||
{"s3.prefix", "1e5",
|
||||
func(cfg *config.Config) string { return cfg.S3.Prefix }},
|
||||
{"s3.bucket", "true",
|
||||
func(cfg *config.Config) string { return cfg.S3.Bucket }},
|
||||
{"s3.region", "FALSE",
|
||||
func(cfg *config.Config) string { return cfg.S3.Region }},
|
||||
{"snapshots.home.exclude.0", "1.10",
|
||||
func(cfg *config.Config) string { return cfg.Snapshots["home"].Exclude[0] }},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.key+"="+tt.value, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := loadAfterConfigSet(t, tt.key, tt.value)
|
||||
|
||||
got := tt.field(cfg)
|
||||
if got != tt.value {
|
||||
t.Errorf("%s = %q after config set %q", tt.key, got, tt.value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigSetNonUTF8Path checks that config set still accepts a value that
|
||||
// is not valid UTF-8, such as a path with a Latin-1 file name, and that
|
||||
// config.Load returns it unchanged.
|
||||
func TestConfigSetNonUTF8Path(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const dir = "/srv/caf\xe9"
|
||||
|
||||
cfg := loadAfterConfigSet(t, "snapshots.home.paths.0", dir)
|
||||
|
||||
got := cfg.Snapshots["home"].Paths[0]
|
||||
if got != dir {
|
||||
t.Errorf("snapshots.home.paths.0 = %q, want %q", got, dir)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigSetNumberStaysNumber checks that a number set for an integer
|
||||
// setting is still read as a number, not as a quoted string.
|
||||
func TestConfigSetNumberStaysNumber(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const level = 9
|
||||
|
||||
cfg := loadAfterConfigSet(t, "compression_level", strconv.Itoa(level))
|
||||
|
||||
if cfg.CompressionLevel != level {
|
||||
t.Errorf("compression_level = %d, want %d", cfg.CompressionLevel, level)
|
||||
}
|
||||
}
|
||||
|
||||
// loadAfterConfigSet writes the file `config init` writes, sets storage_url
|
||||
// to a local directory so that the file passes validation, applies
|
||||
// `config set key value` and returns what config.Load reads back.
|
||||
func loadAfterConfigSet(t *testing.T, key, value string) *config.Config {
|
||||
t.Helper()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
|
||||
err := os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode)
|
||||
if err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
out := ui.NewWithColor(&bytes.Buffer{}, false)
|
||||
|
||||
err = writeConfigSet(out, path, "storage_url", "file:///mnt/backups")
|
||||
if err != nil {
|
||||
t.Fatalf("config set storage_url: %v", err)
|
||||
}
|
||||
|
||||
err = writeConfigSet(out, path, key, value)
|
||||
if err != nil {
|
||||
t.Fatalf("config set %s: %v", key, err)
|
||||
}
|
||||
|
||||
cfg, err := config.Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("config.Load: %v", err)
|
||||
}
|
||||
|
||||
return cfg
|
||||
}
|
||||
|
||||
const testYAML = `# top comment
|
||||
compression_level: 3
|
||||
age_recipients:
|
||||
- age1aaa
|
||||
s3:
|
||||
bucket: oldbucket # inline comment
|
||||
region: us-east-1
|
||||
snapshots:
|
||||
home:
|
||||
paths:
|
||||
- "~"
|
||||
`
|
||||
|
||||
func parseTestYAML(t *testing.T) *yaml.Node {
|
||||
t.Helper()
|
||||
|
||||
var root yaml.Node
|
||||
|
||||
err := yaml.Unmarshal([]byte(testYAML), &root)
|
||||
if err != nil {
|
||||
t.Fatalf("parsing test yaml: %v", err)
|
||||
}
|
||||
|
||||
return &root
|
||||
}
|
||||
|
||||
func TestYAMLPathGet(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := parseTestYAML(t)
|
||||
|
||||
tests := []struct {
|
||||
path string
|
||||
want string
|
||||
err bool
|
||||
}{
|
||||
{"compression_level", "3", false},
|
||||
{"s3.bucket", "oldbucket", false},
|
||||
{"s3.region", "us-east-1", false},
|
||||
{"age_recipients.0", "age1aaa", false},
|
||||
{"age_recipients.5", "", true},
|
||||
{"age_recipients.notanumber", "", true},
|
||||
{"s3.nonexistent", "", true},
|
||||
{"nonexistent", "", true},
|
||||
{"compression_level.sub", "", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.path, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
node, err := yamlPathGet(root, splitPath(tt.path))
|
||||
if tt.err {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for %q", tt.path)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if node.Value != tt.want {
|
||||
t.Errorf("get %q = %q, want %q", tt.path, node.Value, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestYAMLPathSet(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := parseTestYAML(t)
|
||||
|
||||
// Overwrite existing nested value
|
||||
err := yamlPathSet(root, splitPath("s3.bucket"), "newbucket")
|
||||
if err != nil {
|
||||
t.Fatalf("set s3.bucket: %v", err)
|
||||
}
|
||||
|
||||
// Create new nested key with intermediate map
|
||||
err = yamlPathSet(root, splitPath("s3.endpoint"), "s3.example.com")
|
||||
if err != nil {
|
||||
t.Fatalf("set s3.endpoint: %v", err)
|
||||
}
|
||||
|
||||
err = yamlPathSet(root, splitPath("newmap.newkey"), "val")
|
||||
if err != nil {
|
||||
t.Fatalf("set newmap.newkey: %v", err)
|
||||
}
|
||||
|
||||
// Overwrite a sequence element and append a new one
|
||||
err = yamlPathSet(root, splitPath("age_recipients.0"), "age1bbb")
|
||||
if err != nil {
|
||||
t.Fatalf("set age_recipients.0: %v", err)
|
||||
}
|
||||
|
||||
err = yamlPathSet(root, splitPath("age_recipients.1"), "age1ccc")
|
||||
if err != nil {
|
||||
t.Fatalf("append age_recipients.1: %v", err)
|
||||
}
|
||||
|
||||
err = yamlPathSet(root, splitPath("age_recipients.5"), "age1ddd")
|
||||
if err == nil {
|
||||
t.Error("expected out-of-range append to fail")
|
||||
}
|
||||
|
||||
// Round-trip and verify values + comment preservation
|
||||
out, err := yaml.Marshal(root)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
|
||||
text := string(out)
|
||||
|
||||
wants := []string{
|
||||
"newbucket", "s3.example.com", "newkey: val",
|
||||
"# top comment", "# inline comment", "age1bbb", "age1ccc",
|
||||
}
|
||||
for _, want := range wants {
|
||||
if !contains(text, want) {
|
||||
t.Errorf("round-tripped YAML missing %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := yamlPathGet(root, splitPath("s3.bucket"))
|
||||
if err != nil {
|
||||
t.Fatalf("get after set: %v", err)
|
||||
}
|
||||
|
||||
if got.Value != "newbucket" {
|
||||
t.Errorf("s3.bucket = %q after set, want newbucket", got.Value)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigSetPreservesFormatting asserts the `config set` write path
|
||||
// (marshalConfigYAML) round-trips a 2-space-indented file without reindenting
|
||||
// it to yaml.Marshal's 4-space default, and keeps comments.
|
||||
func TestConfigSetPreservesFormatting(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := parseTestYAML(t)
|
||||
|
||||
err := yamlPathSet(root, splitPath("s3.bucket"), "newbucket")
|
||||
if err != nil {
|
||||
t.Fatalf("set s3.bucket: %v", err)
|
||||
}
|
||||
|
||||
out, err := marshalConfigYAML(root)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
|
||||
text := string(out)
|
||||
|
||||
for _, want := range []string{"# top comment", "# inline comment"} {
|
||||
if !contains(text, want) {
|
||||
t.Errorf("round-tripped YAML dropped comment %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
|
||||
// Nested map keys stay at 2-space indent; the bug reindented them to 4.
|
||||
if !contains(text, "\n bucket: newbucket") {
|
||||
t.Errorf("expected 2-space indent for s3.bucket, got:\n%s", text)
|
||||
}
|
||||
|
||||
if contains(text, "\n bucket:") {
|
||||
t.Errorf("s3.bucket reindented to 4 spaces:\n%s", text)
|
||||
}
|
||||
|
||||
// Sequence items under a key also stay at 2 spaces.
|
||||
if !contains(text, "\n - age1aaa") {
|
||||
t.Errorf("expected 2-space indent for sequence item, got:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteConfigSetHidesSecret checks that setting a secret key prints
|
||||
// only the key name, never the value, to the confirmation output.
|
||||
func TestWriteConfigSetHidesSecret(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const secret = "SUPERSECRETVALUE"
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
|
||||
err := os.WriteFile(path, []byte("version: 1\n"), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("seed config: %v", err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
err = writeConfigSet(ui.NewWithColor(&buf, false), path,
|
||||
"s3.secret_access_key", secret)
|
||||
if err != nil {
|
||||
t.Fatalf("writeConfigSet: %v", err)
|
||||
}
|
||||
|
||||
if strings.Contains(buf.String(), secret) {
|
||||
t.Errorf("output echoed the secret value: %q", buf.String())
|
||||
}
|
||||
|
||||
if !strings.Contains(buf.String(), "s3.secret_access_key") {
|
||||
t.Errorf("output did not confirm the key name: %q", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteConfigSetTightensMode checks that a pre-existing group- or
|
||||
// world-readable config is tightened to owner-only after a set, since
|
||||
// os.WriteFile leaves an existing file's mode untouched.
|
||||
func TestWriteConfigSetTightensMode(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yaml")
|
||||
|
||||
// Seed a world-readable config; the loose mode is the condition under
|
||||
// test, so gosec's G306 is expected here.
|
||||
err := os.WriteFile(path, []byte("version: 1\n"), 0o644) //nolint:gosec // G306
|
||||
if err != nil {
|
||||
t.Fatalf("seed config: %v", err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
err = writeConfigSet(ui.NewWithColor(&buf, false), path,
|
||||
"compression_level", "9")
|
||||
if err != nil {
|
||||
t.Fatalf("writeConfigSet: %v", err)
|
||||
}
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("stat config: %v", err)
|
||||
}
|
||||
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Errorf("config mode = %04o, want 0600", info.Mode().Perm())
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigEditEditorWithArgument checks that an $EDITOR carrying an
|
||||
// argument works and that a config path with a space reaches it whole.
|
||||
// The "editor" is cp with the replacement file as its argument, so the
|
||||
// config ends up with the replacement's content only if both arrived.
|
||||
func TestConfigEditEditorWithArgument(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := seedFile(t, dir, "my config.yml", "compression_level: 3\n")
|
||||
edited := seedFile(t, dir, "edited.yml", "compression_level: 9\n")
|
||||
|
||||
setRootFlags(t, RootFlags{ConfigPath: path})
|
||||
t.Setenv("EDITOR", "cp '"+edited+"'")
|
||||
|
||||
mustExecute(t, newConfigEditCommand())
|
||||
|
||||
data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
|
||||
if err != nil {
|
||||
t.Fatalf("reading config back: %v", err)
|
||||
}
|
||||
|
||||
if string(data) != "compression_level: 9\n" {
|
||||
t.Errorf("config after edit = %q, want the edited content", data)
|
||||
}
|
||||
}
|
||||
|
||||
func splitPath(s string) []string {
|
||||
return strings.Split(s, ".")
|
||||
}
|
||||
|
||||
func contains(haystack, needle string) bool {
|
||||
return strings.Contains(haystack, needle)
|
||||
}
|
||||
+45
-23
@@ -4,9 +4,9 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"github.com/spf13/cobra"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
// NewDatabaseCommand creates the database command group
|
||||
@@ -15,35 +15,46 @@ func NewDatabaseCommand() *cobra.Command {
|
||||
Use: "database",
|
||||
Short: "Manage the local state database",
|
||||
Long: `Commands for managing the local SQLite state database.`,
|
||||
// cobra checks Args only on a command that has a Run: without
|
||||
// one, a mistyped subcommand printed help and exited 0.
|
||||
Args: cobra.NoArgs,
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
_ = cmd.Help()
|
||||
},
|
||||
}
|
||||
|
||||
cmd.AddCommand(
|
||||
newDatabasePurgeCommand(),
|
||||
newDatabaseDeleteCommand(),
|
||||
)
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// newDatabasePurgeCommand creates the database purge command
|
||||
func newDatabasePurgeCommand() *cobra.Command {
|
||||
// newDatabaseDeleteCommand creates the database delete command.
|
||||
// (Renamed from "purge"; the operation removes the SQLite file
|
||||
// entirely, which is a delete, not a purge of content.)
|
||||
func newDatabaseDeleteCommand() *cobra.Command {
|
||||
var force bool
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "purge",
|
||||
Short: "Delete the local state database",
|
||||
Use: "delete",
|
||||
Short: "Delete the local state database file",
|
||||
Long: `Completely removes the local SQLite state database.
|
||||
|
||||
This will erase all local tracking of:
|
||||
- File metadata and change detection state
|
||||
- Chunk and blob mappings
|
||||
- Local snapshot records
|
||||
- The storage-binding record
|
||||
|
||||
The remote storage is NOT affected. After purging, the next backup will
|
||||
perform a full scan and re-deduplicate against existing remote blobs.
|
||||
The remote storage is NOT affected. After deletion, the next backup
|
||||
will perform a full scan and re-deduplicate against existing remote
|
||||
blobs, and the local index will re-bind to the currently configured
|
||||
storage destination on that run.
|
||||
|
||||
Use --force to skip the confirmation prompt.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
// Resolve config path
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
@@ -57,26 +68,40 @@ Use --force to skip the confirmation prompt.`,
|
||||
}
|
||||
|
||||
dbPath := cfg.IndexPath
|
||||
out := commandUI(cmd)
|
||||
|
||||
// Check if database exists
|
||||
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
|
||||
fmt.Printf("Database does not exist: %s\n", dbPath)
|
||||
_, err = os.Stat(dbPath)
|
||||
if os.IsNotExist(err) {
|
||||
out.Infof("Local state database does not exist: %s.", dbPath)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Confirm unless --force
|
||||
// Confirm unless --force. The prompt and its immediate result
|
||||
// are an interactive exchange the operator must see, so they go
|
||||
// straight to stdout rather than through the UI and --quiet does
|
||||
// not silence them.
|
||||
if !force {
|
||||
fmt.Printf("This will delete the local state database at:\n %s\n\n", dbPath)
|
||||
fmt.Print("Are you sure? Type 'yes' to confirm: ")
|
||||
w := cmd.OutOrStdout()
|
||||
_, _ = fmt.Fprintf(w,
|
||||
"This will delete the local state database at:\n %s\n\n", dbPath)
|
||||
_, _ = fmt.Fprint(w, "Are you sure? Type 'yes' to confirm: ")
|
||||
|
||||
var confirm string
|
||||
if _, err := fmt.Scanln(&confirm); err != nil || confirm != "yes" {
|
||||
fmt.Println("Aborted.")
|
||||
|
||||
_, err = fmt.Scanln(&confirm)
|
||||
if err != nil || confirm != "yes" {
|
||||
_, _ = fmt.Fprintln(w, "Aborted.")
|
||||
|
||||
//nolint:nilerr // a failed/aborted confirmation is a clean abort
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the database file
|
||||
if err := os.Remove(dbPath); err != nil {
|
||||
err = os.Remove(dbPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete database: %w", err)
|
||||
}
|
||||
|
||||
@@ -86,12 +111,9 @@ Use --force to skip the confirmation prompt.`,
|
||||
_ = os.Remove(walPath) // Ignore errors - files may not exist
|
||||
_ = os.Remove(shmPath)
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
if !rootFlags.Quiet {
|
||||
fmt.Printf("Database purged: %s\n", dbPath)
|
||||
}
|
||||
out.Infof("Local state database deleted: %s.", dbPath)
|
||||
log.Info("Local state database deleted", "path", dbPath)
|
||||
|
||||
log.Info("Local state database purged", "path", dbPath)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// parseDuration parses duration strings. Supports standard Go duration format
|
||||
// (e.g., "3h30m", "1h45m30s") as well as extended units:
|
||||
// - d: days (e.g., "30d", "7d")
|
||||
// - w: weeks (e.g., "2w", "4w")
|
||||
// - mo: months (30 days) (e.g., "6mo", "1mo")
|
||||
// - y: years (365 days) (e.g., "1y", "2y")
|
||||
//
|
||||
// Can combine units: "1y6mo", "2w3d", "1d12h30m"
|
||||
func parseDuration(s string) (time.Duration, error) {
|
||||
// First try standard Go duration parsing
|
||||
if d, err := time.ParseDuration(s); err == nil {
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// Extended duration parsing
|
||||
// Check for negative values
|
||||
if strings.HasPrefix(strings.TrimSpace(s), "-") {
|
||||
return 0, fmt.Errorf("negative durations are not supported")
|
||||
}
|
||||
|
||||
// Pattern matches: number + unit, repeated
|
||||
re := regexp.MustCompile(`(\d+(?:\.\d+)?)\s*([a-zA-Z]+)`)
|
||||
matches := re.FindAllStringSubmatch(s, -1)
|
||||
|
||||
if len(matches) == 0 {
|
||||
return 0, fmt.Errorf("invalid duration format: %q", s)
|
||||
}
|
||||
|
||||
var total time.Duration
|
||||
|
||||
for _, match := range matches {
|
||||
valueStr := match[1]
|
||||
unit := strings.ToLower(match[2])
|
||||
|
||||
value, err := strconv.ParseFloat(valueStr, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid number %q: %w", valueStr, err)
|
||||
}
|
||||
|
||||
var d time.Duration
|
||||
switch unit {
|
||||
// Standard time units
|
||||
case "ns", "nanosecond", "nanoseconds":
|
||||
d = time.Duration(value)
|
||||
case "us", "µs", "microsecond", "microseconds":
|
||||
d = time.Duration(value * float64(time.Microsecond))
|
||||
case "ms", "millisecond", "milliseconds":
|
||||
d = time.Duration(value * float64(time.Millisecond))
|
||||
case "s", "sec", "second", "seconds":
|
||||
d = time.Duration(value * float64(time.Second))
|
||||
case "m", "min", "minute", "minutes":
|
||||
d = time.Duration(value * float64(time.Minute))
|
||||
case "h", "hr", "hour", "hours":
|
||||
d = time.Duration(value * float64(time.Hour))
|
||||
// Extended units
|
||||
case "d", "day", "days":
|
||||
d = time.Duration(value * float64(24*time.Hour))
|
||||
case "w", "week", "weeks":
|
||||
d = time.Duration(value * float64(7*24*time.Hour))
|
||||
case "mo", "month", "months":
|
||||
// Using 30 days as approximation
|
||||
d = time.Duration(value * float64(30*24*time.Hour))
|
||||
case "y", "year", "years":
|
||||
// Using 365 days as approximation
|
||||
d = time.Duration(value * float64(365*24*time.Hour))
|
||||
default:
|
||||
// Try parsing as standard Go duration unit
|
||||
testStr := fmt.Sprintf("1%s", unit)
|
||||
if _, err := time.ParseDuration(testStr); err == nil {
|
||||
// It's a valid Go duration unit, parse the full value
|
||||
fullStr := fmt.Sprintf("%g%s", value, unit)
|
||||
if d, err = time.ParseDuration(fullStr); err != nil {
|
||||
return 0, fmt.Errorf("invalid duration %q: %w", fullStr, err)
|
||||
}
|
||||
} else {
|
||||
return 0, fmt.Errorf("unknown time unit %q", unit)
|
||||
}
|
||||
}
|
||||
|
||||
total += d
|
||||
}
|
||||
|
||||
return total, nil
|
||||
}
|
||||
@@ -1,263 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestParseDuration(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected time.Duration
|
||||
wantErr bool
|
||||
}{
|
||||
// Standard Go durations
|
||||
{
|
||||
name: "standard seconds",
|
||||
input: "30s",
|
||||
expected: 30 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "standard minutes",
|
||||
input: "45m",
|
||||
expected: 45 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "standard hours",
|
||||
input: "2h",
|
||||
expected: 2 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "standard combined",
|
||||
input: "3h30m",
|
||||
expected: 3*time.Hour + 30*time.Minute,
|
||||
},
|
||||
{
|
||||
name: "standard complex",
|
||||
input: "1h45m30s",
|
||||
expected: 1*time.Hour + 45*time.Minute + 30*time.Second,
|
||||
},
|
||||
{
|
||||
name: "standard with milliseconds",
|
||||
input: "1s500ms",
|
||||
expected: 1*time.Second + 500*time.Millisecond,
|
||||
},
|
||||
// Extended units - days
|
||||
{
|
||||
name: "single day",
|
||||
input: "1d",
|
||||
expected: 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "multiple days",
|
||||
input: "7d",
|
||||
expected: 7 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "fractional days",
|
||||
input: "1.5d",
|
||||
expected: 36 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "days spelled out",
|
||||
input: "3days",
|
||||
expected: 3 * 24 * time.Hour,
|
||||
},
|
||||
// Extended units - weeks
|
||||
{
|
||||
name: "single week",
|
||||
input: "1w",
|
||||
expected: 7 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "multiple weeks",
|
||||
input: "4w",
|
||||
expected: 4 * 7 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "weeks spelled out",
|
||||
input: "2weeks",
|
||||
expected: 2 * 7 * 24 * time.Hour,
|
||||
},
|
||||
// Extended units - months
|
||||
{
|
||||
name: "single month",
|
||||
input: "1mo",
|
||||
expected: 30 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "multiple months",
|
||||
input: "6mo",
|
||||
expected: 6 * 30 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "months spelled out",
|
||||
input: "3months",
|
||||
expected: 3 * 30 * 24 * time.Hour,
|
||||
},
|
||||
// Extended units - years
|
||||
{
|
||||
name: "single year",
|
||||
input: "1y",
|
||||
expected: 365 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "multiple years",
|
||||
input: "2y",
|
||||
expected: 2 * 365 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
name: "years spelled out",
|
||||
input: "1year",
|
||||
expected: 365 * 24 * time.Hour,
|
||||
},
|
||||
// Combined extended units
|
||||
{
|
||||
name: "weeks and days",
|
||||
input: "2w3d",
|
||||
expected: 2*7*24*time.Hour + 3*24*time.Hour,
|
||||
},
|
||||
{
|
||||
name: "years and months",
|
||||
input: "1y6mo",
|
||||
expected: 365*24*time.Hour + 6*30*24*time.Hour,
|
||||
},
|
||||
{
|
||||
name: "days and hours",
|
||||
input: "1d12h",
|
||||
expected: 24*time.Hour + 12*time.Hour,
|
||||
},
|
||||
{
|
||||
name: "complex combination",
|
||||
input: "1y2mo3w4d5h6m7s",
|
||||
expected: 365*24*time.Hour + 2*30*24*time.Hour + 3*7*24*time.Hour + 4*24*time.Hour + 5*time.Hour + 6*time.Minute + 7*time.Second,
|
||||
},
|
||||
{
|
||||
name: "with spaces",
|
||||
input: "1d 12h 30m",
|
||||
expected: 24*time.Hour + 12*time.Hour + 30*time.Minute,
|
||||
},
|
||||
// Edge cases
|
||||
{
|
||||
name: "zero duration",
|
||||
input: "0s",
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "large duration",
|
||||
input: "10y",
|
||||
expected: 10 * 365 * 24 * time.Hour,
|
||||
},
|
||||
// Error cases
|
||||
{
|
||||
name: "empty string",
|
||||
input: "",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid format",
|
||||
input: "abc",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unknown unit",
|
||||
input: "5x",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid number",
|
||||
input: "xyzd",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "negative not supported",
|
||||
input: "-5d",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := parseDuration(tt.input)
|
||||
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err, "expected error for input %q", tt.input)
|
||||
return
|
||||
}
|
||||
|
||||
assert.NoError(t, err, "unexpected error for input %q", tt.input)
|
||||
assert.Equal(t, tt.expected, got, "duration mismatch for input %q", tt.input)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDurationSpecialCases(t *testing.T) {
|
||||
// Test that standard Go durations work exactly as expected
|
||||
standardDurations := []string{
|
||||
"300ms",
|
||||
"1.5h",
|
||||
"2h45m",
|
||||
"72h",
|
||||
"1us",
|
||||
"1µs",
|
||||
"1ns",
|
||||
}
|
||||
|
||||
for _, d := range standardDurations {
|
||||
expected, err := time.ParseDuration(d)
|
||||
assert.NoError(t, err)
|
||||
|
||||
got, err := parseDuration(d)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, expected, got, "standard duration %q should parse identically", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDurationRealWorldExamples(t *testing.T) {
|
||||
// Test real-world snapshot purge scenarios
|
||||
tests := []struct {
|
||||
description string
|
||||
input string
|
||||
olderThan time.Duration
|
||||
}{
|
||||
{
|
||||
description: "keep snapshots from last 30 days",
|
||||
input: "30d",
|
||||
olderThan: 30 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
description: "keep snapshots from last 6 months",
|
||||
input: "6mo",
|
||||
olderThan: 6 * 30 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
description: "keep snapshots from last year",
|
||||
input: "1y",
|
||||
olderThan: 365 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
description: "keep snapshots from last week and a half",
|
||||
input: "1w3d",
|
||||
olderThan: 10 * 24 * time.Hour,
|
||||
},
|
||||
{
|
||||
description: "keep snapshots from last 90 days",
|
||||
input: "90d",
|
||||
olderThan: 90 * 24 * time.Hour,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.description, func(t *testing.T) {
|
||||
got, err := parseDuration(tt.input)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.olderThan, got)
|
||||
|
||||
// Verify the duration makes sense for snapshot purging
|
||||
assert.Greater(t, got, time.Hour, "snapshot purge duration should be at least an hour")
|
||||
})
|
||||
}
|
||||
}
|
||||
+121
-6
@@ -1,15 +1,130 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/globals"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
)
|
||||
|
||||
// CLIEntry is the main entry point for the CLI application.
|
||||
// It creates the root command, executes it, and exits with status 1
|
||||
// if an error occurs. This function should be called from main().
|
||||
func CLIEntry() {
|
||||
// shortCommitLen is the number of git commit hash characters shown in
|
||||
// the startup banner.
|
||||
const shortCommitLen = 12
|
||||
|
||||
// exitCodeInterrupted is the exit status of a command that SIGINT or
|
||||
// SIGTERM stopped. It is 128 plus SIGINT's number, 2, which is what a
|
||||
// shell reports for a command stopped by Ctrl-C.
|
||||
const exitCodeInterrupted = 130
|
||||
|
||||
// Entry is the main entry point for the CLI application.
|
||||
// It prints the startup banner to stderr (unless a banner-suppressing
|
||||
// flag is present in os.Args — see bannerSuppressedInArgs), executes the
|
||||
// root cobra command, and routes any returned error through the
|
||||
// ui.Writer so the user sees a properly formatted "🛑 ERROR:" line.
|
||||
// The banner goes to stderr because stdout carries only the output the
|
||||
// user asked for, such as a completion script or a `config get` value.
|
||||
//
|
||||
// It returns the process exit code (0 on success, 130 when interrupted,
|
||||
// 1 on any other error) rather than calling os.Exit, so that main's
|
||||
// deferred profile writers run before the process ends. See run in
|
||||
// cmd/vaultik/main.go.
|
||||
func Entry() int {
|
||||
emitStartupBanner(os.Args[1:], os.Stderr)
|
||||
|
||||
rootCmd := NewRootCommand()
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
os.Exit(1)
|
||||
rootCmd.SilenceErrors = true
|
||||
|
||||
err := rootCmd.Execute()
|
||||
if err != nil {
|
||||
// An operation that ran inside the fx app has already reported
|
||||
// its own failure (`snapshot verify --json` puts it in the
|
||||
// document instead); errReported says so. Printing it again
|
||||
// here would double the error line.
|
||||
// Every other error — bad arguments, a config that would not
|
||||
// load, an interrupt — reaches Entry unreported, so it is shown
|
||||
// here.
|
||||
if !errors.Is(err, errReported) {
|
||||
ReportErrorf("%s", err.Error())
|
||||
}
|
||||
|
||||
if errors.Is(err, errInterrupted) {
|
||||
return exitCodeInterrupted
|
||||
}
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// emitStartupBanner writes the startup banner to w unless args (the
|
||||
// argument vector with the program name already stripped) contains a
|
||||
// flag that suppresses it. Split out of Entry so that the decision is
|
||||
// reachable from a test without running the whole CLI.
|
||||
func emitStartupBanner(args []string, w io.Writer) {
|
||||
if bannerSuppressedInArgs(args) {
|
||||
return
|
||||
}
|
||||
|
||||
short := globals.Commit
|
||||
if len(short) > shortCommitLen {
|
||||
short = short[:shortCommitLen]
|
||||
}
|
||||
|
||||
writeStartupBanner(ui.New(w), time.Now().UTC(), short)
|
||||
}
|
||||
|
||||
// ReportErrorf emits a user-facing error to stderr in the standard
|
||||
// 🛑 ERROR: format. Use it from goroutine error paths (where returning
|
||||
// an error to cobra isn't an option) and anywhere else a CLI command
|
||||
// must surface a failure outside the normal RunE return path.
|
||||
func ReportErrorf(format string, args ...any) {
|
||||
ui.New(os.Stderr).Errorf(format, args...)
|
||||
}
|
||||
|
||||
// bannerSuppressedInArgs reports whether any of args is a flag that
|
||||
// should suppress the startup banner (--quiet/-q/--cron/--json). Stops
|
||||
// at the "--" argument terminator. Recognizes both long forms and short
|
||||
// -q, including combined short flags like "-qv".
|
||||
//
|
||||
// This scans the raw argument vector because the banner is printed
|
||||
// before cobra parses anything — deliberately, so that it still appears
|
||||
// when cobra rejects the arguments and on --help. The consequence is
|
||||
// that a flag is matched wherever it occurs in the vector, including
|
||||
// positions where the command it belongs to would not accept it.
|
||||
// --json is a subcommand flag rather than a persistent one, but so is
|
||||
// --cron (it exists only on `snapshot create`), so this adds no new
|
||||
// class of imprecision. The only cost of a false positive is a missing
|
||||
// decorative banner.
|
||||
func bannerSuppressedInArgs(args []string) bool {
|
||||
for _, a := range args {
|
||||
if a == "--" {
|
||||
return false
|
||||
}
|
||||
|
||||
switch a {
|
||||
case "--quiet", "-q", "--cron", "--json":
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasPrefix(a, "--quiet=") ||
|
||||
strings.HasPrefix(a, "--cron=") ||
|
||||
strings.HasPrefix(a, "--json=") {
|
||||
return true
|
||||
}
|
||||
// Combined short flags like -qv or -vq.
|
||||
if len(a) > 1 && a[0] == '-' && a[1] != '-' {
|
||||
for _, c := range a[1:] {
|
||||
if c == 'q' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
package cli //nolint:testpackage // needs access to unexported emitStartupBanner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/adrg/xdg"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Command words and flags used to build argument vectors below. They are
|
||||
// constants rather than repeated literals so that a rename shows up as a
|
||||
// compile error in one place.
|
||||
const (
|
||||
cmdSnapshot = "snapshot"
|
||||
cmdList = "list"
|
||||
cmdCreate = "create"
|
||||
cmdVerify = "verify"
|
||||
cmdRemove = "remove"
|
||||
cmdPrune = "prune"
|
||||
cmdRemote = "remote"
|
||||
cmdInfo = "info"
|
||||
|
||||
flagJSON = "--json"
|
||||
flagQuiet = "--quiet"
|
||||
flagConfig = "--config"
|
||||
|
||||
// programName is argv[0] as the real process receives it. Entry
|
||||
// strips it before scanning, so it has to be present.
|
||||
programName = "vaultik"
|
||||
|
||||
// someSnapshotID only fills the positional argument; no test needs
|
||||
// the snapshot to exist.
|
||||
someSnapshotID = "host_2026-01-01T00:00:00Z"
|
||||
)
|
||||
|
||||
// jsonArgumentVectors are the argument vectors of every --json
|
||||
// invocation the CLI accepts, with the program name stripped exactly as
|
||||
// Entry strips it. Each one must suppress the banner.
|
||||
//
|
||||
//nolint:gochecknoglobals // read-only test fixture shared by two tests
|
||||
var jsonArgumentVectors = map[string][]string{
|
||||
"snapshot list": {cmdSnapshot, cmdList, flagJSON},
|
||||
"snapshot verify": {cmdSnapshot, cmdVerify, someSnapshotID, flagJSON},
|
||||
"snapshot remove": {cmdSnapshot, cmdRemove, someSnapshotID, flagJSON},
|
||||
"prune": {cmdPrune, flagJSON},
|
||||
"remote info": {cmdRemote, cmdInfo, flagJSON},
|
||||
|
||||
// --json before the subcommand, and with an explicit value: the
|
||||
// scan is positional, so both forms have to be recognized.
|
||||
"json first": {flagJSON, cmdSnapshot, cmdList},
|
||||
"json with value": {cmdSnapshot, cmdList, flagJSON + "=true"},
|
||||
|
||||
// A --json invocation that also carries a flag with a value, so the
|
||||
// scan cannot be fooled by an argument that consumes the next one.
|
||||
"json with config": {
|
||||
flagConfig, "/nonexistent/vaultik.yml", cmdSnapshot, cmdList, flagJSON,
|
||||
},
|
||||
}
|
||||
|
||||
// TestJSONInvocationSuppressesBanner checks that every --json
|
||||
// invocation suppresses the startup banner, as the README says --json
|
||||
// does along with --quiet and --cron. The scan is over the raw argument
|
||||
// vector, so each position and spelling of --json is listed.
|
||||
func TestJSONInvocationSuppressesBanner(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for name, argv := range jsonArgumentVectors {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var banner bytes.Buffer
|
||||
|
||||
emitStartupBanner(argv, &banner)
|
||||
|
||||
assert.Empty(t, banner.String(),
|
||||
"--json suppresses the banner")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBannerStillPrintedWithoutSuppressingFlag pins the other half of
|
||||
// the contract. Without it, deleting the banner outright would satisfy
|
||||
// the test above, and the banner is wanted on interactive invocations.
|
||||
func TestBannerStillPrintedWithoutSuppressingFlag(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for name, argv := range map[string][]string{
|
||||
"no flags": {cmdSnapshot, cmdList},
|
||||
"verbose": {cmdSnapshot, cmdList, "--verbose"},
|
||||
"after the terminator": {
|
||||
cmdSnapshot, "restore", "--", flagJSON,
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var banner bytes.Buffer
|
||||
|
||||
emitStartupBanner(argv, &banner)
|
||||
|
||||
assert.Contains(t, banner.String(), "starting up at",
|
||||
"the banner belongs on invocations that did not opt out")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestBannerSuppressedInArgs covers the suppression scan directly,
|
||||
// including the flags that suppressed the banner before --json joined
|
||||
// them, so that adding --json cannot regress them.
|
||||
func TestBannerSuppressedInArgs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for name, testCase := range map[string]struct {
|
||||
args []string
|
||||
suppressed bool
|
||||
}{
|
||||
"quiet long": {[]string{cmdSnapshot, cmdCreate, flagQuiet}, true},
|
||||
"quiet short": {[]string{cmdSnapshot, cmdCreate, "-q"}, true},
|
||||
"quiet combined": {[]string{cmdSnapshot, cmdCreate, "-qv"}, true},
|
||||
"cron": {[]string{cmdSnapshot, cmdCreate, "--cron"}, true},
|
||||
"json": {[]string{cmdSnapshot, cmdList, flagJSON}, true},
|
||||
"nothing": {[]string{cmdSnapshot, cmdList}, false},
|
||||
"empty": {nil, false},
|
||||
"json after dashes": {
|
||||
[]string{cmdSnapshot, cmdList, "--", flagJSON}, false,
|
||||
},
|
||||
"quiet after dashes": {
|
||||
[]string{cmdSnapshot, cmdCreate, "--", "-q"}, false,
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.Equal(t, testCase.suppressed,
|
||||
bannerSuppressedInArgs(testCase.args))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// hermeticConfig is a complete, valid config that needs no network and
|
||||
// no credentials: file:// storage is exempt from the S3 credential
|
||||
// checks. A test that lists the destination must create its directory
|
||||
// first, because listing a directory that does not exist is an error.
|
||||
// Chunk, blob and compression settings are filled in by config.Load.
|
||||
const hermeticConfig = `age_recipients:
|
||||
- age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj
|
||||
snapshots:
|
||||
test:
|
||||
paths:
|
||||
- %s
|
||||
storage_url: file://%s
|
||||
index_path: %s
|
||||
hostname: test-host
|
||||
`
|
||||
|
||||
// TestEntryJSONStdoutIsExactlyOneDocument runs the real thing: Entry,
|
||||
// with a real argument vector, over the process's real stdout file
|
||||
// descriptor, all the way through cobra and the fx graph to the
|
||||
// document. It is the assertion the issue asks for — `vaultik snapshot
|
||||
// list --json | jq .` with no other flags — with the pipe replaced by a
|
||||
// decoder.
|
||||
//
|
||||
// `snapshot list` is the command chosen because it is the only --json
|
||||
// command that reaches its document without a populated destination
|
||||
// store: it reads the local index, streams `metadata/` (empty here),
|
||||
// and treats an empty destination directory as an empty list rather
|
||||
// than a failure.
|
||||
//
|
||||
// Not parallel: it replaces os.Args, os.Stdout and the xdg globals.
|
||||
func TestEntryJSONStdoutIsExactlyOneDocument(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
configPath := filepath.Join(dir, "config.yml")
|
||||
storeDir := filepath.Join(dir, "store")
|
||||
|
||||
contents := fmt.Sprintf(hermeticConfig,
|
||||
filepath.Join(dir, "source"),
|
||||
storeDir,
|
||||
filepath.Join(dir, "index.sqlite"))
|
||||
|
||||
require.NoError(t,
|
||||
os.WriteFile(configPath, []byte(contents), configFileMode))
|
||||
|
||||
require.NoError(t, os.Mkdir(storeDir, 0o750))
|
||||
|
||||
// The PID lock lives under xdg.DataHome, which xdg resolves at
|
||||
// package init; point it at the temp dir so the test neither
|
||||
// touches nor collides with the real one.
|
||||
t.Setenv("XDG_DATA_HOME", filepath.Join(dir, "data"))
|
||||
xdg.Reload()
|
||||
t.Cleanup(xdg.Reload)
|
||||
|
||||
previousArgs := os.Args
|
||||
|
||||
t.Cleanup(func() {
|
||||
os.Args = previousArgs
|
||||
rootFlags = RootFlags{}
|
||||
})
|
||||
|
||||
os.Args = []string{
|
||||
programName, flagConfig, configPath, cmdSnapshot, cmdList, flagJSON,
|
||||
}
|
||||
|
||||
stdout := captureProcessStdout(t, func() { _ = Entry() })
|
||||
|
||||
requireExactlyOneJSONDocument(t, stdout)
|
||||
|
||||
var snapshots []any
|
||||
|
||||
require.NoError(t, json.Unmarshal([]byte(stdout), &snapshots))
|
||||
assert.Empty(t, snapshots,
|
||||
"a destination store with no snapshots lists none")
|
||||
}
|
||||
|
||||
// captureProcessStdout redirects the process's own stdout to a pipe for
|
||||
// the duration of fn and returns what was written to it. The redirection
|
||||
// has to be at the file-descriptor level rather than through an injected
|
||||
// writer, because the commands Entry runs reach os.Stdout directly.
|
||||
//
|
||||
// Not parallel-safe: os.Stdout is process-global.
|
||||
func captureProcessStdout(t *testing.T, fn func()) string {
|
||||
t.Helper()
|
||||
|
||||
reader, writer, err := os.Pipe()
|
||||
require.NoError(t, err)
|
||||
|
||||
previous := os.Stdout
|
||||
os.Stdout = writer
|
||||
|
||||
captured := make(chan string, 1)
|
||||
|
||||
go func() {
|
||||
var buf bytes.Buffer
|
||||
|
||||
_, _ = io.Copy(&buf, reader)
|
||||
captured <- buf.String()
|
||||
}()
|
||||
|
||||
fn()
|
||||
|
||||
os.Stdout = previous
|
||||
|
||||
require.NoError(t, writer.Close())
|
||||
|
||||
out := <-captured
|
||||
|
||||
require.NoError(t, reader.Close())
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// requireExactlyOneJSONDocument fails unless stdout decodes as a single
|
||||
// JSON value with nothing before or after it — the property that makes
|
||||
// `| jq` work.
|
||||
func requireExactlyOneJSONDocument(t *testing.T, stdout string) {
|
||||
t.Helper()
|
||||
|
||||
decoder := json.NewDecoder(strings.NewReader(stdout))
|
||||
|
||||
var document any
|
||||
|
||||
err := decoder.Decode(&document)
|
||||
require.NoError(t, err,
|
||||
"stdout must parse as JSON, got:\n%s", stdout)
|
||||
|
||||
_, err = decoder.Token()
|
||||
require.ErrorIs(t, err, io.EOF,
|
||||
"stdout must hold exactly one JSON document, got:\n%s", stdout)
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package cli //nolint:testpackage // shares runEntry and the argument constants
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/adrg/xdg"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// stalledStoreConfig is hermeticConfig with an s3:// destination store
|
||||
// in place of the file:// one. The server behind it accepts any
|
||||
// credentials.
|
||||
const stalledStoreConfig = `age_recipients:
|
||||
- age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj
|
||||
snapshots:
|
||||
test:
|
||||
paths:
|
||||
- %s
|
||||
storage_url: s3://bucket?endpoint=%s&ssl=false
|
||||
s3:
|
||||
access_key_id: key
|
||||
secret_access_key: secret
|
||||
index_path: %s
|
||||
hostname: test-host
|
||||
`
|
||||
|
||||
// interruptRepeat is how often interruptOnFirstRequest sends SIGINT.
|
||||
const interruptRepeat = 50 * time.Millisecond
|
||||
|
||||
// TestEntryInterruptedRun sends SIGINT to the test process while a
|
||||
// command waits on the destination store, and checks that Entry returns
|
||||
// 130 and prints one line on stderr saying the run was interrupted. The
|
||||
// store is a local HTTP server that holds every request open, so the
|
||||
// command is always mid-operation when the signal arrives. The two
|
||||
// cases cover --cron and --json, which silence other output.
|
||||
//
|
||||
// Not parallel: it signals the process and replaces os.Args, os.Stdout,
|
||||
// os.Stderr and the xdg globals.
|
||||
//
|
||||
//nolint:paralleltest // signals the process and replaces process globals
|
||||
func TestEntryInterruptedRun(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
args []string
|
||||
}{
|
||||
{
|
||||
name: "snapshot create --cron",
|
||||
args: []string{cmdSnapshot, cmdCreate, "--cron"},
|
||||
},
|
||||
{
|
||||
name: "snapshot verify --json",
|
||||
args: []string{cmdSnapshot, cmdVerify, someSnapshotID, flagJSON},
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
endpoint, requestArrived := startStalledStore(t)
|
||||
configPath := writeStalledStoreConfig(t, endpoint)
|
||||
interruptOnFirstRequest(t, requestArrived)
|
||||
|
||||
code, _, stderr := runEntry(t,
|
||||
append([]string{flagConfig, configPath}, testCase.args...)...)
|
||||
|
||||
assert.Equal(t, 130, code)
|
||||
assert.Equal(t, 1,
|
||||
strings.Count(stderr, errInterrupted.Error()), stderr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// interruptOnFirstRequest sends SIGINT to the test process every
|
||||
// interruptRepeat, from the first request to the destination store until
|
||||
// the test ends. One signal is not enough: the command can reach the
|
||||
// store before fx has started catching signals. The test catches SIGINT
|
||||
// too, so that a signal fx is not catching does not kill the test
|
||||
// binary.
|
||||
func interruptOnFirstRequest(t *testing.T, requestArrived <-chan struct{}) {
|
||||
t.Helper()
|
||||
|
||||
self, err := os.FindProcess(os.Getpid())
|
||||
require.NoError(t, err)
|
||||
|
||||
caught := make(chan os.Signal, 1)
|
||||
signal.Notify(caught, os.Interrupt)
|
||||
|
||||
testEnded := make(chan struct{})
|
||||
senderDone := make(chan struct{})
|
||||
|
||||
// Stop catching SIGINT only after the sender has returned. The sender
|
||||
// waits for each SIGINT it sends to arrive on caught; one still on
|
||||
// its way after signal.Stop would kill the test binary.
|
||||
t.Cleanup(func() {
|
||||
close(testEnded)
|
||||
<-senderDone
|
||||
signal.Stop(caught)
|
||||
})
|
||||
|
||||
go func() {
|
||||
defer close(senderDone)
|
||||
|
||||
select {
|
||||
case <-requestArrived:
|
||||
case <-testEnded:
|
||||
return
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(interruptRepeat)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
// Empty caught, so that the receive below waits for this
|
||||
// SIGINT rather than an earlier one.
|
||||
select {
|
||||
case <-caught:
|
||||
default:
|
||||
}
|
||||
|
||||
sendErr := self.Signal(os.Interrupt)
|
||||
if sendErr != nil {
|
||||
t.Errorf("sending SIGINT: %v", sendErr)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
<-caught
|
||||
|
||||
select {
|
||||
case <-testEnded:
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// startStalledStore starts an HTTP server that never answers: each
|
||||
// request is held until the client gives up on it or the test ends.
|
||||
// It returns the server's host:port and a channel that receives a value
|
||||
// when the first request arrives.
|
||||
func startStalledStore(t *testing.T) (string, <-chan struct{}) {
|
||||
t.Helper()
|
||||
|
||||
requestArrived := make(chan struct{}, 1)
|
||||
release := make(chan struct{})
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(
|
||||
func(_ http.ResponseWriter, r *http.Request) {
|
||||
select {
|
||||
case requestArrived <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
case <-release:
|
||||
}
|
||||
}))
|
||||
|
||||
// Cleanups run last-registered first, so release lets any held
|
||||
// request return before Close waits for it.
|
||||
t.Cleanup(server.Close)
|
||||
t.Cleanup(func() { close(release) })
|
||||
|
||||
return server.Listener.Addr().String(), requestArrived
|
||||
}
|
||||
|
||||
// writeStalledStoreConfig writes a config whose destination store is the
|
||||
// server at endpoint and whose snapshot source holds one small file, so
|
||||
// that `snapshot create` has a blob to upload. Returns the config path.
|
||||
func writeStalledStoreConfig(t *testing.T, endpoint string) string {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
configPath := filepath.Join(dir, "config.yml")
|
||||
sourceDir := filepath.Join(dir, "source")
|
||||
|
||||
require.NoError(t, os.Mkdir(sourceDir, 0o750))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "file.txt"),
|
||||
[]byte("contents"), 0o600))
|
||||
|
||||
contents := fmt.Sprintf(stalledStoreConfig,
|
||||
sourceDir, endpoint, filepath.Join(dir, "index.sqlite"))
|
||||
|
||||
require.NoError(t,
|
||||
os.WriteFile(configPath, []byte(contents), configFileMode))
|
||||
|
||||
// The PID lock lives under xdg.DataHome, which xdg resolves at
|
||||
// package init; point it at the temp dir so the test neither
|
||||
// touches nor collides with the real one.
|
||||
t.Setenv("XDG_DATA_HOME", filepath.Join(dir, "data"))
|
||||
xdg.Reload()
|
||||
t.Cleanup(xdg.Reload)
|
||||
|
||||
return configPath
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package cli //nolint:testpackage // shares the prune fixtures and capture helpers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// staleRecordLogMessage is the local-cleanup audit line CleanupLocalSnapshots
|
||||
// logs for each stale record. It is exactly the signal issue #112 says a
|
||||
// machine consumer lost under --json: gated off stdout, and pinned below
|
||||
// the log level on stderr because --json used to force Quiet.
|
||||
const staleRecordLogMessage = "Removing stale local snapshot record"
|
||||
|
||||
// TestEntryPruneJSONStderrHonoursVerbosity is the end-to-end regression
|
||||
// guard for issue #112. Under --json the log level must still follow
|
||||
// --verbose/--debug rather than being pinned to WARN, so the
|
||||
// local-cleanup records reach stderr under --verbose while stdout stays
|
||||
// exactly one JSON document; without --verbose they stay below the
|
||||
// level, as they do without --json.
|
||||
//
|
||||
// Both halves are asserted together on the same run, because the fix has
|
||||
// to keep the document clean (issue #108) while freeing stderr.
|
||||
//
|
||||
// Not parallel: it replaces os.Args, os.Stdout, os.Stderr and the xdg
|
||||
// globals.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntryPruneJSONStderrHonoursVerbosity(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
verbose bool
|
||||
wantOnStderr bool
|
||||
}{
|
||||
{
|
||||
name: "verbose json surfaces the cleanup record on stderr",
|
||||
verbose: true,
|
||||
wantOnStderr: true,
|
||||
},
|
||||
{
|
||||
name: "json alone keeps the cleanup record below the level",
|
||||
verbose: false,
|
||||
wantOnStderr: false,
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
configPath := writeHermeticPruneConfig(t, true)
|
||||
|
||||
previousArgs := os.Args
|
||||
|
||||
t.Cleanup(func() {
|
||||
os.Args = previousArgs
|
||||
rootFlags = RootFlags{}
|
||||
})
|
||||
|
||||
args := []string{
|
||||
programName, flagConfig, configPath, cmdPrune, flagJSON,
|
||||
}
|
||||
if testCase.verbose {
|
||||
args = append(args, "--verbose")
|
||||
}
|
||||
|
||||
os.Args = args
|
||||
|
||||
stdout, stderr := captureProcessStdoutAndStderr(t,
|
||||
func() { _ = Entry() })
|
||||
|
||||
// The document stays clean in both cases: freeing stderr must
|
||||
// not regress issue #108.
|
||||
requireExactlyOneJSONDocument(t, stdout)
|
||||
|
||||
if testCase.wantOnStderr {
|
||||
assert.Contains(t, stderr, staleRecordLogMessage,
|
||||
"--verbose --json must emit the cleanup record on stderr")
|
||||
assert.Contains(t, stderr, stalePruneSnapshotID,
|
||||
"the record must name the snapshot it removed")
|
||||
} else {
|
||||
assert.NotContains(t, stderr, staleRecordLogMessage,
|
||||
"without --verbose the record stays below the log level")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// captureProcessStdoutAndStderr redirects both of the process's own
|
||||
// standard streams to pipes for the duration of fn and returns what was
|
||||
// written to each. The redirection is at the file-descriptor level
|
||||
// because the logger binds os.Stderr when it initializes inside fn, and
|
||||
// the JSON document reaches os.Stdout independently; the point is to see
|
||||
// where each actually lands.
|
||||
//
|
||||
// Not parallel-safe: os.Stdout and os.Stderr are process-global.
|
||||
func captureProcessStdoutAndStderr(t *testing.T, fn func()) (string, string) {
|
||||
t.Helper()
|
||||
|
||||
outReader, outWriter, err := os.Pipe()
|
||||
require.NoError(t, err)
|
||||
|
||||
errReader, errWriter, err := os.Pipe()
|
||||
require.NoError(t, err)
|
||||
|
||||
previousOut, previousErr := os.Stdout, os.Stderr
|
||||
os.Stdout, os.Stderr = outWriter, errWriter
|
||||
|
||||
capturedOut := drain(outReader)
|
||||
capturedErr := drain(errReader)
|
||||
|
||||
fn()
|
||||
|
||||
os.Stdout, os.Stderr = previousOut, previousErr
|
||||
|
||||
require.NoError(t, outWriter.Close())
|
||||
require.NoError(t, errWriter.Close())
|
||||
|
||||
out, errOut := <-capturedOut, <-capturedErr
|
||||
|
||||
require.NoError(t, outReader.Close())
|
||||
require.NoError(t, errReader.Close())
|
||||
|
||||
return out, errOut
|
||||
}
|
||||
|
||||
// drain copies a reader to a string on a goroutine and delivers the
|
||||
// result once the writer end is closed.
|
||||
func drain(reader io.Reader) <-chan string {
|
||||
captured := make(chan string, 1)
|
||||
|
||||
go func() {
|
||||
var buf bytes.Buffer
|
||||
|
||||
_, _ = io.Copy(&buf, reader)
|
||||
captured <- buf.String()
|
||||
}()
|
||||
|
||||
return captured
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package cli //nolint:testpackage // shares hermeticConfig and the capture helpers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/adrg/xdg"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// pruneJSONDocument is the shape `prune --json` writes: the
|
||||
// PruneBlobsResult document, and nothing else.
|
||||
//
|
||||
//nolint:tagliatelle // snake_case is the established JSON output format
|
||||
type pruneJSONDocument struct {
|
||||
BlobsFound int `json:"blobs_found"`
|
||||
BlobsDeleted int `json:"blobs_deleted"`
|
||||
BytesFreed int64 `json:"bytes_freed"`
|
||||
}
|
||||
|
||||
// stalePruneSnapshotID is seeded into the local index with no manifest
|
||||
// on the destination store, which is exactly what makes it stale.
|
||||
const stalePruneSnapshotID = "test-host_test_2026-04-01T09:00:00Z"
|
||||
|
||||
// TestEntryPruneJSONStdoutIsExactlyOneDocument is the end-to-end
|
||||
// regression guard for issue #108: `vaultik prune --json | jq .` must
|
||||
// work with no other flags.
|
||||
//
|
||||
// It runs Entry over the process's real stdout descriptor, through
|
||||
// cobra and the fx graph, against a hermetic file:// destination store
|
||||
// — the same construction TestEntryJSONStdoutIsExactlyOneDocument uses
|
||||
// for `snapshot list`, with the pipe to jq replaced by a decoder.
|
||||
//
|
||||
// Both branches of the local-snapshot reconciliation are exercised
|
||||
// because the three stdout writes that broke this covered all of them:
|
||||
// one line per stale record and a summary when there were any, and a
|
||||
// "No stale local snapshots found." line when there were none. No input
|
||||
// avoided the contamination, so no single branch demonstrates the fix.
|
||||
//
|
||||
// Not parallel: it replaces os.Args, os.Stdout and the xdg globals.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout and the xdg globals
|
||||
func TestEntryPruneJSONStdoutIsExactlyOneDocument(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
seedStale bool
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "no stale local records",
|
||||
seedStale: false,
|
||||
description: "the empty-index branch used to print a 'No stale' line",
|
||||
},
|
||||
{
|
||||
name: "stale local records present",
|
||||
seedStale: true,
|
||||
description: "the removal branch used to print a line per record " +
|
||||
"plus a summary",
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
configPath := writeHermeticPruneConfig(t, testCase.seedStale)
|
||||
|
||||
previousArgs := os.Args
|
||||
|
||||
t.Cleanup(func() {
|
||||
os.Args = previousArgs
|
||||
rootFlags = RootFlags{}
|
||||
})
|
||||
|
||||
os.Args = []string{
|
||||
programName, flagConfig, configPath, cmdPrune, flagJSON,
|
||||
}
|
||||
|
||||
stdout := captureProcessStdout(t, func() { _ = Entry() })
|
||||
|
||||
requireExactlyOneJSONDocument(t, stdout)
|
||||
|
||||
var document pruneJSONDocument
|
||||
|
||||
require.NoError(t, json.Unmarshal([]byte(stdout), &document),
|
||||
testCase.description)
|
||||
|
||||
// A destination store with no blobs has none to prune. The
|
||||
// assertion that matters is the one above; this one keeps the
|
||||
// test honest about which document it decoded.
|
||||
assert.Equal(t, 0, document.BlobsFound)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// writeHermeticPruneConfig builds a config over a temp directory with an
|
||||
// empty destination directory and, if seedStale is set, creates the
|
||||
// index database up front with one snapshot record that has no
|
||||
// counterpart on the destination store, and gives the destination the
|
||||
// marker a backup writes. Returns the config path.
|
||||
func writeHermeticPruneConfig(t *testing.T, seedStale bool) string {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
configPath := filepath.Join(dir, "config.yml")
|
||||
indexPath := filepath.Join(dir, "index.sqlite")
|
||||
storeDir := filepath.Join(dir, "store")
|
||||
|
||||
contents := fmt.Sprintf(hermeticConfig,
|
||||
filepath.Join(dir, "source"),
|
||||
storeDir,
|
||||
indexPath)
|
||||
|
||||
require.NoError(t,
|
||||
os.WriteFile(configPath, []byte(contents), configFileMode))
|
||||
|
||||
// prune fails on a destination directory that does not exist, so the
|
||||
// empty store is created here rather than left to a first backup.
|
||||
require.NoError(t, os.Mkdir(storeDir, 0o750))
|
||||
|
||||
// The PID lock lives under xdg.DataHome, which xdg resolves at
|
||||
// package init; point it at the temp dir so the test neither
|
||||
// touches nor collides with the real one.
|
||||
t.Setenv("XDG_DATA_HOME", filepath.Join(dir, "data"))
|
||||
xdg.Reload()
|
||||
t.Cleanup(xdg.Reload)
|
||||
|
||||
if seedStale {
|
||||
seedStaleSnapshotRecord(t, indexPath)
|
||||
|
||||
// While the local index holds a snapshot, prune fails on a
|
||||
// destination that answers that it has no marker.
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(storeDir, "vaultik-destination.txt"), nil, 0o600))
|
||||
}
|
||||
|
||||
return configPath
|
||||
}
|
||||
|
||||
// seedStaleSnapshotRecord creates the index database at path and
|
||||
// inserts one completed snapshot into it. Nothing is written to the
|
||||
// destination store, so `prune` finds the record stale and removes it —
|
||||
// the branch that printed a line per record.
|
||||
func seedStaleSnapshotRecord(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
db, err := database.New(ctx, path)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { require.NoError(t, db.Close()) }()
|
||||
|
||||
startedAt := time.Date(2026, 4, 1, 9, 0, 0, 0, time.UTC)
|
||||
completedAt := startedAt.Add(time.Minute)
|
||||
|
||||
snap := &database.Snapshot{
|
||||
ID: types.SnapshotID(stalePruneSnapshotID),
|
||||
Hostname: "test-host",
|
||||
VaultikVersion: "test",
|
||||
StartedAt: startedAt,
|
||||
CompletedAt: &completedAt,
|
||||
}
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
return repos.Snapshots.Create(ctx, tx, snap)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// flagCron is used to build the argument vectors below.
|
||||
const flagCron = "--cron"
|
||||
|
||||
// plantUnreferencedBlob writes a blob that no snapshot references to the
|
||||
// destination store at storeDir, dated long before any prune grace period,
|
||||
// so that a prune deletes it. Returns its path.
|
||||
func plantUnreferencedBlob(t *testing.T, storeDir string) string {
|
||||
t.Helper()
|
||||
|
||||
hash := strings.Repeat("b", 64)
|
||||
path := filepath.Join(storeDir, "blobs", hash[:2], hash[2:4], hash)
|
||||
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o750))
|
||||
require.NoError(t, os.WriteFile(path, []byte("unreferenced"), 0o600))
|
||||
|
||||
longAgo := time.Unix(0, 0)
|
||||
require.NoError(t, os.Chtimes(path, longAgo, longAgo))
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
// TestEntrySnapshotCreatePruneIsSilent checks that a successful
|
||||
// `snapshot create --prune` under --cron or --quiet prints nothing on
|
||||
// stdout or stderr, whether or not the prune finds a blob to delete.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotCreatePruneIsSilent(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
args []string
|
||||
plantBlob bool
|
||||
}{
|
||||
{
|
||||
name: "cron, no blob to delete",
|
||||
args: []string{cmdSnapshot, cmdCreate, flagCron, flagPrune},
|
||||
},
|
||||
{
|
||||
name: "cron, blob to delete",
|
||||
args: []string{cmdSnapshot, cmdCreate, flagCron, flagPrune},
|
||||
plantBlob: true,
|
||||
},
|
||||
{
|
||||
name: "cron with keep-newer-than",
|
||||
args: []string{
|
||||
cmdSnapshot, cmdCreate, flagCron, flagPrune,
|
||||
flagKeepNewerThan, "4w",
|
||||
},
|
||||
plantBlob: true,
|
||||
},
|
||||
{
|
||||
name: "quiet",
|
||||
args: []string{flagQuiet, cmdSnapshot, cmdCreate, flagPrune},
|
||||
plantBlob: true,
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
configPath, storeDir := writeBackupConfig(t)
|
||||
|
||||
var blob string
|
||||
if testCase.plantBlob {
|
||||
blob = plantUnreferencedBlob(t, storeDir)
|
||||
}
|
||||
|
||||
code, stdout, stderr := runEntry(t,
|
||||
append([]string{flagConfig, configPath}, testCase.args...)...)
|
||||
|
||||
assert.Equal(t, 0, code)
|
||||
assert.Empty(t, stdout)
|
||||
assert.Empty(t, stderr)
|
||||
|
||||
if testCase.plantBlob {
|
||||
assert.NoFileExists(t, blob, "the prune must delete the blob")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestEntryPruneQuietIsSilent checks that a successful `prune --force`
|
||||
// under --quiet prints nothing on stdout or stderr, both when there is
|
||||
// nothing to remove and when it removes a stale local snapshot record
|
||||
// and an unreferenced blob.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntryPruneQuietIsSilent(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
staleRecordAndBlob bool
|
||||
}{
|
||||
{name: "nothing to remove"},
|
||||
{name: "stale record and blob to remove", staleRecordAndBlob: true},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
configPath := writeHermeticPruneConfig(t, testCase.staleRecordAndBlob)
|
||||
|
||||
var blob string
|
||||
if testCase.staleRecordAndBlob {
|
||||
// writeHermeticPruneConfig puts the store beside the config.
|
||||
blob = plantUnreferencedBlob(t,
|
||||
filepath.Join(filepath.Dir(configPath), "store"))
|
||||
}
|
||||
|
||||
code, stdout, stderr := runEntry(t,
|
||||
flagConfig, configPath, flagQuiet, cmdPrune, flagForce)
|
||||
|
||||
assert.Equal(t, 0, code)
|
||||
assert.Empty(t, stdout)
|
||||
assert.Empty(t, stderr)
|
||||
|
||||
if testCase.staleRecordAndBlob {
|
||||
assert.NoFileExists(t, blob, "the prune must delete the blob")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
)
|
||||
|
||||
// Flags of `snapshot create` used to build argument vectors below.
|
||||
const (
|
||||
flagPrune = "--prune"
|
||||
flagKeepNewerThan = "--keep-newer-than"
|
||||
)
|
||||
|
||||
// TestEntrySnapshotCreateRejectsKeepNewerThanBeforeBackup checks that a
|
||||
// --keep-newer-than the post-backup prune could not use stops `snapshot
|
||||
// create` before the backup: the command fails and the destination
|
||||
// store stays empty. It used to be read only by the prune, after the
|
||||
// snapshot was uploaded, and without --prune it was ignored.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotCreateRejectsKeepNewerThanBeforeBackup(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
args []string
|
||||
}{
|
||||
{
|
||||
name: "decimal in a day count",
|
||||
args: []string{flagPrune, flagKeepNewerThan, "1.5d"},
|
||||
},
|
||||
{
|
||||
name: "unknown unit",
|
||||
args: []string{flagPrune, flagKeepNewerThan, "4x"},
|
||||
},
|
||||
{
|
||||
name: "without --prune",
|
||||
args: []string{flagKeepNewerThan, "4w"},
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
configPath, storeDir := writeBackupConfig(t)
|
||||
|
||||
code, _, _ := runEntry(t, append(
|
||||
[]string{flagConfig, configPath, cmdSnapshot, cmdCreate},
|
||||
testCase.args...)...)
|
||||
|
||||
assert.Equal(t, 1, code)
|
||||
|
||||
stored, err := os.ReadDir(storeDir)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, stored, "nothing may be uploaded")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestEntrySnapshotCreateReportsPostBackupPruneFailure makes the
|
||||
// post-backup prune fail after a good backup, by planting a manifest
|
||||
// that cannot be decoded, and checks that the failure is reported as the
|
||||
// prune's and not as a failed snapshot creation.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotCreateReportsPostBackupPruneFailure(t *testing.T) {
|
||||
configPath, storeDir := writeBackupConfig(t)
|
||||
|
||||
// A remote key is 64 lowercase hex characters. The prune reads every
|
||||
// manifest on the store and fails on one it cannot decode.
|
||||
plantedDir := filepath.Join(storeDir, "metadata", strings.Repeat("a", 64))
|
||||
require.NoError(t, os.MkdirAll(plantedDir, 0o750))
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(plantedDir, "manifest.json.zst"),
|
||||
[]byte("not a valid manifest"), 0o600))
|
||||
|
||||
code, _, stderr := runEntry(t,
|
||||
flagConfig, configPath, cmdSnapshot, cmdCreate, flagPrune)
|
||||
|
||||
assert.Equal(t, 1, code)
|
||||
assert.Contains(t, stderr, "Backup completed, but post-backup prune failed")
|
||||
assert.NotContains(t, stderr, "Snapshot creation failed")
|
||||
|
||||
snapshots, err := os.ReadDir(filepath.Join(storeDir, "metadata"))
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, snapshots, 2,
|
||||
"the new snapshot is on the store beside the planted one")
|
||||
}
|
||||
|
||||
// TestEntrySnapshotCreatePruneExitsNonZeroWhenSourceIsEmpty backs up,
|
||||
// empties the source directory, as a volume that is not mounted leaves
|
||||
// its mount point, and runs `snapshot create --prune`. The command exits
|
||||
// non-zero and reports the backup as completed, not as a failed snapshot
|
||||
// creation. See https://git.eeqj.de/sneak/vaultik/issues/388.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotCreatePruneExitsNonZeroWhenSourceIsEmpty(t *testing.T) {
|
||||
configPath, storeDir := writeBackupConfig(t)
|
||||
|
||||
code, _, _ := runEntry(t, flagConfig, configPath, cmdSnapshot, cmdCreate)
|
||||
require.Equal(t, 0, code)
|
||||
|
||||
// writeBackupConfig puts the source directory beside the config file.
|
||||
require.NoError(t, os.Remove(
|
||||
filepath.Join(filepath.Dir(configPath), "source", "file.txt")))
|
||||
|
||||
code, _, stderr := runEntry(t,
|
||||
flagConfig, configPath, cmdSnapshot, cmdCreate, flagPrune)
|
||||
|
||||
assert.Equal(t, 1, code)
|
||||
assert.Contains(t, stderr,
|
||||
"Backup completed, but 1 source path(s) were empty")
|
||||
assert.NotContains(t, stderr, "Snapshot creation failed")
|
||||
|
||||
snapshots, err := os.ReadDir(filepath.Join(storeDir, "metadata"))
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, snapshots, 2, "the older snapshot is kept")
|
||||
}
|
||||
|
||||
// TestEntrySnapshotCreatePruneExitsNonZeroWhenMountPointIsMissing backs
|
||||
// up, records in the local index that this backup found the source
|
||||
// directory as a mount point, and runs `snapshot create --prune`, which
|
||||
// does not find it as one. The command exits non-zero, reports the
|
||||
// backup as completed, not as a failed snapshot creation, and keeps the
|
||||
// older snapshot. See https://git.eeqj.de/sneak/vaultik/issues/395.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotCreatePruneExitsNonZeroWhenMountPointIsMissing(
|
||||
t *testing.T,
|
||||
) {
|
||||
configPath, storeDir := writeBackupConfig(t)
|
||||
|
||||
code, _, _ := runEntry(t, flagConfig, configPath, cmdSnapshot, cmdCreate)
|
||||
require.Equal(t, 0, code)
|
||||
|
||||
// writeBackupConfig puts the source directory and the local index
|
||||
// beside the config file.
|
||||
dir := filepath.Dir(configPath)
|
||||
ctx := context.Background()
|
||||
|
||||
db, err := database.New(ctx, filepath.Join(dir, "index.sqlite"))
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
backedUp, err := repos.Snapshots.GetOnlySnapshot(ctx)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, repos.Snapshots.AddMountPoint(ctx, nil,
|
||||
backedUp.ID.String(), filepath.Join(dir, "source")))
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
code, _, stderr := runEntry(t,
|
||||
flagConfig, configPath, cmdSnapshot, cmdCreate, flagPrune)
|
||||
|
||||
assert.Equal(t, 1, code)
|
||||
assert.Contains(t, stderr, "Backup completed, but 1 mount point(s) "+
|
||||
"an earlier snapshot recorded were not found")
|
||||
assert.NotContains(t, stderr, "Snapshot creation failed")
|
||||
|
||||
snapshots, err := os.ReadDir(filepath.Join(storeDir, "metadata"))
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, snapshots, 2, "the older snapshot is kept")
|
||||
}
|
||||
|
||||
// writeBackupConfig builds a hermetic config with one small file in its
|
||||
// source directory and an empty destination store directory. Returns the
|
||||
// config path and the store directory.
|
||||
func writeBackupConfig(t *testing.T) (string, string) {
|
||||
t.Helper()
|
||||
|
||||
configPath := writeHermeticPruneConfig(t, false)
|
||||
|
||||
// writeHermeticPruneConfig puts the source and store directories
|
||||
// beside the config file.
|
||||
dir := filepath.Dir(configPath)
|
||||
sourceDir := filepath.Join(dir, "source")
|
||||
|
||||
require.NoError(t, os.Mkdir(sourceDir, 0o750))
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(sourceDir, "file.txt"), []byte("hello"), 0o600))
|
||||
|
||||
return configPath, filepath.Join(dir, "store")
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// TestEntrySnapshotCreateSkipErrorsExitsNonZeroWhenItSkips backs up,
|
||||
// under --skip-errors, a source directory that does not exist, which the
|
||||
// walk skips. The command exits non-zero and reports the backup as
|
||||
// completed, not as a failed snapshot creation. See
|
||||
// https://git.eeqj.de/sneak/vaultik/issues/310.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotCreateSkipErrorsExitsNonZeroWhenItSkips(t *testing.T) {
|
||||
// writeBackupConfig would create the source directory.
|
||||
configPath := writeHermeticPruneConfig(t, false)
|
||||
|
||||
code, stdout, stderr := runEntry(t,
|
||||
flagConfig, configPath, "--skip-errors", cmdSnapshot, cmdCreate)
|
||||
|
||||
assert.Equal(t, 1, code)
|
||||
assert.Contains(t, stdout,
|
||||
"Files: 1 examined, 0 backed up, 0 unchanged, 1 failed.")
|
||||
assert.NotContains(t, stdout, "Finished successfully.")
|
||||
assert.Contains(t, stderr,
|
||||
"Backup completed, but 1 file(s) could not be read")
|
||||
assert.NotContains(t, stderr, "Snapshot creation failed")
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
)
|
||||
|
||||
// Words and flags of `snapshot purge` used to build argument vectors
|
||||
// below.
|
||||
const (
|
||||
cmdPurge = "purge"
|
||||
flagOlderThan = "--older-than"
|
||||
flagForce = "--force"
|
||||
)
|
||||
|
||||
// TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails makes
|
||||
// the local index refuse to delete an old snapshot and checks that
|
||||
// `snapshot purge` exits non-zero and leaves that snapshot's metadata on
|
||||
// the destination store. It used to log the error, delete the metadata
|
||||
// and exit 0.
|
||||
//
|
||||
// `snapshot create --prune` runs the same purge, but cannot be driven
|
||||
// this way: the refusal also breaks the backup's metadata export, which
|
||||
// deletes the other snapshots from its copy of the local index.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails(t *testing.T) {
|
||||
configPath := writeHermeticPruneConfig(t, false)
|
||||
dir := filepath.Dir(configPath)
|
||||
indexPath := filepath.Join(dir, "index.sqlite")
|
||||
|
||||
// seedStaleSnapshotRecord writes an old snapshot to the local index
|
||||
// only; the manifest planted here puts it on the destination store
|
||||
// too, so the purge selects it rather than dropping it as stale.
|
||||
seedStaleSnapshotRecord(t, indexPath)
|
||||
|
||||
// While the local index holds a snapshot, purge fails on a
|
||||
// destination that answers that it has no marker.
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(dir, "store", "vaultik-destination.txt"), nil, 0o600))
|
||||
|
||||
manifestDir := filepath.Join(dir, "store", "metadata",
|
||||
snapshot.RemoteSnapshotKey(stalePruneSnapshotID))
|
||||
require.NoError(t, os.MkdirAll(manifestDir, 0o750))
|
||||
|
||||
manifestPath := filepath.Join(manifestDir, "manifest.json.zst")
|
||||
require.NoError(t, os.WriteFile(manifestPath, []byte("stub"), 0o600))
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
db, err := database.New(ctx, indexPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = db.Conn().ExecContext(ctx, `
|
||||
CREATE TRIGGER refuse_snapshot_delete BEFORE DELETE ON snapshots
|
||||
BEGIN SELECT RAISE(ABORT, 'simulated index error'); END`)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
code, _, stderr := runEntry(t, flagConfig, configPath,
|
||||
cmdSnapshot, cmdPurge, flagOlderThan, "30d", flagForce)
|
||||
|
||||
assert.Equal(t, 1, code)
|
||||
assert.Contains(t, stderr, "simulated index error",
|
||||
"the purge must fail at the local-index delete")
|
||||
assert.FileExists(t, manifestPath,
|
||||
"the snapshot is still in the local index, so its metadata must stay")
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package cli //nolint:testpackage // shares programName and the capture helpers
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// TestEntryReturnsStatusCode pins the contract main() relies on for
|
||||
// issue #75: Entry reports success or failure through its return value
|
||||
// and never calls os.Exit. An os.Exit from inside Entry would skip
|
||||
// main's deferred profile writers and truncate the profile of a failing
|
||||
// command. main turns this code into os.Exit only after those defers
|
||||
// run, so a failing command must come back with a non-zero code rather
|
||||
// than ending the process here.
|
||||
//
|
||||
// Stdout and stderr are captured only to keep the banner and command
|
||||
// output off the test log; the assertion is on the returned code.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args and rootFlags
|
||||
func TestEntryReturnsStatusCode(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
args []string
|
||||
want int
|
||||
}{
|
||||
{
|
||||
// version is self-contained: it needs no config and no
|
||||
// destination store, so it exercises the success path.
|
||||
name: "successful command returns zero",
|
||||
args: []string{programName, "version"},
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "unknown command returns one",
|
||||
args: []string{programName, "no-such-command"},
|
||||
want: 1,
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
previousArgs := os.Args
|
||||
|
||||
t.Cleanup(func() {
|
||||
os.Args = previousArgs
|
||||
rootFlags = RootFlags{}
|
||||
})
|
||||
|
||||
os.Args = testCase.args
|
||||
|
||||
var code int
|
||||
|
||||
_, _ = captureProcessStdoutAndStderr(t, func() { code = Entry() })
|
||||
|
||||
assert.Equal(t, testCase.want, code)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package cli //nolint:testpackage // shares hermeticConfig and the capture helpers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/adrg/xdg"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
)
|
||||
|
||||
// TestEntryCompletionStdoutIsTheScript runs `vaultik completion bash`,
|
||||
// whose stdout the README tells the user to source. The script has to
|
||||
// start on the first line.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout and os.Stderr
|
||||
func TestEntryCompletionStdoutIsTheScript(t *testing.T) {
|
||||
code, stdout, _ := runEntry(t, "completion", "bash")
|
||||
|
||||
require.Equal(t, 0, code)
|
||||
|
||||
firstLine, _, _ := strings.Cut(stdout, "\n")
|
||||
assert.True(t, strings.HasPrefix(firstLine, "# bash completion"),
|
||||
"the first line of stdout must be the script's, got %q", firstLine)
|
||||
}
|
||||
|
||||
// TestEntryConfigGetStdoutIsTheValue runs `vaultik config get`, whose
|
||||
// stdout a script reads as the value and nothing else.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout and os.Stderr
|
||||
func TestEntryConfigGetStdoutIsTheValue(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||
require.NoError(t, os.WriteFile(configPath,
|
||||
[]byte("hostname: test-host\n"), configFileMode))
|
||||
|
||||
code, stdout, _ := runEntry(t,
|
||||
flagConfig, configPath, "config", "get", "hostname")
|
||||
|
||||
require.Equal(t, 0, code)
|
||||
assert.Equal(t, "test-host\n", stdout)
|
||||
}
|
||||
|
||||
// TestEntryJSONFailureIsReportedOnStderr runs each --json command that
|
||||
// writes no document when it fails, against a destination it cannot
|
||||
// use. The error must reach stderr, and stdout must stay empty.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntryJSONFailureIsReportedOnStderr(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
name string
|
||||
args []string
|
||||
wantOnStderr string
|
||||
}{
|
||||
{
|
||||
name: "remote info",
|
||||
args: []string{cmdRemote, cmdInfo, flagJSON},
|
||||
wantOnStderr: "Failed to get remote info",
|
||||
},
|
||||
{
|
||||
name: "prune",
|
||||
args: []string{cmdPrune, flagJSON},
|
||||
wantOnStderr: "Prune failed",
|
||||
},
|
||||
{
|
||||
name: "snapshot remove",
|
||||
args: []string{cmdSnapshot, cmdRemove, someSnapshotID, flagJSON},
|
||||
wantOnStderr: "Failed to remove snapshot",
|
||||
},
|
||||
} {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
configPath := writeUnusableDestinationConfig(t)
|
||||
|
||||
code, stdout, stderr := runEntry(t,
|
||||
append([]string{flagConfig, configPath}, testCase.args...)...)
|
||||
|
||||
assert.Equal(t, 1, code)
|
||||
assert.Empty(t, stdout,
|
||||
"a failed --json command has no document to write")
|
||||
assert.Contains(t, stderr, testCase.wantOnStderr,
|
||||
"the failure must be reported on stderr")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestEntrySnapshotRemoveJSONWarningIsOnStderr runs `snapshot remove
|
||||
// --json` on a snapshot in the local index, against a destination
|
||||
// directory that does not exist. The command removes the snapshot from
|
||||
// the local index and still exits 0. Its stdout must hold the document
|
||||
// alone, with the warning about the destination store on stderr: the
|
||||
// command to run again once it is reachable, and the snapshot's ID in
|
||||
// the record's snapshot_id field.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||
func TestEntrySnapshotRemoveJSONWarningIsOnStderr(t *testing.T) {
|
||||
configPath, indexPath := writeMissingDestinationConfig(t)
|
||||
seedStaleSnapshotRecord(t, indexPath)
|
||||
|
||||
code, stdout, stderr := runEntry(t, flagConfig, configPath,
|
||||
cmdSnapshot, cmdRemove, stalePruneSnapshotID, flagJSON)
|
||||
|
||||
require.Equal(t, 0, code)
|
||||
requireExactlyOneJSONDocument(t, stdout)
|
||||
|
||||
// stderr is a pipe here, so the logger writes one JSON record a line.
|
||||
var warning map[string]any
|
||||
|
||||
for line := range strings.Lines(stderr) {
|
||||
if strings.Contains(line,
|
||||
"Could not remove snapshot metadata from remote storage") {
|
||||
require.NoError(t, json.Unmarshal([]byte(line), &warning))
|
||||
}
|
||||
}
|
||||
|
||||
require.NotNil(t, warning, "the warning must reach stderr")
|
||||
assert.Contains(t, warning[slog.MessageKey],
|
||||
"run 'vaultik snapshot remove' with the snapshot's ID again")
|
||||
assert.Equal(t, stalePruneSnapshotID, warning["snapshot_id"])
|
||||
}
|
||||
|
||||
// writeMissingDestinationConfig builds a config whose destination
|
||||
// directory does not exist. Returns the config path and the path of
|
||||
// its local index, which is not created here.
|
||||
func writeMissingDestinationConfig(t *testing.T) (string, string) {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
configPath := filepath.Join(dir, "config.yml")
|
||||
indexPath := filepath.Join(dir, "index.sqlite")
|
||||
|
||||
contents := fmt.Sprintf(hermeticConfig,
|
||||
filepath.Join(dir, "source"),
|
||||
filepath.Join(dir, "missing-store"),
|
||||
indexPath)
|
||||
|
||||
require.NoError(t,
|
||||
os.WriteFile(configPath, []byte(contents), configFileMode))
|
||||
|
||||
// The PID lock lives under xdg.DataHome, which xdg resolves at
|
||||
// package init; point it at the temp dir so the test neither
|
||||
// touches nor collides with the real one.
|
||||
t.Setenv("XDG_DATA_HOME", filepath.Join(dir, "data"))
|
||||
xdg.Reload()
|
||||
t.Cleanup(xdg.Reload)
|
||||
|
||||
return configPath, indexPath
|
||||
}
|
||||
|
||||
// writeUnusableDestinationConfig builds a config whose destination
|
||||
// directory does not exist, which fails `remote info`, and whose local
|
||||
// index is bound to another destination, which fails `prune` and
|
||||
// `snapshot remove` (a missing destination alone only makes `snapshot
|
||||
// remove` warn). Returns the config path.
|
||||
func writeUnusableDestinationConfig(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
configPath, indexPath := writeMissingDestinationConfig(t)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
db, err := database.New(ctx, indexPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { require.NoError(t, db.Close()) }()
|
||||
|
||||
require.NoError(t, database.NewRepositories(db).LocalMeta.Set(ctx,
|
||||
database.LocalMetaKeyStorageURL, "file://"+t.TempDir()))
|
||||
|
||||
return configPath
|
||||
}
|
||||
|
||||
// runEntry runs Entry with args after the program name and returns its
|
||||
// exit code and what it wrote to stdout and stderr.
|
||||
//
|
||||
// Not parallel-safe: it replaces os.Args, os.Stdout and os.Stderr.
|
||||
func runEntry(t *testing.T, args ...string) (int, string, string) {
|
||||
t.Helper()
|
||||
|
||||
previousArgs := os.Args
|
||||
|
||||
t.Cleanup(func() {
|
||||
os.Args = previousArgs
|
||||
rootFlags = RootFlags{}
|
||||
})
|
||||
|
||||
os.Args = append([]string{programName}, args...)
|
||||
|
||||
var code int
|
||||
|
||||
stdout, stderr := captureProcessStdoutAndStderr(t,
|
||||
func() { code = Entry() })
|
||||
|
||||
return code, stdout, stderr
|
||||
}
|
||||
@@ -1,14 +1,18 @@
|
||||
package cli
|
||||
package cli_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/cli"
|
||||
)
|
||||
|
||||
// TestCLIEntry ensures the CLI can be imported and basic initialization works
|
||||
func TestCLIEntry(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// This test primarily serves as a compilation test
|
||||
// to ensure all imports resolve correctly
|
||||
cmd := NewRootCommand()
|
||||
cmd := cli.NewRootCommand()
|
||||
if cmd == nil {
|
||||
t.Fatal("NewRootCommand() returned nil")
|
||||
}
|
||||
@@ -18,15 +22,20 @@ func TestCLIEntry(t *testing.T) {
|
||||
}
|
||||
|
||||
// Verify all subcommands are registered
|
||||
expectedCommands := []string{"snapshot", "store", "restore", "prune", "verify", "info", "version"}
|
||||
expectedCommands := []string{
|
||||
"config", "snapshot", "prune", "info", "version", "remote", "database",
|
||||
}
|
||||
for _, expected := range expectedCommands {
|
||||
found := false
|
||||
|
||||
for _, cmd := range cmd.Commands() {
|
||||
if cmd.Use == expected || cmd.Name() == expected {
|
||||
found = true
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Errorf("Expected command '%s' not found", expected)
|
||||
}
|
||||
@@ -38,15 +47,20 @@ func TestCLIEntry(t *testing.T) {
|
||||
t.Errorf("Failed to find snapshot command: %v", err)
|
||||
} else {
|
||||
// Check snapshot subcommands
|
||||
expectedSubCommands := []string{"create", "list", "purge", "verify"}
|
||||
expectedSubCommands := []string{
|
||||
"create", "list", "purge", "verify", "remove", "restore",
|
||||
}
|
||||
for _, expected := range expectedSubCommands {
|
||||
found := false
|
||||
|
||||
for _, subcmd := range snapshotCmd.Commands() {
|
||||
if subcmd.Use == expected || subcmd.Name() == expected {
|
||||
found = true
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Errorf("Expected snapshot subcommand '%s' not found", expected)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package cli //nolint:testpackage // shares runEntry and the command-word constants
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// TestEntryGroupRejectsUnknownSubcommand runs each group command with a
|
||||
// mistyped subcommand, as a crontab line might carry one. It must fail
|
||||
// with exit 1 and name the word, as `vaultik bogus` does at the top
|
||||
// level, instead of printing the group's help and exiting 0 having done
|
||||
// nothing. The group name alone still prints its help.
|
||||
//
|
||||
//nolint:paralleltest // replaces os.Args, os.Stdout and os.Stderr
|
||||
func TestEntryGroupRejectsUnknownSubcommand(t *testing.T) {
|
||||
for _, testCase := range []struct {
|
||||
group string
|
||||
mistyped string
|
||||
}{
|
||||
{group: cmdSnapshot, mistyped: "craete"},
|
||||
{group: "config", mistyped: "inti"},
|
||||
{group: cmdRemote, mistyped: "ifno"},
|
||||
{group: "database", mistyped: "delte"},
|
||||
{group: "completion", mistyped: "nosuchshell"},
|
||||
} {
|
||||
t.Run(testCase.group, func(t *testing.T) {
|
||||
code, stdout, stderr := runEntry(t,
|
||||
testCase.group, testCase.mistyped)
|
||||
|
||||
assert.Equal(t, 1, code)
|
||||
assert.Empty(t, stdout)
|
||||
assert.Contains(t, stderr,
|
||||
`unknown command "`+testCase.mistyped+`"`)
|
||||
|
||||
code, stdout, _ = runEntry(t, testCase.group)
|
||||
|
||||
assert.Equal(t, 0, code)
|
||||
assert.Contains(t, stdout,
|
||||
programName+" "+testCase.group+" [command]")
|
||||
})
|
||||
}
|
||||
}
|
||||
+12
-34
@@ -1,13 +1,9 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// NewInfoCommand creates the info command
|
||||
@@ -22,7 +18,7 @@ func NewInfoCommand() *cobra.Command {
|
||||
- Encryption configuration (recipients)
|
||||
- Local database statistics`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
@@ -31,38 +27,20 @@ func NewInfoCommand() *cobra.Command {
|
||||
|
||||
// Use the app framework
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
|
||||
return RunOperation(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if err := v.ShowInfo(); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Failed to show info", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
Mode: readOnly,
|
||||
}, func(v *vaultik.Vaultik) error {
|
||||
return v.ShowInfo()
|
||||
}, func(err error) {
|
||||
log.Error("Failed to show info", "error", err)
|
||||
ReportErrorf("Failed to show info: %v", err)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package cli //nolint:testpackage // exercises the unexported command constructor
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// TestJSONHelpSaysConfirmationPromptIsSkipped checks that the --json help
|
||||
// of `snapshot remove` and `prune` says the flag skips the confirmation
|
||||
// prompt. Both delete without asking under --json, since a prompt on
|
||||
// stdout would break the JSON document.
|
||||
func TestJSONHelpSaysConfirmationPromptIsSkipped(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, cmd := range []*cobra.Command{
|
||||
newSnapshotRemoveCommand(),
|
||||
NewPruneCommand(),
|
||||
} {
|
||||
assert.Contains(t, cmd.Flags().Lookup("json").Usage,
|
||||
"skips the confirmation prompt", cmd.Name())
|
||||
}
|
||||
}
|
||||
+43
-49
@@ -1,13 +1,9 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// NewPruneCommand creates the prune command
|
||||
@@ -16,16 +12,37 @@ func NewPruneCommand() *cobra.Command {
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "prune",
|
||||
Short: "Remove unreferenced blobs",
|
||||
Long: `Removes blobs that are not referenced by any snapshot.
|
||||
Short: "Tidy local database and remote storage",
|
||||
Long: `Removes orphaned data from the local index database, and
|
||||
unreferenced blobs and directories under metadata/ with no manifest
|
||||
from the backup destination store.
|
||||
|
||||
This command scans all snapshots and their manifests to build a list of
|
||||
referenced blobs, then removes any blobs in storage that are not in this list.
|
||||
Needs permission to list, read and delete objects on the destination
|
||||
store. It does not need the age private key.
|
||||
|
||||
Use this command after deleting snapshots with 'vaultik purge' to reclaim
|
||||
storage space.`,
|
||||
Local cleanup drops snapshots whose manifest is gone from the
|
||||
destination store, incomplete snapshots, and any files, chunks, or
|
||||
blobs no longer referenced by a completed snapshot. Remote cleanup
|
||||
scans every snapshot manifest in the destination store, builds the
|
||||
set of still-referenced blob hashes, and deletes any blob not in that
|
||||
set that is older than prune_grace_period (default 24h). A newer one
|
||||
is kept: it may belong to a backup still running, on this host or
|
||||
another sharing the destination store, that has not yet uploaded its
|
||||
manifest. The grace period does not protect a blob such a backup
|
||||
reuses from an earlier run that stopped before its manifest: that
|
||||
blob keeps the time of its first upload, so it can be deleted.
|
||||
Remote cleanup also deletes each directory under metadata/ with no
|
||||
manifest, as a backup that stopped before its manifest leaves, once
|
||||
all its objects are older than prune_grace_period and its snapshot is
|
||||
not in the local index.
|
||||
|
||||
Snapshot create --prune runs the same cleanup automatically; this
|
||||
command is the manual entry point for the same work (e.g. after a
|
||||
crashed backup or to reclaim storage). Snapshot remove leaves blobs in
|
||||
place; run this command afterwards to delete the ones no longer
|
||||
referenced.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
@@ -34,51 +51,28 @@ storage space.`,
|
||||
|
||||
// Use the app framework like other commands
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
|
||||
return RunOperation(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet || opts.JSON,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
// Start the prune operation in a goroutine
|
||||
go func() {
|
||||
// Run the prune operation
|
||||
if err := v.PruneBlobs(opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
if !opts.JSON {
|
||||
log.Error("Prune operation failed", "error", err)
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when prune completes
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping prune operation")
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
Quiet: rootFlags.Quiet,
|
||||
JSON: opts.JSON,
|
||||
},
|
||||
Mode: mutating,
|
||||
}, func(v *vaultik.Vaultik) error {
|
||||
return v.Prune(opts)
|
||||
}, func(err error) {
|
||||
log.Error("Prune operation failed", "error", err)
|
||||
ReportErrorf("Prune failed: %v", err)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt")
|
||||
cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output pruning stats as JSON")
|
||||
cmd.Flags().BoolVar(&opts.JSON, "json", false,
|
||||
"Output pruning stats as JSON; skips the confirmation prompt, as --force does")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// PurgeOptions contains options for the purge command
|
||||
type PurgeOptions struct {
|
||||
KeepLatest bool
|
||||
OlderThan string
|
||||
Force bool
|
||||
}
|
||||
|
||||
// NewPurgeCommand creates the purge command
|
||||
func NewPurgeCommand() *cobra.Command {
|
||||
opts := &PurgeOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "purge",
|
||||
Short: "Purge old snapshots",
|
||||
Long: `Removes snapshots based on age or count criteria.
|
||||
|
||||
This command allows you to:
|
||||
- Keep only the latest snapshot (--keep-latest)
|
||||
- Remove snapshots older than a specific duration (--older-than)
|
||||
|
||||
Config is located at /etc/vaultik/config.yml by default, but can be overridden by
|
||||
specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Validate flags
|
||||
if !opts.KeepLatest && opts.OlderThan == "" {
|
||||
return fmt.Errorf("must specify either --keep-latest or --older-than")
|
||||
}
|
||||
if opts.KeepLatest && opts.OlderThan != "" {
|
||||
return fmt.Errorf("cannot specify both --keep-latest and --older-than")
|
||||
}
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Use the app framework like other commands
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
// Start the purge operation in a goroutine
|
||||
go func() {
|
||||
// Run the purge operation
|
||||
if err := v.PurgeSnapshots(opts.KeepLatest, opts.OlderThan, opts.Force); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Purge operation failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when purge completes
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping purge operation")
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, "Keep only the latest snapshot")
|
||||
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "", "Remove snapshots older than duration (e.g. 30d, 6m, 1y)")
|
||||
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompts")
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
package cli //nolint:testpackage // sets the unexported rootFlags directly
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// setRootFlags overrides the global rootFlags for the duration of one
|
||||
// test and restores it afterward. These tests must not run in parallel:
|
||||
// the flags are process-global, so the whole struct is saved and put
|
||||
// back rather than left mutated for the next test.
|
||||
func setRootFlags(t *testing.T, f RootFlags) {
|
||||
t.Helper()
|
||||
|
||||
old := rootFlags
|
||||
rootFlags = f
|
||||
|
||||
t.Cleanup(func() { rootFlags = old })
|
||||
}
|
||||
|
||||
// seedFile writes content to a fresh file and returns its path.
|
||||
func seedFile(t *testing.T, dir, name, content string) string {
|
||||
t.Helper()
|
||||
|
||||
path := filepath.Join(dir, name)
|
||||
|
||||
err := os.WriteFile(path, []byte(content), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("seeding %s: %v", name, err)
|
||||
}
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
// mustExecute runs a command with its output captured and fails the test
|
||||
// if it errors, returning what the command printed.
|
||||
func mustExecute(t *testing.T, cmd *cobra.Command, args ...string) string {
|
||||
t.Helper()
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
cmd.SetOut(&out)
|
||||
cmd.SetArgs(args)
|
||||
|
||||
err := cmd.Execute()
|
||||
if err != nil {
|
||||
t.Fatalf("%s failed: %v", cmd.Name(), err)
|
||||
}
|
||||
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// TestVersionQuietSuppressesReport checks that --quiet silences the whole
|
||||
// version report: it is human-facing output, not a scriptable value.
|
||||
//
|
||||
//nolint:paralleltest // mutates the process-global rootFlags
|
||||
func TestVersionQuietSuppressesReport(t *testing.T) {
|
||||
setRootFlags(t, RootFlags{Quiet: true})
|
||||
|
||||
out := mustExecute(t, NewVersionCommand())
|
||||
|
||||
if out != "" {
|
||||
t.Errorf("--quiet version printed %q, want nothing", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigGetIgnoresQuiet checks that a config value is printed even
|
||||
// under --quiet: it is scriptable output a caller depends on, so --quiet
|
||||
// must not suppress it, and it stays machine-plain (no marker, no color).
|
||||
//
|
||||
//nolint:paralleltest // mutates the process-global rootFlags
|
||||
func TestConfigGetIgnoresQuiet(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := seedFile(t, dir, "config.yml", "storage_url: file:///mnt/x\n")
|
||||
|
||||
setRootFlags(t, RootFlags{Quiet: true, ConfigPath: path})
|
||||
|
||||
out := mustExecute(t, newConfigGetCommand(), "storage_url")
|
||||
|
||||
if out != "file:///mnt/x\n" {
|
||||
t.Errorf("config get --quiet = %q, want the plain value", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigSetQuietSuppressesConfirmation checks that --quiet silences
|
||||
// the confirmation line while still writing the value to the file.
|
||||
//
|
||||
//nolint:paralleltest // mutates the process-global rootFlags
|
||||
func TestConfigSetQuietSuppressesConfirmation(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := seedFile(t, dir, "config.yml", "compression_level: 3\n")
|
||||
|
||||
setRootFlags(t, RootFlags{Quiet: true, ConfigPath: path})
|
||||
|
||||
out := mustExecute(t, newConfigSetCommand(), "compression_level", "9")
|
||||
|
||||
if out != "" {
|
||||
t.Errorf("--quiet config set printed %q, want nothing", out)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled path
|
||||
if err != nil {
|
||||
t.Fatalf("reading config back: %v", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(string(data), "compression_level: 9") {
|
||||
t.Errorf("config set did not write the value under --quiet:\n%s", data)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigSetConfirmsWhenNotQuiet checks that the confirmation names
|
||||
// the key (styled) when --quiet is not set.
|
||||
//
|
||||
//nolint:paralleltest // mutates the process-global rootFlags
|
||||
func TestConfigSetConfirmsWhenNotQuiet(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := seedFile(t, dir, "config.yml", "compression_level: 3\n")
|
||||
|
||||
setRootFlags(t, RootFlags{ConfigPath: path})
|
||||
|
||||
out := mustExecute(t, newConfigSetCommand(), "compression_level", "9")
|
||||
|
||||
if !strings.Contains(out, "compression_level") {
|
||||
t.Errorf("config set did not confirm the key: %q", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigInitQuietSuppressesConfirmation checks that --quiet silences
|
||||
// the "config written" confirmation while still writing the file.
|
||||
//
|
||||
//nolint:paralleltest // mutates the process-global rootFlags
|
||||
func TestConfigInitQuietSuppressesConfirmation(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "new-config.yml")
|
||||
|
||||
setRootFlags(t, RootFlags{Quiet: true, ConfigPath: path})
|
||||
|
||||
out := mustExecute(t, newConfigInitCommand())
|
||||
|
||||
if out != "" {
|
||||
t.Errorf("--quiet config init printed %q, want nothing", out)
|
||||
}
|
||||
|
||||
_, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Errorf("config init did not write the file under --quiet: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// seedDatabaseDeleteConfig writes a valid config whose index_path is a
|
||||
// seeded database file, and returns both paths.
|
||||
func seedDatabaseDeleteConfig(t *testing.T, dir string) (string, string) {
|
||||
t.Helper()
|
||||
|
||||
dbPath := seedFile(t, dir, "index.sqlite", "not-a-real-db")
|
||||
cfg := fmt.Sprintf(hermeticConfig,
|
||||
filepath.Join(dir, "source"), filepath.Join(dir, "store"), dbPath)
|
||||
cfgPath := seedFile(t, dir, "config.yml", cfg)
|
||||
|
||||
return dbPath, cfgPath
|
||||
}
|
||||
|
||||
// TestDatabaseDeleteQuietSuppressesMessage checks that --quiet silences
|
||||
// the "database deleted" line while still removing the file.
|
||||
//
|
||||
//nolint:paralleltest // mutates the process-global rootFlags
|
||||
func TestDatabaseDeleteQuietSuppressesMessage(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dbPath, cfgPath := seedDatabaseDeleteConfig(t, dir)
|
||||
|
||||
setRootFlags(t, RootFlags{Quiet: true, ConfigPath: cfgPath})
|
||||
|
||||
out := mustExecute(t, newDatabaseDeleteCommand(), "--force")
|
||||
|
||||
if out != "" {
|
||||
t.Errorf("--quiet database delete printed %q, want nothing", out)
|
||||
}
|
||||
|
||||
_, err := os.Stat(dbPath)
|
||||
if !os.IsNotExist(err) {
|
||||
t.Errorf("database delete did not remove the file: stat err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDatabaseDeleteReportsWhenNotQuiet checks that the deletion is
|
||||
// reported when --quiet is not set.
|
||||
//
|
||||
//nolint:paralleltest // mutates the process-global rootFlags
|
||||
func TestDatabaseDeleteReportsWhenNotQuiet(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
_, cfgPath := seedDatabaseDeleteConfig(t, dir)
|
||||
|
||||
setRootFlags(t, RootFlags{ConfigPath: cfgPath})
|
||||
|
||||
out := mustExecute(t, newDatabaseDeleteCommand(), "--force")
|
||||
|
||||
if !strings.Contains(out, "deleted") {
|
||||
t.Errorf("database delete did not report the deletion: %q", out)
|
||||
}
|
||||
}
|
||||
+63
-37
@@ -1,25 +1,67 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"errors"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// errNukeNeedsForce guards the destructive 'remote nuke' subcommand.
|
||||
var errNukeNeedsForce = errors.New(
|
||||
"remote nuke requires --force (this deletes ALL remote snapshots and blobs)")
|
||||
|
||||
// NewRemoteCommand creates the remote command and subcommands
|
||||
func NewRemoteCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "remote",
|
||||
Short: "Remote storage management commands",
|
||||
Long: "Commands for inspecting and managing remote storage",
|
||||
// cobra checks Args only on a command that has a Run: without
|
||||
// one, a mistyped subcommand printed help and exited 0.
|
||||
Args: cobra.NoArgs,
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
_ = cmd.Help()
|
||||
},
|
||||
}
|
||||
|
||||
// Add subcommands
|
||||
cmd.AddCommand(newRemoteInfoCommand())
|
||||
cmd.AddCommand(newRemoteNukeCommand())
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// newRemoteNukeCommand creates the 'remote nuke' subcommand.
|
||||
func newRemoteNukeCommand() *cobra.Command {
|
||||
var force bool
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "nuke",
|
||||
Short: "Delete ALL snapshot metadata and blobs from the backup destination store",
|
||||
Long: `Removes every snapshot's metadata, every blob and the destination
|
||||
marker from the backup destination store, and each removed snapshot from
|
||||
the local index. After this command completes successfully the bucket
|
||||
prefix is empty and the next backup starts from scratch.
|
||||
|
||||
This is destructive and irreversible. Requires --force, and permission
|
||||
to list and delete objects on the destination store.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if !force {
|
||||
return errNukeNeedsForce
|
||||
}
|
||||
|
||||
return runVaultikApp(cmd, mutating, false, "Remote nuke failed",
|
||||
func(v *vaultik.Vaultik) error {
|
||||
return v.NukeRemote(true)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&force, "force", false,
|
||||
"Required: confirm destruction of ALL remote data")
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -35,9 +77,12 @@ func newRemoteInfoCommand() *cobra.Command {
|
||||
- Size of all snapshot metadata (per snapshot and total)
|
||||
- Count and total size of all blobs
|
||||
- Count and size of referenced blobs (from all manifests)
|
||||
- Count and size of orphaned blobs (not referenced by any manifest)`,
|
||||
- Count and size of orphaned blobs (not referenced by any manifest)
|
||||
|
||||
Needs permission to list and read the destination store. It does not
|
||||
need the age private key.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
@@ -45,40 +90,21 @@ func newRemoteInfoCommand() *cobra.Command {
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
|
||||
return RunOperation(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet || jsonOutput,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if err := v.RemoteInfo(jsonOutput); err != nil {
|
||||
if err != context.Canceled {
|
||||
if !jsonOutput {
|
||||
log.Error("Failed to get remote info", "error", err)
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
Quiet: rootFlags.Quiet,
|
||||
JSON: jsonOutput,
|
||||
},
|
||||
Mode: readOnly,
|
||||
}, func(v *vaultik.Vaultik) error {
|
||||
return v.RemoteInfo(jsonOutput)
|
||||
}, func(err error) {
|
||||
log.Error("Failed to get remote info", "error", err)
|
||||
ReportErrorf("Failed to get remote info: %v", err)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// RestoreOptions contains options for the restore command
|
||||
type RestoreOptions struct {
|
||||
TargetDir string
|
||||
Paths []string // Optional paths to restore (empty = all)
|
||||
Verify bool // Verify restored files after restore
|
||||
}
|
||||
|
||||
// RestoreApp contains all dependencies needed for restore
|
||||
type RestoreApp struct {
|
||||
Globals *globals.Globals
|
||||
Config *config.Config
|
||||
Storage storage.Storer
|
||||
Vaultik *vaultik.Vaultik
|
||||
Shutdowner fx.Shutdowner
|
||||
}
|
||||
|
||||
// NewRestoreCommand creates the restore command
|
||||
func NewRestoreCommand() *cobra.Command {
|
||||
opts := &RestoreOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "restore <snapshot-id> <target-dir> [paths...]",
|
||||
Short: "Restore files from backup",
|
||||
Long: `Download and decrypt files from a backup snapshot.
|
||||
|
||||
This command will restore files from the specified snapshot to the target directory.
|
||||
If no paths are specified, all files are restored.
|
||||
If paths are specified, only matching files/directories are restored.
|
||||
|
||||
Requires the VAULTIK_AGE_SECRET_KEY environment variable to be set with the age private key.
|
||||
|
||||
Examples:
|
||||
# Restore entire snapshot
|
||||
vaultik restore myhost_docs_2025-01-01T12:00:00Z /restore
|
||||
|
||||
# Restore specific file
|
||||
vaultik restore myhost_docs_2025-01-01T12:00:00Z /restore /home/user/important.txt
|
||||
|
||||
# Restore specific directory
|
||||
vaultik restore myhost_docs_2025-01-01T12:00:00Z /restore /home/user/documents/
|
||||
|
||||
# Restore and verify all files
|
||||
vaultik restore --verify myhost_docs_2025-01-01T12:00:00Z /restore`,
|
||||
Args: cobra.MinimumNArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
snapshotID := args[0]
|
||||
opts.TargetDir = args[1]
|
||||
if len(args) > 2 {
|
||||
opts.Paths = args[2:]
|
||||
}
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Use the app framework like other commands
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
},
|
||||
Modules: []fx.Option{
|
||||
fx.Provide(fx.Annotate(
|
||||
func(g *globals.Globals, cfg *config.Config,
|
||||
storer storage.Storer, v *vaultik.Vaultik, shutdowner fx.Shutdowner) *RestoreApp {
|
||||
return &RestoreApp{
|
||||
Globals: g,
|
||||
Config: cfg,
|
||||
Storage: storer,
|
||||
Vaultik: v,
|
||||
Shutdowner: shutdowner,
|
||||
}
|
||||
},
|
||||
)),
|
||||
},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(app *RestoreApp, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
// Start the restore operation in a goroutine
|
||||
go func() {
|
||||
// Run the restore operation
|
||||
restoreOpts := &vaultik.RestoreOptions{
|
||||
SnapshotID: snapshotID,
|
||||
TargetDir: opts.TargetDir,
|
||||
Paths: opts.Paths,
|
||||
Verify: opts.Verify,
|
||||
}
|
||||
if err := app.Vaultik.Restore(restoreOpts); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Restore operation failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown the app when restore completes
|
||||
if err := app.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping restore operation")
|
||||
app.Vaultik.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.Verify, "verify", false, "Verify restored files by checking chunk hashes")
|
||||
|
||||
return cmd
|
||||
}
|
||||
+121
-26
@@ -1,12 +1,20 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/adrg/xdg"
|
||||
"github.com/spf13/cobra"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
)
|
||||
|
||||
// errConfigNotFound is wrapped by all config-resolution failures.
|
||||
var errConfigNotFound = errors.New("config file not found")
|
||||
|
||||
// RootFlags holds global flags that apply to all commands.
|
||||
// These flags are defined on the root command and inherited by all subcommands.
|
||||
type RootFlags struct {
|
||||
@@ -14,8 +22,10 @@ type RootFlags struct {
|
||||
Verbose bool
|
||||
Debug bool
|
||||
Quiet bool
|
||||
SkipErrors bool
|
||||
}
|
||||
|
||||
//nolint:gochecknoglobals // cobra persistent flags bind to package state
|
||||
var rootFlags RootFlags
|
||||
|
||||
// NewRootCommand creates the root cobra command for the vaultik CLI.
|
||||
@@ -25,24 +35,41 @@ func NewRootCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "vaultik",
|
||||
Short: "Secure incremental backup tool with asymmetric encryption",
|
||||
Long: `vaultik is a secure incremental backup daemon that encrypts data using age
|
||||
public keys and uploads to S3-compatible storage. No private keys are needed
|
||||
on the source system.`,
|
||||
Long: `vaultik is an incremental backup tool that encrypts data to age public
|
||||
keys and uploads it to an S3-compatible object store, a local directory or
|
||||
an rclone remote. The backed-up machine needs no private key: only
|
||||
'snapshot restore' and 'snapshot verify --deep' need one.`,
|
||||
SilenceUsage: true,
|
||||
// Bare 'vaultik' (no subcommand): print help. The banner is
|
||||
// printed once at process startup by Entry, before cobra
|
||||
// parses arguments, so it appears even when cobra rejects
|
||||
// args (e.g. "requires at least 2 arg(s)") and on --help.
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
_ = cmd.Help()
|
||||
},
|
||||
}
|
||||
|
||||
// Add global flags
|
||||
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "", "Path to config file (default: $VAULTIK_CONFIG or /etc/vaultik/config.yml)")
|
||||
cmd.PersistentFlags().BoolVarP(&rootFlags.Verbose, "verbose", "v", false, "Enable verbose output")
|
||||
cmd.PersistentFlags().BoolVar(&rootFlags.Debug, "debug", false, "Enable debug output")
|
||||
cmd.PersistentFlags().BoolVarP(&rootFlags.Quiet, "quiet", "q", false, "Suppress non-error output")
|
||||
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "",
|
||||
"Path to config file (default: $VAULTIK_CONFIG, then the platform "+
|
||||
"config directory, then /etc/vaultik/config.yml)")
|
||||
cmd.PersistentFlags().BoolVarP(&rootFlags.Verbose, "verbose", "v", false,
|
||||
"Enable verbose output")
|
||||
cmd.PersistentFlags().BoolVar(&rootFlags.Debug, "debug", false,
|
||||
"Enable debug output")
|
||||
cmd.PersistentFlags().BoolVarP(&rootFlags.Quiet, "quiet", "q", false,
|
||||
"Suppress status output (warnings, errors, JSON documents, "+
|
||||
"'config get' values and confirmation prompts are still printed)")
|
||||
cmd.PersistentFlags().BoolVar(&rootFlags.SkipErrors, "skip-errors", false,
|
||||
"Skip files that cannot be read when creating a snapshot, or "+
|
||||
"that cannot be restored when restoring, instead of aborting "+
|
||||
"(packing and storage errors while creating a snapshot still "+
|
||||
"abort)")
|
||||
|
||||
// Add subcommands
|
||||
cmd.AddCommand(
|
||||
NewRestoreCommand(),
|
||||
NewConfigCommand(),
|
||||
NewPruneCommand(),
|
||||
NewVerifyCommand(),
|
||||
NewStoreCommand(),
|
||||
NewSnapshotCommand(),
|
||||
NewInfoCommand(),
|
||||
NewVersionCommand(),
|
||||
@@ -50,6 +77,21 @@ on the source system.`,
|
||||
NewDatabaseCommand(),
|
||||
)
|
||||
|
||||
// cobra adds the completion command only when Execute runs, and with
|
||||
// no Run, so it skipped its own cobra.NoArgs and a mistyped shell name
|
||||
// printed help and exited 0. Adding it here lets it have a Run. Its
|
||||
// scripts go to os.Stdout as it is now: a later SetOut does not
|
||||
// redirect them.
|
||||
cmd.InitDefaultCompletionCmd()
|
||||
|
||||
for _, sub := range cmd.Commands() {
|
||||
if sub.Name() == "completion" {
|
||||
sub.Run = func(completion *cobra.Command, _ []string) {
|
||||
_ = completion.Help()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
@@ -59,26 +101,79 @@ func GetRootFlags() RootFlags {
|
||||
return rootFlags
|
||||
}
|
||||
|
||||
// ResolveConfigPath resolves the config file path from flags, environment, or default.
|
||||
// It checks in order: 1) --config flag, 2) VAULTIK_CONFIG environment variable,
|
||||
// 3) default location /etc/vaultik/config.yml. Returns an error if no valid
|
||||
// config file can be found through any of these methods.
|
||||
func ResolveConfigPath() (string, error) {
|
||||
// First check global flag
|
||||
if rootFlags.ConfigPath != "" {
|
||||
return rootFlags.ConfigPath, nil
|
||||
// commandUI returns a UI writer for a command's stdout, in quiet mode
|
||||
// when the global --quiet flag is set. This is how the pure-cli
|
||||
// commands (version, config, database) reach internal/ui: color follows
|
||||
// the writer (a TTY gets color, a captured test buffer does not), and
|
||||
// --quiet silences the same message classes it silences everywhere else.
|
||||
func commandUI(cmd *cobra.Command) *ui.Writer {
|
||||
w := ui.New(cmd.OutOrStdout())
|
||||
if GetRootFlags().Quiet {
|
||||
w.SetQuiet(true)
|
||||
}
|
||||
|
||||
// Then check environment variable
|
||||
if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" {
|
||||
return envPath, nil
|
||||
return w
|
||||
}
|
||||
|
||||
// ResolveConfigPath resolves the config file path from flags, environment, or default.
|
||||
// Search order: --config flag, VAULTIK_CONFIG env, XDG config dir,
|
||||
// /etc/vaultik/config.yml.
|
||||
// Explicit paths from --config and $VAULTIK_CONFIG are checked for existence
|
||||
// so the user gets a clear error instead of a downstream YAML parser failure.
|
||||
func ResolveConfigPath() (string, error) {
|
||||
if path := rootFlags.ConfigPath; path != "" {
|
||||
_, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf(
|
||||
"%w: from --config: %s (run 'vaultik config init --config %s' to create it)",
|
||||
errConfigNotFound, path, path)
|
||||
}
|
||||
|
||||
return path, nil
|
||||
}
|
||||
|
||||
if path := os.Getenv("VAULTIK_CONFIG"); path != "" {
|
||||
_, err := os.Stat(path) //nolint:gosec // G703: path is operator-supplied by design
|
||||
if err != nil {
|
||||
return "", fmt.Errorf(
|
||||
"%w: from $VAULTIK_CONFIG: %s (unset VAULTIK_CONFIG, point it at "+
|
||||
"an existing file, or run 'vaultik config init')",
|
||||
errConfigNotFound, path)
|
||||
}
|
||||
|
||||
return path, nil
|
||||
}
|
||||
|
||||
for _, path := range defaultConfigPaths() {
|
||||
_, err := os.Stat(path)
|
||||
if err == nil {
|
||||
return path, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf(
|
||||
"%w: searched %s (run 'vaultik config init' to create the default "+
|
||||
"config, or pass --config <path>)",
|
||||
errConfigNotFound, strings.Join(defaultConfigPaths(), " or "))
|
||||
}
|
||||
|
||||
// defaultConfigPaths returns the ordered list of config paths to search.
|
||||
// On macOS: ~/Library/Application Support/vaultik/config.yml
|
||||
// On Linux: ~/.config/vaultik/config.yml
|
||||
// Fallback: /etc/vaultik/config.yml
|
||||
func defaultConfigPaths() []string {
|
||||
return []string{
|
||||
filepath.Join(xdg.ConfigHome, "vaultik", "config.yml"),
|
||||
"/etc/vaultik/config.yml",
|
||||
}
|
||||
}
|
||||
|
||||
// Finally check default location
|
||||
defaultPath := "/etc/vaultik/config.yml"
|
||||
if _, err := os.Stat(defaultPath); err == nil {
|
||||
return defaultPath, nil
|
||||
// DefaultConfigPath returns the platform-appropriate default config path.
|
||||
// Used by the init command and in help text.
|
||||
func DefaultConfigPath() string {
|
||||
if os.Getuid() == 0 {
|
||||
return "/etc/vaultik/config.yml"
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("no config file specified, VAULTIK_CONFIG not set, and %s not found", defaultPath)
|
||||
return filepath.Join(xdg.ConfigHome, "vaultik", "config.yml")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package cli_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/cli"
|
||||
)
|
||||
|
||||
// TestRunAppWaitsForOperationCleanupOnShutdown drives RunApp with an fx app
|
||||
// wired the way RunOperation wires a command: a single lifecycle hook whose
|
||||
// OnStart launches the operation in its own goroutine and whose OnStop cancels
|
||||
// it and blocks until that goroutine returns. The operation stands in for a
|
||||
// restore blocked mid-download — it holds a decrypted "scratch" file and only
|
||||
// removes it as it unwinds on cancellation.
|
||||
//
|
||||
// The app is asked to stop once the operation is running (standing in for an
|
||||
// OS interrupt; fx delivers a real signal and Shutdowner.Shutdown() on the
|
||||
// same app.Wait channel, so both drive the identical shutdown path). RunApp
|
||||
// must not return until app.Stop has run the OnStop hook, so the scratch file
|
||||
// must be gone by the time RunApp returns. Before the fix RunApp returned as
|
||||
// soon as the app.Wait/Done channel fired, without running app.Stop, so the
|
||||
// cleanup never ran and this file would still be on disk (issue #159).
|
||||
func TestRunAppWaitsForOperationCleanupOnShutdown(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
scratch := filepath.Join(t.TempDir(), "decrypted-scratch")
|
||||
require.NoError(t, os.WriteFile(scratch, []byte("secret"), 0o600))
|
||||
|
||||
// Cancel and reap the operation even if RunApp returns without doing so
|
||||
// (the buggy path), so the goroutine cannot leak past the test.
|
||||
opCtx, opCancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(opCancel)
|
||||
|
||||
var stop func(context.Context) bool
|
||||
|
||||
app := fx.New(
|
||||
fx.NopLogger,
|
||||
fx.Invoke(func(lc fx.Lifecycle, sh fx.Shutdowner) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(_ context.Context) error {
|
||||
done := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
defer close(done)
|
||||
|
||||
// Blocked mid-operation until cancelled, then run the
|
||||
// cleanup an interrupted restore would run.
|
||||
<-opCtx.Done()
|
||||
|
||||
_ = os.Remove(scratch)
|
||||
}()
|
||||
|
||||
stop = func(ctx context.Context) bool {
|
||||
opCancel()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Ask the app to stop now that the operation is running.
|
||||
go func() { _ = sh.Shutdown() }()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
stop(ctx)
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- cli.RunApp(context.Background(), app) }()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
require.NoError(t, err)
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("RunApp did not return after shutdown was requested")
|
||||
}
|
||||
|
||||
_, err := os.Stat(scratch)
|
||||
require.True(t, os.IsNotExist(err),
|
||||
"RunApp returned before the operation removed its decrypted scratch file")
|
||||
}
|
||||
+230
-326
@@ -1,22 +1,52 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
var (
|
||||
errSnapshotIDRequired = errors.New("snapshot ID required")
|
||||
errWrongArgCount = errors.New("wrong argument count")
|
||||
errPurgeCriteriaNeeded = errors.New(
|
||||
"must specify either --keep-latest or --older-than")
|
||||
errPurgeCriteriaBoth = errors.New(
|
||||
"cannot specify both --keep-latest and --older-than")
|
||||
)
|
||||
|
||||
// requireSnapshotIDArg validates that exactly one positional argument
|
||||
// (the snapshot ID) was supplied, printing help otherwise.
|
||||
func requireSnapshotIDArg(cmd *cobra.Command, args []string) error {
|
||||
if len(args) != 1 {
|
||||
_ = cmd.Help()
|
||||
|
||||
if len(args) == 0 {
|
||||
return errSnapshotIDRequired
|
||||
}
|
||||
|
||||
return fmt.Errorf("%w: expected 1 argument, got %d",
|
||||
errWrongArgCount, len(args))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewSnapshotCommand creates the snapshot command and subcommands
|
||||
func NewSnapshotCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "snapshot",
|
||||
Short: "Snapshot management commands",
|
||||
Long: "Commands for creating, listing, and managing snapshots",
|
||||
// cobra checks Args only on a command that has a Run: without
|
||||
// one, a mistyped subcommand printed help and exited 0.
|
||||
Args: cobra.NoArgs,
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
_ = cmd.Help()
|
||||
},
|
||||
}
|
||||
|
||||
// Add subcommands
|
||||
@@ -25,11 +55,26 @@ func NewSnapshotCommand() *cobra.Command {
|
||||
cmd.AddCommand(newSnapshotPurgeCommand())
|
||||
cmd.AddCommand(newSnapshotVerifyCommand())
|
||||
cmd.AddCommand(newSnapshotRemoveCommand())
|
||||
cmd.AddCommand(newSnapshotPruneCommand())
|
||||
cmd.AddCommand(newSnapshotRestoreCommand())
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// snapshotCreateLong is the long help text of 'snapshot create'.
|
||||
const snapshotCreateLong = `Creates new snapshots of the configured directories.
|
||||
|
||||
If snapshot names are provided, only those snapshots are created.
|
||||
If no names are provided, all configured snapshots are created.
|
||||
|
||||
Needs at least one public key in age_recipients, and permission to write
|
||||
objects at the backup destination store. It does not need the age private
|
||||
key. With --prune it also needs permission to list, read and delete objects
|
||||
there.
|
||||
|
||||
The config is read from the path given by --config or VAULTIK_CONFIG;
|
||||
otherwise from the platform config directory (~/.config/vaultik/config.yml
|
||||
on Linux), then /etc/vaultik/config.yml.`
|
||||
|
||||
// newSnapshotCreateCommand creates the 'snapshot create' subcommand
|
||||
func newSnapshotCreateCommand() *cobra.Command {
|
||||
opts := &vaultik.SnapshotCreateOptions{}
|
||||
@@ -37,17 +82,18 @@ func newSnapshotCreateCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "create [snapshot-names...]",
|
||||
Short: "Create new snapshots",
|
||||
Long: `Creates new snapshots of the configured directories.
|
||||
|
||||
If snapshot names are provided, only those snapshots are created.
|
||||
If no names are provided, all configured snapshots are created.
|
||||
|
||||
Config is located at /etc/vaultik/config.yml by default, but can be overridden by
|
||||
specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
Args: cobra.ArbitraryArgs,
|
||||
Long: snapshotCreateLong,
|
||||
Args: cobra.ArbitraryArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
err := opts.Validate()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Pass snapshot names from args
|
||||
opts.Snapshots = args
|
||||
// --skip-errors is a global flag on the root command.
|
||||
opts.SkipErrors = rootFlags.SkipErrors
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
@@ -56,52 +102,56 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
|
||||
// Use the backup functionality from cli package
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
|
||||
// --cron suppression is wired through v.UI by setupGlobals.
|
||||
return RunOperation(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Cron: opts.Cron,
|
||||
Quiet: rootFlags.Quiet,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
// Start the snapshot creation in a goroutine
|
||||
go func() {
|
||||
// Run the snapshot creation
|
||||
if err := v.CreateSnapshot(opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Snapshot creation failed", "error", err)
|
||||
}
|
||||
}
|
||||
Mode: mutating,
|
||||
}, func(v *vaultik.Vaultik) error {
|
||||
return v.CreateSnapshot(opts)
|
||||
}, func(err error) {
|
||||
if errors.Is(err, vaultik.ErrPostBackupPruneFailed) {
|
||||
log.Error("Post-backup prune failed", "error", err)
|
||||
ReportErrorf("Backup completed, but %v", err)
|
||||
|
||||
// Shutdown the app when snapshot completes
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping snapshot creation")
|
||||
// Cancel the Vaultik context
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
return
|
||||
}
|
||||
|
||||
if errors.Is(err, vaultik.ErrFilesUnreadable) ||
|
||||
errors.Is(err, vaultik.ErrSourcePathEmpty) ||
|
||||
errors.Is(err, vaultik.ErrVolumeNotMounted) {
|
||||
log.Error("Backup completed with an error", "error", err)
|
||||
ReportErrorf("Backup completed, but %v", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
log.Error("Snapshot creation failed", "error", err)
|
||||
ReportErrorf("Snapshot creation failed: %v", err)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.Daemon, "daemon", false, "Run in daemon mode with inotify monitoring")
|
||||
cmd.Flags().BoolVar(&opts.Cron, "cron", false, "Run in cron mode (silent unless error)")
|
||||
cmd.Flags().BoolVar(&opts.Prune, "prune", false, "Delete all previous snapshots and unreferenced blobs after backup")
|
||||
cmd.Flags().BoolVar(&opts.SkipErrors, "skip-errors", false, "Skip file read errors (log them loudly but continue)")
|
||||
cmd.Flags().BoolVar(&opts.Cron, "cron", false,
|
||||
"Run in cron mode (silent unless warning or error)")
|
||||
cmd.Flags().BoolVar(&opts.Prune, "prune", false,
|
||||
"After backup, drop older snapshots of the same name, remove "+
|
||||
"orphaned blobs older than prune_grace_period, and delete each "+
|
||||
"directory under metadata/ with no manifest once all its "+
|
||||
"objects are older than prune_grace_period and its snapshot is "+
|
||||
"not in the local index (not after a backup that skipped "+
|
||||
"unreadable files, found a configured path empty, or did not "+
|
||||
"find a mount point an earlier snapshot found)")
|
||||
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "",
|
||||
"With --prune: keep snapshots newer than this duration "+
|
||||
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+
|
||||
"instead of only the latest")
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -114,49 +164,21 @@ func newSnapshotListCommand() *cobra.Command {
|
||||
Use: "list",
|
||||
Aliases: []string{"ls"},
|
||||
Short: "List all snapshots",
|
||||
Long: "Lists all snapshots with their ID, timestamp, and compressed size",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
Long: `Lists the snapshots in the local index and on the backup destination
|
||||
store, with their ID, timestamp, and sizes.
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if err := v.ListSnapshots(jsonOutput); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Failed to list snapshots", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
Needs permission to list and read the destination store. It does not need
|
||||
the age private key. A snapshot found only on the destination store is
|
||||
listed by its remote key, because its ID is stored only in the local index
|
||||
and in the snapshot's encrypted database. If the destination store cannot
|
||||
be listed, the command warns and lists the local index alone.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
return runVaultikApp(cmd, readOnly, false,
|
||||
"Failed to list snapshots",
|
||||
func(v *vaultik.Vaultik) error {
|
||||
return v.ListSnapshots(jsonOutput)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
@@ -167,70 +189,51 @@ func newSnapshotListCommand() *cobra.Command {
|
||||
|
||||
// newSnapshotPurgeCommand creates the 'snapshot purge' subcommand
|
||||
func newSnapshotPurgeCommand() *cobra.Command {
|
||||
var keepLatest bool
|
||||
var olderThan string
|
||||
var force bool
|
||||
opts := &vaultik.SnapshotPurgeOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "purge",
|
||||
Short: "Purge old snapshots",
|
||||
Long: "Removes snapshots based on age or count criteria",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
Long: `Removes snapshots based on age or count criteria. Exactly one of
|
||||
--keep-latest and --older-than is required.
|
||||
|
||||
Retention is per-snapshot-name: --keep-latest keeps the latest snapshot of
|
||||
each name, not the latest globally. Use --snapshot to restrict the
|
||||
operation to specific snapshot names.
|
||||
|
||||
Only snapshots in the local index are removed. Each one is removed from
|
||||
the local index and its metadata is deleted from the backup destination
|
||||
store. Blobs are not touched: run 'vaultik prune' afterwards.
|
||||
|
||||
Needs permission to list and delete objects on the destination store. If
|
||||
the destination store cannot be listed, nothing is removed.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
// Validate flags
|
||||
if !keepLatest && olderThan == "" {
|
||||
return fmt.Errorf("must specify either --keep-latest or --older-than")
|
||||
}
|
||||
if keepLatest && olderThan != "" {
|
||||
return fmt.Errorf("cannot specify both --keep-latest and --older-than")
|
||||
if !opts.KeepLatest && opts.OlderThan == "" {
|
||||
return errPurgeCriteriaNeeded
|
||||
}
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
if opts.KeepLatest && opts.OlderThan != "" {
|
||||
return errPurgeCriteriaBoth
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if err := v.PurgeSnapshots(keepLatest, olderThan, force); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Failed to purge snapshots", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
return runVaultikApp(cmd, mutating, false,
|
||||
"Failed to purge snapshots",
|
||||
func(v *vaultik.Vaultik) error {
|
||||
return v.PurgeSnapshotsWithOptions(opts)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&keepLatest, "keep-latest", false, "Keep only the latest snapshot")
|
||||
cmd.Flags().StringVar(&olderThan, "older-than", "", "Remove snapshots older than duration (e.g., 30d, 6m, 1y)")
|
||||
cmd.Flags().BoolVar(&force, "force", false, "Skip confirmation prompt")
|
||||
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false,
|
||||
"Keep only the latest snapshot of each name")
|
||||
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "",
|
||||
"Remove snapshots older than duration "+
|
||||
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months)")
|
||||
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt")
|
||||
cmd.Flags().StringArrayVar(&opts.Names, "snapshot", nil,
|
||||
"Restrict to snapshots with these names (repeat for multiple)")
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -241,18 +244,31 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "verify <snapshot-id>",
|
||||
Short: "Verify snapshot integrity",
|
||||
Long: "Verifies that all blobs referenced in a snapshot exist",
|
||||
Args: func(cmd *cobra.Command, args []string) error {
|
||||
if len(args) != 1 {
|
||||
_ = cmd.Help()
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("snapshot ID required")
|
||||
}
|
||||
return fmt.Errorf("expected 1 argument, got %d", len(args))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Short: "Check a snapshot's blobs are present with the listed size",
|
||||
Long: `Checks that every blob the snapshot's manifest lists is present
|
||||
in storage with the size the manifest records, and that the
|
||||
snapshot's encrypted database is present. It does not read blob
|
||||
contents and does not need the age private key.
|
||||
|
||||
Needs permission to read the backup destination store, and to list it
|
||||
when a remote key is given.
|
||||
|
||||
Use --deep to download, decrypt, and re-hash every blob to detect
|
||||
corruption -- integrity, not who wrote it.
|
||||
|
||||
--deep requires the age private key, in the VAULTIK_AGE_SECRET_KEY
|
||||
environment variable or as age_secret_key in the config file; the
|
||||
environment variable wins. The variable may hold the whole age-keygen
|
||||
file, with its comments and all of its keys. Read the key from the
|
||||
file rather than typing it, so that it does not land in your shell
|
||||
history:
|
||||
|
||||
export VAULTIK_AGE_SECRET_KEY="$(cat vaultik_backup_private_key.txt)"
|
||||
|
||||
The snapshot may be named by its ID or, on a host with no local
|
||||
index, by the remote key that 'snapshot list' prints for a
|
||||
remote-only snapshot (an unambiguous leading part is enough).`,
|
||||
Args: requireSnapshotIDArg,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
snapshotID := args[0]
|
||||
|
||||
@@ -263,51 +279,33 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
|
||||
return RunOperation(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet || opts.JSON,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
var err error
|
||||
if opts.Deep {
|
||||
err = v.RunDeepVerify(snapshotID, opts)
|
||||
} else {
|
||||
err = v.VerifySnapshotWithOptions(snapshotID, opts)
|
||||
}
|
||||
if err != nil {
|
||||
if err != context.Canceled {
|
||||
if !opts.JSON {
|
||||
log.Error("Verification failed", "error", err)
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
Quiet: rootFlags.Quiet,
|
||||
JSON: opts.JSON,
|
||||
},
|
||||
Mode: readOnly,
|
||||
}, func(v *vaultik.Vaultik) error {
|
||||
return v.VerifySnapshotWithOptions(snapshotID, opts)
|
||||
}, func(err error) {
|
||||
if opts.JSON {
|
||||
return
|
||||
}
|
||||
|
||||
log.Error("Verification failed", "error", err)
|
||||
ReportErrorf("Verification failed: %v", err)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.Deep, "deep", false, "Download and verify blob hashes")
|
||||
cmd.Flags().BoolVar(&opts.Deep, "deep", false,
|
||||
"Download, decrypt and re-hash every blob; requires the age private "+
|
||||
"key in VAULTIK_AGE_SECRET_KEY, or as age_secret_key in the config "+
|
||||
"file")
|
||||
cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output verification results as JSON")
|
||||
|
||||
return cmd
|
||||
@@ -318,150 +316,56 @@ func newSnapshotRemoveCommand() *cobra.Command {
|
||||
opts := &vaultik.RemoveOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "remove [snapshot-id]",
|
||||
Use: "remove <snapshot-id>",
|
||||
Aliases: []string{"rm"},
|
||||
Short: "Remove a snapshot from the local database",
|
||||
Long: `Removes a snapshot from the local database.
|
||||
Short: "Remove a snapshot from local index and remote metadata",
|
||||
Long: `Removes a snapshot.
|
||||
|
||||
By default, only removes from the local database. Use --remote to also remove
|
||||
the snapshot metadata from remote storage.
|
||||
By default, this removes the snapshot from the local index database and
|
||||
strips the snapshot's metadata from the backup destination store. Blobs
|
||||
are NOT touched: deleting them requires reading every remaining remote
|
||||
manifest (the destination store may hold snapshots this host doesn't
|
||||
know about), which is what 'vaultik prune' does. The command prints the
|
||||
'vaultik prune' invocation to run as a follow-up only when it removed the
|
||||
snapshot's metadata from the destination store, and never under --json.
|
||||
|
||||
Note: This does NOT remove blobs. Use 'vaultik prune' to remove orphaned blobs
|
||||
after removing snapshots.
|
||||
The argument must be a snapshot ID: a remote key is not accepted. An ID
|
||||
that names no snapshot in the local index or on the destination store is
|
||||
an error. Needs permission to list and delete objects on the destination
|
||||
store, unless --local-only is given.
|
||||
|
||||
Use --all --force to remove all snapshots.`,
|
||||
Args: func(cmd *cobra.Command, args []string) error {
|
||||
all, _ := cmd.Flags().GetBool("all")
|
||||
if all {
|
||||
if len(args) > 0 {
|
||||
_ = cmd.Help()
|
||||
return fmt.Errorf("--all cannot be used with a snapshot ID")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if len(args) != 1 {
|
||||
_ = cmd.Help()
|
||||
if len(args) == 0 {
|
||||
return fmt.Errorf("snapshot ID required (or use --all --force)")
|
||||
}
|
||||
return fmt.Errorf("expected 1 argument, got %d", len(args))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Use --local-only to skip the remote half (e.g. when you want to forget a
|
||||
snapshot locally without touching the destination store); the snapshot
|
||||
must then be in the local index.
|
||||
|
||||
If removing the snapshot's metadata from the destination store fails (for
|
||||
example because the remote is unreachable), the local-database removal
|
||||
still completes and a warning is emitted; run 'vaultik snapshot remove
|
||||
<snapshot-id>' again once the destination store is reachable to remove
|
||||
the snapshot's metadata from it ('vaultik prune' does not). For a
|
||||
snapshot that is only on the destination store, that failure is an error.
|
||||
|
||||
To wipe the entire destination store and start over, use 'vaultik remote
|
||||
nuke --force' — it is the single supported entry point for that.`,
|
||||
Args: requireSnapshotIDArg,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return runVaultikApp(cmd, mutating, opts.JSON,
|
||||
"Failed to remove snapshot",
|
||||
func(v *vaultik.Vaultik) error {
|
||||
_, err := v.RemoveSnapshot(args[0], opts)
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet || opts.JSON,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
var err error
|
||||
if opts.All {
|
||||
_, err = v.RemoveAllSnapshots(opts)
|
||||
} else {
|
||||
_, err = v.RemoveSnapshot(args[0], opts)
|
||||
}
|
||||
if err != nil {
|
||||
if err != context.Canceled {
|
||||
if !opts.JSON {
|
||||
log.Error("Failed to remove snapshot", "error", err)
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
return err
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVarP(&opts.Force, "force", "f", false, "Skip confirmation prompt")
|
||||
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false, "Show what would be removed without removing")
|
||||
cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output result as JSON")
|
||||
cmd.Flags().BoolVar(&opts.Remote, "remote", false, "Also remove snapshot metadata from remote storage")
|
||||
cmd.Flags().BoolVar(&opts.All, "all", false, "Remove all snapshots (requires --force)")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// newSnapshotPruneCommand creates the 'snapshot prune' subcommand
|
||||
func newSnapshotPruneCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "prune",
|
||||
Short: "Remove orphaned data from local database",
|
||||
Long: `Removes orphaned files, chunks, and blobs from the local database.
|
||||
|
||||
This cleans up data that is no longer referenced by any snapshot, which can
|
||||
accumulate from incomplete backups or deleted snapshots.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if _, err := v.PruneDatabase(); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Failed to prune database", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
cmd.Flags().BoolVar(&opts.DryRun, "dry-run", false,
|
||||
"Show what would be removed without removing")
|
||||
cmd.Flags().BoolVar(&opts.JSON, "json", false,
|
||||
"Output result as JSON; skips the confirmation prompt, as --force does")
|
||||
cmd.Flags().BoolVar(&opts.LocalOnly, "local-only", false,
|
||||
"Skip remote cleanup; only touch the local index")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// restoreMinArgs is the minimum positional argument count of
|
||||
// `snapshot restore <snapshot-id> <target-dir> [paths...]`.
|
||||
const restoreMinArgs = 2
|
||||
|
||||
// RestoreOptions contains options for the restore command
|
||||
type RestoreOptions struct {
|
||||
TargetDir string
|
||||
Paths []string // Optional paths to restore (empty = all)
|
||||
Verify bool // Verify restored files after restore
|
||||
}
|
||||
|
||||
// newSnapshotRestoreCommand creates the 'snapshot restore' subcommand
|
||||
func newSnapshotRestoreCommand() *cobra.Command {
|
||||
opts := &RestoreOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "restore <snapshot-id> <target-dir> [paths...]",
|
||||
Short: "Restore files from a snapshot",
|
||||
Long: `Download and decrypt files from a backup snapshot.
|
||||
|
||||
This command will restore files from the specified snapshot to the
|
||||
target directory.
|
||||
If no paths are specified, all files are restored.
|
||||
If paths are specified, only matching files/directories are restored.
|
||||
A path that matches nothing in the snapshot is named in a warning, and
|
||||
the command exits non-zero after restoring the paths that match.
|
||||
|
||||
The snapshot may be named by its ID or, when restoring on a host with no
|
||||
local index, by the remote key that 'snapshot list' prints for a
|
||||
remote-only snapshot (an unambiguous leading part is enough).
|
||||
|
||||
Each file is restored under its original absolute path inside the target
|
||||
directory. File owners are restored only when the command runs as root.
|
||||
|
||||
Requires the age private key, in the VAULTIK_AGE_SECRET_KEY environment
|
||||
variable or as age_secret_key in the config file; the environment
|
||||
variable wins. The variable may hold the whole age-keygen file, with its
|
||||
comments and all of its keys. Read the key from the file rather than
|
||||
typing it, so that it does not land in your shell history:
|
||||
|
||||
export VAULTIK_AGE_SECRET_KEY="$(cat vaultik_backup_private_key.txt)"
|
||||
|
||||
Also needs permission to read the backup destination store (and to list
|
||||
it when a remote key is given), and free space in the temporary directory
|
||||
for the decrypted blobs.
|
||||
|
||||
Examples:
|
||||
# Restore entire snapshot
|
||||
vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore
|
||||
|
||||
# Restore specific file
|
||||
vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore \
|
||||
/home/user/important.txt
|
||||
|
||||
# Restore specific directory
|
||||
vaultik snapshot restore myhost_docs_2025-01-01T12:00:00Z /restore \
|
||||
/home/user/documents/
|
||||
|
||||
# Restore and verify all files
|
||||
vaultik snapshot restore --verify myhost_docs_2025-01-01T12:00:00Z /restore`,
|
||||
Args: cobra.MinimumNArgs(restoreMinArgs),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runRestore(cmd, args, opts)
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.Verify, "verify", false,
|
||||
"Verify restored files by checking chunk hashes")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// runRestore parses arguments and runs the restore operation through the
|
||||
// app framework.
|
||||
func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
|
||||
snapshotID := args[0]
|
||||
|
||||
opts.TargetDir = args[1]
|
||||
if len(args) > restoreMinArgs {
|
||||
opts.Paths = args[restoreMinArgs:]
|
||||
}
|
||||
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
return RunOperation(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.Options{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
},
|
||||
Mode: readOnly,
|
||||
}, func(v *vaultik.Vaultik) error {
|
||||
return v.Restore(&vaultik.RestoreOptions{
|
||||
SnapshotID: snapshotID,
|
||||
TargetDir: opts.TargetDir,
|
||||
Paths: opts.Paths,
|
||||
Verify: opts.Verify,
|
||||
SkipErrors: rootFlags.SkipErrors,
|
||||
})
|
||||
}, func(err error) {
|
||||
log.Error("Restore operation failed", "error", err)
|
||||
ReportErrorf("Restore failed: %v", err)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package cli //nolint:testpackage // exercises the unexported command constructor
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// TestRestoreCommandDoesNotTakeKeyAsArgument guards the fix for the age
|
||||
// key being echoed on the command line: restore must take the key only
|
||||
// from the environment, never as a flag value, and its help must show the
|
||||
// file-based form rather than a literal key that would land in shell
|
||||
// history.
|
||||
func TestRestoreCommandDoesNotTakeKeyAsArgument(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cmd := newSnapshotRestoreCommand()
|
||||
|
||||
cmd.Flags().VisitAll(func(f *pflag.Flag) {
|
||||
lower := strings.ToLower(f.Name)
|
||||
for _, banned := range []string{"key", "secret", "age", "identity"} {
|
||||
if strings.Contains(lower, banned) {
|
||||
t.Errorf("restore must not accept the key as a flag; found --%s", f.Name)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
help := cmd.Long
|
||||
if strings.Contains(help, "AGE-SECRET-KEY-") {
|
||||
t.Error("restore help must not show a literal age private key to type")
|
||||
}
|
||||
|
||||
if !strings.Contains(help, "$(cat ") {
|
||||
t.Error("restore help should read the key from a file, e.g. $(cat ...)")
|
||||
}
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// StoreApp contains dependencies for store commands
|
||||
type StoreApp struct {
|
||||
Storage storage.Storer
|
||||
Shutdowner fx.Shutdowner
|
||||
}
|
||||
|
||||
// NewStoreCommand creates the store command and subcommands
|
||||
func NewStoreCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "store",
|
||||
Short: "Storage information commands",
|
||||
Long: "Commands for viewing information about the storage backend",
|
||||
}
|
||||
|
||||
// Add subcommands
|
||||
cmd.AddCommand(newStoreInfoCommand())
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// newStoreInfoCommand creates the 'store info' subcommand
|
||||
func newStoreInfoCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "info",
|
||||
Short: "Display storage information",
|
||||
Long: "Shows storage configuration and statistics including snapshots and blobs",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runWithApp(cmd.Context(), func(app *StoreApp) error {
|
||||
return app.Info(cmd.Context())
|
||||
})
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Info displays storage information
|
||||
func (app *StoreApp) Info(ctx context.Context) error {
|
||||
// Get storage info
|
||||
storageInfo := app.Storage.Info()
|
||||
|
||||
fmt.Printf("Storage Information\n")
|
||||
fmt.Printf("==================\n\n")
|
||||
fmt.Printf("Storage Configuration:\n")
|
||||
fmt.Printf(" Type: %s\n", storageInfo.Type)
|
||||
fmt.Printf(" Location: %s\n\n", storageInfo.Location)
|
||||
|
||||
// Count snapshots by listing metadata/ prefix
|
||||
snapshotCount := 0
|
||||
snapshotCh := app.Storage.ListStream(ctx, "metadata/")
|
||||
snapshotDirs := make(map[string]bool)
|
||||
|
||||
for object := range snapshotCh {
|
||||
if object.Err != nil {
|
||||
return fmt.Errorf("listing snapshots: %w", object.Err)
|
||||
}
|
||||
// Extract snapshot ID from path like metadata/2024-01-15-143052-hostname/
|
||||
parts := strings.Split(object.Key, "/")
|
||||
if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" {
|
||||
snapshotDirs[parts[1]] = true
|
||||
}
|
||||
}
|
||||
snapshotCount = len(snapshotDirs)
|
||||
|
||||
// Count blobs and calculate total size by listing blobs/ prefix
|
||||
blobCount := 0
|
||||
var totalSize int64
|
||||
|
||||
blobCh := app.Storage.ListStream(ctx, "blobs/")
|
||||
for object := range blobCh {
|
||||
if object.Err != nil {
|
||||
return fmt.Errorf("listing blobs: %w", object.Err)
|
||||
}
|
||||
if !strings.HasSuffix(object.Key, "/") { // Skip directories
|
||||
blobCount++
|
||||
totalSize += object.Size
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("Storage Statistics:\n")
|
||||
fmt.Printf(" Snapshots: %d\n", snapshotCount)
|
||||
fmt.Printf(" Blobs: %d\n", blobCount)
|
||||
fmt.Printf(" Total Size: %s\n", formatBytes(totalSize))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// formatBytes formats bytes into human-readable format
|
||||
func formatBytes(bytes int64) string {
|
||||
const unit = 1024
|
||||
if bytes < unit {
|
||||
return fmt.Sprintf("%d B", bytes)
|
||||
}
|
||||
div, exp := int64(unit), 0
|
||||
for n := bytes / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
// runWithApp creates the FX app and runs the given function
|
||||
func runWithApp(ctx context.Context, fn func(*StoreApp) error) error {
|
||||
var result error
|
||||
rootFlags := GetRootFlags()
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = RunWithApp(ctx, AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
},
|
||||
Modules: []fx.Option{
|
||||
fx.Provide(func(storer storage.Storer, shutdowner fx.Shutdowner) *StoreApp {
|
||||
return &StoreApp{
|
||||
Storage: storer,
|
||||
Shutdowner: shutdowner,
|
||||
}
|
||||
}),
|
||||
},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(app *StoreApp, shutdowner fx.Shutdowner) {
|
||||
result = fn(app)
|
||||
// Shutdown after command completes
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond) // Brief delay to ensure clean shutdown
|
||||
if err := shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
}),
|
||||
},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
package cli
|
||||
|
||||
import "time"
|
||||
|
||||
// SnapshotInfo represents snapshot information for listing
|
||||
type SnapshotInfo struct {
|
||||
ID string `json:"id"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
CompressedSize int64 `json:"compressed_size"`
|
||||
}
|
||||
@@ -1,98 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// NewVerifyCommand creates the verify command
|
||||
func NewVerifyCommand() *cobra.Command {
|
||||
opts := &vaultik.VerifyOptions{}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "verify <snapshot-id>",
|
||||
Short: "Verify snapshot integrity",
|
||||
Long: `Verifies that all blobs referenced in a snapshot exist and optionally verifies their contents.
|
||||
|
||||
Shallow verification (default):
|
||||
- Downloads and decompresses manifest
|
||||
- Checks existence of all blobs in S3
|
||||
- Reports missing blobs
|
||||
|
||||
Deep verification (--deep):
|
||||
- Downloads and decrypts database
|
||||
- Verifies blob lists match between manifest and database
|
||||
- Downloads, decrypts, and decompresses each blob
|
||||
- Verifies SHA256 hash of each chunk matches database
|
||||
- Ensures chunks are ordered correctly
|
||||
|
||||
The command will fail immediately on any verification error and exit with non-zero status.`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
snapshotID := args[0]
|
||||
|
||||
// Use unified config resolution
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Use the app framework for all verification
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet || opts.JSON, // Suppress log output in JSON mode
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
// Run the verify operation directly
|
||||
go func() {
|
||||
var err error
|
||||
if opts.Deep {
|
||||
err = v.RunDeepVerify(snapshotID, opts)
|
||||
} else {
|
||||
err = v.VerifySnapshotWithOptions(snapshotID, opts)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if err != context.Canceled {
|
||||
if !opts.JSON {
|
||||
log.Error("Verification failed", "error", err)
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
log.Debug("Stopping verify operation")
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
cmd.Flags().BoolVar(&opts.Deep, "deep", false, "Perform deep verification by downloading and verifying all blob contents")
|
||||
cmd.Flags().BoolVar(&opts.JSON, "json", false, "Output verification results as JSON")
|
||||
|
||||
return cmd
|
||||
}
|
||||
+38
-6
@@ -4,8 +4,9 @@ import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"github.com/spf13/cobra"
|
||||
"sneak.berlin/go/vaultik/internal/globals"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
)
|
||||
|
||||
// NewVersionCommand creates the version command
|
||||
@@ -15,13 +16,44 @@ func NewVersionCommand() *cobra.Command {
|
||||
Short: "Print version information",
|
||||
Long: `Print version, git commit, and build information for vaultik.`,
|
||||
Args: cobra.NoArgs,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Printf("vaultik %s\n", globals.Version)
|
||||
fmt.Printf(" commit: %s\n", globals.Commit)
|
||||
fmt.Printf(" go: %s\n", runtime.Version())
|
||||
fmt.Printf(" os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
|
||||
Run: func(cmd *cobra.Command, _ []string) {
|
||||
writeVersion(commandUI(cmd))
|
||||
},
|
||||
}
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// writeVersion prints the version report through the UI writer. The
|
||||
// report is the output this command exists to produce, so it is written
|
||||
// plain (markers would corrupt the aligned report) via the writer's
|
||||
// underlying stdout; --quiet silences it like any other non-error
|
||||
// output.
|
||||
func writeVersion(out *ui.Writer) {
|
||||
if out.Quiet() {
|
||||
return
|
||||
}
|
||||
|
||||
w := out.Out()
|
||||
|
||||
_, _ = fmt.Fprintf(w, "vaultik %s\n", globals.Version)
|
||||
_, _ = fmt.Fprintf(w, " commit: %s\n", globals.Commit)
|
||||
_, _ = fmt.Fprintf(w, " build date: %s\n", globals.CommitDate)
|
||||
_, _ = fmt.Fprintf(w, " go: %s\n", runtime.Version())
|
||||
_, _ = fmt.Fprintf(w, " os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH)
|
||||
_, _ = fmt.Fprintf(w, " author: %s\n", globals.Author)
|
||||
_, _ = fmt.Fprintf(w, " homepage: %s\n", globals.Homepage)
|
||||
_, _ = fmt.Fprintf(w, " license: %s\n", globals.License)
|
||||
|
||||
if globals.IsDevVersion(globals.Version) {
|
||||
_, _ = fmt.Fprintln(w)
|
||||
_, _ = fmt.Fprintln(w,
|
||||
"This is a development build: it was not built from a tagged")
|
||||
_, _ = fmt.Fprintln(w,
|
||||
"commit, so it carries no release version. Released binaries")
|
||||
_, _ = fmt.Fprintf(w,
|
||||
"are published at %s\n", globals.ReleasesURL)
|
||||
_, _ = fmt.Fprintln(w,
|
||||
"and report their tag on the first line above.")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package cli_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/cli"
|
||||
"sneak.berlin/go/vaultik/internal/globals"
|
||||
)
|
||||
|
||||
// runVersionCommand executes `vaultik version` with its output
|
||||
// captured, and returns what it printed.
|
||||
func runVersionCommand(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
cmd := cli.NewVersionCommand()
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
cmd.SetOut(&out)
|
||||
cmd.SetErr(&out)
|
||||
cmd.SetArgs([]string{})
|
||||
|
||||
err := cmd.Execute()
|
||||
if err != nil {
|
||||
t.Fatalf("version command failed: %v", err)
|
||||
}
|
||||
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// TestVersionCommandReportsBuildVersion checks that the first line of
|
||||
// the report is the version the binary was actually built with. The
|
||||
// test binary carries no -ldflags, so that is the "dev" default -- the
|
||||
// same string an untagged `make vaultik` build stamps a prefix of.
|
||||
//
|
||||
//nolint:paralleltest // executes a command that reads the global rootFlags
|
||||
func TestVersionCommandReportsBuildVersion(t *testing.T) {
|
||||
out := runVersionCommand(t)
|
||||
|
||||
wantFirst := "vaultik " + globals.Version
|
||||
if first, _, _ := strings.Cut(out, "\n"); first != wantFirst {
|
||||
t.Errorf("first line = %q, want %q", first, wantFirst)
|
||||
}
|
||||
|
||||
if !strings.Contains(out, "commit:") {
|
||||
t.Error("output does not report the commit")
|
||||
}
|
||||
}
|
||||
|
||||
// TestVersionCommandFlagsDevelopmentBuild is the regression test for
|
||||
// the thing this command exists to prevent: a build that is not a
|
||||
// release must say so. The notice used to be gated on the version
|
||||
// being exactly "dev", so once untagged builds started carrying their
|
||||
// commit sha it would have gone silent and an unreleased binary would
|
||||
// have looked like a release.
|
||||
//
|
||||
//nolint:paralleltest // executes a command that reads the global rootFlags
|
||||
func TestVersionCommandFlagsDevelopmentBuild(t *testing.T) {
|
||||
if !globals.IsDevVersion(globals.Version) {
|
||||
t.Skipf("test binary was stamped with release version %q",
|
||||
globals.Version)
|
||||
}
|
||||
|
||||
out := runVersionCommand(t)
|
||||
|
||||
if !strings.Contains(out, "development build") {
|
||||
t.Errorf("dev build did not print the development-build notice:\n%s",
|
||||
out)
|
||||
}
|
||||
|
||||
if !strings.Contains(out, globals.ReleasesURL) {
|
||||
t.Errorf("development-build notice does not point at %s:\n%s",
|
||||
globals.ReleasesURL, out)
|
||||
}
|
||||
}
|
||||
+308
-111
@@ -1,6 +1,11 @@
|
||||
// Package config loads, validates, and provides the vaultik YAML
|
||||
// configuration, including snapshot definitions, encryption recipients,
|
||||
// and storage settings.
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -10,24 +15,84 @@ import (
|
||||
|
||||
"filippo.io/age"
|
||||
"git.eeqj.de/sneak/smartconfig"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"github.com/adrg/xdg"
|
||||
"go.uber.org/fx"
|
||||
"gopkg.in/yaml.v3"
|
||||
"sneak.berlin/go/vaultik/internal/chunker"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
const appName = "berlin.sneak.app.vaultik"
|
||||
const appName = "vaultik"
|
||||
|
||||
// secretKeyPrefix marks an age secret (private) key. It is compared
|
||||
// case-insensitively so a recipient entry that is actually a private key is
|
||||
// caught and never passed to age or echoed back.
|
||||
//
|
||||
//nolint:gosec // G101: marker for detecting a pasted secret key, not a credential
|
||||
const secretKeyPrefix = "AGE-SECRET-KEY-"
|
||||
|
||||
// Defaults and validation bounds for tunable settings. MB and GB are
|
||||
// 1000-based, as the size parser reads them; the S3 part sizes are S3's own
|
||||
// limits, which are 1024-based (MiB, GiB).
|
||||
const (
|
||||
defaultBlobSizeLimit = Size(10 * 1000 * 1000 * 1000) // 10GB
|
||||
defaultChunkSize = Size(10 * 1000 * 1000) // 10MB
|
||||
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB
|
||||
defaultCompressionLevel = 3
|
||||
defaultPruneGracePeriod = 24 * time.Hour
|
||||
minChunkSize = 1000 * 1000 // 1MB
|
||||
minCompressionLevel = 1
|
||||
maxCompressionLevel = 19
|
||||
// S3 accepts a multipart upload part from 5MiB to 5GiB.
|
||||
minS3PartSize = 5 * 1024 * 1024
|
||||
maxS3PartSize = 5 * 1024 * 1024 * 1024
|
||||
)
|
||||
|
||||
// Sentinel validation errors.
|
||||
var (
|
||||
errNoConfigPath = errors.New("config path not provided")
|
||||
errRecipientIsSecretKey = errors.New(
|
||||
"an age secret key was given where a public key (age1...) belongs")
|
||||
errRecipientNotX25519 = errors.New(
|
||||
"not a valid recipient; only X25519 age1... public keys are supported")
|
||||
errNoSnapshots = errors.New(
|
||||
"at least one snapshot must be configured (see config.example.yml)")
|
||||
errSnapshotNoPaths = errors.New("snapshot must have at least one path")
|
||||
errChunkSizeTooSmall = errors.New("chunk_size must be at least 1MB")
|
||||
errBlobSizeTooSmall = errors.New(
|
||||
"blob_size_limit must be at least the largest chunk the chunker can " +
|
||||
"emit (chunk_size times the FastCDC size spread)")
|
||||
errBadCompression = errors.New("compression_level must be between 1 and 19")
|
||||
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
|
||||
errBadStorageScheme = errors.New(
|
||||
"storage_url must start with s3://, file://, or rclone://")
|
||||
errFileURLNotAbsolute = errors.New(
|
||||
"a file:// storage_url needs an absolute path, " +
|
||||
"as in file:///path/to/backup")
|
||||
errNegativePruneGracePeriod = errors.New(
|
||||
"prune_grace_period must not be negative")
|
||||
errStorageNotConfigured = errors.New(
|
||||
"storage not configured; set storage_url or provide s3.endpoint + " +
|
||||
"s3.bucket + credentials")
|
||||
errS3BucketRequired = errors.New("s3.bucket is required (or set storage_url)")
|
||||
errS3KeyIDRequired = errors.New("s3.access_key_id is required")
|
||||
errS3SecretRequired = errors.New("s3.secret_access_key is required")
|
||||
)
|
||||
|
||||
// expandTilde expands ~ at the start of a path to the user's home directory.
|
||||
func expandTilde(path string) string {
|
||||
if path == "~" {
|
||||
home, _ := os.UserHomeDir()
|
||||
|
||||
return home
|
||||
}
|
||||
|
||||
if strings.HasPrefix(path, "~/") {
|
||||
home, _ := os.UserHomeDir()
|
||||
|
||||
return filepath.Join(home, path[2:])
|
||||
}
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
@@ -35,8 +100,10 @@ func expandTilde(path string) string {
|
||||
func expandTildeInURL(url string) string {
|
||||
if strings.HasPrefix(url, "file://~/") {
|
||||
home, _ := os.UserHomeDir()
|
||||
|
||||
return "file://" + filepath.Join(home, url[9:])
|
||||
}
|
||||
|
||||
return url
|
||||
}
|
||||
|
||||
@@ -64,6 +131,7 @@ func (c *Config) GetExcludes(snapshotName string) []string {
|
||||
combined := make([]string, 0, len(c.Exclude)+len(snap.Exclude))
|
||||
combined = append(combined, c.Exclude...)
|
||||
combined = append(combined, snap.Exclude...)
|
||||
|
||||
return combined
|
||||
}
|
||||
|
||||
@@ -75,40 +143,78 @@ func (c *Config) SnapshotNames() []string {
|
||||
}
|
||||
// Sort for deterministic order
|
||||
sort.Strings(names)
|
||||
|
||||
return names
|
||||
}
|
||||
|
||||
// Names of the two places the age secret key can be configured, used by
|
||||
// AgeSecretKeySourceName for error messages that must not echo the value.
|
||||
//
|
||||
//nolint:gosec // G101: these are the names of the config sources, not a key
|
||||
const (
|
||||
ageSecretKeySourceEnv = "VAULTIK_AGE_SECRET_KEY"
|
||||
ageSecretKeySourceConfig = "age_secret_key"
|
||||
)
|
||||
|
||||
// AgeSecretKeySourceName returns the human name of where AgeSecretKey was
|
||||
// configured. A Config built directly (as in tests) has no recorded
|
||||
// source, so it reports the config-file field name.
|
||||
func (c *Config) AgeSecretKeySourceName() string {
|
||||
if c.AgeSecretKeySource != "" {
|
||||
return c.AgeSecretKeySource
|
||||
}
|
||||
|
||||
return ageSecretKeySourceConfig
|
||||
}
|
||||
|
||||
// Config represents the application configuration for Vaultik.
|
||||
// It defines all settings for backup operations, including source directories,
|
||||
// encryption recipients, storage configuration, and performance tuning parameters.
|
||||
// Configuration is typically loaded from a YAML file.
|
||||
//
|
||||
//nolint:tagliatelle // snake_case is the established config-file format
|
||||
type Config struct {
|
||||
AgeRecipients []string `yaml:"age_recipients"`
|
||||
AgeSecretKey string `yaml:"age_secret_key"`
|
||||
BackupInterval time.Duration `yaml:"backup_interval"`
|
||||
BlobSizeLimit Size `yaml:"blob_size_limit"`
|
||||
ChunkSize Size `yaml:"chunk_size"`
|
||||
Exclude []string `yaml:"exclude"` // Global excludes applied to all snapshots
|
||||
FullScanInterval time.Duration `yaml:"full_scan_interval"`
|
||||
Hostname string `yaml:"hostname"`
|
||||
IndexPath string `yaml:"index_path"`
|
||||
MinTimeBetweenRun time.Duration `yaml:"min_time_between_run"`
|
||||
S3 S3Config `yaml:"s3"`
|
||||
Snapshots map[string]SnapshotConfig `yaml:"snapshots"`
|
||||
CompressionLevel int `yaml:"compression_level"`
|
||||
AgeRecipients []string `yaml:"age_recipients"`
|
||||
AgeSecretKey string `yaml:"age_secret_key"`
|
||||
// AgeSecretKeySource names where AgeSecretKey was configured
|
||||
// ("VAULTIK_AGE_SECRET_KEY" or "age_secret_key") so a later parse
|
||||
// failure can name the source without echoing the secret value. It is
|
||||
// set by Load and never read from or written to the config file.
|
||||
AgeSecretKeySource string `yaml:"-"`
|
||||
BlobSizeLimit Size `yaml:"blob_size_limit"`
|
||||
ChunkSize Size `yaml:"chunk_size"`
|
||||
// Exclude holds global excludes applied to all snapshots.
|
||||
Exclude []string `yaml:"exclude"`
|
||||
Hostname string `yaml:"hostname"`
|
||||
IndexPath string `yaml:"index_path"`
|
||||
S3 S3Config `yaml:"s3"`
|
||||
Snapshots map[string]SnapshotConfig `yaml:"snapshots"`
|
||||
CompressionLevel int `yaml:"compression_level"`
|
||||
|
||||
// PruneGracePeriod is how old a blob no manifest lists must be before
|
||||
// a prune deletes it, and how old every object in a directory under
|
||||
// metadata/ with no manifest must be before a prune deletes that
|
||||
// directory. A backup writes its manifest only once it has uploaded
|
||||
// all its blobs and its db.zst.age, so a younger blob or directory may
|
||||
// belong to a backup still running, on this host or another sharing
|
||||
// the destination store.
|
||||
PruneGracePeriod time.Duration `yaml:"prune_grace_period"`
|
||||
|
||||
// StorageURL specifies the storage backend using a URL format.
|
||||
// Takes precedence over S3Config if set.
|
||||
// Supported formats:
|
||||
// - s3://bucket/prefix?endpoint=host®ion=us-east-1
|
||||
// - file:///path/to/backup
|
||||
// For S3 URLs, credentials are still read from s3.access_key_id and s3.secret_access_key.
|
||||
// For S3 URLs, credentials are still read from s3.access_key_id
|
||||
// and s3.secret_access_key.
|
||||
StorageURL string `yaml:"storage_url"`
|
||||
}
|
||||
|
||||
// S3Config represents S3 storage configuration for backup storage.
|
||||
// It supports both AWS S3 and S3-compatible storage services.
|
||||
// All fields except UseSSL and PartSize are required.
|
||||
//
|
||||
//nolint:tagliatelle // snake_case is the established config-file format
|
||||
type S3Config struct {
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Bucket string `yaml:"bucket"`
|
||||
@@ -116,21 +222,23 @@ type S3Config struct {
|
||||
AccessKeyID string `yaml:"access_key_id"`
|
||||
SecretAccessKey string `yaml:"secret_access_key"`
|
||||
Region string `yaml:"region"`
|
||||
UseSSL bool `yaml:"use_ssl"`
|
||||
PartSize Size `yaml:"part_size"`
|
||||
// UseSSL selects HTTPS for a scheme-less endpoint. Omitted (nil) means
|
||||
// the default, TLS; set it to false only to force plain HTTP.
|
||||
UseSSL *bool `yaml:"use_ssl"`
|
||||
PartSize Size `yaml:"part_size"`
|
||||
}
|
||||
|
||||
// ConfigPath wraps the config file path for fx dependency injection.
|
||||
// Path wraps the config file path for fx dependency injection.
|
||||
// This type allows the config file path to be injected as a distinct type
|
||||
// rather than a plain string, avoiding conflicts with other string dependencies.
|
||||
type ConfigPath string
|
||||
type Path string
|
||||
|
||||
// New creates a new Config instance by loading from the specified path.
|
||||
// This function is used by the fx dependency injection framework.
|
||||
// Returns an error if the path is empty or if loading fails.
|
||||
func New(path ConfigPath) (*Config, error) {
|
||||
func New(path Path) (*Config, error) {
|
||||
if path == "" {
|
||||
return nil, fmt.Errorf("config path not provided")
|
||||
return nil, errNoConfigPath
|
||||
}
|
||||
|
||||
cfg, err := Load(string(path))
|
||||
@@ -141,37 +249,72 @@ func New(path ConfigPath) (*Config, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// Load reads and parses the configuration file from the specified path.
|
||||
// It applies default values for optional fields, performs environment variable
|
||||
// substitution using smartconfig, and validates the configuration.
|
||||
// The configuration file should be in YAML format. Returns an error if the file
|
||||
// cannot be read, parsed, or if validation fails.
|
||||
// Load reads the configuration file at path, parses it with Parse, warns
|
||||
// when others can read the file, and validates the configuration.
|
||||
// Returns an error if the file cannot be read, parsed, or if validation
|
||||
// fails.
|
||||
func Load(path string) (*Config, error) {
|
||||
data, err := os.ReadFile(path) //nolint:gosec // G304: config path is operator-supplied
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load config file: %w", err)
|
||||
}
|
||||
|
||||
cfg, err := Parse(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check config file permissions (warn if world or group readable)
|
||||
//nolint:gosec // G703: config path is operator-supplied by design
|
||||
info, statErr := os.Stat(path)
|
||||
if statErr == nil {
|
||||
mode := info.Mode().Perm()
|
||||
if mode&0044 != 0 { // group or world readable
|
||||
log.Warn(cfg.readableByOthersWarning(),
|
||||
"path", path,
|
||||
"mode", fmt.Sprintf("%04o", mode),
|
||||
"recommendation", "chmod 600 "+path)
|
||||
}
|
||||
}
|
||||
|
||||
err = cfg.Validate()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid config: %w", err)
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// Parse reads YAML config data as Load does, without validating it. It
|
||||
// applies default values for optional fields, fills in ${...} references
|
||||
// using smartconfig, and expands ~ in paths.
|
||||
func Parse(data []byte) (*Config, error) {
|
||||
// Load config using smartconfig for interpolation
|
||||
sc, err := smartconfig.NewFromConfigPath(path)
|
||||
sc, err := smartconfig.NewFromReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load config file: %w", err)
|
||||
}
|
||||
|
||||
cfg := &Config{
|
||||
// Set defaults
|
||||
BlobSizeLimit: Size(10 * 1024 * 1024 * 1024), // 10GB
|
||||
ChunkSize: Size(10 * 1024 * 1024), // 10MB
|
||||
BackupInterval: 1 * time.Hour,
|
||||
FullScanInterval: 24 * time.Hour,
|
||||
MinTimeBetweenRun: 15 * time.Minute,
|
||||
IndexPath: filepath.Join(xdg.DataHome, appName, "index.sqlite"),
|
||||
CompressionLevel: 3,
|
||||
BlobSizeLimit: defaultBlobSizeLimit,
|
||||
ChunkSize: defaultChunkSize,
|
||||
IndexPath: filepath.Join(xdg.DataHome, appName, "index.sqlite"),
|
||||
CompressionLevel: defaultCompressionLevel,
|
||||
PruneGracePeriod: defaultPruneGracePeriod,
|
||||
S3: S3Config{PartSize: defaultS3PartSize},
|
||||
}
|
||||
|
||||
// Convert smartconfig data to YAML then unmarshal
|
||||
configData := sc.Data()
|
||||
|
||||
yamlBytes, err := yaml.Marshal(configData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal config data: %w", err)
|
||||
}
|
||||
|
||||
if err := yaml.Unmarshal(yamlBytes, cfg); err != nil {
|
||||
err = yaml.Unmarshal(yamlBytes, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
@@ -184,6 +327,7 @@ func Load(path string) (*Config, error) {
|
||||
for i, path := range snap.Paths {
|
||||
snap.Paths[i] = expandTilde(path)
|
||||
}
|
||||
|
||||
cfg.Snapshots[name] = snap
|
||||
}
|
||||
|
||||
@@ -192,10 +336,7 @@ func Load(path string) (*Config, error) {
|
||||
cfg.IndexPath = expandTilde(envIndexPath)
|
||||
}
|
||||
|
||||
// Check for environment variable override for AgeSecretKey
|
||||
if envAgeSecretKey := os.Getenv("VAULTIK_AGE_SECRET_KEY"); envAgeSecretKey != "" {
|
||||
cfg.AgeSecretKey = extractAgeSecretKey(envAgeSecretKey)
|
||||
}
|
||||
cfg.setAgeSecretKey()
|
||||
|
||||
// Get hostname if not set
|
||||
if cfg.Hostname == "" {
|
||||
@@ -203,6 +344,7 @@ func Load(path string) (*Config, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get hostname: %w", err)
|
||||
}
|
||||
|
||||
cfg.Hostname = hostname
|
||||
}
|
||||
|
||||
@@ -210,137 +352,192 @@ func Load(path string) (*Config, error) {
|
||||
if cfg.S3.Region == "" {
|
||||
cfg.S3.Region = "us-east-1"
|
||||
}
|
||||
if cfg.S3.PartSize == 0 {
|
||||
cfg.S3.PartSize = Size(5 * 1024 * 1024) // 5MB
|
||||
}
|
||||
|
||||
// Check config file permissions (warn if world or group readable)
|
||||
if info, err := os.Stat(path); err == nil {
|
||||
mode := info.Mode().Perm()
|
||||
if mode&0044 != 0 { // group or world readable
|
||||
log.Warn("Config file has insecure permissions (contains S3 credentials)",
|
||||
"path", path,
|
||||
"mode", fmt.Sprintf("%04o", mode),
|
||||
"recommendation", "chmod 600 "+path)
|
||||
}
|
||||
}
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("invalid config: %w", err)
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// Validate checks if the configuration is valid and complete.
|
||||
// It ensures all required fields are present and have valid values:
|
||||
// - At least one age recipient must be specified
|
||||
// - At least one snapshot must be configured with at least one path
|
||||
// - Storage must be configured (either storage_url or s3.* fields)
|
||||
// - Chunk size must be at least 1MB
|
||||
// - Blob size limit must be at least the chunk size
|
||||
// - Compression level must be between 1 and 19
|
||||
// - Every age recipient must parse as an X25519 age1... public key (so a
|
||||
// bad entry fails at load, not mid-backup); errors name the position,
|
||||
// never the value. An empty list is accepted, because only snapshot
|
||||
// create needs a recipient and it checks for one itself
|
||||
// - At least one snapshot must be configured with at least one path
|
||||
// - Storage must be configured (either storage_url or s3.* fields)
|
||||
// - Chunk size must be at least 1MB
|
||||
// - Blob size limit must be at least the largest chunk the chunker can emit
|
||||
// (chunk_size times chunker.ChunkSizeSpread), so a single-chunk blob never
|
||||
// exceeds the configured limit
|
||||
// - Compression level must be between 1 and 19
|
||||
// - S3 part size must be between 5MiB and 5GiB, the part sizes S3 accepts
|
||||
// - Prune grace period must not be negative
|
||||
//
|
||||
// Returns an error describing the first validation failure encountered.
|
||||
func (c *Config) Validate() error {
|
||||
if len(c.AgeRecipients) == 0 {
|
||||
return fmt.Errorf("at least one age_recipient is required")
|
||||
for i, recipient := range c.AgeRecipients {
|
||||
err := validateAgeRecipient(recipient)
|
||||
if err != nil {
|
||||
return fmt.Errorf("age_recipients[%d]: %w", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(c.Snapshots) == 0 {
|
||||
return fmt.Errorf("at least one snapshot must be configured")
|
||||
return errNoSnapshots
|
||||
}
|
||||
|
||||
for name, snap := range c.Snapshots {
|
||||
if len(snap.Paths) == 0 {
|
||||
return fmt.Errorf("snapshot %q must have at least one path", name)
|
||||
return fmt.Errorf("%w: %q", errSnapshotNoPaths, name)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate storage configuration
|
||||
if err := c.validateStorage(); err != nil {
|
||||
err := c.validateStorage()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if c.ChunkSize.Int64() < 1024*1024 { // 1MB minimum
|
||||
return fmt.Errorf("chunk_size must be at least 1MB")
|
||||
if c.ChunkSize.Int64() < minChunkSize {
|
||||
return errChunkSizeTooSmall
|
||||
}
|
||||
|
||||
if c.BlobSizeLimit.Int64() < c.ChunkSize.Int64() {
|
||||
return fmt.Errorf("blob_size_limit must be at least chunk_size")
|
||||
// The chunker can emit chunks up to chunk_size * ChunkSizeSpread, and the
|
||||
// packer places a single such chunk into an otherwise empty blob. A limit
|
||||
// below that bound would let a blob exceed it, so reject it.
|
||||
largestChunk := c.ChunkSize.Int64() * chunker.ChunkSizeSpread
|
||||
if c.BlobSizeLimit.Int64() < largestChunk {
|
||||
return fmt.Errorf("%w: need at least %d bytes",
|
||||
errBlobSizeTooSmall, largestChunk)
|
||||
}
|
||||
|
||||
if c.CompressionLevel < 1 || c.CompressionLevel > 19 {
|
||||
return fmt.Errorf("compression_level must be between 1 and 19")
|
||||
if c.CompressionLevel < minCompressionLevel ||
|
||||
c.CompressionLevel > maxCompressionLevel {
|
||||
return errBadCompression
|
||||
}
|
||||
|
||||
if c.S3.PartSize.Int64() < minS3PartSize ||
|
||||
c.S3.PartSize.Int64() > maxS3PartSize {
|
||||
return errBadS3PartSize
|
||||
}
|
||||
|
||||
if c.PruneGracePeriod < 0 {
|
||||
return errNegativePruneGracePeriod
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateAgeRecipient parses one age_recipients entry with the age library
|
||||
// and returns a value-free error on failure. A recipient string can be
|
||||
// sensitive (an operator may paste a secret key by mistake), so neither the
|
||||
// entry nor age's own error (which quotes its input) is ever included.
|
||||
func validateAgeRecipient(recipient string) error {
|
||||
if strings.HasPrefix(strings.ToUpper(recipient), secretKeyPrefix) {
|
||||
return errRecipientIsSecretKey
|
||||
}
|
||||
|
||||
_, err := age.ParseX25519Recipient(recipient)
|
||||
if err != nil {
|
||||
return errRecipientNotX25519
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// setAgeSecretKey records the age secret key and where it came from. The
|
||||
// value is stored raw and parsed only where decryption happens
|
||||
// (internal/vaultik), so backup, list and prune keep working whatever the
|
||||
// field holds. The environment variable overrides the config-file field.
|
||||
func (c *Config) setAgeSecretKey() {
|
||||
if c.AgeSecretKey != "" {
|
||||
c.AgeSecretKeySource = ageSecretKeySourceConfig
|
||||
}
|
||||
|
||||
if env := os.Getenv("VAULTIK_AGE_SECRET_KEY"); env != "" {
|
||||
c.AgeSecretKey = env
|
||||
c.AgeSecretKeySource = ageSecretKeySourceEnv
|
||||
}
|
||||
}
|
||||
|
||||
// readableByOthersWarning is the warning Load logs when others can read
|
||||
// the config file. It says "may contain" because the S3 credentials are
|
||||
// seen only after smartconfig has replaced any ${...} reference in the
|
||||
// file with its value, so a set credential need not be in the file.
|
||||
func (c *Config) readableByOthersWarning() string {
|
||||
if c.S3.AccessKeyID != "" || c.S3.SecretAccessKey != "" {
|
||||
return "Config file is readable by others and may contain S3 credentials"
|
||||
}
|
||||
|
||||
return "Config file is readable by others"
|
||||
}
|
||||
|
||||
// validateStorage validates storage configuration.
|
||||
// If StorageURL is set, it takes precedence. S3 URLs require credentials.
|
||||
// File URLs don't require any S3 configuration.
|
||||
// If StorageURL is not set, legacy S3 configuration is required.
|
||||
func (c *Config) validateStorage() error {
|
||||
if c.StorageURL != "" {
|
||||
// URL-based configuration
|
||||
if strings.HasPrefix(c.StorageURL, "file://") {
|
||||
// File storage doesn't need S3 credentials
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(c.StorageURL, "s3://") {
|
||||
// S3 storage needs credentials
|
||||
if c.S3.AccessKeyID == "" {
|
||||
return fmt.Errorf("s3.access_key_id is required for s3:// URLs")
|
||||
}
|
||||
if c.S3.SecretAccessKey == "" {
|
||||
return fmt.Errorf("s3.secret_access_key is required for s3:// URLs")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(c.StorageURL, "rclone://") {
|
||||
// Rclone storage uses rclone's own config
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("storage_url must start with s3://, file://, or rclone://")
|
||||
return c.validateStorageURL()
|
||||
}
|
||||
|
||||
// Legacy S3 configuration
|
||||
if c.S3.Endpoint == "" {
|
||||
return fmt.Errorf("s3.endpoint is required (or set storage_url)")
|
||||
return errStorageNotConfigured
|
||||
}
|
||||
|
||||
if c.S3.Bucket == "" {
|
||||
return fmt.Errorf("s3.bucket is required (or set storage_url)")
|
||||
return errS3BucketRequired
|
||||
}
|
||||
|
||||
if c.S3.AccessKeyID == "" {
|
||||
return fmt.Errorf("s3.access_key_id is required")
|
||||
return errS3KeyIDRequired
|
||||
}
|
||||
|
||||
if c.S3.SecretAccessKey == "" {
|
||||
return fmt.Errorf("s3.secret_access_key is required")
|
||||
return errS3SecretRequired
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// extractAgeSecretKey extracts the AGE-SECRET-KEY from the input using
|
||||
// the age library's parser, which handles comments and whitespace.
|
||||
func extractAgeSecretKey(input string) string {
|
||||
identities, err := age.ParseIdentities(strings.NewReader(input))
|
||||
if err != nil || len(identities) == 0 {
|
||||
// Fall back to trimmed input if parsing fails
|
||||
return strings.TrimSpace(input)
|
||||
// validateStorageURL validates URL-based storage configuration. File and
|
||||
// rclone URLs need no credentials; S3 URLs require the legacy s3.*
|
||||
// credential fields. A file URL must name an absolute path: a relative one
|
||||
// would resolve against the working directory, so the destination would
|
||||
// move with it.
|
||||
func (c *Config) validateStorageURL() error {
|
||||
switch {
|
||||
case strings.HasPrefix(c.StorageURL, "file://"):
|
||||
// storage.ParseStorageURL applies the same rule, but config
|
||||
// cannot import storage, which imports config.
|
||||
if !strings.HasPrefix(c.StorageURL, "file:///") {
|
||||
return errFileURLNotAbsolute
|
||||
}
|
||||
|
||||
return nil
|
||||
case strings.HasPrefix(c.StorageURL, "rclone://"):
|
||||
// Rclone storage uses rclone's own config
|
||||
return nil
|
||||
case strings.HasPrefix(c.StorageURL, "s3://"):
|
||||
// S3 storage needs credentials
|
||||
if c.S3.AccessKeyID == "" {
|
||||
return fmt.Errorf("%w for s3:// URLs", errS3KeyIDRequired)
|
||||
}
|
||||
|
||||
if c.S3.SecretAccessKey == "" {
|
||||
return fmt.Errorf("%w for s3:// URLs", errS3SecretRequired)
|
||||
}
|
||||
|
||||
return nil
|
||||
default:
|
||||
return errBadStorageScheme
|
||||
}
|
||||
// Return the string representation of the first identity
|
||||
if id, ok := identities[0].(*age.X25519Identity); ok {
|
||||
return id.String()
|
||||
}
|
||||
return strings.TrimSpace(input)
|
||||
}
|
||||
|
||||
// Module exports the config module for fx dependency injection.
|
||||
// It provides the Config type to other modules in the application.
|
||||
//
|
||||
//nolint:gochecknoglobals // fx module definitions are package globals
|
||||
var Module = fx.Module("config",
|
||||
fx.Provide(New),
|
||||
)
|
||||
|
||||
+663
-53
@@ -1,21 +1,32 @@
|
||||
package config
|
||||
package config //nolint:testpackage // exercises unexported source constants
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/chunker"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
const (
|
||||
TEST_SNEAK_AGE_PUBLIC_KEY = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
|
||||
TEST_INTEGRATION_AGE_PUBLIC_KEY = "age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg"
|
||||
TEST_INTEGRATION_AGE_PRIVATE_KEY = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5"
|
||||
testSneakAgePublicKey = "age1278m9q7dp3chsh2dcy82qk27v047zywyvt" +
|
||||
"xwnj4cvt0z65jw6a7q5dqhfj"
|
||||
testIntegrationAgePublicKey = "age1ezrjmfpwsc95svdg0y54mums3zevgzu" +
|
||||
"0x0ecq2f7tp8a05gl0sjq9q9wjg"
|
||||
testIntegrationAgePrivateKey = "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GX" +
|
||||
"VEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// Set up test environment
|
||||
testConfigPath := filepath.Join("..", "..", "test", "config.yaml")
|
||||
if absPath, err := filepath.Abs(testConfigPath); err == nil {
|
||||
|
||||
absPath, err := filepath.Abs(testConfigPath)
|
||||
if err == nil {
|
||||
_ = os.Setenv("VAULTIK_CONFIG", absPath)
|
||||
}
|
||||
|
||||
@@ -23,8 +34,11 @@ func TestMain(m *testing.M) {
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
// TestConfigLoad ensures the config package can be imported and basic functionality works
|
||||
// TestConfigLoad ensures the config package can be imported and basic
|
||||
// functionality works.
|
||||
func TestConfigLoad(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Use the test config file
|
||||
configPath := os.Getenv("VAULTIK_CONFIG")
|
||||
if configPath == "" {
|
||||
@@ -41,8 +55,10 @@ func TestConfigLoad(t *testing.T) {
|
||||
if len(cfg.AgeRecipients) != 2 {
|
||||
t.Errorf("Expected 2 age recipients, got %d", len(cfg.AgeRecipients))
|
||||
}
|
||||
if cfg.AgeRecipients[0] != TEST_SNEAK_AGE_PUBLIC_KEY {
|
||||
t.Errorf("Expected first age recipient to be %s, got '%s'", TEST_SNEAK_AGE_PUBLIC_KEY, cfg.AgeRecipients[0])
|
||||
|
||||
if cfg.AgeRecipients[0] != testSneakAgePublicKey {
|
||||
t.Errorf("Expected first age recipient to be %s, got '%s'",
|
||||
testSneakAgePublicKey, cfg.AgeRecipients[0])
|
||||
}
|
||||
|
||||
if len(cfg.Snapshots) != 1 {
|
||||
@@ -59,11 +75,13 @@ func TestConfigLoad(t *testing.T) {
|
||||
}
|
||||
|
||||
if testSnap.Paths[0] != "/tmp/vaultik-test-source" {
|
||||
t.Errorf("Expected first path to be '/tmp/vaultik-test-source', got '%s'", testSnap.Paths[0])
|
||||
t.Errorf("Expected first path to be '/tmp/vaultik-test-source', got '%s'",
|
||||
testSnap.Paths[0])
|
||||
}
|
||||
|
||||
if cfg.S3.Bucket != "vaultik-test-bucket" {
|
||||
t.Errorf("Expected S3 bucket to be 'vaultik-test-bucket', got '%s'", cfg.S3.Bucket)
|
||||
t.Errorf("Expected S3 bucket to be 'vaultik-test-bucket', got '%s'",
|
||||
cfg.S3.Bucket)
|
||||
}
|
||||
|
||||
if cfg.Hostname != "test-host" {
|
||||
@@ -71,76 +89,668 @@ func TestConfigLoad(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestExampleConfigIsScrubbedAndLoads checks that the shipped
|
||||
// config.example.yml carries only neutral placeholders (no real credentials,
|
||||
// private addresses, or internal host names) and still parses.
|
||||
func TestExampleConfigIsScrubbedAndLoads(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
examplePath := filepath.Join("..", "..", "config.example.yml")
|
||||
|
||||
cfg, err := Load(examplePath)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to load config.example.yml: %v", err)
|
||||
}
|
||||
|
||||
if cfg.StorageURL != "rclone://myremote/path/to/backups" {
|
||||
t.Errorf("Expected neutral storage_url, got '%s'", cfg.StorageURL)
|
||||
}
|
||||
|
||||
//nolint:gosec // G304: examplePath is a fixed in-repo path, not user input
|
||||
raw, err := os.ReadFile(examplePath)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read config.example.yml: %v", err)
|
||||
}
|
||||
|
||||
text := string(raw)
|
||||
|
||||
wantSubstrings := []string{
|
||||
"YOUR_ACCESS_KEY",
|
||||
"YOUR_SECRET_KEY",
|
||||
"endpoint: https://",
|
||||
}
|
||||
for _, want := range wantSubstrings {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("Expected config.example.yml to contain %q", want)
|
||||
}
|
||||
}
|
||||
|
||||
// A raw "http://" scheme would mean a plaintext, likely private endpoint.
|
||||
if strings.Contains(text, "http://") {
|
||||
t.Error("config.example.yml should not contain an http:// endpoint")
|
||||
}
|
||||
}
|
||||
|
||||
// TestConfigFromEnv tests loading config path from environment variable
|
||||
func TestConfigFromEnv(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
configPath := os.Getenv("VAULTIK_CONFIG")
|
||||
if configPath == "" {
|
||||
t.Skip("VAULTIK_CONFIG not set")
|
||||
}
|
||||
|
||||
// Verify the file exists
|
||||
if _, err := os.Stat(configPath); os.IsNotExist(err) {
|
||||
t.Errorf("Config file does not exist at path from VAULTIK_CONFIG: %s", configPath)
|
||||
//nolint:gosec // G703: test config path comes from the test environment
|
||||
_, err := os.Stat(configPath)
|
||||
if os.IsNotExist(err) {
|
||||
t.Errorf("Config file does not exist at path from VAULTIK_CONFIG: %s",
|
||||
configPath)
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractAgeSecretKey tests extraction of AGE-SECRET-KEY from various inputs
|
||||
func TestExtractAgeSecretKey(t *testing.T) {
|
||||
// TestValidateBlobSizeLimit checks the blob_size_limit boundary: it must be at
|
||||
// least the largest chunk the chunker can emit (chunk_size times
|
||||
// chunker.ChunkSizeSpread), because the packer places a single such chunk into
|
||||
// an otherwise empty blob. A limit between chunk_size and that bound is rejected.
|
||||
func TestValidateBlobSizeLimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const chunkSize = Size(10 * 1024 * 1024) // 10MB
|
||||
|
||||
largestChunk := chunkSize.Int64() * chunker.ChunkSizeSpread
|
||||
|
||||
newConfig := func(blobLimit Size) *Config {
|
||||
return &Config{
|
||||
AgeRecipients: []string{testSneakAgePublicKey},
|
||||
Snapshots: map[string]SnapshotConfig{"test": {Paths: []string{"/tmp/src"}}},
|
||||
StorageURL: "file:///tmp/vaultik-test-store",
|
||||
ChunkSize: chunkSize,
|
||||
BlobSizeLimit: blobLimit,
|
||||
CompressionLevel: 3,
|
||||
S3: S3Config{PartSize: defaultS3PartSize},
|
||||
}
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
name string
|
||||
blobLimit Size
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "plain key",
|
||||
input: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
|
||||
expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
|
||||
name: "at chunk_size but below largest chunk is rejected",
|
||||
blobLimit: chunkSize,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "key with trailing newline",
|
||||
input: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5\n",
|
||||
expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
|
||||
name: "between chunk_size and largest chunk is rejected",
|
||||
blobLimit: Size(chunkSize.Int64() * 2),
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "full age-keygen output",
|
||||
input: `# created: 2025-01-14T12:00:00Z
|
||||
# public key: age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg
|
||||
AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5
|
||||
`,
|
||||
expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
|
||||
name: "one byte below largest chunk is rejected",
|
||||
blobLimit: Size(largestChunk - 1),
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "age-keygen output with extra blank lines",
|
||||
input: `# created: 2025-01-14T12:00:00Z
|
||||
# public key: age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg
|
||||
|
||||
AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5
|
||||
|
||||
`,
|
||||
expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
|
||||
name: "exactly at largest chunk is accepted",
|
||||
blobLimit: Size(largestChunk),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "key with leading whitespace",
|
||||
input: " AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5 ",
|
||||
expected: "AGE-SECRET-KEY-19CR5YSFW59HM4TLD6GXVEDMZFTVVF7PPHKUT68TXSFPK7APHXA2QS2NJA5",
|
||||
},
|
||||
{
|
||||
name: "empty input",
|
||||
input: "",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "only comments",
|
||||
input: "# this is a comment\n# another comment",
|
||||
expected: "# this is a comment\n# another comment",
|
||||
name: "above largest chunk is accepted",
|
||||
blobLimit: Size(largestChunk * 100),
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := extractAgeSecretKey(tt.input)
|
||||
if result != tt.expected {
|
||||
t.Errorf("extractAgeSecretKey(%q) = %q, want %q", tt.input, result, tt.expected)
|
||||
t.Parallel()
|
||||
|
||||
err := newConfig(tt.blobLimit).Validate()
|
||||
if tt.wantErr {
|
||||
if !errors.Is(err, errBlobSizeTooSmall) {
|
||||
t.Fatalf("Validate() error = %v, want errBlobSizeTooSmall", err)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Validate() unexpected error: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidateS3PartSize checks that s3.part_size is held to the part sizes
|
||||
// S3 accepts, 5MiB to 5GiB, by changing only the part size of the test
|
||||
// config. "5MB" in the config file is 5,000,000 bytes, below the minimum.
|
||||
func TestValidateS3PartSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
base, err := Load(os.Getenv("VAULTIK_CONFIG"))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to load config: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
partSize Size
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "5MB is rejected",
|
||||
partSize: 5_000_000,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "one byte below 5MiB is rejected",
|
||||
partSize: minS3PartSize - 1,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "5MiB is accepted",
|
||||
partSize: minS3PartSize,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "5GiB is accepted",
|
||||
partSize: maxS3PartSize,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "one byte above 5GiB is rejected",
|
||||
partSize: maxS3PartSize + 1,
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := *base
|
||||
cfg.S3.PartSize = tt.partSize
|
||||
|
||||
err := cfg.Validate()
|
||||
if tt.wantErr {
|
||||
if !errors.Is(err, errBadS3PartSize) {
|
||||
t.Fatalf("Validate() error = %v, want errBadS3PartSize", err)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Validate() unexpected error: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadS3PartSize checks that a config file without s3.part_size loads
|
||||
// with the 5MiB default, and that an explicit 0 fails at load like any other
|
||||
// part size S3 refuses.
|
||||
func TestLoadS3PartSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const withoutPartSize = "snapshots:\n" +
|
||||
" test:\n" +
|
||||
" paths: [/tmp/vaultik-test-source]\n" +
|
||||
"storage_url: file:///tmp/vaultik-test-storage\n"
|
||||
|
||||
writeConfig := func(t *testing.T, text string) string {
|
||||
t.Helper()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
|
||||
err := os.WriteFile(path, []byte(text), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
t.Run("absent loads as 5MiB", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg, err := Load(writeConfig(t, withoutPartSize))
|
||||
if err != nil {
|
||||
t.Fatalf("Load() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.S3.PartSize != defaultS3PartSize {
|
||||
t.Errorf("s3.part_size = %d, want %d",
|
||||
cfg.S3.PartSize, defaultS3PartSize)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("0 is rejected", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := Load(writeConfig(t, withoutPartSize+"s3:\n part_size: 0\n"))
|
||||
if !errors.Is(err, errBadS3PartSize) {
|
||||
t.Fatalf("Load() error = %v, want errBadS3PartSize", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestDocumentedSizeDefaults checks that the defaults are the documented
|
||||
// chunk_size 10MB and blob_size_limit 10GB as the parser reads those values
|
||||
// (1000-based), so writing a default out in the config file changes nothing.
|
||||
func TestDocumentedSizeDefaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg, err := Parse([]byte("chunk_size: 10MB\nblob_size_limit: 10GB\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.ChunkSize != defaultChunkSize {
|
||||
t.Errorf("chunk_size: 10MB is %d bytes, the default is %d",
|
||||
cfg.ChunkSize, defaultChunkSize)
|
||||
}
|
||||
|
||||
if cfg.BlobSizeLimit != defaultBlobSizeLimit {
|
||||
t.Errorf("blob_size_limit: 10GB is %d bytes, the default is %d",
|
||||
cfg.BlobSizeLimit, defaultBlobSizeLimit)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDocumentedSizeExamples checks the 1MB minimum chunk_size and that
|
||||
// every size the README, config.example.yml and the config init template
|
||||
// list as an example is accepted.
|
||||
func TestDocumentedSizeExamples(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const withoutSizes = "snapshots:\n" +
|
||||
" test:\n" +
|
||||
" paths: [/tmp/vaultik-test-source]\n" +
|
||||
"storage_url: file:///tmp/vaultik-test-storage\n"
|
||||
|
||||
tests := []struct {
|
||||
setting string
|
||||
wantErr error
|
||||
}{
|
||||
{"chunk_size: 1MB", nil},
|
||||
{"chunk_size: 999999", errChunkSizeTooSmall},
|
||||
{"chunk_size: 5M", nil},
|
||||
{"chunk_size: 10M", nil},
|
||||
{"chunk_size: 10MB", nil},
|
||||
{"chunk_size: 64MiB", nil},
|
||||
{"chunk_size: 1GB", nil},
|
||||
// Four times the default chunk_size, the documented minimum.
|
||||
{"blob_size_limit: 40MB", nil},
|
||||
{"blob_size_limit: 500MB", nil},
|
||||
{"blob_size_limit: 1GB", nil},
|
||||
{"blob_size_limit: 1GiB", nil},
|
||||
{"blob_size_limit: 10G", nil},
|
||||
{"blob_size_limit: 10GB", nil},
|
||||
{"s3: {part_size: 5MiB}", nil},
|
||||
{"s3: {part_size: 10MB}", nil},
|
||||
{"s3: {part_size: 16MiB}", nil},
|
||||
{"s3: {part_size: 100MiB}", nil},
|
||||
{"s3: {part_size: 5GiB}", nil},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.setting, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg, err := Parse([]byte(withoutSizes + tt.setting + "\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
err = cfg.Validate()
|
||||
if !errors.Is(err, tt.wantErr) {
|
||||
t.Fatalf("Validate() error = %v, want %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidatePruneGracePeriod checks that a negative prune_grace_period
|
||||
// is refused rather than acting like 0s, which deletes unreferenced blobs
|
||||
// and directories under metadata/ with no manifest at once, and that 0s
|
||||
// and the documented examples are accepted.
|
||||
func TestValidatePruneGracePeriod(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const withoutGracePeriod = "snapshots:\n" +
|
||||
" test:\n" +
|
||||
" paths: [/tmp/vaultik-test-source]\n" +
|
||||
"storage_url: file:///tmp/vaultik-test-storage\n"
|
||||
|
||||
tests := []struct {
|
||||
setting string
|
||||
wantErr error
|
||||
}{
|
||||
{"prune_grace_period: 0s", nil},
|
||||
{"prune_grace_period: 90m", nil},
|
||||
{"prune_grace_period: 36h", nil},
|
||||
{"prune_grace_period: -1h", errNegativePruneGracePeriod},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.setting, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg, err := Parse([]byte(withoutGracePeriod + tt.setting + "\n"))
|
||||
if err != nil {
|
||||
t.Fatalf("Parse() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
err = cfg.Validate()
|
||||
if !errors.Is(err, tt.wantErr) {
|
||||
t.Fatalf("Validate() error = %v, want %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadPruneGracePeriod checks that a config file without
|
||||
// prune_grace_period loads with the documented 24h. The value is written out
|
||||
// here rather than taken from defaultPruneGracePeriod, so that a change of the
|
||||
// default fails this test.
|
||||
func TestLoadPruneGracePeriod(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const withoutGracePeriod = "snapshots:\n" +
|
||||
" test:\n" +
|
||||
" paths: [/tmp/vaultik-test-source]\n" +
|
||||
"storage_url: file:///tmp/vaultik-test-storage\n"
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
|
||||
err := os.WriteFile(path, []byte(withoutGracePeriod), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Load() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.PruneGracePeriod != 24*time.Hour {
|
||||
t.Errorf("prune_grace_period = %s, want 24h", cfg.PruneGracePeriod)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a
|
||||
// relative path fails at load. Accepted, it would resolve against the
|
||||
// working directory, so a backup run from another directory would write
|
||||
// to a different destination and still succeed.
|
||||
func TestLoadRejectsRelativeFileURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const relativeConfig = "snapshots:\n" +
|
||||
" test:\n" +
|
||||
" paths: [/tmp/vaultik-test-source]\n" +
|
||||
"storage_url: file://dest\n"
|
||||
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
|
||||
err := os.WriteFile(path, []byte(relativeConfig), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
_, err = Load(path)
|
||||
if !errors.Is(err, errFileURLNotAbsolute) {
|
||||
t.Fatalf("Load() error = %v, want errFileURLNotAbsolute", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(err.Error(), "absolute path") {
|
||||
t.Errorf("error %q does not say to use an absolute path", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidateAgeRecipients checks that recipients are parsed at config load
|
||||
// (a bad entry fails immediately, not mid-backup) and that no invalid entry —
|
||||
// least of all a pasted secret key — is echoed in the error. An empty list
|
||||
// loads, because only snapshot create needs a recipient.
|
||||
func TestValidateAgeRecipients(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
baseConfig := func(recipients []string) *Config {
|
||||
return &Config{
|
||||
AgeRecipients: recipients,
|
||||
Snapshots: map[string]SnapshotConfig{"test": {Paths: []string{"/tmp/src"}}},
|
||||
StorageURL: "file:///tmp/vaultik-test-store",
|
||||
ChunkSize: Size(10 * 1024 * 1024),
|
||||
BlobSizeLimit: Size(10 * 1024 * 1024 * 1024),
|
||||
CompressionLevel: 3,
|
||||
S3: S3Config{PartSize: defaultS3PartSize},
|
||||
}
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
recipients []string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "no recipients is accepted",
|
||||
recipients: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "placeholder recipient is rejected",
|
||||
recipients: []string{"age1REPLACE_WITH_YOUR_PUBLIC_KEY"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "ssh-ed25519 recipient is rejected",
|
||||
recipients: []string{"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIexamplekeydata"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "truncated age1 string is rejected",
|
||||
recipients: []string{"age1short"},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "secret key passed as recipient is rejected",
|
||||
recipients: []string{testIntegrationAgePrivateKey},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "two valid recipients are accepted",
|
||||
recipients: []string{testSneakAgePublicKey, testIntegrationAgePublicKey},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
err := baseConfig(tt.recipients).Validate()
|
||||
if !tt.wantErr {
|
||||
if err != nil {
|
||||
t.Fatalf("Validate() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Validate() returned nil, want error")
|
||||
}
|
||||
|
||||
// The entry itself must never appear in the error, since a
|
||||
// recipient string can be a secret key.
|
||||
for _, recipient := range tt.recipients {
|
||||
if strings.Contains(err.Error(), recipient) {
|
||||
t.Fatalf("Validate() error echoed the recipient value: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestAgeSecretKeySourceName checks the name reported for the configured
|
||||
// age secret key: the recorded source when Load set one, and the
|
||||
// config-file field name for a Config built directly (as in tests).
|
||||
func TestAgeSecretKeySourceName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
source string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "unset defaults to config field",
|
||||
source: "",
|
||||
want: ageSecretKeySourceConfig,
|
||||
},
|
||||
{
|
||||
name: "environment source",
|
||||
source: ageSecretKeySourceEnv,
|
||||
want: ageSecretKeySourceEnv,
|
||||
},
|
||||
{
|
||||
name: "config-file source",
|
||||
source: ageSecretKeySourceConfig,
|
||||
want: ageSecretKeySourceConfig,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := &Config{AgeSecretKeySource: tt.source}
|
||||
if got := cfg.AgeSecretKeySourceName(); got != tt.want {
|
||||
t.Errorf("AgeSecretKeySourceName() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// loadReadableConfig writes configYAML to a file that others can read,
|
||||
// loads it, and returns what the logger wrote to stderr meanwhile. The
|
||||
// logger writes to the os.Stderr it finds when it is initialized, so
|
||||
// os.Stderr is pointed at a file first. Not parallel-safe: os.Stderr and
|
||||
// the logger are process-global.
|
||||
func loadReadableConfig(t *testing.T, configYAML string) string {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
configPath := filepath.Join(dir, "config.yml")
|
||||
stderrPath := filepath.Join(dir, "stderr")
|
||||
|
||||
err := os.WriteFile(configPath, []byte(configYAML), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("writing config: %v", err)
|
||||
}
|
||||
|
||||
//nolint:gosec // G302: the test needs a config file others can read
|
||||
err = os.Chmod(configPath, 0o644)
|
||||
if err != nil {
|
||||
t.Fatalf("chmod config: %v", err)
|
||||
}
|
||||
|
||||
stderrFile, err := os.Create(stderrPath) //nolint:gosec // G304: test temp path
|
||||
if err != nil {
|
||||
t.Fatalf("creating stderr file: %v", err)
|
||||
}
|
||||
|
||||
previous := os.Stderr
|
||||
os.Stderr = stderrFile
|
||||
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
_, loadErr := Load(configPath)
|
||||
|
||||
os.Stderr = previous
|
||||
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
_ = stderrFile.Close()
|
||||
|
||||
if loadErr != nil {
|
||||
t.Fatalf("Load() error = %v", loadErr)
|
||||
}
|
||||
|
||||
captured, err := os.ReadFile(stderrPath) //nolint:gosec // G304: test temp path
|
||||
if err != nil {
|
||||
t.Fatalf("reading stderr file: %v", err)
|
||||
}
|
||||
|
||||
return string(captured)
|
||||
}
|
||||
|
||||
// TestLoadWarnsReadableConfigWithoutS3Credentials checks that a config
|
||||
// file others can read, holding no S3 credentials, is warned about
|
||||
// without a claim that it holds them.
|
||||
//
|
||||
//nolint:paralleltest // loadReadableConfig replaces os.Stderr
|
||||
func TestLoadWarnsReadableConfigWithoutS3Credentials(t *testing.T) {
|
||||
stderr := loadReadableConfig(t, `
|
||||
storage_url: file:///var/backups/vaultik
|
||||
snapshots:
|
||||
home:
|
||||
paths:
|
||||
- /home
|
||||
`)
|
||||
|
||||
if !strings.Contains(stderr, "Config file is readable by others") {
|
||||
t.Errorf("expected a warning that the file is readable by others, got %q",
|
||||
stderr)
|
||||
}
|
||||
|
||||
if strings.Contains(stderr, "S3 credentials") {
|
||||
t.Errorf("warning names S3 credentials the file does not set: %q", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadWarnsReadableConfigWithS3Credentials checks that a config file
|
||||
// others can read and that sets S3 credentials, as values or as ${ENV:...}
|
||||
// references, is warned about as one that may contain them.
|
||||
//
|
||||
//nolint:paralleltest // loadReadableConfig replaces os.Stderr
|
||||
func TestLoadWarnsReadableConfigWithS3Credentials(t *testing.T) {
|
||||
t.Setenv("VAULTIK_TEST_ACCESS_KEY_ID", "test-access-key")
|
||||
t.Setenv("VAULTIK_TEST_SECRET_ACCESS_KEY", "test-secret-key")
|
||||
|
||||
configs := map[string]string{
|
||||
"values": `
|
||||
storage_url: s3://bucket/prefix?endpoint=s3.example.com
|
||||
s3:
|
||||
access_key_id: test-access-key
|
||||
secret_access_key: test-secret-key
|
||||
snapshots:
|
||||
home:
|
||||
paths:
|
||||
- /home
|
||||
`,
|
||||
"references": `
|
||||
storage_url: s3://bucket/prefix?endpoint=s3.example.com
|
||||
s3:
|
||||
access_key_id: ${ENV:VAULTIK_TEST_ACCESS_KEY_ID}
|
||||
secret_access_key: ${ENV:VAULTIK_TEST_SECRET_ACCESS_KEY}
|
||||
snapshots:
|
||||
home:
|
||||
paths:
|
||||
- /home
|
||||
`,
|
||||
}
|
||||
|
||||
for name, configYAML := range configs {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
stderr := loadReadableConfig(t, configYAML)
|
||||
|
||||
if !strings.Contains(stderr,
|
||||
"Config file is readable by others and may contain S3 credentials") {
|
||||
t.Errorf("expected a warning naming the S3 credentials, got %q",
|
||||
stderr)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+27
-4
@@ -1,11 +1,18 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
)
|
||||
|
||||
var (
|
||||
errSizeType = errors.New("size must be a number or string")
|
||||
errSizeTooLarge = errors.New("size exceeds maximum supported value")
|
||||
)
|
||||
|
||||
// Size represents a byte size that can be specified in configuration files.
|
||||
// It can unmarshal from both numeric values (interpreted as bytes) and
|
||||
// human-readable strings like "10MB", "2.5GB", or "1TB".
|
||||
@@ -14,18 +21,23 @@ type Size int64
|
||||
// UnmarshalYAML implements yaml.Unmarshaler for Size, allowing it to be
|
||||
// parsed from YAML configuration files. It accepts both numeric values
|
||||
// (interpreted as bytes) and string values with units (e.g., "10MB").
|
||||
func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
func (s *Size) UnmarshalYAML(unmarshal func(any) error) error {
|
||||
// Try to unmarshal as int64 first
|
||||
var intVal int64
|
||||
if err := unmarshal(&intVal); err == nil {
|
||||
|
||||
err := unmarshal(&intVal)
|
||||
if err == nil {
|
||||
*s = Size(intVal)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Try to unmarshal as string
|
||||
var strVal string
|
||||
if err := unmarshal(&strVal); err != nil {
|
||||
return fmt.Errorf("size must be a number or string")
|
||||
|
||||
err = unmarshal(&strVal)
|
||||
if err != nil {
|
||||
return errSizeType
|
||||
}
|
||||
|
||||
// Parse the string using go-humanize
|
||||
@@ -34,7 +46,12 @@ func (s *Size) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
return fmt.Errorf("invalid size format: %w", err)
|
||||
}
|
||||
|
||||
if bytes > math.MaxInt64 {
|
||||
return fmt.Errorf("%w: %s", errSizeTooLarge, strVal)
|
||||
}
|
||||
|
||||
*s = Size(bytes)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -49,6 +66,7 @@ func (s Size) Int64() int64 {
|
||||
// For example, 1048576 bytes would be formatted as "1.0 MB".
|
||||
// This implements the fmt.Stringer interface.
|
||||
func (s Size) String() string {
|
||||
//nolint:gosec // G115: sizes are non-negative by construction
|
||||
return humanize.Bytes(uint64(s))
|
||||
}
|
||||
|
||||
@@ -58,5 +76,10 @@ func ParseSize(s string) (Size, error) {
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid size format: %w", err)
|
||||
}
|
||||
|
||||
if bytes > math.MaxInt64 {
|
||||
return 0, fmt.Errorf("%w: %s", errSizeTooLarge, s)
|
||||
}
|
||||
|
||||
return Size(bytes), nil
|
||||
}
|
||||
|
||||
@@ -1,209 +0,0 @@
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"filippo.io/age"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// Encryptor provides thread-safe encryption using the age encryption library.
|
||||
// It supports encrypting data for multiple recipients simultaneously, allowing
|
||||
// any of the corresponding private keys to decrypt the data. This is useful
|
||||
// for backup scenarios where multiple parties should be able to decrypt the data.
|
||||
type Encryptor struct {
|
||||
recipients []age.Recipient
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewEncryptor creates a new encryptor with the given age public keys.
|
||||
// Each public key should be a valid age X25519 recipient string (e.g., "age1...")
|
||||
// At least one recipient must be provided. Returns an error if any of the
|
||||
// public keys are invalid or if no recipients are specified.
|
||||
func NewEncryptor(publicKeys []string) (*Encryptor, error) {
|
||||
if len(publicKeys) == 0 {
|
||||
return nil, fmt.Errorf("at least one recipient is required")
|
||||
}
|
||||
|
||||
recipients := make([]age.Recipient, 0, len(publicKeys))
|
||||
for _, key := range publicKeys {
|
||||
recipient, err := age.ParseX25519Recipient(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing age recipient %s: %w", key, err)
|
||||
}
|
||||
recipients = append(recipients, recipient)
|
||||
}
|
||||
|
||||
return &Encryptor{
|
||||
recipients: recipients,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Encrypt encrypts data using age encryption for all configured recipients.
|
||||
// The encrypted data can be decrypted by any of the corresponding private keys.
|
||||
// This method is suitable for small to medium amounts of data that fit in memory.
|
||||
// For large data streams, use EncryptStream or EncryptWriter instead.
|
||||
func (e *Encryptor) Encrypt(data []byte) ([]byte, error) {
|
||||
e.mu.RLock()
|
||||
recipients := e.recipients
|
||||
e.mu.RUnlock()
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
// Create encrypted writer for all recipients
|
||||
w, err := age.Encrypt(&buf, recipients...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating encrypted writer: %w", err)
|
||||
}
|
||||
|
||||
// Write data
|
||||
if _, err := w.Write(data); err != nil {
|
||||
return nil, fmt.Errorf("writing encrypted data: %w", err)
|
||||
}
|
||||
|
||||
// Close to flush
|
||||
if err := w.Close(); err != nil {
|
||||
return nil, fmt.Errorf("closing encrypted writer: %w", err)
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// EncryptStream encrypts data from reader to writer using age encryption.
|
||||
// This method is suitable for encrypting large files or streams as it processes
|
||||
// data in a streaming fashion without loading everything into memory.
|
||||
// The encrypted data is written directly to the destination writer.
|
||||
func (e *Encryptor) EncryptStream(dst io.Writer, src io.Reader) error {
|
||||
e.mu.RLock()
|
||||
recipients := e.recipients
|
||||
e.mu.RUnlock()
|
||||
|
||||
// Create encrypted writer for all recipients
|
||||
w, err := age.Encrypt(dst, recipients...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating encrypted writer: %w", err)
|
||||
}
|
||||
|
||||
// Copy data
|
||||
if _, err := io.Copy(w, src); err != nil {
|
||||
return fmt.Errorf("copying encrypted data: %w", err)
|
||||
}
|
||||
|
||||
// Close to flush
|
||||
if err := w.Close(); err != nil {
|
||||
return fmt.Errorf("closing encrypted writer: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncryptWriter creates a writer that encrypts data written to it.
|
||||
// All data written to the returned WriteCloser will be encrypted and written
|
||||
// to the destination writer. The caller must call Close() on the returned
|
||||
// writer to ensure all encrypted data is properly flushed and finalized.
|
||||
// This is useful for integrating encryption into existing writer-based pipelines.
|
||||
func (e *Encryptor) EncryptWriter(dst io.Writer) (io.WriteCloser, error) {
|
||||
e.mu.RLock()
|
||||
recipients := e.recipients
|
||||
e.mu.RUnlock()
|
||||
|
||||
// Create encrypted writer for all recipients
|
||||
w, err := age.Encrypt(dst, recipients...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating encrypted writer: %w", err)
|
||||
}
|
||||
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// UpdateRecipients updates the recipients for future encryption operations.
|
||||
// This method is thread-safe and can be called while other encryption operations
|
||||
// are in progress. Existing encryption operations will continue with the old
|
||||
// recipients. At least one recipient must be provided. Returns an error if any
|
||||
// of the public keys are invalid or if no recipients are specified.
|
||||
func (e *Encryptor) UpdateRecipients(publicKeys []string) error {
|
||||
if len(publicKeys) == 0 {
|
||||
return fmt.Errorf("at least one recipient is required")
|
||||
}
|
||||
|
||||
recipients := make([]age.Recipient, 0, len(publicKeys))
|
||||
for _, key := range publicKeys {
|
||||
recipient, err := age.ParseX25519Recipient(key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing age recipient %s: %w", key, err)
|
||||
}
|
||||
recipients = append(recipients, recipient)
|
||||
}
|
||||
|
||||
e.mu.Lock()
|
||||
e.recipients = recipients
|
||||
e.mu.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Decryptor provides thread-safe decryption using the age encryption library.
|
||||
// It uses a private key to decrypt data that was encrypted for the corresponding
|
||||
// public key.
|
||||
type Decryptor struct {
|
||||
identity age.Identity
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewDecryptor creates a new decryptor with the given age private key.
|
||||
// The private key should be a valid age X25519 identity string.
|
||||
// Returns an error if the private key is invalid.
|
||||
func NewDecryptor(privateKey string) (*Decryptor, error) {
|
||||
identity, err := age.ParseX25519Identity(privateKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing age identity: %w", err)
|
||||
}
|
||||
|
||||
return &Decryptor{
|
||||
identity: identity,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Decrypt decrypts data using age decryption.
|
||||
// This method is suitable for small to medium amounts of data that fit in memory.
|
||||
// For large data streams, use DecryptStream instead.
|
||||
func (d *Decryptor) Decrypt(data []byte) ([]byte, error) {
|
||||
d.mu.RLock()
|
||||
identity := d.identity
|
||||
d.mu.RUnlock()
|
||||
|
||||
r, err := age.Decrypt(bytes.NewReader(data), identity)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating decrypted reader: %w", err)
|
||||
}
|
||||
|
||||
decrypted, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading decrypted data: %w", err)
|
||||
}
|
||||
|
||||
return decrypted, nil
|
||||
}
|
||||
|
||||
// DecryptStream returns a reader that decrypts data from the provided reader.
|
||||
// This method is suitable for decrypting large files or streams as it processes
|
||||
// data in a streaming fashion without loading everything into memory.
|
||||
// The caller should close the input reader when done.
|
||||
func (d *Decryptor) DecryptStream(src io.Reader) (io.Reader, error) {
|
||||
d.mu.RLock()
|
||||
identity := d.identity
|
||||
d.mu.RUnlock()
|
||||
|
||||
r, err := age.Decrypt(src, identity)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating decrypted reader: %w", err)
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Module exports the crypto module for fx dependency injection.
|
||||
var Module = fx.Module("crypto")
|
||||
@@ -1,157 +0,0 @@
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
)
|
||||
|
||||
func TestEncryptor(t *testing.T) {
|
||||
// Generate a test key pair
|
||||
identity, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate identity: %v", err)
|
||||
}
|
||||
|
||||
publicKey := identity.Recipient().String()
|
||||
|
||||
// Create encryptor
|
||||
enc, err := NewEncryptor([]string{publicKey})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create encryptor: %v", err)
|
||||
}
|
||||
|
||||
// Test data
|
||||
plaintext := []byte("Hello, World! This is a test message.")
|
||||
|
||||
// Encrypt
|
||||
ciphertext, err := enc.Encrypt(plaintext)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to encrypt: %v", err)
|
||||
}
|
||||
|
||||
// Verify it's actually encrypted (should be larger and different)
|
||||
if bytes.Equal(plaintext, ciphertext) {
|
||||
t.Error("ciphertext equals plaintext")
|
||||
}
|
||||
|
||||
// Decrypt to verify
|
||||
r, err := age.Decrypt(bytes.NewReader(ciphertext), identity)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to decrypt: %v", err)
|
||||
}
|
||||
|
||||
var decrypted bytes.Buffer
|
||||
if _, err := decrypted.ReadFrom(r); err != nil {
|
||||
t.Fatalf("failed to read decrypted data: %v", err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(plaintext, decrypted.Bytes()) {
|
||||
t.Error("decrypted data doesn't match original")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptorMultipleRecipients(t *testing.T) {
|
||||
// Generate three test key pairs
|
||||
identity1, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate identity1: %v", err)
|
||||
}
|
||||
identity2, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate identity2: %v", err)
|
||||
}
|
||||
identity3, err := age.GenerateX25519Identity()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to generate identity3: %v", err)
|
||||
}
|
||||
|
||||
publicKeys := []string{
|
||||
identity1.Recipient().String(),
|
||||
identity2.Recipient().String(),
|
||||
identity3.Recipient().String(),
|
||||
}
|
||||
|
||||
// Create encryptor with multiple recipients
|
||||
enc, err := NewEncryptor(publicKeys)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create encryptor: %v", err)
|
||||
}
|
||||
|
||||
// Test data
|
||||
plaintext := []byte("Secret message for multiple recipients")
|
||||
|
||||
// Encrypt
|
||||
ciphertext, err := enc.Encrypt(plaintext)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to encrypt: %v", err)
|
||||
}
|
||||
|
||||
// Verify each recipient can decrypt
|
||||
identities := []age.Identity{identity1, identity2, identity3}
|
||||
for i, identity := range identities {
|
||||
r, err := age.Decrypt(bytes.NewReader(ciphertext), identity)
|
||||
if err != nil {
|
||||
t.Fatalf("recipient %d failed to decrypt: %v", i+1, err)
|
||||
}
|
||||
|
||||
var decrypted bytes.Buffer
|
||||
if _, err := decrypted.ReadFrom(r); err != nil {
|
||||
t.Fatalf("recipient %d failed to read decrypted data: %v", i+1, err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(plaintext, decrypted.Bytes()) {
|
||||
t.Errorf("recipient %d: decrypted data doesn't match original", i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptorUpdateRecipients(t *testing.T) {
|
||||
// Generate two identities
|
||||
identity1, _ := age.GenerateX25519Identity()
|
||||
identity2, _ := age.GenerateX25519Identity()
|
||||
|
||||
publicKey1 := identity1.Recipient().String()
|
||||
publicKey2 := identity2.Recipient().String()
|
||||
|
||||
// Create encryptor with first key
|
||||
enc, err := NewEncryptor([]string{publicKey1})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create encryptor: %v", err)
|
||||
}
|
||||
|
||||
// Encrypt with first key
|
||||
plaintext := []byte("test data")
|
||||
ciphertext1, err := enc.Encrypt(plaintext)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to encrypt: %v", err)
|
||||
}
|
||||
|
||||
// Update to second key
|
||||
if err := enc.UpdateRecipients([]string{publicKey2}); err != nil {
|
||||
t.Fatalf("failed to update recipients: %v", err)
|
||||
}
|
||||
|
||||
// Encrypt with second key
|
||||
ciphertext2, err := enc.Encrypt(plaintext)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to encrypt: %v", err)
|
||||
}
|
||||
|
||||
// First ciphertext should only decrypt with first identity
|
||||
if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity1); err != nil {
|
||||
t.Error("failed to decrypt with identity1")
|
||||
}
|
||||
if _, err := age.Decrypt(bytes.NewReader(ciphertext1), identity2); err == nil {
|
||||
t.Error("should not decrypt with identity2")
|
||||
}
|
||||
|
||||
// Second ciphertext should only decrypt with second identity
|
||||
if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity2); err != nil {
|
||||
t.Error("failed to decrypt with identity2")
|
||||
}
|
||||
if _, err := age.Decrypt(bytes.NewReader(ciphertext2), identity1); err == nil {
|
||||
t.Error("should not decrypt with identity1")
|
||||
}
|
||||
}
|
||||
@@ -3,18 +3,25 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// BlobChunkRepository provides access to the blob_chunks table, which maps
|
||||
// blobs to the chunks they contain (with offset and length).
|
||||
type BlobChunkRepository struct {
|
||||
db *DB
|
||||
}
|
||||
|
||||
// NewBlobChunkRepository creates a BlobChunkRepository backed by db.
|
||||
func NewBlobChunkRepository(db *DB) *BlobChunkRepository {
|
||||
return &BlobChunkRepository{db: db}
|
||||
}
|
||||
|
||||
func (r *BlobChunkRepository) Create(ctx context.Context, tx *sql.Tx, bc *BlobChunk) error {
|
||||
// Create inserts a blob_chunks row, using tx when non-nil.
|
||||
func (r *BlobChunkRepository) Create(
|
||||
ctx context.Context, tx *sql.Tx, bc *BlobChunk,
|
||||
) error {
|
||||
query := `
|
||||
INSERT INTO blob_chunks (blob_id, chunk_hash, offset, length)
|
||||
VALUES (?, ?, ?, ?)
|
||||
@@ -34,7 +41,11 @@ func (r *BlobChunkRepository) Create(ctx context.Context, tx *sql.Tx, bc *BlobCh
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([]*BlobChunk, error) {
|
||||
// GetByBlobID returns all chunks contained in the given blob, ordered by
|
||||
// their offset within the blob.
|
||||
func (r *BlobChunkRepository) GetByBlobID(
|
||||
ctx context.Context, blobID string,
|
||||
) ([]*BlobChunk, error) {
|
||||
query := `
|
||||
SELECT blob_id, chunk_hash, offset, length
|
||||
FROM blob_chunks
|
||||
@@ -46,22 +57,30 @@ func (r *BlobChunkRepository) GetByBlobID(ctx context.Context, blobID string) ([
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying blob chunks: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var blobChunks []*BlobChunk
|
||||
|
||||
for rows.Next() {
|
||||
var bc BlobChunk
|
||||
|
||||
err := rows.Scan(&bc.BlobID, &bc.ChunkHash, &bc.Offset, &bc.Length)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning blob chunk: %w", err)
|
||||
}
|
||||
|
||||
blobChunks = append(blobChunks, &bc)
|
||||
}
|
||||
|
||||
return blobChunks, rows.Err()
|
||||
}
|
||||
|
||||
func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash string) (*BlobChunk, error) {
|
||||
// GetByChunkHash returns one blob_chunks row containing the given chunk,
|
||||
// or nil if the chunk is not packed in any blob.
|
||||
func (r *BlobChunkRepository) GetByChunkHash(
|
||||
ctx context.Context, chunkHash string,
|
||||
) (*BlobChunk, error) {
|
||||
query := `
|
||||
SELECT blob_id, chunk_hash, offset, length
|
||||
FROM blob_chunks
|
||||
@@ -70,7 +89,9 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
|
||||
`
|
||||
|
||||
LogSQL("GetByChunkHash", query, chunkHash)
|
||||
|
||||
var bc BlobChunk
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, chunkHash).Scan(
|
||||
&bc.BlobID,
|
||||
&bc.ChunkHash,
|
||||
@@ -78,21 +99,27 @@ func (r *BlobChunkRepository) GetByChunkHash(ctx context.Context, chunkHash stri
|
||||
&bc.Length,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
LogSQL("GetByChunkHash", "No rows found", chunkHash)
|
||||
return nil, nil
|
||||
|
||||
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
LogSQL("GetByChunkHash", "Error", chunkHash, err)
|
||||
|
||||
return nil, fmt.Errorf("querying blob chunk: %w", err)
|
||||
}
|
||||
|
||||
LogSQL("GetByChunkHash", "Found blob", chunkHash, "blob", bc.BlobID)
|
||||
|
||||
return &bc, nil
|
||||
}
|
||||
|
||||
// GetByChunkHashTx retrieves a blob chunk within a transaction
|
||||
func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx, chunkHash string) (*BlobChunk, error) {
|
||||
func (r *BlobChunkRepository) GetByChunkHashTx(
|
||||
ctx context.Context, tx *sql.Tx, chunkHash string,
|
||||
) (*BlobChunk, error) {
|
||||
query := `
|
||||
SELECT blob_id, chunk_hash, offset, length
|
||||
FROM blob_chunks
|
||||
@@ -101,7 +128,9 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
|
||||
`
|
||||
|
||||
LogSQL("GetByChunkHashTx", query, chunkHash)
|
||||
|
||||
var bc BlobChunk
|
||||
|
||||
err := tx.QueryRowContext(ctx, query, chunkHash).Scan(
|
||||
&bc.BlobID,
|
||||
&bc.ChunkHash,
|
||||
@@ -109,42 +138,51 @@ func (r *BlobChunkRepository) GetByChunkHashTx(ctx context.Context, tx *sql.Tx,
|
||||
&bc.Length,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
LogSQL("GetByChunkHashTx", "No rows found", chunkHash)
|
||||
return nil, nil
|
||||
|
||||
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
LogSQL("GetByChunkHashTx", "Error", chunkHash, err)
|
||||
|
||||
return nil, fmt.Errorf("querying blob chunk: %w", err)
|
||||
}
|
||||
|
||||
LogSQL("GetByChunkHashTx", "Found blob", chunkHash, "blob", bc.BlobID)
|
||||
|
||||
return &bc, nil
|
||||
}
|
||||
|
||||
// DeleteOrphaned deletes blob_chunks entries where either the blob or chunk no longer exists
|
||||
// DeleteOrphaned deletes blob_chunks entries where either the blob or the
|
||||
// chunk no longer exists.
|
||||
func (r *BlobChunkRepository) DeleteOrphaned(ctx context.Context) error {
|
||||
// Delete blob_chunks where the blob doesn't exist
|
||||
query1 := `
|
||||
DELETE FROM blob_chunks
|
||||
DELETE FROM blob_chunks
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM blobs
|
||||
SELECT 1 FROM blobs
|
||||
WHERE blobs.id = blob_chunks.blob_id
|
||||
)
|
||||
`
|
||||
if _, err := r.db.ExecWithLog(ctx, query1); err != nil {
|
||||
|
||||
_, err := r.db.ExecWithLog(ctx, query1)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting blob_chunks with missing blobs: %w", err)
|
||||
}
|
||||
|
||||
// Delete blob_chunks where the chunk doesn't exist
|
||||
query2 := `
|
||||
DELETE FROM blob_chunks
|
||||
DELETE FROM blob_chunks
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM chunks
|
||||
SELECT 1 FROM chunks
|
||||
WHERE chunks.chunk_hash = blob_chunks.chunk_hash
|
||||
)
|
||||
`
|
||||
if _, err := r.db.ExecWithLog(ctx, query2); err != nil {
|
||||
|
||||
_, err = r.db.ExecWithLog(ctx, query2)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting blob_chunks with missing chunks: %w", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package database
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -6,71 +6,107 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func TestBlobChunkRepository(t *testing.T) {
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
// Chunk hashes used across the blob_chunks tests.
|
||||
const (
|
||||
chunk1Hash = "chunk1"
|
||||
chunk2Hash = "chunk2"
|
||||
chunk3Hash = "chunk3"
|
||||
)
|
||||
|
||||
// mustCreateChunks registers the given chunk hashes (1024 bytes each).
|
||||
func mustCreateChunks(
|
||||
t *testing.T,
|
||||
repos *database.Repositories,
|
||||
hashes ...types.ChunkHash,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
// Create blob first
|
||||
blob := &Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("blob1-hash"),
|
||||
CreatedTS: time.Now(),
|
||||
}
|
||||
err := repos.Blobs.Create(ctx, nil, blob)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob: %v", err)
|
||||
}
|
||||
|
||||
// Create chunks
|
||||
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
|
||||
for _, chunkHash := range chunks {
|
||||
chunk := &Chunk{
|
||||
for _, chunkHash := range hashes {
|
||||
chunk := &database.Chunk{
|
||||
ChunkHash: chunkHash,
|
||||
Size: 1024,
|
||||
}
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
|
||||
err := repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mustCreateBlob creates a blob row with the given hash.
|
||||
func mustCreateBlob(
|
||||
t *testing.T,
|
||||
repos *database.Repositories,
|
||||
hash types.BlobHash,
|
||||
) *database.Blob {
|
||||
t.Helper()
|
||||
|
||||
blob := &database.Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: hash,
|
||||
CreatedTS: time.Now(),
|
||||
}
|
||||
|
||||
err := repos.Blobs.Create(context.Background(), nil, blob)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob %s: %v", hash, err)
|
||||
}
|
||||
|
||||
return blob
|
||||
}
|
||||
|
||||
func TestBlobChunkRepository(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
blob := mustCreateBlob(t, repos, "blob1-hash")
|
||||
mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash)
|
||||
|
||||
// Test Create
|
||||
bc1 := &BlobChunk{
|
||||
bc1 := &database.BlobChunk{
|
||||
BlobID: blob.ID,
|
||||
ChunkHash: types.ChunkHash("chunk1"),
|
||||
ChunkHash: types.ChunkHash(chunk1Hash),
|
||||
Offset: 0,
|
||||
Length: 1024,
|
||||
}
|
||||
|
||||
err = repos.BlobChunks.Create(ctx, nil, bc1)
|
||||
err := repos.BlobChunks.Create(ctx, nil, bc1)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob chunk: %v", err)
|
||||
}
|
||||
|
||||
// Add more chunks to the same blob
|
||||
bc2 := &BlobChunk{
|
||||
bc2 := &database.BlobChunk{
|
||||
BlobID: blob.ID,
|
||||
ChunkHash: types.ChunkHash("chunk2"),
|
||||
ChunkHash: types.ChunkHash(chunk2Hash),
|
||||
Offset: 1024,
|
||||
Length: 2048,
|
||||
}
|
||||
|
||||
err = repos.BlobChunks.Create(ctx, nil, bc2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create second blob chunk: %v", err)
|
||||
}
|
||||
|
||||
bc3 := &BlobChunk{
|
||||
bc3 := &database.BlobChunk{
|
||||
BlobID: blob.ID,
|
||||
ChunkHash: types.ChunkHash("chunk3"),
|
||||
ChunkHash: types.ChunkHash(chunk3Hash),
|
||||
Offset: 3072,
|
||||
Length: 512,
|
||||
}
|
||||
|
||||
err = repos.BlobChunks.Create(ctx, nil, bc3)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create third blob chunk: %v", err)
|
||||
@@ -81,6 +117,7 @@ func TestBlobChunkRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(blobChunks) != 3 {
|
||||
t.Errorf("expected 3 chunks, got %d", len(blobChunks))
|
||||
}
|
||||
@@ -89,92 +126,97 @@ func TestBlobChunkRepository(t *testing.T) {
|
||||
expectedOffsets := []int64{0, 1024, 3072}
|
||||
for i, bc := range blobChunks {
|
||||
if bc.Offset != expectedOffsets[i] {
|
||||
t.Errorf("wrong chunk order: expected offset %d, got %d", expectedOffsets[i], bc.Offset)
|
||||
t.Errorf("wrong chunk order: expected offset %d, got %d",
|
||||
expectedOffsets[i], bc.Offset)
|
||||
}
|
||||
}
|
||||
|
||||
// Test GetByChunkHash
|
||||
bc, err := repos.BlobChunks.GetByChunkHash(ctx, "chunk2")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob chunk by chunk hash: %v", err)
|
||||
}
|
||||
if bc == nil {
|
||||
t.Fatal("expected blob chunk, got nil")
|
||||
}
|
||||
if bc.BlobID != blob.ID {
|
||||
t.Errorf("wrong blob ID: expected %s, got %s", blob.ID, bc.BlobID)
|
||||
}
|
||||
if bc.Offset != 1024 {
|
||||
t.Errorf("wrong offset: expected 1024, got %d", bc.Offset)
|
||||
}
|
||||
|
||||
// Test duplicate insert (should fail due to primary key constraint)
|
||||
err = repos.BlobChunks.Create(ctx, nil, bc1)
|
||||
if err == nil {
|
||||
t.Fatal("duplicate blob_chunk insert should fail due to primary key constraint")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "UNIQUE") && !strings.Contains(err.Error(), "constraint") {
|
||||
|
||||
if !strings.Contains(err.Error(), "UNIQUE") &&
|
||||
!strings.Contains(err.Error(), "constraint") {
|
||||
t.Fatalf("expected constraint error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobChunkRepositoryGetByChunkHash(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
blob := mustCreateBlob(t, repos, "blob-gbch-hash")
|
||||
mustCreateChunks(t, repos, chunk2Hash)
|
||||
|
||||
bc2 := &database.BlobChunk{
|
||||
BlobID: blob.ID,
|
||||
ChunkHash: types.ChunkHash(chunk2Hash),
|
||||
Offset: 1024,
|
||||
Length: 2048,
|
||||
}
|
||||
|
||||
err := repos.BlobChunks.Create(ctx, nil, bc2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob chunk: %v", err)
|
||||
}
|
||||
|
||||
// Test GetByChunkHash
|
||||
bc, err := repos.BlobChunks.GetByChunkHash(ctx, chunk2Hash)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob chunk by chunk hash: %v", err)
|
||||
}
|
||||
|
||||
if bc == nil {
|
||||
t.Fatal("expected blob chunk, got nil")
|
||||
}
|
||||
|
||||
if bc.BlobID != blob.ID {
|
||||
t.Errorf("wrong blob ID: expected %s, got %s", blob.ID, bc.BlobID)
|
||||
}
|
||||
|
||||
if bc.Offset != 1024 {
|
||||
t.Errorf("wrong offset: expected 1024, got %d", bc.Offset)
|
||||
}
|
||||
|
||||
// Test non-existent chunk
|
||||
bc, err = repos.BlobChunks.GetByChunkHash(ctx, "nonexistent")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if bc != nil {
|
||||
t.Error("expected nil for non-existent chunk")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Create blobs
|
||||
blob1 := &Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("blob1-hash"),
|
||||
CreatedTS: time.Now(),
|
||||
}
|
||||
blob2 := &Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("blob2-hash"),
|
||||
CreatedTS: time.Now(),
|
||||
}
|
||||
|
||||
err := repos.Blobs.Create(ctx, nil, blob1)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob1: %v", err)
|
||||
}
|
||||
err = repos.Blobs.Create(ctx, nil, blob2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob2: %v", err)
|
||||
}
|
||||
|
||||
// Create chunks
|
||||
chunkHashes := []types.ChunkHash{"chunk1", "chunk2", "chunk3"}
|
||||
for _, chunkHash := range chunkHashes {
|
||||
chunk := &Chunk{
|
||||
ChunkHash: chunkHash,
|
||||
Size: 1024,
|
||||
}
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||
}
|
||||
}
|
||||
blob1 := mustCreateBlob(t, repos, "blob1-hash")
|
||||
blob2 := mustCreateBlob(t, repos, "blob2-hash")
|
||||
mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash)
|
||||
|
||||
// Create chunks across multiple blobs
|
||||
// Some chunks are shared between blobs (deduplication scenario)
|
||||
blobChunks := []BlobChunk{
|
||||
{BlobID: blob1.ID, ChunkHash: types.ChunkHash("chunk1"), Offset: 0, Length: 1024},
|
||||
{BlobID: blob1.ID, ChunkHash: types.ChunkHash("chunk2"), Offset: 1024, Length: 1024},
|
||||
{BlobID: blob2.ID, ChunkHash: types.ChunkHash("chunk2"), Offset: 0, Length: 1024}, // chunk2 is shared
|
||||
{BlobID: blob2.ID, ChunkHash: types.ChunkHash("chunk3"), Offset: 1024, Length: 1024},
|
||||
blobChunks := []database.BlobChunk{
|
||||
{BlobID: blob1.ID, ChunkHash: chunk1Hash, Offset: 0, Length: 1024},
|
||||
{BlobID: blob1.ID, ChunkHash: chunk2Hash, Offset: 1024, Length: 1024},
|
||||
// chunk2 is shared between the blobs
|
||||
{BlobID: blob2.ID, ChunkHash: chunk2Hash, Offset: 0, Length: 1024},
|
||||
{BlobID: blob2.ID, ChunkHash: chunk3Hash, Offset: 1024, Length: 1024},
|
||||
}
|
||||
|
||||
for _, bc := range blobChunks {
|
||||
@@ -189,6 +231,7 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob1 chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(chunks) != 2 {
|
||||
t.Errorf("expected 2 chunks for blob1, got %d", len(chunks))
|
||||
}
|
||||
@@ -198,15 +241,17 @@ func TestBlobChunkRepositoryMultipleBlobs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob2 chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(chunks) != 2 {
|
||||
t.Errorf("expected 2 chunks for blob2, got %d", len(chunks))
|
||||
}
|
||||
|
||||
// Verify shared chunk
|
||||
bc, err := repos.BlobChunks.GetByChunkHash(ctx, "chunk2")
|
||||
bc, err := repos.BlobChunks.GetByChunkHash(ctx, chunk2Hash)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get shared chunk: %v", err)
|
||||
}
|
||||
|
||||
if bc == nil {
|
||||
t.Fatal("expected shared chunk, got nil")
|
||||
}
|
||||
|
||||
+155
-75
@@ -3,31 +3,39 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
// BlobRepository provides access to the blobs table, which tracks the
|
||||
// packed, encrypted storage units uploaded to the destination.
|
||||
type BlobRepository struct {
|
||||
db *DB
|
||||
}
|
||||
|
||||
// NewBlobRepository creates a BlobRepository backed by db.
|
||||
func NewBlobRepository(db *DB) *BlobRepository {
|
||||
return &BlobRepository{db: db}
|
||||
}
|
||||
|
||||
// Create inserts a blob row, using tx when non-nil.
|
||||
func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) error {
|
||||
query := `
|
||||
INSERT INTO blobs (id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts)
|
||||
INSERT INTO blobs (id, blob_hash, created_ts, finished_ts,
|
||||
uncompressed_size, compressed_size, uploaded_ts)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
var finishedTS, uploadedTS *int64
|
||||
|
||||
if blob.FinishedTS != nil {
|
||||
ts := blob.FinishedTS.Unix()
|
||||
finishedTS = &ts
|
||||
}
|
||||
|
||||
if blob.UploadedTS != nil {
|
||||
ts := blob.UploadedTS.Unix()
|
||||
uploadedTS = &ts
|
||||
@@ -49,89 +57,81 @@ func (r *BlobRepository) Create(ctx context.Context, tx *sql.Tx, blob *Blob) err
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetByHash returns the blob with the given content hash, or nil if no
|
||||
// such blob exists.
|
||||
func (r *BlobRepository) GetByHash(ctx context.Context, hash string) (*Blob, error) {
|
||||
query := `
|
||||
SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts
|
||||
FROM blobs
|
||||
WHERE blob_hash = ?
|
||||
`
|
||||
|
||||
var blob Blob
|
||||
var createdTSUnix int64
|
||||
var finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, hash).Scan(
|
||||
&blob.ID,
|
||||
&blob.Hash,
|
||||
&createdTSUnix,
|
||||
&finishedTSUnix,
|
||||
&blob.UncompressedSize,
|
||||
&blob.CompressedSize,
|
||||
&uploadedTSUnix,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying blob: %w", err)
|
||||
}
|
||||
|
||||
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
|
||||
if finishedTSUnix.Valid {
|
||||
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
|
||||
blob.FinishedTS = &ts
|
||||
}
|
||||
if uploadedTSUnix.Valid {
|
||||
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
|
||||
blob.UploadedTS = &ts
|
||||
}
|
||||
return &blob, nil
|
||||
return r.getOne(ctx, "blob_hash", hash)
|
||||
}
|
||||
|
||||
// GetByID retrieves a blob by its ID
|
||||
func (r *BlobRepository) GetByID(ctx context.Context, id string) (*Blob, error) {
|
||||
return r.getOne(ctx, "id", id)
|
||||
}
|
||||
|
||||
// GetAll returns every blob row keyed by blob ID. Useful at restore
|
||||
// start to translate the per-chunk blob_id references in chunkToBlobMap
|
||||
// into blob hashes without doing one GetByID query per chunk.
|
||||
func (r *BlobRepository) GetAll(ctx context.Context) (map[string]*Blob, error) {
|
||||
query := `
|
||||
SELECT id, blob_hash, created_ts, finished_ts, uncompressed_size, compressed_size, uploaded_ts
|
||||
SELECT id, blob_hash, created_ts, finished_ts,
|
||||
uncompressed_size, compressed_size, uploaded_ts
|
||||
FROM blobs
|
||||
WHERE id = ?
|
||||
`
|
||||
|
||||
var blob Blob
|
||||
var createdTSUnix int64
|
||||
var finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, id).Scan(
|
||||
&blob.ID,
|
||||
&blob.Hash,
|
||||
&createdTSUnix,
|
||||
&finishedTSUnix,
|
||||
&blob.UncompressedSize,
|
||||
&blob.CompressedSize,
|
||||
&uploadedTSUnix,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := r.db.conn.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying blob: %w", err)
|
||||
return nil, fmt.Errorf("querying blobs: %w", err)
|
||||
}
|
||||
|
||||
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
|
||||
if finishedTSUnix.Valid {
|
||||
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
|
||||
blob.FinishedTS = &ts
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
out := make(map[string]*Blob)
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
blob Blob
|
||||
createdTSUnix int64
|
||||
finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||
)
|
||||
|
||||
err := rows.Scan(
|
||||
&blob.ID,
|
||||
&blob.Hash,
|
||||
&createdTSUnix,
|
||||
&finishedTSUnix,
|
||||
&blob.UncompressedSize,
|
||||
&blob.CompressedSize,
|
||||
&uploadedTSUnix,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning blob: %w", err)
|
||||
}
|
||||
|
||||
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
|
||||
if finishedTSUnix.Valid {
|
||||
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
|
||||
blob.FinishedTS = &ts
|
||||
}
|
||||
|
||||
if uploadedTSUnix.Valid {
|
||||
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
|
||||
blob.UploadedTS = &ts
|
||||
}
|
||||
|
||||
out[blob.ID.String()] = &blob
|
||||
}
|
||||
if uploadedTSUnix.Valid {
|
||||
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
|
||||
blob.UploadedTS = &ts
|
||||
}
|
||||
return &blob, nil
|
||||
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// UpdateFinished updates a blob when it's finalized
|
||||
func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id string, hash string, uncompressedSize, compressedSize int64) error {
|
||||
func (r *BlobRepository) UpdateFinished(
|
||||
ctx context.Context,
|
||||
tx *sql.Tx,
|
||||
id string,
|
||||
hash string,
|
||||
uncompressedSize, compressedSize int64,
|
||||
) error {
|
||||
query := `
|
||||
UPDATE blobs
|
||||
SET blob_hash = ?, finished_ts = ?, uncompressed_size = ?, compressed_size = ?
|
||||
@@ -139,6 +139,7 @@ func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id stri
|
||||
`
|
||||
|
||||
now := time.Now().UTC().Unix()
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, hash, now, uncompressedSize, compressedSize, id)
|
||||
@@ -153,20 +154,26 @@ func (r *BlobRepository) UpdateFinished(ctx context.Context, tx *sql.Tx, id stri
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateUploaded marks a blob as uploaded
|
||||
func (r *BlobRepository) UpdateUploaded(ctx context.Context, tx *sql.Tx, id string) error {
|
||||
// UpdateUploaded marks a blob as uploaded and records the size of the
|
||||
// stored object as its compressed size. That size can differ from the
|
||||
// local copy's when an identical blob was already stored with another
|
||||
// compression level or other recipients.
|
||||
func (r *BlobRepository) UpdateUploaded(
|
||||
ctx context.Context, tx *sql.Tx, id string, storedSize int64,
|
||||
) error {
|
||||
query := `
|
||||
UPDATE blobs
|
||||
SET uploaded_ts = ?
|
||||
UPDATE blobs
|
||||
SET uploaded_ts = ?, compressed_size = ?
|
||||
WHERE id = ?
|
||||
`
|
||||
|
||||
now := time.Now().UTC().Unix()
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, now, id)
|
||||
_, err = tx.ExecContext(ctx, query, now, storedSize, id)
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, now, id)
|
||||
_, err = r.db.ExecWithLog(ctx, query, now, storedSize, id)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -198,3 +205,76 @@ func (r *BlobRepository) DeleteOrphaned(ctx context.Context) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteUnuploaded deletes blob rows whose upload never completed
|
||||
// (uploaded_ts IS NULL) and returns how many were removed. Their
|
||||
// blob_chunks rows are removed by the ON DELETE CASCADE foreign key.
|
||||
// A blob is only ever attached to a snapshot once its upload has been
|
||||
// recorded, so an un-uploaded blob is never referenced by a completed
|
||||
// snapshot: dropping it discards chunk rows that point at data which
|
||||
// was never stored remotely, so the affected content is re-chunked and
|
||||
// re-uploaded on the next run.
|
||||
func (r *BlobRepository) DeleteUnuploaded(ctx context.Context) (int64, error) {
|
||||
query := `DELETE FROM blobs WHERE uploaded_ts IS NULL`
|
||||
|
||||
result, err := r.db.ExecWithLog(ctx, query)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("deleting un-uploaded blobs: %w", err)
|
||||
}
|
||||
|
||||
rowsAffected, _ := result.RowsAffected()
|
||||
if rowsAffected > 0 {
|
||||
log.Debug("Deleted un-uploaded blobs", "count", rowsAffected)
|
||||
}
|
||||
|
||||
return rowsAffected, nil
|
||||
}
|
||||
|
||||
// getOne fetches a single blob row matched on the given column, or
|
||||
// (nil, nil) when no row matches.
|
||||
func (r *BlobRepository) getOne(
|
||||
ctx context.Context, column, value string,
|
||||
) (*Blob, error) {
|
||||
query := `
|
||||
SELECT id, blob_hash, created_ts, finished_ts,
|
||||
uncompressed_size, compressed_size, uploaded_ts
|
||||
FROM blobs
|
||||
WHERE ` + column + ` = ?`
|
||||
|
||||
var (
|
||||
blob Blob
|
||||
createdTSUnix int64
|
||||
finishedTSUnix, uploadedTSUnix sql.NullInt64
|
||||
)
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, value).Scan(
|
||||
&blob.ID,
|
||||
&blob.Hash,
|
||||
&createdTSUnix,
|
||||
&finishedTSUnix,
|
||||
&blob.UncompressedSize,
|
||||
&blob.CompressedSize,
|
||||
&uploadedTSUnix,
|
||||
)
|
||||
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying blob: %w", err)
|
||||
}
|
||||
|
||||
blob.CreatedTS = time.Unix(createdTSUnix, 0).UTC()
|
||||
if finishedTSUnix.Valid {
|
||||
ts := time.Unix(finishedTSUnix.Int64, 0).UTC()
|
||||
blob.FinishedTS = &ts
|
||||
}
|
||||
|
||||
if uploadedTSUnix.Valid {
|
||||
ts := time.Unix(uploadedTSUnix.Int64, 0).UTC()
|
||||
blob.UploadedTS = &ts
|
||||
}
|
||||
|
||||
return &blob, nil
|
||||
}
|
||||
|
||||
@@ -1,22 +1,25 @@
|
||||
package database
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func TestBlobRepository(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewBlobRepository(db)
|
||||
repo := database.NewBlobRepository(db)
|
||||
|
||||
// Test Create
|
||||
blob := &Blob{
|
||||
blob := &database.Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("blobhash123"),
|
||||
CreatedTS: time.Now().Truncate(time.Second),
|
||||
@@ -32,14 +35,18 @@ func TestBlobRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob: %v", err)
|
||||
}
|
||||
|
||||
if retrieved == nil {
|
||||
t.Fatal("expected blob, got nil")
|
||||
}
|
||||
|
||||
if retrieved.Hash != blob.Hash {
|
||||
t.Errorf("blob hash mismatch: got %s, want %s", retrieved.Hash, blob.Hash)
|
||||
}
|
||||
|
||||
if !retrieved.CreatedTS.Equal(blob.CreatedTS) {
|
||||
t.Errorf("created timestamp mismatch: got %v, want %v", retrieved.CreatedTS, blob.CreatedTS)
|
||||
t.Errorf("created timestamp mismatch: got %v, want %v",
|
||||
retrieved.CreatedTS, blob.CreatedTS)
|
||||
}
|
||||
|
||||
// Test GetByID
|
||||
@@ -47,26 +54,51 @@ func TestBlobRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get blob by ID: %v", err)
|
||||
}
|
||||
|
||||
if retrievedByID == nil {
|
||||
t.Fatal("expected blob, got nil")
|
||||
}
|
||||
|
||||
if retrievedByID.ID != blob.ID {
|
||||
t.Errorf("blob ID mismatch: got %s, want %s", retrievedByID.ID, blob.ID)
|
||||
}
|
||||
|
||||
// Test with second blob
|
||||
blob2 := &Blob{
|
||||
blob2 := &database.Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("blobhash456"),
|
||||
CreatedTS: time.Now().Truncate(time.Second),
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, blob2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create second blob: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobRepositoryUpdates(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := database.NewBlobRepository(db)
|
||||
|
||||
blob := &database.Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("blobhash123"),
|
||||
CreatedTS: time.Now().Truncate(time.Second),
|
||||
}
|
||||
|
||||
err := repo.Create(ctx, nil, blob)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create blob: %v", err)
|
||||
}
|
||||
|
||||
// Test UpdateFinished
|
||||
now := time.Now()
|
||||
|
||||
err = repo.UpdateFinished(ctx, nil, blob.ID.String(), blob.Hash.String(), 1000, 500)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to update blob as finished: %v", err)
|
||||
@@ -77,18 +109,21 @@ func TestBlobRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get updated blob: %v", err)
|
||||
}
|
||||
|
||||
if updated.FinishedTS == nil {
|
||||
t.Fatal("expected finished timestamp to be set")
|
||||
}
|
||||
|
||||
if updated.UncompressedSize != 1000 {
|
||||
t.Errorf("expected uncompressed size 1000, got %d", updated.UncompressedSize)
|
||||
}
|
||||
|
||||
if updated.CompressedSize != 500 {
|
||||
t.Errorf("expected compressed size 500, got %d", updated.CompressedSize)
|
||||
}
|
||||
|
||||
// Test UpdateUploaded
|
||||
err = repo.UpdateUploaded(ctx, nil, blob.ID.String())
|
||||
// Test UpdateUploaded, with a stored object larger than the local copy
|
||||
err = repo.UpdateUploaded(ctx, nil, blob.ID.String(), 600)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to update blob as uploaded: %v", err)
|
||||
}
|
||||
@@ -98,6 +133,7 @@ func TestBlobRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get uploaded blob: %v", err)
|
||||
}
|
||||
|
||||
if uploaded.UploadedTS == nil {
|
||||
t.Fatal("expected uploaded timestamp to be set")
|
||||
}
|
||||
@@ -105,16 +141,22 @@ func TestBlobRepository(t *testing.T) {
|
||||
if uploaded.UploadedTS.Before(now.Add(-1 * time.Second)) {
|
||||
t.Error("uploaded timestamp should be around test time")
|
||||
}
|
||||
|
||||
if uploaded.CompressedSize != 600 {
|
||||
t.Errorf("expected the stored size 600, got %d", uploaded.CompressedSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobRepositoryDuplicate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewBlobRepository(db)
|
||||
repo := database.NewBlobRepository(db)
|
||||
|
||||
blob := &Blob{
|
||||
blob := &database.Blob{
|
||||
ID: types.NewBlobID(),
|
||||
Hash: types.BlobHash("duplicate_blob"),
|
||||
CreatedTS: time.Now().Truncate(time.Second),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
//nolint:testpackage // inspects the unexported database connection
|
||||
package database
|
||||
|
||||
import (
|
||||
@@ -6,47 +7,39 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// TestCascadeDeleteDebug tests cascade delete with debug output
|
||||
func TestCascadeDeleteDebug(t *testing.T) {
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
// createCascadeFixtures creates a file with three chunk mappings for the
|
||||
// cascade-delete test.
|
||||
func createCascadeFixtures(t *testing.T, repos *Repositories) *File {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
// Check if foreign keys are enabled
|
||||
var fkEnabled int
|
||||
err := db.conn.QueryRow("PRAGMA foreign_keys").Scan(&fkEnabled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("Foreign keys enabled: %d", fkEnabled)
|
||||
|
||||
// Create a file
|
||||
file := &File{
|
||||
Path: "/cascade-test.txt",
|
||||
MTime: time.Now().Truncate(time.Second),
|
||||
CTime: time.Now().Truncate(time.Second),
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
}
|
||||
err = repos.Files.Create(ctx, nil, file)
|
||||
|
||||
err := repos.Files.Create(ctx, nil, file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("Created file with ID: %s", file.ID)
|
||||
|
||||
// Create chunks and file-chunk mappings
|
||||
for i := 0; i < 3; i++ {
|
||||
for i := range 3 {
|
||||
chunk := &Chunk{
|
||||
ChunkHash: types.ChunkHash(fmt.Sprintf("cascade-chunk-%d", i)),
|
||||
Size: 1024,
|
||||
}
|
||||
|
||||
err = repos.Chunks.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk: %v", err)
|
||||
@@ -57,33 +50,73 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
||||
Idx: i,
|
||||
ChunkHash: chunk.ChunkHash,
|
||||
}
|
||||
|
||||
err = repos.FileChunks.Create(ctx, nil, fc)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file chunk: %v", err)
|
||||
}
|
||||
t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash)
|
||||
|
||||
t.Logf("Created file chunk mapping: file_id=%s, idx=%d, chunk=%s",
|
||||
fc.FileID, fc.Idx, fc.ChunkHash)
|
||||
}
|
||||
|
||||
return file
|
||||
}
|
||||
|
||||
// logCascadeDebugInfo logs foreign-key state and the file_chunks table
|
||||
// definition for cascade-delete debugging.
|
||||
func logCascadeDebugInfo(ctx context.Context, t *testing.T, db *DB) {
|
||||
t.Helper()
|
||||
|
||||
// Check if foreign keys are enabled
|
||||
var fkEnabled int
|
||||
|
||||
err := db.conn.QueryRowContext(ctx, "PRAGMA foreign_keys").Scan(&fkEnabled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("Foreign keys enabled: %d", fkEnabled)
|
||||
|
||||
// Check the foreign key constraint
|
||||
var fkInfo string
|
||||
|
||||
err = db.conn.QueryRowContext(ctx, `
|
||||
SELECT sql FROM sqlite_master
|
||||
WHERE type='table' AND name='file_chunks'
|
||||
`).Scan(&fkInfo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("file_chunks table definition:\n%s", fkInfo)
|
||||
}
|
||||
|
||||
// TestCascadeDeleteDebug tests cascade delete with debug output
|
||||
func TestCascadeDeleteDebug(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repos := NewRepositories(db)
|
||||
|
||||
logCascadeDebugInfo(ctx, t, db)
|
||||
|
||||
file := createCascadeFixtures(t, repos)
|
||||
|
||||
// Verify file chunks exist
|
||||
fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("File chunks before delete: %d", len(fileChunks))
|
||||
|
||||
// Check the foreign key constraint
|
||||
var fkInfo string
|
||||
err = db.conn.QueryRow(`
|
||||
SELECT sql FROM sqlite_master
|
||||
WHERE type='table' AND name='file_chunks'
|
||||
`).Scan(&fkInfo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("file_chunks table definition:\n%s", fkInfo)
|
||||
t.Logf("File chunks before delete: %d", len(fileChunks))
|
||||
|
||||
// Delete the file
|
||||
t.Log("Deleting file...")
|
||||
|
||||
err = repos.Files.DeleteByID(ctx, nil, file.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to delete file: %v", err)
|
||||
@@ -94,6 +127,7 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if deletedFile != nil {
|
||||
t.Error("file should have been deleted")
|
||||
} else {
|
||||
@@ -105,21 +139,27 @@ func TestCascadeDeleteDebug(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("File chunks after delete: %d", len(fileChunks))
|
||||
|
||||
// Manually check the database
|
||||
var count int
|
||||
err = db.conn.QueryRow("SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID).Scan(&count)
|
||||
|
||||
err = db.conn.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM file_chunks WHERE file_id = ?", file.ID,
|
||||
).Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("Manual count of file_chunks for deleted file: %d", count)
|
||||
|
||||
if len(fileChunks) != 0 {
|
||||
t.Errorf("expected 0 file chunks after cascade delete, got %d", len(fileChunks))
|
||||
// List the remaining chunks
|
||||
for _, fc := range fileChunks {
|
||||
t.Logf("Remaining chunk: file_id=%s, idx=%d, chunk=%s", fc.FileID, fc.Idx, fc.ChunkHash)
|
||||
t.Logf("Remaining chunk: file_id=%s, idx=%d, chunk=%s",
|
||||
fc.FileID, fc.Idx, fc.ChunkHash)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,19 +4,26 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// ChunkFileRepository provides access to the chunk_files table, the
|
||||
// reverse mapping from chunks to the files that contain them.
|
||||
type ChunkFileRepository struct {
|
||||
db *DB
|
||||
}
|
||||
|
||||
// NewChunkFileRepository creates a ChunkFileRepository backed by db.
|
||||
func NewChunkFileRepository(db *DB) *ChunkFileRepository {
|
||||
return &ChunkFileRepository{db: db}
|
||||
}
|
||||
|
||||
func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkFile) error {
|
||||
// Create inserts a chunk_files row (idempotently), using tx when non-nil.
|
||||
func (r *ChunkFileRepository) Create(
|
||||
ctx context.Context, tx *sql.Tx, cf *ChunkFile,
|
||||
) error {
|
||||
query := `
|
||||
INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length)
|
||||
VALUES (?, ?, ?, ?)
|
||||
@@ -25,9 +32,11 @@ func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkF
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
|
||||
_, err = tx.ExecContext(ctx, query,
|
||||
cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
|
||||
_, err = r.db.ExecWithLog(ctx, query,
|
||||
cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -37,7 +46,10 @@ func (r *ChunkFileRepository) Create(ctx context.Context, tx *sql.Tx, cf *ChunkF
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ChunkFileRepository) GetByChunkHash(ctx context.Context, chunkHash types.ChunkHash) ([]*ChunkFile, error) {
|
||||
// GetByChunkHash returns all chunk_files rows for the given chunk hash.
|
||||
func (r *ChunkFileRepository) GetByChunkHash(
|
||||
ctx context.Context, chunkHash types.ChunkHash,
|
||||
) ([]*ChunkFile, error) {
|
||||
query := `
|
||||
SELECT chunk_hash, file_id, file_offset, length
|
||||
FROM chunk_files
|
||||
@@ -48,12 +60,16 @@ func (r *ChunkFileRepository) GetByChunkHash(ctx context.Context, chunkHash type
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying chunk files: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
return r.scanChunkFiles(rows)
|
||||
}
|
||||
|
||||
func (r *ChunkFileRepository) GetByFilePath(ctx context.Context, filePath string) ([]*ChunkFile, error) {
|
||||
// GetByFilePath returns all chunk_files rows for the file at the given path.
|
||||
func (r *ChunkFileRepository) GetByFilePath(
|
||||
ctx context.Context, filePath string,
|
||||
) ([]*ChunkFile, error) {
|
||||
query := `
|
||||
SELECT cf.chunk_hash, cf.file_id, cf.file_offset, cf.length
|
||||
FROM chunk_files cf
|
||||
@@ -65,13 +81,16 @@ func (r *ChunkFileRepository) GetByFilePath(ctx context.Context, filePath string
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying chunk files: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
return r.scanChunkFiles(rows)
|
||||
}
|
||||
|
||||
// GetByFileID retrieves chunk files by file ID
|
||||
func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.FileID) ([]*ChunkFile, error) {
|
||||
func (r *ChunkFileRepository) GetByFileID(
|
||||
ctx context.Context, fileID types.FileID,
|
||||
) ([]*ChunkFile, error) {
|
||||
query := `
|
||||
SELECT chunk_hash, file_id, file_offset, length
|
||||
FROM chunk_files
|
||||
@@ -82,34 +101,16 @@ func (r *ChunkFileRepository) GetByFileID(ctx context.Context, fileID types.File
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying chunk files: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
return r.scanChunkFiles(rows)
|
||||
}
|
||||
|
||||
// scanChunkFiles is a helper that scans chunk file rows
|
||||
func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) {
|
||||
var chunkFiles []*ChunkFile
|
||||
for rows.Next() {
|
||||
var cf ChunkFile
|
||||
var chunkHashStr, fileIDStr string
|
||||
err := rows.Scan(&chunkHashStr, &fileIDStr, &cf.FileOffset, &cf.Length)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning chunk file: %w", err)
|
||||
}
|
||||
cf.ChunkHash = types.ChunkHash(chunkHashStr)
|
||||
cf.FileID, err = types.ParseFileID(fileIDStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||
}
|
||||
chunkFiles = append(chunkFiles, &cf)
|
||||
}
|
||||
|
||||
return chunkFiles, rows.Err()
|
||||
}
|
||||
|
||||
// DeleteByFileID deletes all chunk_files entries for a given file ID
|
||||
func (r *ChunkFileRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fileID types.FileID) error {
|
||||
func (r *ChunkFileRepository) DeleteByFileID(
|
||||
ctx context.Context, tx *sql.Tx, fileID types.FileID,
|
||||
) error {
|
||||
query := `DELETE FROM chunk_files WHERE file_id = ?`
|
||||
|
||||
var err error
|
||||
@@ -127,7 +128,11 @@ func (r *ChunkFileRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fi
|
||||
}
|
||||
|
||||
// DeleteByFileIDs deletes all chunk_files for multiple files in a single statement.
|
||||
func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, fileIDs []types.FileID) error {
|
||||
//
|
||||
//nolint:dupl // symmetric implementation for a parallel association table
|
||||
func (r *ChunkFileRepository) DeleteByFileIDs(
|
||||
ctx context.Context, tx *sql.Tx, fileIDs []types.FileID,
|
||||
) error {
|
||||
if len(fileIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -136,14 +141,15 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
const batchSize = 500
|
||||
|
||||
for i := 0; i < len(fileIDs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(fileIDs) {
|
||||
end = len(fileIDs)
|
||||
}
|
||||
end := min(i+batchSize, len(fileIDs))
|
||||
|
||||
batch := fileIDs[i:end]
|
||||
|
||||
query := "DELETE FROM chunk_files WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
|
||||
args := make([]interface{}, len(batch))
|
||||
//nolint:gosec // G202: concatenates constant SQL and "?" placeholders only
|
||||
query := "DELETE FROM chunk_files WHERE file_id IN (?" +
|
||||
repeatPlaceholder(len(batch)-1) + ")"
|
||||
|
||||
args := make([]any, len(batch))
|
||||
for j, id := range batch {
|
||||
args[j] = id.String()
|
||||
}
|
||||
@@ -154,6 +160,7 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch deleting chunk_files: %w", err)
|
||||
}
|
||||
@@ -163,30 +170,43 @@ func (r *ChunkFileRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
}
|
||||
|
||||
// CreateBatch inserts multiple chunk_files in a single statement for efficiency.
|
||||
func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs []ChunkFile) error {
|
||||
func (r *ChunkFileRepository) CreateBatch(
|
||||
ctx context.Context, tx *sql.Tx, cfs []ChunkFile,
|
||||
) error {
|
||||
if len(cfs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Each ChunkFile has 4 values, so batch at 200 to be safe with SQLite's variable limit
|
||||
// Each chunk_files row binds this many SQL variables.
|
||||
const chunkFileCols = 4
|
||||
|
||||
// Batch at 200 rows to be safe with SQLite's variable limit.
|
||||
const batchSize = 200
|
||||
|
||||
for i := 0; i < len(cfs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(cfs) {
|
||||
end = len(cfs)
|
||||
}
|
||||
end := min(i+batchSize, len(cfs))
|
||||
|
||||
batch := cfs[i:end]
|
||||
|
||||
query := "INSERT INTO chunk_files (chunk_hash, file_id, file_offset, length) VALUES "
|
||||
args := make([]interface{}, 0, len(batch)*4)
|
||||
|
||||
args := make([]any, 0, len(batch)*chunkFileCols)
|
||||
|
||||
var querySb183 strings.Builder
|
||||
|
||||
for j, cf := range batch {
|
||||
if j > 0 {
|
||||
query += ", "
|
||||
querySb183.WriteString(", ")
|
||||
}
|
||||
query += "(?, ?, ?, ?)"
|
||||
args = append(args, cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
|
||||
|
||||
querySb183.WriteString("(?, ?, ?, ?)")
|
||||
|
||||
args = append(args,
|
||||
cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
|
||||
}
|
||||
|
||||
query += querySb183.String()
|
||||
|
||||
query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING"
|
||||
|
||||
var err error
|
||||
@@ -195,6 +215,7 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch inserting chunk_files: %w", err)
|
||||
}
|
||||
@@ -202,3 +223,31 @@ func (r *ChunkFileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, cfs [
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// scanChunkFiles is a helper that scans chunk file rows.
|
||||
func (r *ChunkFileRepository) scanChunkFiles(rows *sql.Rows) ([]*ChunkFile, error) {
|
||||
var chunkFiles []*ChunkFile
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
cf ChunkFile
|
||||
chunkHashStr, fileIDStr string
|
||||
)
|
||||
|
||||
err := rows.Scan(&chunkHashStr, &fileIDStr, &cf.FileOffset, &cf.Length)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning chunk file: %w", err)
|
||||
}
|
||||
|
||||
cf.ChunkHash = types.ChunkHash(chunkHashStr)
|
||||
|
||||
cf.FileID, err = types.ParseFileID(fileIDStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||
}
|
||||
|
||||
chunkFiles = append(chunkFiles, &cf)
|
||||
}
|
||||
|
||||
return chunkFiles, rows.Err()
|
||||
}
|
||||
|
||||
@@ -1,122 +1,139 @@
|
||||
package database
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
const chunk4Hash = "chunk4"
|
||||
|
||||
// verifyChunkFilePair asserts that the chunk-file rows cover both test
|
||||
// files at their expected offsets.
|
||||
func verifyChunkFilePair(
|
||||
t *testing.T, chunkFiles []*database.ChunkFile,
|
||||
file1ID, file2ID types.FileID,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
foundFile1 := false
|
||||
foundFile2 := false
|
||||
|
||||
for _, cf := range chunkFiles {
|
||||
if cf.FileID == file1ID && cf.FileOffset == 0 {
|
||||
foundFile1 = true
|
||||
}
|
||||
|
||||
if cf.FileID == file2ID && cf.FileOffset == 2048 {
|
||||
foundFile2 = true
|
||||
}
|
||||
}
|
||||
|
||||
if !foundFile1 || !foundFile2 {
|
||||
t.Error("not all expected files found")
|
||||
}
|
||||
}
|
||||
|
||||
// createChunkFileTestFiles creates the two files used by the chunk-file
|
||||
// repository tests.
|
||||
func createChunkFileTestFiles(
|
||||
t *testing.T, fileRepo *database.FileRepository,
|
||||
) (*database.File, *database.File) {
|
||||
t.Helper()
|
||||
|
||||
testTime := time.Now().Truncate(time.Second)
|
||||
file1 := &database.File{
|
||||
Path: testFilePath1,
|
||||
MTime: testTime,
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
LinkTarget: "",
|
||||
}
|
||||
file2 := &database.File{
|
||||
Path: testFilePath2,
|
||||
MTime: testTime,
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
LinkTarget: "",
|
||||
}
|
||||
|
||||
mustCreateFile(t, fileRepo, file1)
|
||||
mustCreateFile(t, fileRepo, file2)
|
||||
|
||||
return file1, file2
|
||||
}
|
||||
|
||||
func TestChunkFileRepository(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewChunkFileRepository(db)
|
||||
fileRepo := NewFileRepository(db)
|
||||
chunksRepo := NewChunkRepository(db)
|
||||
repo := database.NewChunkFileRepository(db)
|
||||
fileRepo := database.NewFileRepository(db)
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Create test files first
|
||||
testTime := time.Now().Truncate(time.Second)
|
||||
file1 := &File{
|
||||
Path: "/file1.txt",
|
||||
MTime: testTime,
|
||||
CTime: testTime,
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
LinkTarget: "",
|
||||
}
|
||||
err := fileRepo.Create(ctx, nil, file1)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file1: %v", err)
|
||||
}
|
||||
|
||||
file2 := &File{
|
||||
Path: "/file2.txt",
|
||||
MTime: testTime,
|
||||
CTime: testTime,
|
||||
Size: 1024,
|
||||
Mode: 0644,
|
||||
UID: 1000,
|
||||
GID: 1000,
|
||||
LinkTarget: "",
|
||||
}
|
||||
err = fileRepo.Create(ctx, nil, file2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create file2: %v", err)
|
||||
}
|
||||
|
||||
// Create chunk first
|
||||
chunk := &Chunk{
|
||||
ChunkHash: types.ChunkHash("chunk1"),
|
||||
Size: 1024,
|
||||
}
|
||||
err = chunksRepo.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk: %v", err)
|
||||
}
|
||||
file1, file2 := createChunkFileTestFiles(t, fileRepo)
|
||||
mustCreateChunks(t, repos, chunk1Hash)
|
||||
|
||||
// Test Create
|
||||
cf1 := &ChunkFile{
|
||||
ChunkHash: types.ChunkHash("chunk1"),
|
||||
cf1 := &database.ChunkFile{
|
||||
ChunkHash: types.ChunkHash(chunk1Hash),
|
||||
FileID: file1.ID,
|
||||
FileOffset: 0,
|
||||
Length: 1024,
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, cf1)
|
||||
err := repo.Create(ctx, nil, cf1)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk file: %v", err)
|
||||
}
|
||||
|
||||
// Add same chunk in different file (deduplication scenario)
|
||||
cf2 := &ChunkFile{
|
||||
ChunkHash: types.ChunkHash("chunk1"),
|
||||
cf2 := &database.ChunkFile{
|
||||
ChunkHash: types.ChunkHash(chunk1Hash),
|
||||
FileID: file2.ID,
|
||||
FileOffset: 2048,
|
||||
Length: 1024,
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, cf2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create second chunk file: %v", err)
|
||||
}
|
||||
|
||||
// Test GetByChunkHash
|
||||
chunkFiles, err := repo.GetByChunkHash(ctx, "chunk1")
|
||||
chunkFiles, err := repo.GetByChunkHash(ctx, chunk1Hash)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunk files: %v", err)
|
||||
}
|
||||
|
||||
if len(chunkFiles) != 2 {
|
||||
t.Errorf("expected 2 files for chunk, got %d", len(chunkFiles))
|
||||
}
|
||||
|
||||
// Verify both files are returned
|
||||
foundFile1 := false
|
||||
foundFile2 := false
|
||||
for _, cf := range chunkFiles {
|
||||
if cf.FileID == file1.ID && cf.FileOffset == 0 {
|
||||
foundFile1 = true
|
||||
}
|
||||
if cf.FileID == file2.ID && cf.FileOffset == 2048 {
|
||||
foundFile2 = true
|
||||
}
|
||||
}
|
||||
if !foundFile1 || !foundFile2 {
|
||||
t.Error("not all expected files found")
|
||||
}
|
||||
verifyChunkFilePair(t, chunkFiles, file1.ID, file2.ID)
|
||||
|
||||
// Test GetByFileID
|
||||
chunkFiles, err = repo.GetByFileID(ctx, file1.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks by file ID: %v", err)
|
||||
}
|
||||
|
||||
if len(chunkFiles) != 1 {
|
||||
t.Errorf("expected 1 chunk for file, got %d", len(chunkFiles))
|
||||
}
|
||||
if chunkFiles[0].ChunkHash != types.ChunkHash("chunk1") {
|
||||
|
||||
if chunkFiles[0].ChunkHash != types.ChunkHash(chunk1Hash) {
|
||||
t.Errorf("wrong chunk hash: expected chunk1, got %s", chunkFiles[0].ChunkHash)
|
||||
}
|
||||
|
||||
@@ -128,60 +145,53 @@ func TestChunkFileRepository(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewChunkFileRepository(db)
|
||||
fileRepo := NewFileRepository(db)
|
||||
chunksRepo := NewChunkRepository(db)
|
||||
repo := database.NewChunkFileRepository(db)
|
||||
fileRepo := database.NewFileRepository(db)
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// Create test files
|
||||
testTime := time.Now().Truncate(time.Second)
|
||||
file1 := &File{Path: "/file1.txt", MTime: testTime, CTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000}
|
||||
file2 := &File{Path: "/file2.txt", MTime: testTime, CTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000}
|
||||
file3 := &File{Path: "/file3.txt", MTime: testTime, CTime: testTime, Size: 2048, Mode: 0644, UID: 1000, GID: 1000}
|
||||
|
||||
if err := fileRepo.Create(ctx, nil, file1); err != nil {
|
||||
t.Fatalf("failed to create file1: %v", err)
|
||||
file1 := &database.File{
|
||||
Path: testFilePath1, MTime: testTime, Size: 3072,
|
||||
Mode: 0644, UID: 1000, GID: 1000,
|
||||
}
|
||||
if err := fileRepo.Create(ctx, nil, file2); err != nil {
|
||||
t.Fatalf("failed to create file2: %v", err)
|
||||
file2 := &database.File{
|
||||
Path: testFilePath2, MTime: testTime, Size: 3072,
|
||||
Mode: 0644, UID: 1000, GID: 1000,
|
||||
}
|
||||
if err := fileRepo.Create(ctx, nil, file3); err != nil {
|
||||
t.Fatalf("failed to create file3: %v", err)
|
||||
file3 := &database.File{
|
||||
Path: "/file3.txt", MTime: testTime, Size: 2048,
|
||||
Mode: 0644, UID: 1000, GID: 1000,
|
||||
}
|
||||
|
||||
// Create chunks first
|
||||
chunks := []types.ChunkHash{"chunk1", "chunk2", "chunk3", "chunk4"}
|
||||
for _, chunkHash := range chunks {
|
||||
chunk := &Chunk{
|
||||
ChunkHash: chunkHash,
|
||||
Size: 1024,
|
||||
}
|
||||
err := chunksRepo.Create(ctx, nil, chunk)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create chunk %s: %v", chunkHash, err)
|
||||
}
|
||||
}
|
||||
mustCreateFile(t, fileRepo, file1)
|
||||
mustCreateFile(t, fileRepo, file2)
|
||||
mustCreateFile(t, fileRepo, file3)
|
||||
mustCreateChunks(t, repos, chunk1Hash, chunk2Hash, chunk3Hash, chunk4Hash)
|
||||
|
||||
// Simulate a scenario where multiple files share chunks
|
||||
// File1: chunk1, chunk2, chunk3
|
||||
// File2: chunk2, chunk3, chunk4
|
||||
// File3: chunk1, chunk4
|
||||
|
||||
chunkFiles := []ChunkFile{
|
||||
chunkFiles := []database.ChunkFile{
|
||||
// File1
|
||||
{ChunkHash: types.ChunkHash("chunk1"), FileID: file1.ID, FileOffset: 0, Length: 1024},
|
||||
{ChunkHash: types.ChunkHash("chunk2"), FileID: file1.ID, FileOffset: 1024, Length: 1024},
|
||||
{ChunkHash: types.ChunkHash("chunk3"), FileID: file1.ID, FileOffset: 2048, Length: 1024},
|
||||
{ChunkHash: chunk1Hash, FileID: file1.ID, FileOffset: 0, Length: 1024},
|
||||
{ChunkHash: chunk2Hash, FileID: file1.ID, FileOffset: 1024, Length: 1024},
|
||||
{ChunkHash: chunk3Hash, FileID: file1.ID, FileOffset: 2048, Length: 1024},
|
||||
// File2
|
||||
{ChunkHash: types.ChunkHash("chunk2"), FileID: file2.ID, FileOffset: 0, Length: 1024},
|
||||
{ChunkHash: types.ChunkHash("chunk3"), FileID: file2.ID, FileOffset: 1024, Length: 1024},
|
||||
{ChunkHash: types.ChunkHash("chunk4"), FileID: file2.ID, FileOffset: 2048, Length: 1024},
|
||||
{ChunkHash: chunk2Hash, FileID: file2.ID, FileOffset: 0, Length: 1024},
|
||||
{ChunkHash: chunk3Hash, FileID: file2.ID, FileOffset: 1024, Length: 1024},
|
||||
{ChunkHash: chunk4Hash, FileID: file2.ID, FileOffset: 2048, Length: 1024},
|
||||
// File3
|
||||
{ChunkHash: types.ChunkHash("chunk1"), FileID: file3.ID, FileOffset: 0, Length: 1024},
|
||||
{ChunkHash: types.ChunkHash("chunk4"), FileID: file3.ID, FileOffset: 1024, Length: 1024},
|
||||
{ChunkHash: chunk1Hash, FileID: file3.ID, FileOffset: 0, Length: 1024},
|
||||
{ChunkHash: chunk4Hash, FileID: file3.ID, FileOffset: 1024, Length: 1024},
|
||||
}
|
||||
|
||||
for _, cf := range chunkFiles {
|
||||
@@ -192,19 +202,21 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
||||
}
|
||||
|
||||
// Test chunk1 (used by file1 and file3)
|
||||
files, err := repo.GetByChunkHash(ctx, "chunk1")
|
||||
files, err := repo.GetByChunkHash(ctx, chunk1Hash)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get files for chunk1: %v", err)
|
||||
}
|
||||
|
||||
if len(files) != 2 {
|
||||
t.Errorf("expected 2 files for chunk1, got %d", len(files))
|
||||
}
|
||||
|
||||
// Test chunk2 (used by file1 and file2)
|
||||
files, err = repo.GetByChunkHash(ctx, "chunk2")
|
||||
files, err = repo.GetByChunkHash(ctx, chunk2Hash)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get files for chunk2: %v", err)
|
||||
}
|
||||
|
||||
if len(files) != 2 {
|
||||
t.Errorf("expected 2 files for chunk2, got %d", len(files))
|
||||
}
|
||||
@@ -214,6 +226,7 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks for file2: %v", err)
|
||||
}
|
||||
|
||||
if len(file2Chunks) != 3 {
|
||||
t.Errorf("expected 3 chunks for file2, got %d", len(file2Chunks))
|
||||
}
|
||||
|
||||
+39
-10
@@ -3,19 +3,25 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
// ChunkRepository provides access to the chunks table, which tracks
|
||||
// content-defined chunks by hash and size.
|
||||
type ChunkRepository struct {
|
||||
db *DB
|
||||
}
|
||||
|
||||
// NewChunkRepository creates a ChunkRepository backed by db.
|
||||
func NewChunkRepository(db *DB) *ChunkRepository {
|
||||
return &ChunkRepository{db: db}
|
||||
}
|
||||
|
||||
// Create inserts a chunk row (idempotently), using tx when non-nil.
|
||||
func (r *ChunkRepository) Create(ctx context.Context, tx *sql.Tx, chunk *Chunk) error {
|
||||
query := `
|
||||
INSERT INTO chunks (chunk_hash, size)
|
||||
@@ -37,6 +43,8 @@ func (r *ChunkRepository) Create(ctx context.Context, tx *sql.Tx, chunk *Chunk)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetByHash returns the chunk with the given hash, or nil if it is not
|
||||
// known to the index.
|
||||
func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, error) {
|
||||
query := `
|
||||
SELECT chunk_hash, size
|
||||
@@ -51,9 +59,10 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e
|
||||
&chunk.Size,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil //nolint:nilnil // nil,nil signals not-found; callers check nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying chunk: %w", err)
|
||||
}
|
||||
@@ -61,7 +70,11 @@ func (r *ChunkRepository) GetByHash(ctx context.Context, hash string) (*Chunk, e
|
||||
return &chunk, nil
|
||||
}
|
||||
|
||||
func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*Chunk, error) {
|
||||
// GetByHashes returns the chunks whose hashes appear in hashes, ordered by
|
||||
// chunk hash. Unknown hashes are silently omitted from the result.
|
||||
func (r *ChunkRepository) GetByHashes(
|
||||
ctx context.Context, hashes []string,
|
||||
) ([]*Chunk, error) {
|
||||
if len(hashes) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -71,23 +84,33 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
|
||||
FROM chunks
|
||||
WHERE chunk_hash IN (`
|
||||
|
||||
args := make([]interface{}, len(hashes))
|
||||
args := make([]any, len(hashes))
|
||||
|
||||
var querySb75 strings.Builder
|
||||
|
||||
for i, hash := range hashes {
|
||||
if i > 0 {
|
||||
query += ", "
|
||||
querySb75.WriteString(", ")
|
||||
}
|
||||
query += "?"
|
||||
|
||||
querySb75.WriteString("?")
|
||||
|
||||
args[i] = hash
|
||||
}
|
||||
|
||||
query += querySb75.String()
|
||||
|
||||
query += ") ORDER BY chunk_hash"
|
||||
|
||||
rows, err := r.db.conn.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying chunks: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var chunks []*Chunk
|
||||
|
||||
for rows.Next() {
|
||||
var chunk Chunk
|
||||
|
||||
@@ -105,7 +128,11 @@ func (r *ChunkRepository) GetByHashes(ctx context.Context, hashes []string) ([]*
|
||||
return chunks, rows.Err()
|
||||
}
|
||||
|
||||
func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk, error) {
|
||||
// ListUnpacked returns up to limit chunks that are not yet stored in any
|
||||
// blob, ordered by chunk hash.
|
||||
func (r *ChunkRepository) ListUnpacked(
|
||||
ctx context.Context, limit int,
|
||||
) ([]*Chunk, error) {
|
||||
query := `
|
||||
SELECT c.chunk_hash, c.size
|
||||
FROM chunks c
|
||||
@@ -119,9 +146,11 @@ func (r *ChunkRepository) ListUnpacked(ctx context.Context, limit int) ([]*Chunk
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying unpacked chunks: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var chunks []*Chunk
|
||||
|
||||
for rows.Next() {
|
||||
var chunk Chunk
|
||||
|
||||
|
||||
@@ -5,20 +5,43 @@ import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// List returns every chunk in the index, ordered by chunk hash.
|
||||
func (r *ChunkRepository) List(ctx context.Context) ([]*Chunk, error) {
|
||||
query := `
|
||||
return r.list(ctx, `
|
||||
SELECT chunk_hash, size
|
||||
FROM chunks
|
||||
ORDER BY chunk_hash
|
||||
`
|
||||
`)
|
||||
}
|
||||
|
||||
// ListInUploadedBlobs returns the chunks that are stored in a blob whose
|
||||
// upload has completed (uploaded_ts set), ordered by chunk hash. These
|
||||
// are the only chunks a backup may safely deduplicate against: a chunk
|
||||
// recorded solely in a blob that was never uploaded refers to data that
|
||||
// is not in remote storage, so trusting it would silently drop that data
|
||||
// from later snapshots.
|
||||
func (r *ChunkRepository) ListInUploadedBlobs(ctx context.Context) ([]*Chunk, error) {
|
||||
return r.list(ctx, `
|
||||
SELECT DISTINCT c.chunk_hash, c.size
|
||||
FROM chunks c
|
||||
JOIN blob_chunks bc ON c.chunk_hash = bc.chunk_hash
|
||||
JOIN blobs b ON bc.blob_id = b.id
|
||||
WHERE b.uploaded_ts IS NOT NULL
|
||||
ORDER BY c.chunk_hash
|
||||
`)
|
||||
}
|
||||
|
||||
// list runs a chunk-selecting query and scans the (chunk_hash, size) rows.
|
||||
func (r *ChunkRepository) list(ctx context.Context, query string) ([]*Chunk, error) {
|
||||
rows, err := r.db.conn.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying chunks: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var chunks []*Chunk
|
||||
|
||||
for rows.Next() {
|
||||
var chunk Chunk
|
||||
|
||||
|
||||
@@ -1,21 +1,24 @@
|
||||
package database
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func TestChunkRepository(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewChunkRepository(db)
|
||||
repo := database.NewChunkRepository(db)
|
||||
|
||||
// Test Create
|
||||
chunk := &Chunk{
|
||||
chunk := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash("chunkhash123"),
|
||||
Size: 4096,
|
||||
}
|
||||
@@ -30,12 +33,15 @@ func TestChunkRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunk: %v", err)
|
||||
}
|
||||
|
||||
if retrieved == nil {
|
||||
t.Fatal("expected chunk, got nil")
|
||||
}
|
||||
|
||||
if retrieved.ChunkHash != chunk.ChunkHash {
|
||||
t.Errorf("chunk hash mismatch: got %s, want %s", retrieved.ChunkHash, chunk.ChunkHash)
|
||||
}
|
||||
|
||||
if retrieved.Size != chunk.Size {
|
||||
t.Errorf("size mismatch: got %d, want %d", retrieved.Size, chunk.Size)
|
||||
}
|
||||
@@ -47,19 +53,23 @@ func TestChunkRepository(t *testing.T) {
|
||||
}
|
||||
|
||||
// Test GetByHashes
|
||||
chunk2 := &Chunk{
|
||||
chunk2 := &database.Chunk{
|
||||
ChunkHash: types.ChunkHash("chunkhash456"),
|
||||
Size: 8192,
|
||||
}
|
||||
|
||||
err = repo.Create(ctx, nil, chunk2)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create second chunk: %v", err)
|
||||
}
|
||||
|
||||
chunks, err := repo.GetByHashes(ctx, []string{chunk.ChunkHash.String(), chunk2.ChunkHash.String()})
|
||||
chunks, err := repo.GetByHashes(ctx, []string{
|
||||
chunk.ChunkHash.String(), chunk2.ChunkHash.String(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to get chunks by hashes: %v", err)
|
||||
}
|
||||
|
||||
if len(chunks) != 2 {
|
||||
t.Errorf("expected 2 chunks, got %d", len(chunks))
|
||||
}
|
||||
@@ -69,23 +79,27 @@ func TestChunkRepository(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to list unpacked chunks: %v", err)
|
||||
}
|
||||
|
||||
if len(unpacked) != 2 {
|
||||
t.Errorf("expected 2 unpacked chunks, got %d", len(unpacked))
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkRepositoryNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, cleanup := setupTestDB(t)
|
||||
defer cleanup()
|
||||
|
||||
ctx := context.Background()
|
||||
repo := NewChunkRepository(db)
|
||||
repo := database.NewChunkRepository(db)
|
||||
|
||||
// Test GetByHash with non-existent hash
|
||||
chunk, err := repo.GetByHash(ctx, "nonexistent")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if chunk != nil {
|
||||
t.Error("expected nil for non-existent chunk")
|
||||
}
|
||||
@@ -95,6 +109,7 @@ func TestChunkRepositoryNotFound(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if chunks != nil {
|
||||
t.Error("expected nil for empty hash list")
|
||||
}
|
||||
|
||||
+429
-94
@@ -3,27 +3,48 @@
|
||||
//
|
||||
// Blobs in Vaultik are the final storage units uploaded to S3. Each blob is a
|
||||
// large (up to 10GB) file containing many compressed and encrypted chunks from
|
||||
// multiple source files. Blobs are content-addressed, meaning their filename
|
||||
// is derived from their SHA256 hash after compression and encryption.
|
||||
// multiple source files. Blobs are content-addressed: the filename in S3 is
|
||||
// hex(SHA256(SHA256(uncompressed blob contents))), computed from the chunk data
|
||||
// before compression and encryption (not from the stored bytes). See
|
||||
// blobgen.DoubleSHA256 and docs/REPOSTRUCTURE.md.
|
||||
//
|
||||
// The database does not support migrations. If the schema changes, delete
|
||||
// the local database and perform a full backup to recreate it.
|
||||
// Schema is managed via numbered SQL migrations embedded in the schema/
|
||||
// directory. Migration 000.sql bootstraps the schema_migrations tracking
|
||||
// table; subsequent migrations (001, 002, …) are applied in order.
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
_ "embed"
|
||||
"embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
// Register the pure-Go sqlite driver.
|
||||
_ "modernc.org/sqlite"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
//go:embed schema.sql
|
||||
var schemaSQL string
|
||||
// errInvalidMigrationFilename is returned when an embedded migration file
|
||||
// does not follow the "<version>[_<description>].sql" naming pattern.
|
||||
var errInvalidMigrationFilename = errors.New("invalid migration filename")
|
||||
|
||||
//go:embed schema/*.sql
|
||||
var schemaFS embed.FS
|
||||
|
||||
// bootstrapVersion is the migration that creates the schema_migrations
|
||||
// table itself. It is applied before the normal migration loop.
|
||||
const bootstrapVersion = 0
|
||||
|
||||
// busyTimeoutMs is how long a connection to the index waits for another
|
||||
// connection's lock before failing with "database is locked".
|
||||
const busyTimeoutMs = 10000
|
||||
|
||||
// DB represents the Vaultik local index database connection.
|
||||
// It uses SQLite to track file metadata, content-defined chunks, and blob associations.
|
||||
@@ -35,10 +56,69 @@ type DB struct {
|
||||
path string
|
||||
}
|
||||
|
||||
// ParseMigrationVersion extracts the numeric version prefix from a migration
|
||||
// filename. Filenames must follow the pattern "<version>.sql" or
|
||||
// "<version>_<description>.sql", where version is a zero-padded numeric
|
||||
// string (e.g. "001", "002"). Returns the version as an integer and an
|
||||
// error if the filename does not match the expected pattern.
|
||||
func ParseMigrationVersion(filename string) (int, error) {
|
||||
name := strings.TrimSuffix(filename, filepath.Ext(filename))
|
||||
if name == "" {
|
||||
return 0, fmt.Errorf("%w %q: empty name", errInvalidMigrationFilename, filename)
|
||||
}
|
||||
|
||||
// Split on underscore to separate version from description.
|
||||
// If there's no underscore, the entire stem is the version.
|
||||
versionStr := name
|
||||
if before, _, ok := strings.Cut(name, "_"); ok {
|
||||
versionStr = before
|
||||
}
|
||||
|
||||
if versionStr == "" {
|
||||
return 0, fmt.Errorf(
|
||||
"%w %q: empty version prefix", errInvalidMigrationFilename, filename,
|
||||
)
|
||||
}
|
||||
|
||||
// Validate the version is purely numeric.
|
||||
for _, ch := range versionStr {
|
||||
if ch < '0' || ch > '9' {
|
||||
return 0, fmt.Errorf(
|
||||
"%w %q: version %q contains non-numeric character %q",
|
||||
errInvalidMigrationFilename, filename, versionStr, string(ch),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
version, err := strconv.Atoi(versionStr)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid migration filename %q: %w", filename, err)
|
||||
}
|
||||
|
||||
return version, nil
|
||||
}
|
||||
|
||||
// indexDSN returns the driver DSN that opens the database at path. The
|
||||
// driver runs each _pragma parameter on every connection it opens and drops
|
||||
// any parameter it does not know without an error, so a setting written in
|
||||
// another form silently does nothing. In WAL mode one connection can write
|
||||
// while others read; the busy timeout makes a connection wait for a lock
|
||||
// instead of failing at once.
|
||||
func indexDSN(path string) string {
|
||||
return fmt.Sprintf(
|
||||
"%s?_pragma=busy_timeout(%d)&_pragma=journal_mode(WAL)"+
|
||||
"&_pragma=synchronous(NORMAL)&_pragma=foreign_keys(1)",
|
||||
path, busyTimeoutMs)
|
||||
}
|
||||
|
||||
// New creates a new database connection at the specified path.
|
||||
// It creates the schema if needed and configures SQLite with WAL mode for
|
||||
// better concurrency. SQLite handles crash recovery automatically when
|
||||
// opening a database with journal/WAL files present.
|
||||
// It creates the schema if needed. Every connection runs in WAL mode with
|
||||
// a busy timeout and foreign keys on (see indexDSN), so a read-only command
|
||||
// can read the index while a backup writes to it. Committed rows can sit in
|
||||
// the -wal file beside the database until a checkpoint, so a copy of the
|
||||
// database file alone may miss them.
|
||||
// SQLite handles crash recovery automatically when opening a database with
|
||||
// journal/WAL files present.
|
||||
// The path parameter can be a file path for persistent storage or ":memory:"
|
||||
// for an in-memory database (useful for testing).
|
||||
func New(ctx context.Context, path string) (*DB, error) {
|
||||
@@ -50,100 +130,244 @@ func New(ctx context.Context, path string) (*DB, error) {
|
||||
|
||||
// First attempt with standard WAL mode
|
||||
log.Debug("Attempting to open database with WAL mode", "path", path)
|
||||
conn, err := sql.Open(
|
||||
"sqlite",
|
||||
path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000&_locking_mode=NORMAL&_foreign_keys=ON",
|
||||
)
|
||||
if err == nil {
|
||||
// Set connection pool settings
|
||||
// SQLite can handle multiple readers but only one writer at a time.
|
||||
// Setting MaxOpenConns to 1 ensures all writes are serialized through
|
||||
// a single connection, preventing SQLITE_BUSY errors.
|
||||
conn.SetMaxOpenConns(1)
|
||||
conn.SetMaxIdleConns(1)
|
||||
|
||||
if err := conn.PingContext(ctx); err == nil {
|
||||
conn, err := sql.Open("sqlite", indexDSN(path))
|
||||
if err == nil {
|
||||
configureConnPool(conn)
|
||||
|
||||
err = conn.PingContext(ctx)
|
||||
if err == nil {
|
||||
// Success on first try
|
||||
log.Debug("Database opened successfully with WAL mode", "path", path)
|
||||
|
||||
// Enable foreign keys explicitly
|
||||
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys = ON"); err != nil {
|
||||
log.Warn("Failed to enable foreign keys", "error", err)
|
||||
}
|
||||
|
||||
db := &DB{conn: conn, path: path}
|
||||
if err := db.createSchema(ctx); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("creating schema: %w", err)
|
||||
}
|
||||
return db, nil
|
||||
return finishOpen(ctx, conn, path)
|
||||
}
|
||||
log.Debug("Failed to ping database, closing connection", "path", path, "error", err)
|
||||
|
||||
log.Debug(
|
||||
"Failed to ping database, closing connection",
|
||||
"path", path, "error", err,
|
||||
)
|
||||
|
||||
_ = conn.Close()
|
||||
}
|
||||
|
||||
// If first attempt failed, try with TRUNCATE mode to clear any locks
|
||||
log.Info(
|
||||
"Database appears locked, attempting recovery with TRUNCATE mode",
|
||||
"path", path,
|
||||
)
|
||||
conn, err = sql.Open(
|
||||
"sqlite",
|
||||
path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000&_foreign_keys=ON",
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening database in recovery mode: %w", err)
|
||||
}
|
||||
// If the first attempt failed, try once more
|
||||
return retryOpen(ctx, path)
|
||||
}
|
||||
|
||||
// Set connection pool settings
|
||||
// SQLite can handle multiple readers but only one writer at a time.
|
||||
// Setting MaxOpenConns to 1 ensures all writes are serialized through
|
||||
// a single connection, preventing SQLITE_BUSY errors.
|
||||
// configureConnPool serializes all database access through one connection.
|
||||
// SQLite can handle multiple readers but only one writer at a time; setting
|
||||
// MaxOpenConns to 1 ensures all writes go through a single connection,
|
||||
// preventing SQLITE_BUSY errors.
|
||||
func configureConnPool(conn *sql.DB) {
|
||||
conn.SetMaxOpenConns(1)
|
||||
conn.SetMaxIdleConns(1)
|
||||
}
|
||||
|
||||
if err := conn.PingContext(ctx); err != nil {
|
||||
log.Debug("Failed to ping database in recovery mode, closing", "path", path, "error", err)
|
||||
// finishOpen wraps the connection and applies any pending migrations. On
|
||||
// migration failure the connection is closed.
|
||||
func finishOpen(ctx context.Context, conn *sql.DB, path string) (*DB, error) {
|
||||
db := &DB{conn: conn, path: path}
|
||||
|
||||
err := applyMigrations(ctx, conn)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
|
||||
return nil, fmt.Errorf("applying migrations: %w", err)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// retryOpen makes a second attempt to open the database, with the same
|
||||
// settings, after the first attempt failed, for example because another
|
||||
// process held a lock for longer than the busy timeout.
|
||||
func retryOpen(ctx context.Context, path string) (*DB, error) {
|
||||
log.Info("Database appears locked, retrying open", "path", path)
|
||||
|
||||
conn, err := sql.Open("sqlite", indexDSN(path))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening database on retry: %w", err)
|
||||
}
|
||||
|
||||
configureConnPool(conn)
|
||||
|
||||
err = conn.PingContext(ctx)
|
||||
if err != nil {
|
||||
log.Debug(
|
||||
"Failed to ping database on retry, closing",
|
||||
"path", path, "error", err,
|
||||
)
|
||||
|
||||
_ = conn.Close()
|
||||
|
||||
return nil, fmt.Errorf(
|
||||
"database still locked after recovery attempt: %w",
|
||||
"database still locked on retry: %w",
|
||||
err,
|
||||
)
|
||||
}
|
||||
|
||||
log.Debug("Database opened in TRUNCATE mode", "path", path)
|
||||
|
||||
// Switch back to WAL mode
|
||||
log.Debug("Switching database back to WAL mode", "path", path)
|
||||
if _, err := conn.ExecContext(ctx, "PRAGMA journal_mode=WAL"); err != nil {
|
||||
log.Warn("Failed to switch back to WAL mode", "path", path, "error", err)
|
||||
}
|
||||
|
||||
// Ensure foreign keys are enabled
|
||||
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=ON"); err != nil {
|
||||
log.Warn("Failed to enable foreign keys", "path", path, "error", err)
|
||||
}
|
||||
|
||||
db := &DB{conn: conn, path: path}
|
||||
if err := db.createSchema(ctx); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("creating schema: %w", err)
|
||||
db, err := finishOpen(ctx, conn, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
log.Debug("Database connection established successfully", "path", path)
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// errUntrustedSnapshotSchema is returned when a downloaded snapshot
|
||||
// database carries schema objects the real schema never defines, or is
|
||||
// missing a table the restore and deep-verify queries read.
|
||||
var errUntrustedSnapshotSchema = errors.New(
|
||||
"downloaded snapshot database has an untrusted schema")
|
||||
|
||||
// snapshotReadOnlyDSN builds the driver DSN that opens a materialized
|
||||
// snapshot database file read-only. mode=ro opens the file read-only at
|
||||
// the OS level, query_only rejects any write the engine is asked to make,
|
||||
// and trusted_schema=OFF refuses to run application code named in the
|
||||
// schema. The file: URI form is required for the driver to honour the
|
||||
// mode parameter.
|
||||
func snapshotReadOnlyDSN(path string) string {
|
||||
u := url.URL{
|
||||
Scheme: "file",
|
||||
Path: path,
|
||||
RawQuery: "mode=ro&_pragma=query_only(true)&_pragma=trusted_schema(false)",
|
||||
}
|
||||
|
||||
return u.String()
|
||||
}
|
||||
|
||||
// OpenReadOnly opens an already-materialized SQLite file for read-only
|
||||
// querying of a snapshot database downloaded from the store, used by
|
||||
// restore and deep verify. Unlike New it never applies schema migrations
|
||||
// and never writes: the connection is opened read-only with query_only
|
||||
// and trusted_schema=OFF. It refuses any file whose schema carries a
|
||||
// trigger, view or virtual table, or lacks an expected table, so a forged
|
||||
// file cannot redefine what the restore queries return. The caller owns
|
||||
// the file and must remove it.
|
||||
func OpenReadOnly(ctx context.Context, path string) (*DB, error) {
|
||||
conn, err := sql.Open("sqlite", snapshotReadOnlyDSN(path))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening read-only database: %w", err)
|
||||
}
|
||||
|
||||
configureConnPool(conn)
|
||||
|
||||
err = conn.PingContext(ctx)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
|
||||
return nil, fmt.Errorf("opening read-only database: %w", err)
|
||||
}
|
||||
|
||||
err = verifySnapshotSchema(ctx, conn)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &DB{conn: conn, path: path}, nil
|
||||
}
|
||||
|
||||
// verifySnapshotSchema rejects a downloaded database whose schema is not
|
||||
// the plain table set the real schema defines. Any trigger, view or
|
||||
// virtual table, or a missing expected table, fails the open.
|
||||
func verifySnapshotSchema(ctx context.Context, conn *sql.DB) error {
|
||||
// expectedSnapshotTables are the tables the restore and deep-verify
|
||||
// queries read. A downloaded database missing any of them is not a
|
||||
// genuine snapshot database and is refused.
|
||||
expectedSnapshotTables := []string{
|
||||
"blob_chunks",
|
||||
"blobs",
|
||||
"chunks",
|
||||
"file_chunks",
|
||||
"files",
|
||||
}
|
||||
|
||||
rows, err := conn.QueryContext(
|
||||
ctx, "SELECT type, name, sql FROM sqlite_master")
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading snapshot schema: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
present := make(map[string]struct{})
|
||||
|
||||
for rows.Next() {
|
||||
var objType, name string
|
||||
|
||||
var objSQL sql.NullString
|
||||
|
||||
err = rows.Scan(&objType, &name, &objSQL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading snapshot schema: %w", err)
|
||||
}
|
||||
|
||||
switch objType {
|
||||
case "trigger", "view":
|
||||
return fmt.Errorf(
|
||||
"%w: unexpected %s %q", errUntrustedSnapshotSchema, objType, name)
|
||||
case "table":
|
||||
if isVirtualTableSQL(objSQL.String) {
|
||||
return fmt.Errorf(
|
||||
"%w: unexpected virtual table %q",
|
||||
errUntrustedSnapshotSchema, name)
|
||||
}
|
||||
|
||||
present[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading snapshot schema: %w", err)
|
||||
}
|
||||
|
||||
for _, table := range expectedSnapshotTables {
|
||||
if _, ok := present[table]; !ok {
|
||||
return fmt.Errorf(
|
||||
"%w: missing table %q", errUntrustedSnapshotSchema, table)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isVirtualTableSQL reports whether a sqlite_master row's SQL defines a
|
||||
// virtual table. Virtual tables are recorded with type 'table' but a
|
||||
// "CREATE VIRTUAL TABLE" definition and can run module code, so they are
|
||||
// refused alongside triggers and views.
|
||||
func isVirtualTableSQL(createSQL string) bool {
|
||||
return strings.HasPrefix(
|
||||
strings.ToUpper(strings.TrimSpace(createSQL)), "CREATE VIRTUAL TABLE")
|
||||
}
|
||||
|
||||
// NewTestDB creates an in-memory SQLite database for testing purposes.
|
||||
// The database is automatically initialized with the schema and is ready
|
||||
// for use. Each call creates a new independent database instance.
|
||||
func NewTestDB() (*DB, error) {
|
||||
return New(context.Background(), ":memory:")
|
||||
}
|
||||
|
||||
// Close closes the database connection.
|
||||
// It ensures all pending operations are completed before closing.
|
||||
// Returns an error if the database connection cannot be closed properly.
|
||||
func (db *DB) Close() error {
|
||||
log.Debug("Closing database connection", "path", db.path)
|
||||
if err := db.conn.Close(); err != nil {
|
||||
|
||||
err := db.conn.Close()
|
||||
if err != nil {
|
||||
log.Error("Failed to close database", "path", db.path, "error", err)
|
||||
|
||||
return fmt.Errorf("failed to close database: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Database connection closed successfully", "path", db.path)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -179,54 +403,165 @@ func (db *DB) BeginTx(
|
||||
func (db *DB) ExecWithLog(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
args ...interface{},
|
||||
args ...any,
|
||||
) (sql.Result, error) {
|
||||
LogSQL("Execute", query, args...)
|
||||
|
||||
return db.conn.ExecContext(ctx, query, args...)
|
||||
}
|
||||
|
||||
// QueryRowWithLog executes a query that returns at most one row with SQL logging.
|
||||
// This is useful for queries that modify data and return values (e.g., INSERT ... RETURNING).
|
||||
// SQLite handles its own locking internally.
|
||||
// The query and args parameters follow the same format as sql.DB.QueryRowContext.
|
||||
// QueryRowWithLog executes a query that returns at most one row with SQL
|
||||
// logging. This is useful for queries that modify data and return values
|
||||
// (e.g., INSERT ... RETURNING). SQLite handles its own locking internally.
|
||||
// The query and args parameters follow the same format as
|
||||
// sql.DB.QueryRowContext.
|
||||
func (db *DB) QueryRowWithLog(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
args ...interface{},
|
||||
args ...any,
|
||||
) *sql.Row {
|
||||
LogSQL("QueryRow", query, args...)
|
||||
|
||||
return db.conn.QueryRowContext(ctx, query, args...)
|
||||
}
|
||||
|
||||
func (db *DB) createSchema(ctx context.Context) error {
|
||||
_, err := db.conn.ExecContext(ctx, schemaSQL)
|
||||
return err
|
||||
// collectMigrations reads the embedded schema directory and returns
|
||||
// migration filenames sorted lexicographically.
|
||||
func collectMigrations() ([]string, error) {
|
||||
entries, err := schemaFS.ReadDir("schema")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read schema directory: %w", err)
|
||||
}
|
||||
|
||||
var migrations []string
|
||||
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".sql") {
|
||||
migrations = append(migrations, entry.Name())
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(migrations)
|
||||
|
||||
return migrations, nil
|
||||
}
|
||||
|
||||
// NewTestDB creates an in-memory SQLite database for testing purposes.
|
||||
// The database is automatically initialized with the schema and is ready for use.
|
||||
// Each call creates a new independent database instance.
|
||||
func NewTestDB() (*DB, error) {
|
||||
return New(context.Background(), ":memory:")
|
||||
// bootstrapMigrationsTable ensures the schema_migrations table exists
|
||||
// by applying 000.sql if the table is missing.
|
||||
func bootstrapMigrationsTable(ctx context.Context, db *sql.DB) error {
|
||||
var tableExists int
|
||||
|
||||
err := db.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'",
|
||||
).Scan(&tableExists)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check for migrations table: %w", err)
|
||||
}
|
||||
|
||||
if tableExists > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
content, err := schemaFS.ReadFile("schema/000.sql")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read bootstrap migration 000.sql: %w", err)
|
||||
}
|
||||
|
||||
log.Info("applying bootstrap migration", "version", bootstrapVersion)
|
||||
|
||||
_, err = db.ExecContext(ctx, string(content))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to apply bootstrap migration: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// repeatPlaceholder generates a string of ", ?" repeated n times for IN clause construction.
|
||||
// For example, repeatPlaceholder(2) returns ", ?, ?".
|
||||
// applyMigrations applies all pending migrations to db. It first bootstraps
|
||||
// the schema_migrations table via 000.sql, then iterates through remaining
|
||||
// migration files in order.
|
||||
func applyMigrations(ctx context.Context, db *sql.DB) error {
|
||||
err := bootstrapMigrationsTable(ctx, db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
migrations, err := collectMigrations()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, migration := range migrations {
|
||||
version, parseErr := ParseMigrationVersion(migration)
|
||||
if parseErr != nil {
|
||||
return parseErr
|
||||
}
|
||||
|
||||
// Check if already applied.
|
||||
var count int
|
||||
|
||||
err := db.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM schema_migrations WHERE version = ?",
|
||||
version,
|
||||
).Scan(&count)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check migration status: %w", err)
|
||||
}
|
||||
|
||||
if count > 0 {
|
||||
log.Debug("migration already applied", "version", version)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// Read and apply migration.
|
||||
content, readErr := schemaFS.ReadFile(filepath.Join("schema", migration))
|
||||
if readErr != nil {
|
||||
return fmt.Errorf("failed to read migration %s: %w", migration, readErr)
|
||||
}
|
||||
|
||||
log.Info("applying migration", "version", version)
|
||||
|
||||
_, execErr := db.ExecContext(ctx, string(content))
|
||||
if execErr != nil {
|
||||
return fmt.Errorf("failed to apply migration %s: %w", migration, execErr)
|
||||
}
|
||||
|
||||
// Record migration as applied.
|
||||
_, recErr := db.ExecContext(ctx,
|
||||
"INSERT INTO schema_migrations (version) VALUES (?)",
|
||||
version,
|
||||
)
|
||||
if recErr != nil {
|
||||
return fmt.Errorf("failed to record migration %s: %w", migration, recErr)
|
||||
}
|
||||
|
||||
log.Info("migration applied successfully", "version", version)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// repeatPlaceholder generates a string of ", ?" repeated n times for IN
|
||||
// clause construction. For example, repeatPlaceholder(2) returns ", ?, ?".
|
||||
func repeatPlaceholder(n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
return strings.Repeat(", ?", n)
|
||||
}
|
||||
|
||||
// LogSQL logs SQL queries and their arguments when debug mode is enabled.
|
||||
// Debug mode is activated by setting the GODEBUG environment variable to include "vaultik".
|
||||
// This is useful for troubleshooting database operations and understanding query patterns.
|
||||
// Debug mode is activated by setting the GODEBUG environment variable to
|
||||
// include "vaultik". This is useful for troubleshooting database operations
|
||||
// and understanding query patterns.
|
||||
//
|
||||
// The operation parameter describes the type of SQL operation (e.g., "Execute", "Query").
|
||||
// The query parameter is the SQL statement being executed.
|
||||
// The args parameter contains the query arguments that will be interpolated.
|
||||
func LogSQL(operation, query string, args ...interface{}) {
|
||||
// The operation parameter describes the type of SQL operation (e.g.,
|
||||
// "Execute", "Query"). The query parameter is the SQL statement being
|
||||
// executed. The args parameter contains the query arguments that will be
|
||||
// interpolated.
|
||||
func LogSQL(operation, query string, args ...any) {
|
||||
if strings.Contains(os.Getenv("GODEBUG"), "vaultik") {
|
||||
log.Debug(
|
||||
"SQL "+operation,
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
//nolint:testpackage // exercises unexported migration internals
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDatabase(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||
|
||||
@@ -15,8 +19,10 @@ func TestDatabase(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -26,16 +32,20 @@ func TestDatabase(t *testing.T) {
|
||||
t.Fatal("database connection is nil")
|
||||
}
|
||||
|
||||
// Test schema creation (already done in New)
|
||||
// Test schema creation (already done in New via migrations)
|
||||
// Verify tables exist
|
||||
tables := []string{
|
||||
"schema_migrations",
|
||||
"files", "file_chunks", "chunks", "blobs",
|
||||
"blob_chunks", "chunk_files", "snapshots",
|
||||
}
|
||||
|
||||
for _, table := range tables {
|
||||
var name string
|
||||
err := db.conn.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name=?", table).Scan(&name)
|
||||
|
||||
err := db.conn.QueryRowContext(ctx,
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name=?", table,
|
||||
).Scan(&name)
|
||||
if err != nil {
|
||||
t.Errorf("table %s does not exist: %v", table, err)
|
||||
}
|
||||
@@ -43,6 +53,8 @@ func TestDatabase(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDatabaseInvalidPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Test with invalid path
|
||||
@@ -53,6 +65,8 @@ func TestDatabaseInvalidPath(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDatabaseConcurrentAccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||
|
||||
@@ -60,8 +74,10 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
@@ -71,18 +87,20 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
|
||||
index int
|
||||
err error
|
||||
}
|
||||
|
||||
results := make(chan result, 10)
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
for i := range 10 {
|
||||
go func(i int) {
|
||||
_, err := db.ExecWithLog(ctx, "INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)",
|
||||
_, err := db.ExecWithLog(ctx,
|
||||
"INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)",
|
||||
fmt.Sprintf("hash%d", i), i*1024)
|
||||
results <- result{index: i, err: err}
|
||||
}(i)
|
||||
}
|
||||
|
||||
// Wait for all goroutines and check results
|
||||
for i := 0; i < 10; i++ {
|
||||
for range 10 {
|
||||
r := <-results
|
||||
if r.err != nil {
|
||||
t.Fatalf("concurrent insert %d failed: %v", r.index, r.err)
|
||||
@@ -91,11 +109,279 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
|
||||
|
||||
// Verify all inserts succeeded
|
||||
var count int
|
||||
|
||||
err = db.conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM chunks").Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to count chunks: %v", err)
|
||||
}
|
||||
|
||||
if count != 10 {
|
||||
t.Errorf("expected 10 chunks, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewSetsJournalModeAndBusyTimeout checks that the connection settings
|
||||
// New passes reach SQLite. The driver drops a setting it does not recognise
|
||||
// without an error, so only reading the value back shows it took effect.
|
||||
func TestNewSetsJournalModeAndBusyTimeout(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
db, err := New(ctx, filepath.Join(t.TempDir(), "index.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create database: %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
var journalMode string
|
||||
|
||||
err = db.conn.QueryRowContext(ctx, "PRAGMA journal_mode").Scan(&journalMode)
|
||||
if err != nil {
|
||||
t.Fatalf("reading journal_mode: %v", err)
|
||||
}
|
||||
|
||||
if journalMode != "wal" {
|
||||
t.Errorf("journal_mode = %q, want %q", journalMode, "wal")
|
||||
}
|
||||
|
||||
var busyTimeout int
|
||||
|
||||
err = db.conn.QueryRowContext(ctx, "PRAGMA busy_timeout").Scan(&busyTimeout)
|
||||
if err != nil {
|
||||
t.Fatalf("reading busy_timeout: %v", err)
|
||||
}
|
||||
|
||||
if busyTimeout != busyTimeoutMs {
|
||||
t.Errorf("busy_timeout = %d, want %d", busyTimeout, busyTimeoutMs)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewWriteSucceedsWhileAnotherHandleReads opens the same index twice,
|
||||
// as a read-only command does while a backup runs, and checks that a write
|
||||
// on one handle commits while the other is in the middle of a read.
|
||||
func TestNewWriteSucceedsWhileAnotherHandleReads(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
dbPath := filepath.Join(t.TempDir(), "index.db")
|
||||
|
||||
reader, err := New(ctx, dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open reading handle: %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
writer, err := New(ctx, dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open writing handle: %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = writer.Close() }()
|
||||
|
||||
// The read lock taken by the SELECT is held until the transaction ends.
|
||||
readTx, err := reader.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("beginning read transaction: %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = readTx.Rollback() }()
|
||||
|
||||
var count int
|
||||
|
||||
err = readTx.QueryRowContext(ctx, "SELECT COUNT(*) FROM chunks").Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatalf("reading chunks: %v", err)
|
||||
}
|
||||
|
||||
_, err = writer.ExecWithLog(ctx,
|
||||
"INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)", "hash", 1024)
|
||||
if err != nil {
|
||||
t.Fatalf("write while another handle reads: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMigrationVersion(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
filename string
|
||||
wantVer int
|
||||
wantError bool
|
||||
}{
|
||||
{name: "valid 000.sql", filename: "000.sql", wantVer: 0, wantError: false},
|
||||
{name: "valid 001.sql", filename: "001.sql", wantVer: 1, wantError: false},
|
||||
{name: "valid 099.sql", filename: "099.sql", wantVer: 99, wantError: false},
|
||||
{
|
||||
name: "valid with description", filename: "001_initial_schema.sql",
|
||||
wantVer: 1, wantError: false,
|
||||
},
|
||||
{
|
||||
name: "valid large version", filename: "123_big_migration.sql",
|
||||
wantVer: 123, wantError: false,
|
||||
},
|
||||
{name: "invalid alpha version", filename: "abc.sql", wantVer: 0, wantError: true},
|
||||
{name: "invalid mixed chars", filename: "12a.sql", wantVer: 0, wantError: true},
|
||||
{name: "invalid no extension", filename: "schema.sql", wantVer: 0, wantError: true},
|
||||
{name: "empty string", filename: "", wantVer: 0, wantError: true},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := ParseMigrationVersion(tc.filename)
|
||||
if tc.wantError {
|
||||
if err == nil {
|
||||
t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error",
|
||||
tc.filename, got)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("ParseMigrationVersion(%q) unexpected error: %v",
|
||||
tc.filename, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if got != tc.wantVer {
|
||||
t.Errorf("ParseMigrationVersion(%q) = %d; want %d",
|
||||
tc.filename, got, tc.wantVer)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyMigrations_Idempotent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open database: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err := conn.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
conn.SetMaxOpenConns(1)
|
||||
conn.SetMaxIdleConns(1)
|
||||
|
||||
// First run: apply all migrations.
|
||||
err = applyMigrations(ctx, conn)
|
||||
if err != nil {
|
||||
t.Fatalf("first applyMigrations failed: %v", err)
|
||||
}
|
||||
|
||||
// Count rows in schema_migrations after first run.
|
||||
var countBefore int
|
||||
|
||||
err = conn.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM schema_migrations",
|
||||
).Scan(&countBefore)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to count schema_migrations after first run: %v", err)
|
||||
}
|
||||
|
||||
// Second run: must be a no-op.
|
||||
err = applyMigrations(ctx, conn)
|
||||
if err != nil {
|
||||
t.Fatalf("second applyMigrations failed: %v", err)
|
||||
}
|
||||
|
||||
// Count rows in schema_migrations after second run — must be unchanged.
|
||||
var countAfter int
|
||||
|
||||
err = conn.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM schema_migrations",
|
||||
).Scan(&countAfter)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to count schema_migrations after second run: %v", err)
|
||||
}
|
||||
|
||||
if countBefore != countAfter {
|
||||
t.Errorf("schema_migrations row count changed: before=%d, after=%d",
|
||||
countBefore, countAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open database: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err := conn.Close()
|
||||
if err != nil {
|
||||
t.Errorf("failed to close database: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
conn.SetMaxOpenConns(1)
|
||||
conn.SetMaxIdleConns(1)
|
||||
|
||||
// Verify schema_migrations does NOT exist yet.
|
||||
var tableBefore int
|
||||
|
||||
err = conn.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'",
|
||||
).Scan(&tableBefore)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to check for table before bootstrap: %v", err)
|
||||
}
|
||||
|
||||
if tableBefore != 0 {
|
||||
t.Fatal("schema_migrations table should not exist before bootstrap")
|
||||
}
|
||||
|
||||
// Run bootstrap.
|
||||
err = bootstrapMigrationsTable(ctx, conn)
|
||||
if err != nil {
|
||||
t.Fatalf("bootstrapMigrationsTable failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify schema_migrations now exists.
|
||||
var tableAfter int
|
||||
|
||||
err = conn.QueryRowContext(ctx,
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'",
|
||||
).Scan(&tableAfter)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to check for table after bootstrap: %v", err)
|
||||
}
|
||||
|
||||
if tableAfter != 1 {
|
||||
t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d",
|
||||
tableAfter)
|
||||
}
|
||||
|
||||
// Verify version 0 row exists.
|
||||
var version int
|
||||
|
||||
err = conn.QueryRowContext(ctx,
|
||||
"SELECT version FROM schema_migrations WHERE version = 0",
|
||||
).Scan(&version)
|
||||
if err != nil {
|
||||
t.Fatalf("version 0 row not found in schema_migrations: %v", err)
|
||||
}
|
||||
|
||||
if version != 0 {
|
||||
t.Errorf("expected version 0, got %d", version)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Fatal prints an error message to stderr and exits with status 1
|
||||
func Fatal(format string, args ...interface{}) {
|
||||
fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// CloseRows closes rows and exits on error
|
||||
func CloseRows(rows *sql.Rows) {
|
||||
if err := rows.Close(); err != nil {
|
||||
Fatal("failed to close rows: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -4,19 +4,26 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// FileChunkRepository provides access to the file_chunks table, which maps
|
||||
// files to their ordered constituent chunks.
|
||||
type FileChunkRepository struct {
|
||||
db *DB
|
||||
}
|
||||
|
||||
// NewFileChunkRepository creates a FileChunkRepository backed by db.
|
||||
func NewFileChunkRepository(db *DB) *FileChunkRepository {
|
||||
return &FileChunkRepository{db: db}
|
||||
}
|
||||
|
||||
func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileChunk) error {
|
||||
// Create inserts a file_chunks row (idempotently), using tx when non-nil.
|
||||
func (r *FileChunkRepository) Create(
|
||||
ctx context.Context, tx *sql.Tx, fc *FileChunk,
|
||||
) error {
|
||||
query := `
|
||||
INSERT INTO file_chunks (file_id, idx, chunk_hash)
|
||||
VALUES (?, ?, ?)
|
||||
@@ -27,7 +34,8 @@ func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileCh
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
|
||||
_, err = r.db.ExecWithLog(ctx, query,
|
||||
fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
@@ -37,7 +45,10 @@ func (r *FileChunkRepository) Create(ctx context.Context, tx *sql.Tx, fc *FileCh
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *FileChunkRepository) GetByPath(ctx context.Context, path string) ([]*FileChunk, error) {
|
||||
// GetByPath returns the ordered chunks of the file at the given path.
|
||||
func (r *FileChunkRepository) GetByPath(
|
||||
ctx context.Context, path string,
|
||||
) ([]*FileChunk, error) {
|
||||
query := `
|
||||
SELECT fc.file_id, fc.idx, fc.chunk_hash
|
||||
FROM file_chunks fc
|
||||
@@ -50,13 +61,16 @@ func (r *FileChunkRepository) GetByPath(ctx context.Context, path string) ([]*Fi
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying file chunks: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
return r.scanFileChunks(rows)
|
||||
}
|
||||
|
||||
// GetByFileID retrieves file chunks by file ID
|
||||
func (r *FileChunkRepository) GetByFileID(ctx context.Context, fileID types.FileID) ([]*FileChunk, error) {
|
||||
func (r *FileChunkRepository) GetByFileID(
|
||||
ctx context.Context, fileID types.FileID,
|
||||
) ([]*FileChunk, error) {
|
||||
query := `
|
||||
SELECT file_id, idx, chunk_hash
|
||||
FROM file_chunks
|
||||
@@ -68,13 +82,16 @@ func (r *FileChunkRepository) GetByFileID(ctx context.Context, fileID types.File
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying file chunks: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
return r.scanFileChunks(rows)
|
||||
}
|
||||
|
||||
// GetByPathTx retrieves file chunks within a transaction
|
||||
func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path string) ([]*FileChunk, error) {
|
||||
func (r *FileChunkRepository) GetByPathTx(
|
||||
ctx context.Context, tx *sql.Tx, path string,
|
||||
) ([]*FileChunk, error) {
|
||||
query := `
|
||||
SELECT fc.file_id, fc.idx, fc.chunk_hash
|
||||
FROM file_chunks fc
|
||||
@@ -84,40 +101,28 @@ func (r *FileChunkRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path
|
||||
`
|
||||
|
||||
LogSQL("GetByPathTx", query, path)
|
||||
|
||||
rows, err := tx.QueryContext(ctx, query, path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying file chunks: %w", err)
|
||||
}
|
||||
defer CloseRows(rows)
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
fileChunks, err := r.scanFileChunks(rows)
|
||||
LogSQL("GetByPathTx", "Complete", path, "count", len(fileChunks))
|
||||
|
||||
return fileChunks, err
|
||||
}
|
||||
|
||||
// scanFileChunks is a helper that scans file chunk rows
|
||||
func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) {
|
||||
var fileChunks []*FileChunk
|
||||
for rows.Next() {
|
||||
var fc FileChunk
|
||||
var fileIDStr, chunkHashStr string
|
||||
err := rows.Scan(&fileIDStr, &fc.Idx, &chunkHashStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning file chunk: %w", err)
|
||||
}
|
||||
fc.FileID, err = types.ParseFileID(fileIDStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||
}
|
||||
fc.ChunkHash = types.ChunkHash(chunkHashStr)
|
||||
fileChunks = append(fileChunks, &fc)
|
||||
}
|
||||
|
||||
return fileChunks, rows.Err()
|
||||
}
|
||||
|
||||
func (r *FileChunkRepository) DeleteByPath(ctx context.Context, tx *sql.Tx, path string) error {
|
||||
query := `DELETE FROM file_chunks WHERE file_id = (SELECT id FROM files WHERE path = ?)`
|
||||
// DeleteByPath deletes all file_chunks rows for the file at the given path.
|
||||
func (r *FileChunkRepository) DeleteByPath(
|
||||
ctx context.Context, tx *sql.Tx, path string,
|
||||
) error {
|
||||
query := `
|
||||
DELETE FROM file_chunks
|
||||
WHERE file_id = (SELECT id FROM files WHERE path = ?)
|
||||
`
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
@@ -134,7 +139,9 @@ func (r *FileChunkRepository) DeleteByPath(ctx context.Context, tx *sql.Tx, path
|
||||
}
|
||||
|
||||
// DeleteByFileID deletes all chunks for a file by its UUID
|
||||
func (r *FileChunkRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fileID types.FileID) error {
|
||||
func (r *FileChunkRepository) DeleteByFileID(
|
||||
ctx context.Context, tx *sql.Tx, fileID types.FileID,
|
||||
) error {
|
||||
query := `DELETE FROM file_chunks WHERE file_id = ?`
|
||||
|
||||
var err error
|
||||
@@ -152,7 +159,11 @@ func (r *FileChunkRepository) DeleteByFileID(ctx context.Context, tx *sql.Tx, fi
|
||||
}
|
||||
|
||||
// DeleteByFileIDs deletes all chunks for multiple files in a single statement.
|
||||
func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, fileIDs []types.FileID) error {
|
||||
//
|
||||
//nolint:dupl // symmetric implementation for a parallel association table
|
||||
func (r *FileChunkRepository) DeleteByFileIDs(
|
||||
ctx context.Context, tx *sql.Tx, fileIDs []types.FileID,
|
||||
) error {
|
||||
if len(fileIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -161,14 +172,15 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
const batchSize = 500
|
||||
|
||||
for i := 0; i < len(fileIDs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(fileIDs) {
|
||||
end = len(fileIDs)
|
||||
}
|
||||
end := min(i+batchSize, len(fileIDs))
|
||||
|
||||
batch := fileIDs[i:end]
|
||||
|
||||
query := "DELETE FROM file_chunks WHERE file_id IN (?" + repeatPlaceholder(len(batch)-1) + ")"
|
||||
args := make([]interface{}, len(batch))
|
||||
//nolint:gosec // G202: concatenates constant SQL and "?" placeholders only
|
||||
query := "DELETE FROM file_chunks WHERE file_id IN (?" +
|
||||
repeatPlaceholder(len(batch)-1) + ")"
|
||||
|
||||
args := make([]any, len(batch))
|
||||
for j, id := range batch {
|
||||
args[j] = id.String()
|
||||
}
|
||||
@@ -179,6 +191,7 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch deleting file_chunks: %w", err)
|
||||
}
|
||||
@@ -189,32 +202,44 @@ func (r *FileChunkRepository) DeleteByFileIDs(ctx context.Context, tx *sql.Tx, f
|
||||
|
||||
// CreateBatch inserts multiple file_chunks in a single statement for efficiency.
|
||||
// Batches are automatically split to stay within SQLite's variable limit.
|
||||
func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs []FileChunk) error {
|
||||
func (r *FileChunkRepository) CreateBatch(
|
||||
ctx context.Context, tx *sql.Tx, fcs []FileChunk,
|
||||
) error {
|
||||
if len(fcs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SQLite has a limit on variables (typically 999 or 32766).
|
||||
// Each FileChunk has 3 values, so batch at 300 to be safe.
|
||||
// Each file_chunks row binds this many SQL variables.
|
||||
const fileChunkCols = 3
|
||||
|
||||
// SQLite has a limit on variables (typically 999 or 32766), so batch
|
||||
// at 300 rows to be safe.
|
||||
const batchSize = 300
|
||||
|
||||
for i := 0; i < len(fcs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(fcs) {
|
||||
end = len(fcs)
|
||||
}
|
||||
end := min(i+batchSize, len(fcs))
|
||||
|
||||
batch := fcs[i:end]
|
||||
|
||||
// Build the query with multiple value sets
|
||||
query := "INSERT INTO file_chunks (file_id, idx, chunk_hash) VALUES "
|
||||
args := make([]interface{}, 0, len(batch)*3)
|
||||
|
||||
args := make([]any, 0, len(batch)*fileChunkCols)
|
||||
|
||||
var querySb211 strings.Builder
|
||||
|
||||
for j, fc := range batch {
|
||||
if j > 0 {
|
||||
query += ", "
|
||||
querySb211.WriteString(", ")
|
||||
}
|
||||
query += "(?, ?, ?)"
|
||||
|
||||
querySb211.WriteString("(?, ?, ?)")
|
||||
|
||||
args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
|
||||
}
|
||||
|
||||
query += querySb211.String()
|
||||
|
||||
query += " ON CONFLICT(file_id, idx) DO NOTHING"
|
||||
|
||||
var err error
|
||||
@@ -223,6 +248,7 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, args...)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("batch inserting file_chunks: %w", err)
|
||||
}
|
||||
@@ -232,17 +258,50 @@ func (r *FileChunkRepository) CreateBatch(ctx context.Context, tx *sql.Tx, fcs [
|
||||
}
|
||||
|
||||
// GetByFile is an alias for GetByPath for compatibility
|
||||
func (r *FileChunkRepository) GetByFile(ctx context.Context, path string) ([]*FileChunk, error) {
|
||||
func (r *FileChunkRepository) GetByFile(
|
||||
ctx context.Context, path string,
|
||||
) ([]*FileChunk, error) {
|
||||
LogSQL("GetByFile", "Starting", path)
|
||||
result, err := r.GetByPath(ctx, path)
|
||||
LogSQL("GetByFile", "Complete", path, "count", len(result))
|
||||
|
||||
return result, err
|
||||
}
|
||||
|
||||
// GetByFileTx retrieves file chunks within a transaction
|
||||
func (r *FileChunkRepository) GetByFileTx(ctx context.Context, tx *sql.Tx, path string) ([]*FileChunk, error) {
|
||||
func (r *FileChunkRepository) GetByFileTx(
|
||||
ctx context.Context, tx *sql.Tx, path string,
|
||||
) ([]*FileChunk, error) {
|
||||
LogSQL("GetByFileTx", "Starting", path)
|
||||
result, err := r.GetByPathTx(ctx, tx, path)
|
||||
LogSQL("GetByFileTx", "Complete", path, "count", len(result))
|
||||
|
||||
return result, err
|
||||
}
|
||||
|
||||
// scanFileChunks is a helper that scans file chunk rows
|
||||
func (r *FileChunkRepository) scanFileChunks(rows *sql.Rows) ([]*FileChunk, error) {
|
||||
var fileChunks []*FileChunk
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
fc FileChunk
|
||||
fileIDStr, chunkHashStr string
|
||||
)
|
||||
|
||||
err := rows.Scan(&fileIDStr, &fc.Idx, &chunkHashStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning file chunk: %w", err)
|
||||
}
|
||||
|
||||
fc.FileID, err = types.ParseFileID(fileIDStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing file ID: %w", err)
|
||||
}
|
||||
|
||||
fc.ChunkHash = types.ChunkHash(chunkHashStr)
|
||||
fileChunks = append(fileChunks, &fc)
|
||||
}
|
||||
|
||||
return fileChunks, rows.Err()
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user