Compare commits
16
Commits
617a2c6966
...
next
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
343129f891 | ||
|
|
a50e3fa038 | ||
|
|
6fcd8e1668 | ||
|
|
aab6a87f8c | ||
|
|
c355ef4d25 | ||
|
|
5927e1aa3d | ||
|
|
9ca962969a | ||
|
|
89ebfc78e2 | ||
|
|
07ef3a1c78 | ||
|
|
c423d13191 | ||
|
|
75a10d3a22 | ||
|
|
3d56dd7eb0 | ||
|
|
bdce350041 | ||
|
|
753bc3ef60 | ||
|
|
d2a0510cb4 | ||
|
|
583f65040a |
@@ -1,9 +1,9 @@
|
|||||||
name: check
|
name: check
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [main]
|
branches: [main, next]
|
||||||
pull_request:
|
pull_request:
|
||||||
branches: [main]
|
branches: [main, next]
|
||||||
jobs:
|
jobs:
|
||||||
check:
|
check:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
@@ -20,33 +20,21 @@ jobs:
|
|||||||
# check.yml runs script/cibuild, which does all of its work inside
|
# check.yml runs script/cibuild, which does all of its work inside
|
||||||
# the digest-pinned Dockerfile images -- so without this step the
|
# the digest-pinned Dockerfile images -- so without this step the
|
||||||
# release either fails at the before-hook or, worse, ships binaries
|
# release either fails at the before-hook or, worse, ships binaries
|
||||||
# built by whatever unpinned Go the runner happens to carry.
|
# built by whatever Go the runner happens to carry.
|
||||||
# REPO_POLICIES.md requires every external reference to be pinned,
|
|
||||||
# and script/release already refuses a goreleaser that is not the
|
|
||||||
# pinned build; the compiler that actually produces the artifacts
|
|
||||||
# is the last thing that should be exempt from that.
|
|
||||||
#
|
#
|
||||||
# go-version-file rather than a literal: go.mod's `go 1.26.1` is
|
# actions/setup-go would pin the action by commit sha, but the Go
|
||||||
# the single source of truth for the toolchain, the same way the
|
# tarball it downloads at runtime is verified against no value in
|
||||||
# Dockerfile FROM line is the single source of truth for the
|
# this repo, and the action exposes no checksum input.
|
||||||
# linter version that script/lint enforces. It is a three-component
|
# REPO_POLICIES.md requires every external reference to be pinned
|
||||||
# version, so setup-go resolves it exactly -- no silent drift onto
|
# by hash with no exceptions, and this is the compiler that
|
||||||
# a newer patch release.
|
# produces the published binaries -- the input where a substituted
|
||||||
#
|
# artifact matters most. So Go is installed the way goreleaser is:
|
||||||
# actions/setup-go v5.6.0, 2025-12-15. Pinned by commit sha, like
|
# script/install-go downloads the exact archive for go.mod's `go`
|
||||||
# the checkout above. v5.x is a node20 action, matching the node20
|
# directive and refuses it unless its sha256 matches the value
|
||||||
# actions/checkout v4 already in use here; the v6/v7 line requires
|
# committed in the script, then puts .tool/go/bin on PATH for the
|
||||||
# a node24 runner, which this Gitea runner has never been asked
|
# steps below.
|
||||||
# for and cannot be assumed to provide.
|
|
||||||
- name: Install Go
|
- name: Install Go
|
||||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff
|
run: script/install-go
|
||||||
with:
|
|
||||||
go-version-file: go.mod
|
|
||||||
# setup-go's module cache needs a runner-side cache backend.
|
|
||||||
# A release is cut rarely and a cold module download costs
|
|
||||||
# seconds; a release failing because a cache service is absent
|
|
||||||
# costs a re-tag. Off, deliberately.
|
|
||||||
cache: false
|
|
||||||
- name: Install goreleaser
|
- name: Install goreleaser
|
||||||
run: script/install-goreleaser
|
run: script/install-goreleaser
|
||||||
- name: Release
|
- name: Release
|
||||||
@@ -58,3 +46,8 @@ jobs:
|
|||||||
# It is deliberately not the runner's automatic token, which is
|
# It is deliberately not the runner's automatic token, which is
|
||||||
# not guaranteed to carry that scope.
|
# not guaranteed to carry that scope.
|
||||||
GITEA_TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
GITEA_TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||||
|
# Build with the toolchain install-go just verified, never a
|
||||||
|
# different one auto-downloaded from a `toolchain` directive:
|
||||||
|
# the point of the hash pin is that this exact compiler makes
|
||||||
|
# the release.
|
||||||
|
GOTOOLCHAIN: local
|
||||||
|
|||||||
@@ -104,7 +104,12 @@ Version: 2025-06-08
|
|||||||
|
|
||||||
13. Pre-1.0: NEVER write database migrations. There are no live databases
|
13. Pre-1.0: NEVER write database migrations. There are no live databases
|
||||||
anywhere — every user's local index can be rebuilt from a fresh full
|
anywhere — every user's local index can be rebuilt from a fresh full
|
||||||
backup. When the schema changes, just change `schema.sql` (and any code
|
backup. To change the schema, edit `internal/database/schema/001.sql`
|
||||||
that touches the affected tables). The local index is disposable until
|
(and any code that touches the affected tables) directly; do not add new
|
||||||
1.0 ships and is tagged.
|
numbered schema files. Those numbered files and the `schema_migrations`
|
||||||
|
table they populate only bootstrap a fresh database — they are not an
|
||||||
|
upgrade path. The local index is disposable until 1.0 ships and is
|
||||||
|
tagged; once 1.0 is tagged that clause expires and the question of
|
||||||
|
upgrading existing indexes returns. See [`docs/DATAMODEL.md`](docs/DATAMODEL.md)
|
||||||
|
for the full explanation.
|
||||||
|
|
||||||
|
|||||||
+7
-1
@@ -366,11 +366,17 @@ bucket/
|
|||||||
│ └── {full-hash} # Compressed+encrypted blob
|
│ └── {full-hash} # Compressed+encrypted blob
|
||||||
│
|
│
|
||||||
└── metadata/
|
└── metadata/
|
||||||
└── {snapshot-id}/
|
└── {remote-key}/
|
||||||
├── db.zst.age # Encrypted binary SQLite database
|
├── db.zst.age # Encrypted binary SQLite database
|
||||||
└── manifest.json.zst # Blob list (for pruning/verification)
|
└── manifest.json.zst # Blob list (for pruning/verification)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The `{remote-key}` directory name is a one-way double SHA-256 hash of the human
|
||||||
|
snapshot ID, so the human ID (hostname, snapshot name, timestamp) is never
|
||||||
|
written to the store as a directory name. See
|
||||||
|
[docs/REPOSTRUCTURE.md](docs/REPOSTRUCTURE.md#remote-key-derivation) for the
|
||||||
|
derivation and a worked example.
|
||||||
|
|
||||||
## Thread Safety
|
## Thread Safety
|
||||||
|
|
||||||
- `Packer`: Thread-safe via mutex. Multiple goroutines can call `AddChunk()`.
|
- `Packer`: Thread-safe via mutex. Multiple goroutines can call `AddChunk()`.
|
||||||
|
|||||||
+29
-6
@@ -20,10 +20,10 @@
|
|||||||
# golang:1.26.1-alpine, 2026-03-17
|
# golang:1.26.1-alpine, 2026-03-17
|
||||||
FROM golang:1.26.1-alpine@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f6c5bd6be49ed82039 AS builder
|
FROM golang:1.26.1-alpine@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f6c5bd6be49ed82039 AS builder
|
||||||
|
|
||||||
ARG VERSION=dev
|
# Build tooling: make, plus a C toolchain because `go test -race` needs cgo.
|
||||||
|
# The sqlite driver is pure Go (modernc.org/sqlite), so no sqlite library or
|
||||||
# Install build dependencies for CGO (mattn/go-sqlite3) and sqlite3 CLI (tests)
|
# CLI is required.
|
||||||
RUN apk add --no-cache make build-base sqlite
|
RUN apk add --no-cache make build-base
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
@@ -64,14 +64,37 @@ RUN [ -n "$CHECK_EPOCH" ] || exit 1
|
|||||||
RUN echo "check epoch: ${CHECK_EPOCH}" && make fmt-check
|
RUN echo "check epoch: ${CHECK_EPOCH}" && make fmt-check
|
||||||
RUN echo "check epoch: ${CHECK_EPOCH}" && make test
|
RUN echo "check epoch: ${CHECK_EPOCH}" && make test
|
||||||
|
|
||||||
|
# Version, commit and build date are computed on the host by
|
||||||
|
# script/docker and script/cibuild (where .git exists) and passed in as
|
||||||
|
# build args. The build context excludes .git (see .dockerignore), so
|
||||||
|
# the build cannot derive them itself: it used to try, with `git
|
||||||
|
# rev-parse` inside this stage, and always got "unknown". VERSION comes
|
||||||
|
# from script/version, the source of truth shared with the Makefile, so
|
||||||
|
# it carries the same tag / dev-<sha> / -dirty rules and a Docker image
|
||||||
|
# reports the same string a local build of the same tree would.
|
||||||
|
#
|
||||||
|
# The defaults are the fallback for a bare `docker build .` that passes
|
||||||
|
# none of them: an unset arg would otherwise stamp an empty string and
|
||||||
|
# produce an image that cannot report its own version, commit or date.
|
||||||
|
# They match what an out-of-git build reports elsewhere.
|
||||||
|
#
|
||||||
|
# These ARGs sit here, after the checks, rather than at the top of the
|
||||||
|
# stage: every commit changes their values, and a value change
|
||||||
|
# invalidates all layers below the ARG. Declared up top they would bust
|
||||||
|
# `go mod download`; here they only rekey this build layer, which the
|
||||||
|
# COPY of the sources above already rebuilds on any change anyway.
|
||||||
|
ARG VERSION=dev
|
||||||
|
ARG COMMIT=unknown
|
||||||
|
ARG COMMIT_DATE=unknown
|
||||||
|
|
||||||
# Build (pure Go, no CGO required since we use modernc.org/sqlite)
|
# Build (pure Go, no CGO required since we use modernc.org/sqlite)
|
||||||
RUN CGO_ENABLED=0 go build -ldflags "-X 'sneak.berlin/go/vaultik/internal/globals.Version=${VERSION}' -X 'sneak.berlin/go/vaultik/internal/globals.Commit=$(git rev-parse HEAD 2>/dev/null || echo unknown)' -X 'sneak.berlin/go/vaultik/internal/globals.CommitDate=$(git show -s --format=%cs HEAD 2>/dev/null || echo unknown)'" -o /vaultik ./cmd/vaultik
|
RUN CGO_ENABLED=0 go build -ldflags "-X 'sneak.berlin/go/vaultik/internal/globals.Version=${VERSION}' -X 'sneak.berlin/go/vaultik/internal/globals.Commit=${COMMIT}' -X 'sneak.berlin/go/vaultik/internal/globals.CommitDate=${COMMIT_DATE}'" -o /vaultik ./cmd/vaultik
|
||||||
|
|
||||||
# Runtime stage
|
# Runtime stage
|
||||||
# alpine:3.21, 2026-02-25
|
# alpine:3.21, 2026-02-25
|
||||||
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
||||||
|
|
||||||
RUN apk add --no-cache ca-certificates sqlite
|
RUN apk add --no-cache ca-certificates
|
||||||
|
|
||||||
# Copy binary from builder
|
# Copy binary from builder
|
||||||
COPY --from=builder /vaultik /usr/local/bin/vaultik
|
COPY --from=builder /vaultik /usr/local/bin/vaultik
|
||||||
|
|||||||
+5
-5
@@ -72,11 +72,11 @@ RUN [ -n "$CHECK_EPOCH" ] || exit 1
|
|||||||
# running, and exits 0 reporting `0 issues.` on a tree the real config
|
# running, and exits 0 reporting `0 issues.` on a tree the real config
|
||||||
# fails. Demonstrated on this repo at this pin, recorded on
|
# fails. Demonstrated on this repo at this pin, recorded on
|
||||||
# https://git.eeqj.de/sneak/vaultik/pulls/114: with a planted
|
# https://git.eeqj.de/sneak/vaultik/pulls/114: with a planted
|
||||||
# over-length line, `script/lint` exits 1 naming the `lll` finding with
|
# over-length line, `script/lint` exits 1 naming the `revive` finding
|
||||||
# `linters:` and exits 0 with `linterz:`. A set-but-ineffective config
|
# with `linters:` and exits 0 with `linterz:`. A set-but-ineffective
|
||||||
# quietly falling back to defaults is precisely the false-green class
|
# config quietly falling back to defaults is precisely the false-green
|
||||||
# this gate exists to eliminate, so it must not sit in the gate's own
|
# class this gate exists to eliminate, so it must not sit in the gate's
|
||||||
# configuration.
|
# own configuration.
|
||||||
#
|
#
|
||||||
# `config verify` catches it, and it does so OFFLINE at this pinned
|
# `config verify` catches it, and it does so OFFLINE at this pinned
|
||||||
# version -- verified, not assumed. Under `docker run --network none`
|
# version -- verified, not assumed. Under `docker run --network none`
|
||||||
|
|||||||
@@ -84,6 +84,57 @@ VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...' vaultik snapshot restore <snapshot-i
|
|||||||
# 0 3 * * * vaultik snapshot create --cron --prune --keep-newer-than 4w
|
# 0 3 * * * vaultik snapshot create --cron --prune --keep-newer-than 4w
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## restoring on another machine
|
||||||
|
|
||||||
|
Restoring on a host that never ran the backup — a replacement machine
|
||||||
|
after the original is gone — is the case vaultik is built for. That host
|
||||||
|
needs only three things: the `vaultik` binary, the age **private** key,
|
||||||
|
and the storage credentials for the destination. It does **not** need the
|
||||||
|
local index, the original config file, or the original hostname.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# install
|
||||||
|
go install sneak.berlin/go/vaultik/cmd/vaultik@latest
|
||||||
|
|
||||||
|
# create a config and point it at the ORIGINAL backup destination
|
||||||
|
vaultik config init
|
||||||
|
vaultik config set storage_url "s3://bucket/prefix?endpoint=https://s3.example.com"
|
||||||
|
vaultik config set s3.access_key_id "..."
|
||||||
|
vaultik config set s3.secret_access_key "..."
|
||||||
|
|
||||||
|
# see what is on the destination store
|
||||||
|
vaultik snapshot list
|
||||||
|
```
|
||||||
|
|
||||||
|
`snapshot list` reads the destination store without the private key. A
|
||||||
|
snapshot that is not in this host's (empty) local index is shown as
|
||||||
|
remote-only: its row is identified by `<remote only:...>` rather than by
|
||||||
|
a `hostname_name_timestamp` name, because the name lives only in the
|
||||||
|
local index and the encrypted database and cannot be recovered from the
|
||||||
|
store. Its timestamp and compressed size are real. (See the `snapshot
|
||||||
|
list` description under [command details](#command-details) for the full
|
||||||
|
explanation.)
|
||||||
|
|
||||||
|
Use that remote key — the hex printed inside `<remote only:...>`, or the
|
||||||
|
full `remote_key` from `snapshot list --json` — to restore and verify:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# restore everything to /tmp/restored, then check every restored file's
|
||||||
|
# chunk hashes
|
||||||
|
VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...' \
|
||||||
|
vaultik snapshot restore --verify <remote-key> /tmp/restored
|
||||||
|
|
||||||
|
# optionally, deep-verify the snapshot against the store (downloads and
|
||||||
|
# cryptographically checks every blob)
|
||||||
|
VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...' \
|
||||||
|
vaultik snapshot verify --deep <remote-key>
|
||||||
|
```
|
||||||
|
|
||||||
|
`age_recipients` (the public key) is not needed to restore — only the
|
||||||
|
private key in `VAULTIK_AGE_SECRET_KEY`. Both the abbreviated key printed
|
||||||
|
in the table and the full 64-character key from `--json` are accepted; a
|
||||||
|
leading part of the key is enough as long as it is unambiguous.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## cli
|
## cli
|
||||||
@@ -245,13 +296,16 @@ local index alone, and still exits zero.
|
|||||||
* Default (shallow): checks that all blobs referenced in the manifest exist in storage
|
* Default (shallow): checks that all blobs referenced in the manifest exist in storage
|
||||||
* `--deep`: Downloads and decrypts each blob, verifies chunk hashes against the
|
* `--deep`: Downloads and decrypts each blob, verifies chunk hashes against the
|
||||||
encrypted metadata database
|
encrypted metadata database
|
||||||
|
* Accepts the same identifiers as `snapshot restore`: a snapshot ID, or a
|
||||||
|
remote-only snapshot's remote key (or an unambiguous leading part of it)
|
||||||
* `--json`: Output results as JSON
|
* `--json`: Output results as JSON
|
||||||
|
|
||||||
**`snapshot purge`**: Remove old snapshots based on criteria. Retention is
|
**`snapshot purge`**: Remove old snapshots based on criteria. Retention is
|
||||||
per-snapshot-name (`--keep-latest` keeps the latest of each name, not the
|
per-snapshot-name (`--keep-latest` keeps the latest of each name, not the
|
||||||
latest globally).
|
latest globally).
|
||||||
* `--keep-latest`: Keep only the most recent snapshot of each name
|
* `--keep-latest`: Keep only the most recent snapshot of each name
|
||||||
* `--older-than <duration>`: Remove snapshots older than duration (e.g. `30d`, `6m`, `1y`)
|
* `--older-than <duration>`: Remove snapshots older than duration (e.g. `30d`,
|
||||||
|
`4w`, `6mo`, `1y`; `m` is minutes, `mo` is months)
|
||||||
* `--snapshot <name>`: Restrict to specific snapshot names (repeat for multiple)
|
* `--snapshot <name>`: Restrict to specific snapshot names (repeat for multiple)
|
||||||
* `--force`: Skip confirmation prompt
|
* `--force`: Skip confirmation prompt
|
||||||
|
|
||||||
@@ -274,6 +328,10 @@ on the destination in one go, use `vaultik remote nuke --force`.
|
|||||||
|
|
||||||
**`snapshot restore`**: Restore files from a backup snapshot.
|
**`snapshot restore`**: Restore files from a backup snapshot.
|
||||||
* Requires `VAULTIK_AGE_SECRET_KEY` environment variable
|
* Requires `VAULTIK_AGE_SECRET_KEY` environment variable
|
||||||
|
* Accepts a snapshot ID, or — for a snapshot only on the destination
|
||||||
|
store — its remote key (or an unambiguous leading part of it) as shown
|
||||||
|
by `snapshot list`. See
|
||||||
|
[restoring on another machine](#restoring-on-another-machine).
|
||||||
* Optional path arguments to restore specific files/directories (default: all)
|
* Optional path arguments to restore specific files/directories (default: all)
|
||||||
* Preserves file permissions, timestamps, ownership (ownership requires root),
|
* Preserves file permissions, timestamps, ownership (ownership requires root),
|
||||||
symlinks, and empty directories
|
symlinks, and empty directories
|
||||||
@@ -344,7 +402,7 @@ both are set.
|
|||||||
├── blobs/
|
├── blobs/
|
||||||
│ └── <aa>/<bb>/<full_blob_hash>
|
│ └── <aa>/<bb>/<full_blob_hash>
|
||||||
└── metadata/
|
└── metadata/
|
||||||
└── <snapshot_id>/
|
└── <remote-key>/
|
||||||
├── db.zst.age # Encrypted binary SQLite database
|
├── db.zst.age # Encrypted binary SQLite database
|
||||||
└── manifest.json.zst # Unencrypted blob list (for pruning)
|
└── manifest.json.zst # Unencrypted blob list (for pruning)
|
||||||
```
|
```
|
||||||
@@ -355,8 +413,18 @@ both are set.
|
|||||||
* `manifest.json.zst` is an unencrypted compressed JSON blob list, enabling
|
* `manifest.json.zst` is an unencrypted compressed JSON blob list, enabling
|
||||||
pruning without the private key
|
pruning without the private key
|
||||||
|
|
||||||
Snapshot IDs follow the format `<hostname>_<snapshot-name>_<RFC3339-timestamp>`
|
Snapshot IDs follow the human-readable format
|
||||||
(e.g. `server1_home_2025-06-01T12:00:00Z`).
|
`<hostname>_<snapshot-name>_<RFC3339-timestamp>` (e.g.
|
||||||
|
`server1_home_2025-06-01T12:00:00Z`), but this ID is never written to the
|
||||||
|
destination store in plaintext. Each snapshot's metadata directory is named
|
||||||
|
with its `<remote-key>`, a one-way double SHA-256 hash of the ID, so a listing
|
||||||
|
of the store reveals no hostname or snapshot name. The backup time is not
|
||||||
|
hidden: manifest.json.zst carries a plaintext timestamp, and object
|
||||||
|
modification times are visible at the storage layer regardless. For example,
|
||||||
|
`server1_home_2025-06-01T12:00:00Z` is stored under
|
||||||
|
`metadata/17f97bcde958748af076b926af59823943db59e80ce7170b40f124dfa28f64aa/`.
|
||||||
|
See [docs/REPOSTRUCTURE.md](docs/REPOSTRUCTURE.md#remote-key-derivation) for the
|
||||||
|
derivation.
|
||||||
|
|
||||||
### data flow
|
### data flow
|
||||||
|
|
||||||
@@ -373,7 +441,7 @@ Snapshot IDs follow the format `<hostname>_<snapshot-name>_<RFC3339-timestamp>`
|
|||||||
|
|
||||||
**restore:**
|
**restore:**
|
||||||
|
|
||||||
1. Download and decrypt `metadata/<snapshot_id>/db.zst.age`
|
1. Download and decrypt `metadata/<remote-key>/db.zst.age`
|
||||||
2. Open the binary SQLite database
|
2. Open the binary SQLite database
|
||||||
3. Query files (optionally filtered by paths)
|
3. Query files (optionally filtered by paths)
|
||||||
4. Download and decrypt required blobs
|
4. Download and decrypt required blobs
|
||||||
@@ -446,9 +514,13 @@ Key fields:
|
|||||||
sequentially. Restore speed is bound by single-stream throughput.
|
sequentially. Restore speed is bound by single-stream throughput.
|
||||||
* **Device nodes, named pipes, and sockets are silently skipped.** Only
|
* **Device nodes, named pipes, and sockets are silently skipped.** Only
|
||||||
regular files, directories, and symlinks are backed up.
|
regular files, directories, and symlinks are backed up.
|
||||||
* **No database migrations.** If the local SQLite schema changes between
|
* **No upgrade path between versions.** There is no supported way to carry
|
||||||
versions, delete the local database (`vaultik database delete`) and run
|
an existing local index across a schema change; if the local SQLite
|
||||||
a full backup. Remote storage is unaffected.
|
schema changes between versions, delete the local database (`vaultik
|
||||||
|
database delete`) and run a full backup. Remote storage is unaffected.
|
||||||
|
(The binary does embed numbered schema files and a `schema_migrations`
|
||||||
|
table to bootstrap a fresh database — see [`docs/DATAMODEL.md`](docs/DATAMODEL.md)
|
||||||
|
— but that is not an upgrade path.)
|
||||||
* **Files that change during backup may be inconsistent.** There is no
|
* **Files that change during backup may be inconsistent.** There is no
|
||||||
filesystem snapshot or freeze. If a file is modified between the scan
|
filesystem snapshot or freeze. If a file is modified between the scan
|
||||||
and chunk phases, the backed-up copy may reflect a partial write.
|
and chunk phases, the backed-up copy may reflect a partial write.
|
||||||
@@ -514,14 +586,12 @@ priority.
|
|||||||
|
|
||||||
### infrastructure
|
### infrastructure
|
||||||
|
|
||||||
* **Cross-machine restore documentation.** The "restore from
|
* **Cross-version schema upgrades.** There is no upgrade path between
|
||||||
another host" workflow works but isn't documented as a
|
released versions — pre-1.0 schema changes are handled by `vaultik
|
||||||
first-class operation in this README. Worth a dedicated section
|
database delete` plus a full re-scan (see
|
||||||
once it's settled.
|
[`docs/DATAMODEL.md`](docs/DATAMODEL.md)). Post-1.0 we'll need a
|
||||||
* **Schema migrations.** Currently nonexistent — pre-1.0 schema
|
migration story to keep existing index databases usable across
|
||||||
changes are handled by `vaultik database delete` plus a full
|
upgrades.
|
||||||
re-scan. Post-1.0 we'll need a migration story to keep existing
|
|
||||||
index databases usable across upgrades.
|
|
||||||
* **Storage backend coverage tests.** S3, file://, and rclone://
|
* **Storage backend coverage tests.** S3, file://, and rclone://
|
||||||
all share the Storer interface but the rclone path is the least
|
all share the Storer interface but the rclone path is the least
|
||||||
exercised in CI.
|
exercised in CI.
|
||||||
@@ -603,7 +673,6 @@ regardless of color setting (emoji are not color).
|
|||||||
and the pre-commit hook both run it. A `golangci-lint` installed on
|
and the pre-commit hook both run it. A `golangci-lint` installed on
|
||||||
`PATH` is not a substitute and is never used on a host, whatever its
|
`PATH` is not a substitute and is never used on a host, whatever its
|
||||||
version.
|
version.
|
||||||
* `sqlite3` CLI, which the test suite shells out to
|
|
||||||
* S3-compatible object storage (or local filesystem, or rclone remote)
|
* S3-compatible object storage (or local filesystem, or rclone remote)
|
||||||
|
|
||||||
## development workflow
|
## development workflow
|
||||||
@@ -634,8 +703,8 @@ standard: normalized scripts in `script/` are the entrypoints for the
|
|||||||
development workflow, and the Makefile targets are thin shims that call
|
development workflow, and the Makefile targets are thin shims that call
|
||||||
them. We provide:
|
them. We provide:
|
||||||
|
|
||||||
* `script/bootstrap` — install all development dependencies (go, sqlite3,
|
* `script/bootstrap` — install all development dependencies (go, Go
|
||||||
Go module download). It deliberately does not install `golangci-lint`;
|
module download). It deliberately does not install `golangci-lint`;
|
||||||
see `script/lint` below.
|
see `script/lint` below.
|
||||||
* `script/setup` — make a fresh clone ready for development: runs
|
* `script/setup` — make a fresh clone ready for development: runs
|
||||||
`script/bootstrap`, then `script/install-precommit`
|
`script/bootstrap`, then `script/install-precommit`
|
||||||
@@ -649,6 +718,14 @@ them. We provide:
|
|||||||
called by `script/bootstrap`; the release workflow calls it directly
|
called by `script/bootstrap`; the release workflow calls it directly
|
||||||
because it needs `goreleaser` but not the Docker daemon
|
because it needs `goreleaser` but not the Docker daemon
|
||||||
`script/bootstrap` insists on.
|
`script/bootstrap` insists on.
|
||||||
|
* `script/install-go` — install the Go toolchain named by `go.mod`'s
|
||||||
|
`go` directive into `.tool/go` from a sha256-verified `go.dev`
|
||||||
|
archive, and put it on `PATH`. Idempotent. Called only by the release
|
||||||
|
workflow, which needs a host Go for `goreleaser` to shell out to;
|
||||||
|
nothing else on the release runner does. `actions/setup-go` is not
|
||||||
|
used because it verifies the downloaded toolchain against no value in
|
||||||
|
this repo. Bumping Go edits `go.mod`, the checksum in this script, and
|
||||||
|
the `Dockerfile` `golang` digest together.
|
||||||
* `script/release` — cross-compile and publish the release artifacts
|
* `script/release` — cross-compile and publish the release artifacts
|
||||||
with the pinned `goreleaser`. Refuses a `goreleaser` on `PATH` whose
|
with the pinned `goreleaser`. Refuses a `goreleaser` on `PATH` whose
|
||||||
version is not the pinned one, on the same reasoning as `script/lint`.
|
version is not the pinned one, on the same reasoning as `script/lint`.
|
||||||
@@ -716,6 +793,8 @@ them. We provide:
|
|||||||
then the product image). Either failing fails the script. It runs the
|
then the product image). Either failing fails the script. It runs the
|
||||||
checks in the same containers CI does, from a clean copy of the tree,
|
checks in the same containers CI does, from a clean copy of the tree,
|
||||||
so it also catches anything that depends on host state.
|
so it also catches anything that depends on host state.
|
||||||
|
`.gitea/workflows/check.yml` runs it on every push to `main` and
|
||||||
|
`next` and on every pull request against either.
|
||||||
|
|
||||||
It passes a fresh `--build-arg CHECK_EPOCH` to each build, unique per
|
It passes a fresh `--build-arg CHECK_EPOCH` to each build, unique per
|
||||||
invocation, which both files declare immediately above their check
|
invocation, which both files declare immediately above their check
|
||||||
|
|||||||
@@ -25,6 +25,77 @@ release" is exactly the contradiction
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-09-21: Stopped `prune` from reporting a failed row count as 0
|
||||||
|
([issue #96](https://git.eeqj.de/sneak/vaultik/issues/96)). The seven
|
||||||
|
`getTableCount` reads in `PruneDatabase` discarded their error, so a
|
||||||
|
query that could not run became a plausible `0` and the before/after
|
||||||
|
delta computed from it looked like real work. Each read now logs at
|
||||||
|
warn on failure and renders as `unknown`, never `0`, so an empty table
|
||||||
|
is distinguishable from one that could not be queried. The counts have
|
||||||
|
no `--json` representation — under `--json` the summary is suppressed
|
||||||
|
entirely — so nothing there can show a false `0`.
|
||||||
|
|
||||||
|
- 2026-09-21: Made the s3 storage backend report a missing object as
|
||||||
|
`storage.ErrNotFound`, like the `file` and `rclone` backends and as the
|
||||||
|
`Storer` interface documents. `S3Storer.Get` and `Stat` returned the raw
|
||||||
|
AWS SDK error, so `errors.Is(err, storage.ErrNotFound)` was false on s3
|
||||||
|
and callers branched differently per backend. Added a small `s3.IsNotFound`
|
||||||
|
helper (reused by `HeadObject`) and a test that a missing key maps to
|
||||||
|
`ErrNotFound`
|
||||||
|
([issue #129](https://git.eeqj.de/sneak/vaultik/issues/129)).
|
||||||
|
- 2026-09-21: Fixed `verify --deep` reporting healthy snapshots as
|
||||||
|
corrupt. Its final blob-integrity check hashed the encrypted
|
||||||
|
downloaded bytes with a single SHA256 and compared that to the blob
|
||||||
|
ID, which is the double SHA256 of the plaintext, so the two could
|
||||||
|
never match. It now hashes the decompressed plaintext and compares the
|
||||||
|
double SHA256. Added a test that backs up a real snapshot, deep-verifies
|
||||||
|
it, then flips a byte in one stored blob and confirms deep verification
|
||||||
|
then fails
|
||||||
|
([issue #131](https://git.eeqj.de/sneak/vaultik/issues/131)).
|
||||||
|
|
||||||
|
- 2026-09-21: Made `snapshot create` VACUUM the per-snapshot metadata
|
||||||
|
database through the `modernc.org/sqlite` driver instead of shelling
|
||||||
|
out to the external `sqlite` command-line binary (issue #120). A
|
||||||
|
backup no longer needs that binary on `PATH`, so `make check` passes
|
||||||
|
on a stock `go install` host; `script/bootstrap` and the `Dockerfile`
|
||||||
|
(both the test-build and the shipped runtime stage) no longer install
|
||||||
|
it, and a new test asserts the uploaded database keeps no pages from
|
||||||
|
deleted rows. Dropped the now-false note on the 2026-08-07 entry below
|
||||||
|
that said bootstrap installs it.
|
||||||
|
- 2026-09-21: Made `.gitea/workflows/check.yml` run on pushes to `main`
|
||||||
|
and `next` and on pull requests against either, so unit PRs (whose
|
||||||
|
base is `next`) and `next` itself get a CI run instead of relying on a
|
||||||
|
local `make check`
|
||||||
|
([issue #122](https://git.eeqj.de/sneak/vaultik/issues/122)).
|
||||||
|
|
||||||
|
- 2026-09-21: Hash-verified the Go toolchain in the release workflow
|
||||||
|
([issue #105](https://git.eeqj.de/sneak/vaultik/issues/105)). New
|
||||||
|
`script/install-go` downloads the exact `go.dev` archive for `go.mod`'s
|
||||||
|
`go` directive and refuses it unless its sha256 matches a value
|
||||||
|
committed in the script; `.gitea/workflows/release.yml` calls it
|
||||||
|
instead of `actions/setup-go`, which verified the downloaded toolchain
|
||||||
|
against nothing in the repo. `GOTOOLCHAIN: local` on the release step
|
||||||
|
keeps that exact compiler from auto-switching. Bumping Go now touches
|
||||||
|
`go.mod`, the checksum, and the `Dockerfile` `golang` digest together.
|
||||||
|
|
||||||
|
- 2026-09-21: Collapsed the two duration parsers into one and fixed the
|
||||||
|
`--older-than` months example
|
||||||
|
([issue #123](https://git.eeqj.de/sneak/vaultik/issues/123)). Two
|
||||||
|
functions named `parseDuration` existed with different grammars;
|
||||||
|
`snapshot purge --older-than` and `--keep-newer-than` both already went
|
||||||
|
through the one in `internal/vaultik`, while the richer copy in
|
||||||
|
`internal/cli/duration.go` was reachable only from its own test. Kept
|
||||||
|
the live-path parser and deleted the unused one, so no flag's accepted
|
||||||
|
grammar changes. The trap the issue was filed over: `README.md`
|
||||||
|
documented `6m` as the months example for `--older-than`, but `m` is
|
||||||
|
minutes, so the documented command deleted every snapshot older than
|
||||||
|
six minutes on a destructive flag. Corrected the doc to `6mo` and put
|
||||||
|
both flags' help text on one example list that states `m` is minutes
|
||||||
|
and `mo` is months. The surviving parser now rejects negatives, which
|
||||||
|
it previously accepted (`-5h`) or silently made positive (`-5d`).
|
||||||
|
Table-driven tests cover every unit, `6m` as six minutes, `6mo` as 180
|
||||||
|
days, and rejection of a bare number, an unknown unit, and a negative.
|
||||||
|
|
||||||
- 2026-08-10: Moved every lint run into its own container, as a build
|
- 2026-08-10: Moved every lint run into its own container, as a build
|
||||||
step ([issue #113](https://git.eeqj.de/sneak/vaultik/issues/113)).
|
step ([issue #113](https://git.eeqj.de/sneak/vaultik/issues/113)).
|
||||||
New root `Dockerfile.lint`, built by `script/lint`, runs
|
New root `Dockerfile.lint`, built by `script/lint`, runs
|
||||||
@@ -53,10 +124,11 @@ release" is exactly the contradiction
|
|||||||
into each check command, and a fresh `$(date +%s%N)$$` per invocation
|
into each check command, and a fresh `$(date +%s%N)$$` per invocation
|
||||||
computed as a bare assignment. `cmd/vaultik/lintdocker_test.go`
|
computed as a bare assignment. `cmd/vaultik/lintdocker_test.go`
|
||||||
parses both Dockerfiles and both scripts and fails if any part of
|
parses both Dockerfiles and both scripts and fails if any part of
|
||||||
that is dropped, because every way of losing it is silent. Its
|
that is dropped, because every way of losing it is silent. No test
|
||||||
host-lint assertion is structural — no script runs `golangci-lint`
|
asserts that no script runs the host linter: `script/lint` is the one
|
||||||
except through `docker` — rather than a search for the one retired
|
lint entry point and runs `golangci-lint` only inside the container,
|
||||||
variable name, which nothing could ever reintroduce.
|
and keeping it that way is a review matter, not something a test
|
||||||
|
proves.
|
||||||
|
|
||||||
The product `Dockerfile` lost its lint stage rather than gaining a
|
The product `Dockerfile` lost its lint stage rather than gaining a
|
||||||
second linter pin: `make lint` is now `docker build`, so the stage
|
second linter pin: `make lint` is now `docker build`, so the stage
|
||||||
@@ -523,7 +595,7 @@ release" is exactly the contradiction
|
|||||||
was green was wrong.
|
was green was wrong.
|
||||||
- 2026-08-07: Added the standard `.golangci.yml` and `.editorconfig`
|
- 2026-08-07: Added the standard `.golangci.yml` and `.editorconfig`
|
||||||
(issue #59); lint findings under the new config are tracked in issue
|
(issue #59); lint findings under the new config are tracked in issue
|
||||||
#61. `script/bootstrap` now installs sqlite3 (needed by tests).
|
#61.
|
||||||
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
|
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
|
||||||
Makefile shims, README Entrypoints section
|
Makefile shims, README Entrypoints section
|
||||||
- 2026-07-02: Consolidated CLI verbs, retired overlapping commands; bound
|
- 2026-07-02: Consolidated CLI verbs, retired overlapping commands; bound
|
||||||
|
|||||||
@@ -0,0 +1,102 @@
|
|||||||
|
package main_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// This file guards the version stamping of the product image (issue
|
||||||
|
// #75). The failure it protects against is silent: the image still
|
||||||
|
// builds and runs, but `vaultik version` inside it reports "commit:
|
||||||
|
// unknown", so an operator cannot tell which source produced a given
|
||||||
|
// backup. .dockerignore excludes .git, so the build cannot derive the
|
||||||
|
// commit itself; the values must be computed on the host and passed in.
|
||||||
|
//
|
||||||
|
// These are parses of the committed files, for the same reason the lint
|
||||||
|
// guards next door are: shelling out to docker would nest a build
|
||||||
|
// inside `make test`. That `vaultik version` in the built image really
|
||||||
|
// prints the host's version is verified by hand and recorded on the
|
||||||
|
// pull request.
|
||||||
|
|
||||||
|
// dockerScript is script/docker, relative to the repository root.
|
||||||
|
const dockerScript = "script/docker"
|
||||||
|
|
||||||
|
// versionArgs are the ldflag targets the build stamps and, matching
|
||||||
|
// them, the build args the host must supply. The names line up so the
|
||||||
|
// same list checks both files.
|
||||||
|
func versionArgs() []string {
|
||||||
|
return []string{"VERSION", "COMMIT", "COMMIT_DATE"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestProductDockerfileTakesVersionAsBuildArgs fails unless the build
|
||||||
|
// declares each version arg and stamps it into the binary by ldflag
|
||||||
|
// reference, rather than computing it in the container.
|
||||||
|
func TestProductDockerfileTakesVersionAsBuildArgs(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
found := instructions(t, productDockerfile)
|
||||||
|
|
||||||
|
for _, arg := range versionArgs() {
|
||||||
|
require.GreaterOrEqual(t, indexOf(found, "ARG "+arg), 0,
|
||||||
|
"%s must declare `ARG %s` so the host can pass it in",
|
||||||
|
productDockerfile, arg)
|
||||||
|
|
||||||
|
assertLdflagReferences(t, found, arg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestProductDockerfileDoesNotDeriveVersionItself is the anti-regression
|
||||||
|
// for the original defect: the container ran `git rev-parse`, but .git
|
||||||
|
// is not in the build context, so it always resolved to "unknown". No
|
||||||
|
// git command may reach into a build that cannot see the history.
|
||||||
|
func TestProductDockerfileDoesNotDeriveVersionItself(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
text := instructionText(readRepoFile(t, productDockerfile))
|
||||||
|
|
||||||
|
assert.NotContains(t, text, "git ",
|
||||||
|
"%s must not run git: .git is excluded from the build context, so"+
|
||||||
|
" any value it derives is wrong. Pass version, commit and date"+
|
||||||
|
" in as build args instead.", productDockerfile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDockerScriptComputesVersionOnTheHost fails unless script/docker
|
||||||
|
// derives each value where .git exists and passes it as a build arg,
|
||||||
|
// with VERSION coming from script/version so a Docker build reports the
|
||||||
|
// same string a local build of the same tree would.
|
||||||
|
func TestDockerScriptComputesVersionOnTheHost(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
script := readRepoFile(t, dockerScript)
|
||||||
|
|
||||||
|
for _, arg := range versionArgs() {
|
||||||
|
assert.Contains(t, script, "--build-arg "+arg+"=",
|
||||||
|
"%s must pass --build-arg %s to the build", dockerScript, arg)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Contains(t, script, "/version",
|
||||||
|
"%s must take VERSION from script/version, the source of truth"+
|
||||||
|
" shared with the Makefile", dockerScript)
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertLdflagReferences fails unless some build instruction stamps the
|
||||||
|
// named variable from the ARG (a ${arg} reference), not from a value
|
||||||
|
// computed inside the container.
|
||||||
|
func assertLdflagReferences(t *testing.T, found []string, arg string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
for _, instruction := range found {
|
||||||
|
if strings.HasPrefix(instruction, "RUN ") &&
|
||||||
|
strings.Contains(instruction, "go build") &&
|
||||||
|
strings.Contains(instruction, "${"+arg+"}") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Fail(t, "version arg is declared but never stamped",
|
||||||
|
"the go build in %s must reference ${%s} in its ldflags, or the"+
|
||||||
|
" arg is passed and discarded", productDockerfile, arg)
|
||||||
|
}
|
||||||
+13
-153
@@ -28,6 +28,11 @@ import (
|
|||||||
// -- that a real finding actually fails the build -- is verified by
|
// -- that a real finding actually fails the build -- is verified by
|
||||||
// hand against a deliberately broken tree, recorded on the pull
|
// hand against a deliberately broken tree, recorded on the pull
|
||||||
// request.
|
// request.
|
||||||
|
//
|
||||||
|
// One property is deliberately NOT tested here: that no script runs the
|
||||||
|
// linter on the host. script/lint is the only lint entry point, and it
|
||||||
|
// runs golangci-lint only inside the container; keeping it that way is a
|
||||||
|
// review matter, not something a test in this file establishes.
|
||||||
|
|
||||||
// The files under guard, relative to the repository root.
|
// The files under guard, relative to the repository root.
|
||||||
const (
|
const (
|
||||||
@@ -37,9 +42,8 @@ const (
|
|||||||
cibuildScript = "script/cibuild"
|
cibuildScript = "script/cibuild"
|
||||||
)
|
)
|
||||||
|
|
||||||
// linterBinary is the linter's command name. Every occurrence of it in
|
// linterBinary is the linter's command name, used to locate the
|
||||||
// executable shell in this repo must be inside a docker invocation; see
|
// config-verify and lint steps in Dockerfile.lint.
|
||||||
// TestNoHostLintPathRemains.
|
|
||||||
const linterBinary = "golangci-lint"
|
const linterBinary = "golangci-lint"
|
||||||
|
|
||||||
// checkEpochARG is the declaration, with no default value. A default
|
// checkEpochARG is the declaration, with no default value. A default
|
||||||
@@ -219,90 +223,6 @@ func TestCibuildBuildsBothDockerfilesWithFreshEpochs(t *testing.T) {
|
|||||||
"%s must build %s", cibuildScript, lintDockerfile)
|
"%s must build %s", cibuildScript, lintDockerfile)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestNoHostLintPathRemains fails if any escape hatch to a host linter
|
|
||||||
// comes back. The owner's ruling is that every lint run happens inside
|
|
||||||
// a container; a PATH binary that happens to match the pinned version
|
|
||||||
// is a different build reached by a different code path, and admitting
|
|
||||||
// it is what lets a local pass disagree with CI.
|
|
||||||
//
|
|
||||||
// This asserts the PROPERTY -- no script invokes the linter except
|
|
||||||
// through docker -- rather than the absence of any particular variable
|
|
||||||
// name. An earlier version of this test looked only for the literal
|
|
||||||
// VAULTIK_LINT_IN_CONTAINER, the name of the hatch that was removed
|
|
||||||
// alongside it, so nothing could ever trip it again: a hatch under any
|
|
||||||
// other name left it passing. A structural test that passes on a broken
|
|
||||||
// tree is worse than no test, because it is what a later reader trusts
|
|
||||||
// instead of re-deriving the invariant.
|
|
||||||
//
|
|
||||||
// script/lint-fix is not exempted. It is the one script that runs the
|
|
||||||
// linter as a container rather than as a build step, but it still runs
|
|
||||||
// it in one, so the same property holds of it.
|
|
||||||
func TestNoHostLintPathRemains(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
root := repoRoot(t)
|
|
||||||
|
|
||||||
entries, err := os.ReadDir(filepath.Join(root, "script"))
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotEmpty(t, entries, "no scripts found to scan")
|
|
||||||
|
|
||||||
for _, entry := range entries {
|
|
||||||
if entry.IsDir() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
name := filepath.Join("script", entry.Name())
|
|
||||||
for _, line := range shellCode(readRepoFile(t, name)) {
|
|
||||||
assertLinterIsContainerised(t, name, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertLinterIsContainerised fails if the line runs the linter without
|
|
||||||
// handing it to docker first. Position matters: docker has to come
|
|
||||||
// before the binary, or the line is running the host linter and merely
|
|
||||||
// mentioning docker afterwards.
|
|
||||||
func assertLinterIsContainerised(t *testing.T, name, line string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
at := strings.Index(line, linterBinary)
|
|
||||||
if at < 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
docker := strings.Index(line, "docker")
|
|
||||||
|
|
||||||
assert.True(t, docker >= 0 && docker < at,
|
|
||||||
"%s runs %s on the host; every lint run happens in a container"+
|
|
||||||
" (line: %s)", name, linterBinary, line)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestShellCodeSeesCodeAndNotProse keeps the scanner above honest. It
|
|
||||||
// has to ignore comments and here-document bodies, because script/lint
|
|
||||||
// and script/bootstrap both NAME golangci-lint in prose -- in comments,
|
|
||||||
// and in the error text they print -- precisely to say that the host
|
|
||||||
// binary is never used. A scanner that went blind, by over-eager
|
|
||||||
// stripping or by failing to join continuation lines, would make
|
|
||||||
// TestNoHostLintPathRemains pass on everything.
|
|
||||||
func TestShellCodeSeesCodeAndNotProse(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
script := strings.Join([]string{
|
|
||||||
"#!/bin/sh",
|
|
||||||
"# a comment naming golangci-lint",
|
|
||||||
"cat >&2 <<EOF",
|
|
||||||
"prose naming golangci-lint, printed not executed",
|
|
||||||
"EOF",
|
|
||||||
"docker run --rm \\",
|
|
||||||
" \"$image\" \\",
|
|
||||||
" golangci-lint run ./...",
|
|
||||||
}, "\n")
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
[]string{"cat >&2 <<EOF", `docker run --rm "$image" golangci-lint run ./...`},
|
|
||||||
shellCode(script))
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertEpochExpandedInto fails unless some instruction runs the named
|
// assertEpochExpandedInto fails unless some instruction runs the named
|
||||||
// command with the epoch expanded into it. Expansion, not mere
|
// command with the epoch expanded into it. Expansion, not mere
|
||||||
// declaration: an ARG that no instruction references is not guaranteed
|
// declaration: an ARG that no instruction references is not guaranteed
|
||||||
@@ -384,10 +304,14 @@ func instructionText(contents string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// indexOf returns the position of the first instruction equal to, or
|
// indexOf returns the position of the first instruction equal to, or
|
||||||
// beginning with, want; -1 if there is none.
|
// beginning with, want; -1 if there is none. An `ARG NAME=default`
|
||||||
|
// counts as beginning with `ARG NAME`, so a declared arg is found
|
||||||
|
// whether or not it carries a default.
|
||||||
func indexOf(found []string, want string) int {
|
func indexOf(found []string, want string) int {
|
||||||
for i, instruction := range found {
|
for i, instruction := range found {
|
||||||
if instruction == want || strings.HasPrefix(instruction, want+" ") {
|
if instruction == want ||
|
||||||
|
strings.HasPrefix(instruction, want+" ") ||
|
||||||
|
strings.HasPrefix(instruction, want+"=") {
|
||||||
return i
|
return i
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -407,70 +331,6 @@ func indexContaining(found []string, want string) int {
|
|||||||
return -1
|
return -1
|
||||||
}
|
}
|
||||||
|
|
||||||
// shellCode returns a POSIX shell script's executable lines: comments
|
|
||||||
// dropped, here-document bodies dropped, and backslash continuations
|
|
||||||
// joined so a multi-line command is a single string. Whitespace is
|
|
||||||
// collapsed, as it is for Dockerfile instructions.
|
|
||||||
//
|
|
||||||
// Both exclusions are load-bearing rather than tidiness. The scripts
|
|
||||||
// name golangci-lint in prose to state that the host binary is never
|
|
||||||
// used, and joining continuations is what lets the one legitimate
|
|
||||||
// container invocation -- script/lint-fix's `docker run`, whose linter
|
|
||||||
// command sits several lines below the word `docker` -- be recognised
|
|
||||||
// as containerised.
|
|
||||||
func shellCode(contents string) []string {
|
|
||||||
var (
|
|
||||||
out []string
|
|
||||||
joined string
|
|
||||||
terminate string
|
|
||||||
)
|
|
||||||
|
|
||||||
for line := range strings.SplitSeq(contents, "\n") {
|
|
||||||
trimmed := strings.TrimSpace(line)
|
|
||||||
|
|
||||||
if terminate != "" {
|
|
||||||
if trimmed == terminate {
|
|
||||||
terminate = ""
|
|
||||||
}
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if joined == "" && (trimmed == "" || strings.HasPrefix(trimmed, "#")) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
joined += strings.TrimSuffix(trimmed, `\`) + " "
|
|
||||||
if strings.HasSuffix(trimmed, `\`) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
joined = strings.Join(strings.Fields(joined), " ")
|
|
||||||
terminate = heredocTerminator(joined)
|
|
||||||
|
|
||||||
out = append(out, joined)
|
|
||||||
joined = ""
|
|
||||||
}
|
|
||||||
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// heredocTerminator returns the terminator of the here-document a
|
|
||||||
// command opens, or "" if it opens none. Only the first on a line is
|
|
||||||
// recognised; nothing in script/ opens two.
|
|
||||||
func heredocTerminator(line string) string {
|
|
||||||
_, after, opens := strings.Cut(line, "<<")
|
|
||||||
if !opens {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// `<<-` strips leading tabs from the body; the terminator word is
|
|
||||||
// the same either way, and callers compare against trimmed lines.
|
|
||||||
word, _, _ := strings.Cut(strings.TrimPrefix(after, "-"), " ")
|
|
||||||
|
|
||||||
return strings.Trim(word, `'"`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// readRepoFile reads a file by its path relative to the repository
|
// readRepoFile reads a file by its path relative to the repository
|
||||||
// root.
|
// root.
|
||||||
func readRepoFile(t *testing.T, name string) string {
|
func readRepoFile(t *testing.T, name string) string {
|
||||||
|
|||||||
+11
-1
@@ -10,6 +10,16 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
os.Exit(run())
|
||||||
|
}
|
||||||
|
|
||||||
|
// run sets up optional profiling, runs the CLI, and returns the process
|
||||||
|
// exit code. os.Exit lives in main so it fires only after run's deferred
|
||||||
|
// profile writers have flushed. cli.Entry returns a status code rather
|
||||||
|
// than calling os.Exit itself: an os.Exit from inside it would skip
|
||||||
|
// these defers and truncate the profile of a failing command -- exactly
|
||||||
|
// the command one most often wants to profile.
|
||||||
|
func run() int {
|
||||||
// CPU profiling: set VAULTIK_CPUPROFILE=/path/to/cpu.prof
|
// CPU profiling: set VAULTIK_CPUPROFILE=/path/to/cpu.prof
|
||||||
if cpuProfile := os.Getenv("VAULTIK_CPUPROFILE"); cpuProfile != "" {
|
if cpuProfile := os.Getenv("VAULTIK_CPUPROFILE"); cpuProfile != "" {
|
||||||
f, err := os.Create(cpuProfile) //nolint:gosec // G304: operator-set path
|
f, err := os.Create(cpuProfile) //nolint:gosec // G304: operator-set path
|
||||||
@@ -46,5 +56,5 @@ func main() {
|
|||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
cli.Entry()
|
return cli.Entry()
|
||||||
}
|
}
|
||||||
|
|||||||
+29
-8
@@ -5,11 +5,30 @@
|
|||||||
Vaultik uses a local SQLite database to track file metadata, chunk mappings, and blob associations during the backup process. This database serves as an index for incremental backups and enables efficient deduplication.
|
Vaultik uses a local SQLite database to track file metadata, chunk mappings, and blob associations during the backup process. This database serves as an index for incremental backups and enables efficient deduplication.
|
||||||
|
|
||||||
**Important Notes:**
|
**Important Notes:**
|
||||||
- **No Migration Support (pre-1.0)**: Vaultik does not support database schema
|
|
||||||
migrations. The local index is treated as disposable — if the schema changes,
|
This section is the authoritative explanation of the schema/migration story;
|
||||||
delete the local SQLite database (`vaultik database delete`) and run a full
|
other documents (the README and `AGENTS.md`) link here.
|
||||||
backup. The remote storage is unaffected; the new index will re-deduplicate
|
|
||||||
against existing remote blobs.
|
- **No upgrade path between versions (pre-1.0)**: Vaultik has no supported way to
|
||||||
|
carry an existing local index across a schema change. The index is disposable
|
||||||
|
— if the on-disk schema changes between versions, delete the local SQLite
|
||||||
|
database (`vaultik database delete`) and run a full backup. Remote storage is
|
||||||
|
unaffected; the new index re-deduplicates against existing remote blobs. This
|
||||||
|
is the standing project policy, and it is separate from the schema bootstrap
|
||||||
|
described next.
|
||||||
|
- **Schema bootstrap**: a fresh database is populated from numbered SQL files
|
||||||
|
embedded in the binary under `internal/database/schema/`. `000.sql` creates the
|
||||||
|
`schema_migrations` table; `001.sql` creates the application tables. On opening
|
||||||
|
a database the code applies each numbered file that has not yet run and records
|
||||||
|
its version in `schema_migrations`. This bootstraps a new database; it does not
|
||||||
|
upgrade an existing one between released versions.
|
||||||
|
- **Changing the schema (pre-1.0)**: edit `internal/database/schema/001.sql` (and
|
||||||
|
the code that touches the affected tables) directly. Do not add new numbered
|
||||||
|
files — there is no installed base to migrate.
|
||||||
|
- **Disposability expires at 1.0**: the index is treated as disposable only until
|
||||||
|
1.0 ships and is tagged. Once 1.0 is tagged that clause expires and the
|
||||||
|
question of upgrading existing indexes returns. It is deliberately left open
|
||||||
|
here.
|
||||||
- **Version Compatibility**: In rare cases, you may need to use the same version
|
- **Version Compatibility**: In rare cases, you may need to use the same version
|
||||||
of Vaultik to restore a backup as was used to create it. This ensures
|
of Vaultik to restore a backup as was used to create it. This ensures
|
||||||
compatibility with the metadata format stored in S3.
|
compatibility with the metadata format stored in S3.
|
||||||
@@ -192,10 +211,12 @@ Tracks blob upload metrics.
|
|||||||
After a snapshot is completed:
|
After a snapshot is completed:
|
||||||
1. Copy database to temporary file
|
1. Copy database to temporary file
|
||||||
2. Clean temporary database to contain only current snapshot data
|
2. Clean temporary database to contain only current snapshot data
|
||||||
3. Export to SQL dump using sqlite3
|
3. VACUUM the trimmed database so deleted rows leave no pages behind
|
||||||
4. Compress with zstd and encrypt with age
|
4. Compress with zstd and encrypt with age
|
||||||
5. Upload to S3 as `metadata/{snapshot-id}/db.zst.age`
|
5. Upload to S3 as `metadata/{remote-key}/db.zst.age`
|
||||||
6. Generate blob manifest and upload as `metadata/{snapshot-id}/manifest.json.zst`
|
6. Generate blob manifest and upload as `metadata/{remote-key}/manifest.json.zst`
|
||||||
|
|
||||||
|
The `{remote-key}` directory name is a one-way hash of the human snapshot ID, so the ID is never written to the store in plaintext; see [REPOSTRUCTURE.md](REPOSTRUCTURE.md#remote-key-derivation).
|
||||||
|
|
||||||
### 4. Restore Process
|
### 4. Restore Process
|
||||||
|
|
||||||
|
|||||||
+37
-17
@@ -17,11 +17,13 @@ Vaultik stores all backup data in an S3-compatible object store. The repository
|
|||||||
│ └── <hash[2:4]>/
|
│ └── <hash[2:4]>/
|
||||||
│ └── <full-hash>
|
│ └── <full-hash>
|
||||||
└── metadata/
|
└── metadata/
|
||||||
└── <snapshot-id>/
|
└── <remote-key>/
|
||||||
├── db.zst.age
|
├── db.zst.age
|
||||||
└── manifest.json.zst
|
└── manifest.json.zst
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The metadata subdirectory is named with the **remote key**, a one-way hash of the snapshot ID, not with the human-readable snapshot ID itself. See [Remote Key Derivation](#remote-key-derivation).
|
||||||
|
|
||||||
## Blobs Directory (`blobs/`)
|
## Blobs Directory (`blobs/`)
|
||||||
|
|
||||||
### Structure
|
### Structure
|
||||||
@@ -40,9 +42,11 @@ Blobs contain the actual file data from backups and must be encrypted for securi
|
|||||||
|
|
||||||
## Metadata Directory (`metadata/`)
|
## Metadata Directory (`metadata/`)
|
||||||
|
|
||||||
Each snapshot has its own subdirectory named with the snapshot ID.
|
Each snapshot has its own subdirectory. The directory is **not** named with the human-readable snapshot ID; it is named with the remote key — a one-way hash of that ID. The human ID is never written to the destination store as a directory name (see [Remote Key Derivation](#remote-key-derivation)).
|
||||||
|
|
||||||
### Snapshot ID Format
|
### Snapshot ID Format
|
||||||
|
|
||||||
|
The human-readable snapshot ID is used in CLI arguments, log lines, and the local database. It is not written to the destination store.
|
||||||
- **Format**: `<hostname>_<snapshot-name>_<RFC3339>` (or `<hostname>_<RFC3339>` if no
|
- **Format**: `<hostname>_<snapshot-name>_<RFC3339>` (or `<hostname>_<RFC3339>` if no
|
||||||
name was specified)
|
name was specified)
|
||||||
- **Example**: `laptop_home_2024-01-15T14:30:52Z`
|
- **Example**: `laptop_home_2024-01-15T14:30:52Z`
|
||||||
@@ -51,6 +55,19 @@ Each snapshot has its own subdirectory named with the snapshot ID.
|
|||||||
- Snapshot name from the configured `snapshots:` map (optional)
|
- Snapshot name from the configured `snapshots:` map (optional)
|
||||||
- RFC3339 UTC timestamp
|
- RFC3339 UTC timestamp
|
||||||
|
|
||||||
|
This ID reveals the hostname, the configured snapshot name, and the backup time, so it is never used as the on-disk directory name — the remote key is used instead.
|
||||||
|
|
||||||
|
### Remote Key Derivation
|
||||||
|
|
||||||
|
The remote key is `hex(SHA256(SHA256("vaultik|" + snapshot-id)))`: a double SHA-256 over the snapshot ID, with a `vaultik|` domain-separation prefix. The result is a 64-character hex string with no structure a remote observer can reverse. Implemented in `internal/snapshot/remotekey.go`.
|
||||||
|
|
||||||
|
Worked example:
|
||||||
|
- Snapshot ID: `server1_home_2025-06-01T12:00:00Z`
|
||||||
|
- Remote key: `17f97bcde958748af076b926af59823943db59e80ce7170b40f124dfa28f64aa`
|
||||||
|
- Directory: `metadata/17f97bcde958748af076b926af59823943db59e80ce7170b40f124dfa28f64aa/`
|
||||||
|
|
||||||
|
Because the hash is one-way, a listing of the destination store reveals neither the hostname nor the snapshot name of any backup. The same remote key is stored in the manifest's `snapshot_id` field.
|
||||||
|
|
||||||
### Files in Each Snapshot Directory
|
### Files in Each Snapshot Directory
|
||||||
|
|
||||||
#### `db.zst.age` - Encrypted Database
|
#### `db.zst.age` - Encrypted Database
|
||||||
@@ -68,16 +85,17 @@ Each snapshot has its own subdirectory named with the snapshot ID.
|
|||||||
- **Structure**:
|
- **Structure**:
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"snapshot_id": "laptop_home_2024-01-15T14:30:52Z",
|
"snapshot_id": "17f97bcde958748af076b926af59823943db59e80ce7170b40f124dfa28f64aa",
|
||||||
"timestamp": "2024-01-15T14:30:52Z",
|
"timestamp": "2025-06-01T12:00:00Z",
|
||||||
"blob_count": 42,
|
"blob_count": 42,
|
||||||
|
"total_compressed_size": 1048576,
|
||||||
"blobs": [
|
"blobs": [
|
||||||
"cafebabe1234567890abcdef1234567890abcdef1234567890abcdef12345678",
|
{ "hash": "cafebabe1234567890abcdef1234567890abcdef1234567890abcdef12345678", "compressed_size": 24576 },
|
||||||
"deadbeef1234567890abcdef1234567890abcdef1234567890abcdef12345678",
|
{ "hash": "deadbeef1234567890abcdef1234567890abcdef1234567890abcdef12345678", "compressed_size": 32768 }
|
||||||
...
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
`snapshot_id` is the remote key (a hash), not the human ID; `timestamp` is written in the clear.
|
||||||
|
|
||||||
### Why Manifest is Unencrypted
|
### Why Manifest is Unencrypted
|
||||||
The manifest must be readable without the private key to enable:
|
The manifest must be readable without the private key to enable:
|
||||||
@@ -86,7 +104,7 @@ The manifest must be readable without the private key to enable:
|
|||||||
3. **Verification** - Checking blob existence without decryption
|
3. **Verification** - Checking blob existence without decryption
|
||||||
4. **Cross-snapshot deduplication analysis** - Finding shared blobs between snapshots
|
4. **Cross-snapshot deduplication analysis** - Finding shared blobs between snapshots
|
||||||
|
|
||||||
The manifest only contains blob hashes, not file names or any other sensitive information.
|
The manifest contains the remote key, the backup timestamp, the blob count and total compressed size, and each blob's hash and compressed size. It contains no file names, paths, or other decrypted metadata.
|
||||||
|
|
||||||
## Security Considerations
|
## Security Considerations
|
||||||
|
|
||||||
@@ -96,19 +114,21 @@ The manifest only contains blob hashes, not file names or any other sensitive in
|
|||||||
- **File-to-chunk mappings** (in db.zst.age)
|
- **File-to-chunk mappings** (in db.zst.age)
|
||||||
|
|
||||||
### What's Not Encrypted
|
### What's Not Encrypted
|
||||||
- **Blob hashes** (in manifest.json.zst)
|
- **The remote key** — directory names and the manifest `snapshot_id`, a one-way hash of the snapshot ID (see [Remote Key Derivation](#remote-key-derivation))
|
||||||
- **Snapshot IDs** (directory names)
|
- **The backup timestamp** (in manifest.json.zst)
|
||||||
- **Blob count per snapshot** (in manifest.json.zst)
|
- **Blob hashes and their compressed sizes** (in manifest.json.zst)
|
||||||
|
- **Blob count and total compressed size per snapshot** (in manifest.json.zst)
|
||||||
|
|
||||||
### Privacy Implications
|
### Privacy Implications
|
||||||
From the unencrypted data, an observer can determine:
|
From the unencrypted data, an observer of the destination store can determine:
|
||||||
- When backups were taken (from snapshot IDs)
|
- **When each backup was taken** — not from the directory name, which is a one-way hash, but from the plaintext `timestamp` field in manifest.json.zst, which is published in the clear
|
||||||
- Which hostname created backups (from snapshot IDs)
|
- How many blobs each snapshot references, and the total compressed size
|
||||||
- How many blobs each snapshot references
|
- The compressed size of each blob, and which blobs are shared between snapshots (deduplication patterns)
|
||||||
- Which blobs are shared between snapshots (deduplication patterns)
|
|
||||||
- The size of each encrypted blob
|
Together these give an observer a timing-and-size profile of every snapshot. This is an accepted, documented property of the format, not a defect: the manifest is unencrypted so that pruning can run without the private key, and the timing channel could not be closed by encrypting it anyway — object creation times and per-object sizes stay visible at the storage layer on both `s3://` and `file://` destinations regardless.
|
||||||
|
|
||||||
An observer cannot determine:
|
An observer cannot determine:
|
||||||
|
- The hostname or snapshot name of any backup (the directory name and the manifest `snapshot_id` are one-way hashes of the human ID)
|
||||||
- File names or paths
|
- File names or paths
|
||||||
- File contents
|
- File contents
|
||||||
- File permissions or ownership
|
- File permissions or ownership
|
||||||
|
|||||||
+90
-39
@@ -11,6 +11,7 @@ import (
|
|||||||
"os/signal"
|
"os/signal"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -196,13 +197,90 @@ func RunApp(ctx context.Context, app *fx.App) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// errReported marks a failure the operation has already shown the user
|
||||||
|
// (and deliberately withheld under --json). Entry turns it into a
|
||||||
|
// non-zero exit status without printing anything further, so the error
|
||||||
|
// line is not doubled. It flows up from RunOperation through cobra to
|
||||||
|
// Entry.
|
||||||
|
var errReported = errors.New("operation failed")
|
||||||
|
|
||||||
|
// RunOperation runs op against the Vaultik instance inside the fx app
|
||||||
|
// and turns a failure into a returned error rather than an os.Exit from
|
||||||
|
// within the goroutine. An os.Exit there skipped main's deferred
|
||||||
|
// profile writers -- so profiling a failing command yielded a truncated
|
||||||
|
// profile (issue #75) -- and RunWithApp's PID-lock release, and denied
|
||||||
|
// the app any graceful shutdown; returning the error to the top runs
|
||||||
|
// all three.
|
||||||
|
//
|
||||||
|
// op runs in a goroutine so OnStart returns promptly and an interrupt
|
||||||
|
// can still cancel through OnStop; when it finishes, success or failure,
|
||||||
|
// it triggers shutdown, which is what lets RunWithApp return. report is
|
||||||
|
// called with a non-canceled failure so the caller can log it (and
|
||||||
|
// suppress it under --json) before it becomes errReported. A context
|
||||||
|
// cancellation is the interrupt path, not a failure: it is neither
|
||||||
|
// reported nor counted as one.
|
||||||
|
func RunOperation(
|
||||||
|
ctx context.Context, opts AppOptions,
|
||||||
|
op func(v *vaultik.Vaultik) error, report func(err error),
|
||||||
|
) error {
|
||||||
|
var (
|
||||||
|
mu sync.Mutex
|
||||||
|
failed bool
|
||||||
|
)
|
||||||
|
|
||||||
|
opts.Invokes = append(opts.Invokes,
|
||||||
|
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||||
|
lc.Append(fx.Hook{
|
||||||
|
OnStart: func(_ context.Context) error {
|
||||||
|
go func() {
|
||||||
|
err := op(v)
|
||||||
|
if err != nil && !errors.Is(err, context.Canceled) {
|
||||||
|
report(err)
|
||||||
|
|
||||||
|
mu.Lock()
|
||||||
|
failed = true
|
||||||
|
mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
stopErr := v.Shutdowner.Shutdown()
|
||||||
|
if stopErr != nil {
|
||||||
|
log.Error("Failed to shutdown", "error", stopErr)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
OnStop: func(_ context.Context) error {
|
||||||
|
v.Cancel()
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}))
|
||||||
|
|
||||||
|
err := RunWithApp(ctx, opts)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// The goroutine sets failed before triggering the shutdown that lets
|
||||||
|
// RunWithApp return, so the write is in place by the time we read it.
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
|
||||||
|
if failed {
|
||||||
|
return errReported
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// runVaultikApp runs the standard single-operation command lifecycle
|
// runVaultikApp runs the standard single-operation command lifecycle
|
||||||
// shared by the list/purge/verify/remove/remote-info subcommands:
|
// shared by the list/purge/verify/remove/remote-info subcommands:
|
||||||
// resolve the config, start the fx app, run op against the Vaultik
|
// resolve the config, then run op against the Vaultik instance through
|
||||||
// instance in a goroutine, report a failure prefixed with failMsg
|
// RunOperation, reporting a failure prefixed with failMsg (suppressed
|
||||||
// (suppressed while suppressErrors is true, e.g. under --json), then
|
// while suppressErrors is true, e.g. under --json). extraQuiet is OR-ed
|
||||||
// trigger shutdown. The operation is cancelled when the app stops.
|
// into LogOptions.Quiet (e.g. --json output modes).
|
||||||
// extraQuiet is OR-ed into LogOptions.Quiet (e.g. --json output modes).
|
|
||||||
func runVaultikApp(
|
func runVaultikApp(
|
||||||
cmd *cobra.Command, extraQuiet, suppressErrors bool,
|
cmd *cobra.Command, extraQuiet, suppressErrors bool,
|
||||||
failMsg string, op func(v *vaultik.Vaultik) error,
|
failMsg string, op func(v *vaultik.Vaultik) error,
|
||||||
@@ -214,47 +292,20 @@ func runVaultikApp(
|
|||||||
|
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
return RunOperation(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.Options{
|
LogOptions: log.Options{
|
||||||
Verbose: rootFlags.Verbose,
|
Verbose: rootFlags.Verbose,
|
||||||
Debug: rootFlags.Debug,
|
Debug: rootFlags.Debug,
|
||||||
Quiet: rootFlags.Quiet || extraQuiet,
|
Quiet: rootFlags.Quiet || extraQuiet,
|
||||||
},
|
},
|
||||||
Modules: []fx.Option{},
|
}, op, func(err error) {
|
||||||
Invokes: []fx.Option{
|
if suppressErrors {
|
||||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
return
|
||||||
lc.Append(fx.Hook{
|
}
|
||||||
OnStart: func(_ context.Context) error {
|
|
||||||
go func() {
|
|
||||||
err := op(v)
|
|
||||||
if err != nil {
|
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
if !suppressErrors {
|
|
||||||
log.Error(failMsg, "error", err)
|
|
||||||
ReportErrorf("%s: %v", failMsg, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
os.Exit(1)
|
log.Error(failMsg, "error", err)
|
||||||
}
|
ReportErrorf("%s: %v", failMsg, err)
|
||||||
}
|
|
||||||
|
|
||||||
err = v.Shutdowner.Shutdown()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
OnStop: func(_ context.Context) error {
|
|
||||||
v.Cancel()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+30
-1
@@ -1,6 +1,7 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
@@ -24,6 +25,11 @@ const configSetArgs = 2
|
|||||||
// parent config dirs (e.g. ~/.config) are conventionally traversable.
|
// parent config dirs (e.g. ~/.config) are conventionally traversable.
|
||||||
const configDirMode = 0o755
|
const configDirMode = 0o755
|
||||||
|
|
||||||
|
// configYAMLIndent matches the 2-space indentation of defaultConfigTemplate,
|
||||||
|
// so `config set` writes the file back with the same indentation rather than
|
||||||
|
// yaml.Marshal's 4-space default.
|
||||||
|
const configYAMLIndent = 2
|
||||||
|
|
||||||
var (
|
var (
|
||||||
errConfigExists = errors.New("config file already exists")
|
errConfigExists = errors.New("config file already exists")
|
||||||
errEmptyConfig = errors.New("empty config file")
|
errEmptyConfig = errors.New("empty config file")
|
||||||
@@ -381,7 +387,7 @@ Examples:
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
out, err := yaml.Marshal(root)
|
out, err := marshalConfigYAML(root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("marshaling config: %w", err)
|
return fmt.Errorf("marshaling config: %w", err)
|
||||||
}
|
}
|
||||||
@@ -405,6 +411,29 @@ Examples:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// marshalConfigYAML renders a config document tree with 2-space indentation,
|
||||||
|
// matching defaultConfigTemplate. yaml.Marshal defaults to 4 spaces, which
|
||||||
|
// would reindent the whole file on the first `config set` despite the promise
|
||||||
|
// to preserve formatting.
|
||||||
|
func marshalConfigYAML(root *yaml.Node) ([]byte, error) {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
|
||||||
|
enc := yaml.NewEncoder(&buf)
|
||||||
|
enc.SetIndent(configYAMLIndent)
|
||||||
|
|
||||||
|
err := enc.Encode(root)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = enc.Close()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return buf.Bytes(), nil
|
||||||
|
}
|
||||||
|
|
||||||
// loadYAMLFile parses a YAML file into a yaml.Node document tree,
|
// loadYAMLFile parses a YAML file into a yaml.Node document tree,
|
||||||
// which preserves comments and ordering for round-tripping.
|
// which preserves comments and ordering for round-tripping.
|
||||||
func loadYAMLFile(path string) (*yaml.Node, error) {
|
func loadYAMLFile(path string) (*yaml.Node, error) {
|
||||||
|
|||||||
@@ -188,6 +188,47 @@ func TestYAMLPathSet(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestConfigSetPreservesFormatting asserts the `config set` write path
|
||||||
|
// (marshalConfigYAML) round-trips a 2-space-indented file without reindenting
|
||||||
|
// it to yaml.Marshal's 4-space default, and keeps comments.
|
||||||
|
func TestConfigSetPreservesFormatting(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
root := parseTestYAML(t)
|
||||||
|
|
||||||
|
err := yamlPathSet(root, splitPath("s3.bucket"), "newbucket")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("set s3.bucket: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := marshalConfigYAML(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
text := string(out)
|
||||||
|
|
||||||
|
for _, want := range []string{"# top comment", "# inline comment"} {
|
||||||
|
if !contains(text, want) {
|
||||||
|
t.Errorf("round-tripped YAML dropped comment %q:\n%s", want, text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nested map keys stay at 2-space indent; the bug reindented them to 4.
|
||||||
|
if !contains(text, "\n bucket: newbucket") {
|
||||||
|
t.Errorf("expected 2-space indent for s3.bucket, got:\n%s", text)
|
||||||
|
}
|
||||||
|
|
||||||
|
if contains(text, "\n bucket:") {
|
||||||
|
t.Errorf("s3.bucket reindented to 4 spaces:\n%s", text)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sequence items under a key also stay at 2 spaces.
|
||||||
|
if !contains(text, "\n - age1aaa") {
|
||||||
|
t.Errorf("expected 2-space indent for sequence item, got:\n%s", text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func splitPath(s string) []string {
|
func splitPath(s string) []string {
|
||||||
return strings.Split(s, ".")
|
return strings.Split(s, ".")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,126 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"regexp"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Approximate lengths of the extended calendar units accepted by
|
|
||||||
// parseDuration.
|
|
||||||
const (
|
|
||||||
durationDay = 24 * time.Hour
|
|
||||||
durationWeek = 7 * durationDay
|
|
||||||
durationMonth = 30 * durationDay
|
|
||||||
durationYear = 365 * durationDay
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
errNegativeDuration = errors.New("negative durations are not supported")
|
|
||||||
errInvalidDuration = errors.New("invalid duration format")
|
|
||||||
errUnknownTimeUnit = errors.New("unknown time unit")
|
|
||||||
)
|
|
||||||
|
|
||||||
// parseDuration parses duration strings. Supports standard Go duration format
|
|
||||||
// (e.g., "3h30m", "1h45m30s") as well as extended units:
|
|
||||||
// - d: days (e.g., "30d", "7d")
|
|
||||||
// - w: weeks (e.g., "2w", "4w")
|
|
||||||
// - mo: months (30 days) (e.g., "6mo", "1mo")
|
|
||||||
// - y: years (365 days) (e.g., "1y", "2y")
|
|
||||||
//
|
|
||||||
// Can combine units: "1y6mo", "2w3d", "1d12h30m"
|
|
||||||
func parseDuration(s string) (time.Duration, error) {
|
|
||||||
// First try standard Go duration parsing
|
|
||||||
d, err := time.ParseDuration(s)
|
|
||||||
if err == nil {
|
|
||||||
return d, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extended duration parsing
|
|
||||||
// Check for negative values
|
|
||||||
if strings.HasPrefix(strings.TrimSpace(s), "-") {
|
|
||||||
return 0, errNegativeDuration
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pattern matches: number + unit, repeated
|
|
||||||
re := regexp.MustCompile(`(\d+(?:\.\d+)?)\s*([a-zA-Z]+)`)
|
|
||||||
matches := re.FindAllStringSubmatch(s, -1)
|
|
||||||
|
|
||||||
if len(matches) == 0 {
|
|
||||||
return 0, fmt.Errorf("%w: %q", errInvalidDuration, s)
|
|
||||||
}
|
|
||||||
|
|
||||||
var total time.Duration
|
|
||||||
|
|
||||||
for _, match := range matches {
|
|
||||||
valueStr := match[1]
|
|
||||||
unit := strings.ToLower(match[2])
|
|
||||||
|
|
||||||
value, err := strconv.ParseFloat(valueStr, 64)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("invalid number %q: %w", valueStr, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
d, err := durationForUnit(value, unit)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
total += d
|
|
||||||
}
|
|
||||||
|
|
||||||
return total, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// durationForUnit converts a value with a (case-normalized) unit suffix
|
|
||||||
// into a time.Duration, accepting Go's standard units plus the extended
|
|
||||||
// calendar units.
|
|
||||||
func durationForUnit(value float64, unit string) (time.Duration, error) {
|
|
||||||
switch unit {
|
|
||||||
// Standard time units
|
|
||||||
case "ns", "nanosecond", "nanoseconds":
|
|
||||||
return time.Duration(value), nil
|
|
||||||
case "us", "µs", "microsecond", "microseconds":
|
|
||||||
return time.Duration(value * float64(time.Microsecond)), nil
|
|
||||||
case "ms", "millisecond", "milliseconds":
|
|
||||||
return time.Duration(value * float64(time.Millisecond)), nil
|
|
||||||
case "s", "sec", "second", "seconds":
|
|
||||||
return time.Duration(value * float64(time.Second)), nil
|
|
||||||
case "m", "min", "minute", "minutes":
|
|
||||||
return time.Duration(value * float64(time.Minute)), nil
|
|
||||||
case "h", "hr", "hour", "hours":
|
|
||||||
return time.Duration(value * float64(time.Hour)), nil
|
|
||||||
// Extended units
|
|
||||||
case "d", "day", "days":
|
|
||||||
return time.Duration(value * float64(durationDay)), nil
|
|
||||||
case "w", "week", "weeks":
|
|
||||||
return time.Duration(value * float64(durationWeek)), nil
|
|
||||||
case "mo", "month", "months":
|
|
||||||
// Using 30 days as approximation
|
|
||||||
return time.Duration(value * float64(durationMonth)), nil
|
|
||||||
case "y", "year", "years":
|
|
||||||
// Using 365 days as approximation
|
|
||||||
return time.Duration(value * float64(durationYear)), nil
|
|
||||||
default:
|
|
||||||
// Try parsing as standard Go duration unit
|
|
||||||
testStr := "1" + unit
|
|
||||||
|
|
||||||
_, err := time.ParseDuration(testStr)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("%w: %q", errUnknownTimeUnit, unit)
|
|
||||||
}
|
|
||||||
|
|
||||||
// It's a valid Go duration unit, parse the full value
|
|
||||||
fullStr := fmt.Sprintf("%g%s", value, unit)
|
|
||||||
|
|
||||||
d, err := time.ParseDuration(fullStr)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("invalid duration %q: %w", fullStr, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return d, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,299 +0,0 @@
|
|||||||
package cli //nolint:testpackage // needs access to unexported parseDuration
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
type parseDurationCase struct {
|
|
||||||
name string
|
|
||||||
input string
|
|
||||||
expected time.Duration
|
|
||||||
wantErr bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// runParseDurationCases executes a table of parseDuration cases as
|
|
||||||
// parallel subtests.
|
|
||||||
func runParseDurationCases(t *testing.T, tests []parseDurationCase) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
got, err := parseDuration(tt.input)
|
|
||||||
|
|
||||||
if tt.wantErr {
|
|
||||||
require.Error(t, err, "expected error for input %q", tt.input)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, err, "unexpected error for input %q", tt.input)
|
|
||||||
assert.Equal(t, tt.expected, got, "duration mismatch for input %q", tt.input)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseDurationStandard(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
runParseDurationCases(t, []parseDurationCase{
|
|
||||||
{
|
|
||||||
name: "standard seconds",
|
|
||||||
input: "30s",
|
|
||||||
expected: 30 * time.Second,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "standard minutes",
|
|
||||||
input: "45m",
|
|
||||||
expected: 45 * time.Minute,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "standard hours",
|
|
||||||
input: "2h",
|
|
||||||
expected: 2 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "standard combined",
|
|
||||||
input: "3h30m",
|
|
||||||
expected: 3*time.Hour + 30*time.Minute,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "standard complex",
|
|
||||||
input: "1h45m30s",
|
|
||||||
expected: 1*time.Hour + 45*time.Minute + 30*time.Second,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "standard with milliseconds",
|
|
||||||
input: "1s500ms",
|
|
||||||
expected: 1*time.Second + 500*time.Millisecond,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseDurationExtendedUnits(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
runParseDurationCases(t, []parseDurationCase{
|
|
||||||
// Extended units - days
|
|
||||||
{
|
|
||||||
name: "single day",
|
|
||||||
input: "1d",
|
|
||||||
expected: 24 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "multiple days",
|
|
||||||
input: "7d",
|
|
||||||
expected: 7 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "fractional days",
|
|
||||||
input: "1.5d",
|
|
||||||
expected: 36 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "days spelled out",
|
|
||||||
input: "3days",
|
|
||||||
expected: 3 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
// Extended units - weeks
|
|
||||||
{
|
|
||||||
name: "single week",
|
|
||||||
input: "1w",
|
|
||||||
expected: 7 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "multiple weeks",
|
|
||||||
input: "4w",
|
|
||||||
expected: 4 * 7 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "weeks spelled out",
|
|
||||||
input: "2weeks",
|
|
||||||
expected: 2 * 7 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
// Extended units - months
|
|
||||||
{
|
|
||||||
name: "single month",
|
|
||||||
input: "1mo",
|
|
||||||
expected: 30 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "multiple months",
|
|
||||||
input: "6mo",
|
|
||||||
expected: 6 * 30 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "months spelled out",
|
|
||||||
input: "3months",
|
|
||||||
expected: 3 * 30 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
// Extended units - years
|
|
||||||
{
|
|
||||||
name: "single year",
|
|
||||||
input: "1y",
|
|
||||||
expected: 365 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "multiple years",
|
|
||||||
input: "2y",
|
|
||||||
expected: 2 * 365 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "years spelled out",
|
|
||||||
input: "1year",
|
|
||||||
expected: 365 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseDurationCombinedAndErrors(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
runParseDurationCases(t, []parseDurationCase{
|
|
||||||
// Combined extended units
|
|
||||||
{
|
|
||||||
name: "weeks and days",
|
|
||||||
input: "2w3d",
|
|
||||||
expected: 2*7*24*time.Hour + 3*24*time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "years and months",
|
|
||||||
input: "1y6mo",
|
|
||||||
expected: 365*24*time.Hour + 6*30*24*time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "days and hours",
|
|
||||||
input: "1d12h",
|
|
||||||
expected: 24*time.Hour + 12*time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "complex combination",
|
|
||||||
input: "1y2mo3w4d5h6m7s",
|
|
||||||
expected: 365*24*time.Hour + 2*30*24*time.Hour +
|
|
||||||
3*7*24*time.Hour + 4*24*time.Hour +
|
|
||||||
5*time.Hour + 6*time.Minute + 7*time.Second,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "with spaces",
|
|
||||||
input: "1d 12h 30m",
|
|
||||||
expected: 24*time.Hour + 12*time.Hour + 30*time.Minute,
|
|
||||||
},
|
|
||||||
// Edge cases
|
|
||||||
{
|
|
||||||
name: "zero duration",
|
|
||||||
input: "0s",
|
|
||||||
expected: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "large duration",
|
|
||||||
input: "10y",
|
|
||||||
expected: 10 * 365 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
// Error cases
|
|
||||||
{
|
|
||||||
name: "empty string",
|
|
||||||
input: "",
|
|
||||||
wantErr: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid format",
|
|
||||||
input: "abc",
|
|
||||||
wantErr: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "unknown unit",
|
|
||||||
input: "5x",
|
|
||||||
wantErr: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid number",
|
|
||||||
input: "xyzd",
|
|
||||||
wantErr: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "negative not supported",
|
|
||||||
input: "-5d",
|
|
||||||
wantErr: true,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseDurationSpecialCases(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Test that standard Go durations work exactly as expected
|
|
||||||
standardDurations := []string{
|
|
||||||
"300ms",
|
|
||||||
"1.5h",
|
|
||||||
"2h45m",
|
|
||||||
"72h",
|
|
||||||
"1us",
|
|
||||||
"1µs",
|
|
||||||
"1ns",
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, d := range standardDurations {
|
|
||||||
expected, err := time.ParseDuration(d)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
got, err := parseDuration(d)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, expected, got, "standard duration %q should parse identically", d)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseDurationRealWorldExamples(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Test real-world snapshot purge scenarios
|
|
||||||
tests := []struct {
|
|
||||||
description string
|
|
||||||
input string
|
|
||||||
olderThan time.Duration
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
description: "keep snapshots from last 30 days",
|
|
||||||
input: "30d",
|
|
||||||
olderThan: 30 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
description: "keep snapshots from last 6 months",
|
|
||||||
input: "6mo",
|
|
||||||
olderThan: 6 * 30 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
description: "keep snapshots from last year",
|
|
||||||
input: "1y",
|
|
||||||
olderThan: 365 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
description: "keep snapshots from last week and a half",
|
|
||||||
input: "1w3d",
|
|
||||||
olderThan: 10 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
description: "keep snapshots from last 90 days",
|
|
||||||
input: "90d",
|
|
||||||
olderThan: 90 * 24 * time.Hour,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.description, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
got, err := parseDuration(tt.input)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, tt.olderThan, got)
|
|
||||||
|
|
||||||
// Verify the duration makes sense for snapshot purging
|
|
||||||
assert.Greater(t, got, time.Hour,
|
|
||||||
"snapshot purge duration should be at least an hour")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+18
-3
@@ -1,6 +1,7 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -19,7 +20,11 @@ const shortCommitLen = 12
|
|||||||
// flag is present in os.Args — see bannerSuppressedInArgs), executes the
|
// flag is present in os.Args — see bannerSuppressedInArgs), executes the
|
||||||
// root cobra command, and routes any returned error through the
|
// root cobra command, and routes any returned error through the
|
||||||
// ui.Writer so the user sees a properly formatted "🛑 ERROR:" line.
|
// ui.Writer so the user sees a properly formatted "🛑 ERROR:" line.
|
||||||
func Entry() {
|
//
|
||||||
|
// It returns the process exit code (0 on success, 1 on error) rather
|
||||||
|
// than calling os.Exit, so that main's deferred profile writers run
|
||||||
|
// before the process ends. See run in cmd/vaultik/main.go.
|
||||||
|
func Entry() int {
|
||||||
emitStartupBanner(os.Args[1:], os.Stdout)
|
emitStartupBanner(os.Args[1:], os.Stdout)
|
||||||
|
|
||||||
rootCmd := NewRootCommand()
|
rootCmd := NewRootCommand()
|
||||||
@@ -27,9 +32,19 @@ func Entry() {
|
|||||||
|
|
||||||
err := rootCmd.Execute()
|
err := rootCmd.Execute()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ReportErrorf("%s", err.Error())
|
// An operation that ran inside the fx app has already reported
|
||||||
os.Exit(1)
|
// its own failure (and suppressed it under --json); errReported
|
||||||
|
// says so. Printing it again here would double the error line.
|
||||||
|
// Every other error — bad arguments, a config that would not
|
||||||
|
// load — reaches Entry unreported, so it is shown here.
|
||||||
|
if !errors.Is(err, errReported) {
|
||||||
|
ReportErrorf("%s", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// emitStartupBanner writes the startup banner to w unless args (the
|
// emitStartupBanner writes the startup banner to w unless args (the
|
||||||
|
|||||||
@@ -230,7 +230,7 @@ func TestEntryJSONStdoutIsExactlyOneDocument(t *testing.T) {
|
|||||||
programName, flagConfig, configPath, cmdSnapshot, cmdList, flagJSON,
|
programName, flagConfig, configPath, cmdSnapshot, cmdList, flagJSON,
|
||||||
}
|
}
|
||||||
|
|
||||||
stdout := captureProcessStdout(t, Entry)
|
stdout := captureProcessStdout(t, func() { _ = Entry() })
|
||||||
|
|
||||||
requireExactlyOneJSONDocument(t, stdout)
|
requireExactlyOneJSONDocument(t, stdout)
|
||||||
|
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ func TestEntryPruneJSONStdoutIsExactlyOneDocument(t *testing.T) {
|
|||||||
programName, flagConfig, configPath, cmdPrune, flagJSON,
|
programName, flagConfig, configPath, cmdPrune, flagJSON,
|
||||||
}
|
}
|
||||||
|
|
||||||
stdout := captureProcessStdout(t, Entry)
|
stdout := captureProcessStdout(t, func() { _ = Entry() })
|
||||||
|
|
||||||
requireExactlyOneJSONDocument(t, stdout)
|
requireExactlyOneJSONDocument(t, stdout)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
package cli //nolint:testpackage // shares programName and the capture helpers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestEntryReturnsStatusCode pins the contract main() relies on for
|
||||||
|
// issue #75: Entry reports success or failure through its return value
|
||||||
|
// and never calls os.Exit. An os.Exit from inside Entry would skip
|
||||||
|
// main's deferred profile writers and truncate the profile of a failing
|
||||||
|
// command. main turns this code into os.Exit only after those defers
|
||||||
|
// run, so a failing command must come back with a non-zero code rather
|
||||||
|
// than ending the process here.
|
||||||
|
//
|
||||||
|
// Stdout is captured only to keep the banner and command output off the
|
||||||
|
// test log; the assertion is on the returned code.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // replaces os.Args and rootFlags
|
||||||
|
func TestEntryReturnsStatusCode(t *testing.T) {
|
||||||
|
for _, testCase := range []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
// version is self-contained: it needs no config and no
|
||||||
|
// destination store, so it exercises the success path.
|
||||||
|
name: "successful command returns zero",
|
||||||
|
args: []string{programName, "version"},
|
||||||
|
want: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown command returns one",
|
||||||
|
args: []string{programName, "no-such-command"},
|
||||||
|
want: 1,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(testCase.name, func(t *testing.T) {
|
||||||
|
previousArgs := os.Args
|
||||||
|
|
||||||
|
t.Cleanup(func() {
|
||||||
|
os.Args = previousArgs
|
||||||
|
rootFlags = RootFlags{}
|
||||||
|
})
|
||||||
|
|
||||||
|
os.Args = testCase.args
|
||||||
|
|
||||||
|
var code int
|
||||||
|
|
||||||
|
_ = captureProcessStdout(t, func() { code = Entry() })
|
||||||
|
|
||||||
|
assert.Equal(t, testCase.want, code)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
+6
-37
@@ -1,12 +1,7 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"go.uber.org/fx"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
)
|
)
|
||||||
@@ -33,44 +28,18 @@ func NewInfoCommand() *cobra.Command {
|
|||||||
// Use the app framework
|
// Use the app framework
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
return RunOperation(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.Options{
|
LogOptions: log.Options{
|
||||||
Verbose: rootFlags.Verbose,
|
Verbose: rootFlags.Verbose,
|
||||||
Debug: rootFlags.Debug,
|
Debug: rootFlags.Debug,
|
||||||
Quiet: rootFlags.Quiet,
|
Quiet: rootFlags.Quiet,
|
||||||
},
|
},
|
||||||
Modules: []fx.Option{},
|
}, func(v *vaultik.Vaultik) error {
|
||||||
Invokes: []fx.Option{
|
return v.ShowInfo()
|
||||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
}, func(err error) {
|
||||||
lc.Append(fx.Hook{
|
log.Error("Failed to show info", "error", err)
|
||||||
OnStart: func(_ context.Context) error {
|
ReportErrorf("Failed to show info: %v", err)
|
||||||
go func() {
|
|
||||||
err := v.ShowInfo()
|
|
||||||
if err != nil {
|
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
log.Error("Failed to show info", "error", err)
|
|
||||||
ReportErrorf("Failed to show info: %v", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err = v.Shutdowner.Shutdown()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
OnStop: func(_ context.Context) error {
|
|
||||||
v.Cancel()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
+9
-43
@@ -1,12 +1,7 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"go.uber.org/fx"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
)
|
)
|
||||||
@@ -41,51 +36,22 @@ work (e.g. after a crashed backup or to reclaim storage).`,
|
|||||||
// Use the app framework like other commands
|
// Use the app framework like other commands
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
return RunOperation(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.Options{
|
LogOptions: log.Options{
|
||||||
Verbose: rootFlags.Verbose,
|
Verbose: rootFlags.Verbose,
|
||||||
Debug: rootFlags.Debug,
|
Debug: rootFlags.Debug,
|
||||||
Quiet: rootFlags.Quiet || opts.JSON,
|
Quiet: rootFlags.Quiet || opts.JSON,
|
||||||
},
|
},
|
||||||
Modules: []fx.Option{},
|
}, func(v *vaultik.Vaultik) error {
|
||||||
Invokes: []fx.Option{
|
return v.Prune(opts)
|
||||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
}, func(err error) {
|
||||||
lc.Append(fx.Hook{
|
if opts.JSON {
|
||||||
OnStart: func(_ context.Context) error {
|
return
|
||||||
// Start the prune operation in a goroutine
|
}
|
||||||
go func() {
|
|
||||||
// Run the prune operation
|
|
||||||
err := v.Prune(opts)
|
|
||||||
if err != nil {
|
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
if !opts.JSON {
|
|
||||||
log.Error("Prune operation failed", "error", err)
|
|
||||||
ReportErrorf("Prune failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
os.Exit(1)
|
log.Error("Prune operation failed", "error", err)
|
||||||
}
|
ReportErrorf("Prune failed: %v", err)
|
||||||
}
|
|
||||||
|
|
||||||
// Shutdown the app when prune completes
|
|
||||||
err = v.Shutdowner.Shutdown()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
OnStop: func(_ context.Context) error {
|
|
||||||
log.Debug("Stopping prune operation")
|
|
||||||
v.Cancel()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
+9
-37
@@ -1,12 +1,9 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"go.uber.org/fx"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
)
|
)
|
||||||
@@ -83,47 +80,22 @@ func newRemoteInfoCommand() *cobra.Command {
|
|||||||
|
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
return RunOperation(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.Options{
|
LogOptions: log.Options{
|
||||||
Verbose: rootFlags.Verbose,
|
Verbose: rootFlags.Verbose,
|
||||||
Debug: rootFlags.Debug,
|
Debug: rootFlags.Debug,
|
||||||
Quiet: rootFlags.Quiet || jsonOutput,
|
Quiet: rootFlags.Quiet || jsonOutput,
|
||||||
},
|
},
|
||||||
Modules: []fx.Option{},
|
}, func(v *vaultik.Vaultik) error {
|
||||||
Invokes: []fx.Option{
|
return v.RemoteInfo(jsonOutput)
|
||||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
}, func(err error) {
|
||||||
lc.Append(fx.Hook{
|
if jsonOutput {
|
||||||
OnStart: func(_ context.Context) error {
|
return
|
||||||
go func() {
|
}
|
||||||
err := v.RemoteInfo(jsonOutput)
|
|
||||||
if err != nil {
|
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
if !jsonOutput {
|
|
||||||
log.Error("Failed to get remote info", "error", err)
|
|
||||||
ReportErrorf("Failed to get remote info: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
os.Exit(1)
|
log.Error("Failed to get remote info", "error", err)
|
||||||
}
|
ReportErrorf("Failed to get remote info: %v", err)
|
||||||
}
|
|
||||||
|
|
||||||
err = v.Shutdowner.Shutdown()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
OnStop: func(_ context.Context) error {
|
|
||||||
v.Cancel()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
+26
-79
@@ -1,13 +1,10 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"go.uber.org/fx"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
)
|
)
|
||||||
@@ -86,7 +83,8 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
|||||||
// Use the backup functionality from cli package
|
// Use the backup functionality from cli package
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
// --cron suppression is wired through v.UI by setupGlobals.
|
||||||
|
return RunOperation(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.Options{
|
LogOptions: log.Options{
|
||||||
Verbose: rootFlags.Verbose,
|
Verbose: rootFlags.Verbose,
|
||||||
@@ -94,54 +92,24 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
|||||||
Cron: opts.Cron,
|
Cron: opts.Cron,
|
||||||
Quiet: rootFlags.Quiet,
|
Quiet: rootFlags.Quiet,
|
||||||
},
|
},
|
||||||
Modules: []fx.Option{},
|
}, func(v *vaultik.Vaultik) error {
|
||||||
Invokes: []fx.Option{
|
return v.CreateSnapshot(opts)
|
||||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
}, func(err error) {
|
||||||
lc.Append(fx.Hook{
|
log.Error("Snapshot creation failed", "error", err)
|
||||||
OnStart: func(_ context.Context) error {
|
ReportErrorf("Snapshot creation failed: %v", err)
|
||||||
// Start the snapshot creation in a goroutine
|
|
||||||
go func() {
|
|
||||||
// --cron suppression is wired through v.UI by setupGlobals.
|
|
||||||
err := v.CreateSnapshot(opts)
|
|
||||||
if err != nil {
|
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
log.Error("Snapshot creation failed", "error", err)
|
|
||||||
ReportErrorf("Snapshot creation failed: %v", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Shutdown the app when snapshot completes
|
|
||||||
err = v.Shutdowner.Shutdown()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
OnStop: func(_ context.Context) error {
|
|
||||||
log.Debug("Stopping snapshot creation")
|
|
||||||
// Cancel the Vaultik context
|
|
||||||
v.Cancel()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd.Flags().BoolVar(&opts.Cron, "cron", false,
|
cmd.Flags().BoolVar(&opts.Cron, "cron", false,
|
||||||
"Run in cron mode (silent unless error)")
|
"Run in cron mode (silent unless warning or error)")
|
||||||
cmd.Flags().BoolVar(&opts.Prune, "prune", false,
|
cmd.Flags().BoolVar(&opts.Prune, "prune", false,
|
||||||
"After backup, drop older snapshots of the same name and remove "+
|
"After backup, drop older snapshots of the same name and remove "+
|
||||||
"orphaned blobs")
|
"orphaned blobs")
|
||||||
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "",
|
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "",
|
||||||
"With --prune: keep snapshots newer than this duration "+
|
"With --prune: keep snapshots newer than this duration "+
|
||||||
"(e.g. 4w, 30d, 6mo) instead of only the latest")
|
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+
|
||||||
|
"instead of only the latest")
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
@@ -204,7 +172,8 @@ restrict the operation to specific snapshot names.`,
|
|||||||
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false,
|
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false,
|
||||||
"Keep only the latest snapshot of each name")
|
"Keep only the latest snapshot of each name")
|
||||||
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "",
|
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "",
|
||||||
"Remove snapshots older than duration (e.g., 30d, 6m, 1y)")
|
"Remove snapshots older than duration "+
|
||||||
|
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months)")
|
||||||
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt")
|
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt")
|
||||||
cmd.Flags().StringArrayVar(&opts.Names, "snapshot", nil,
|
cmd.Flags().StringArrayVar(&opts.Names, "snapshot", nil,
|
||||||
"Restrict to snapshots with these names (repeat for multiple)")
|
"Restrict to snapshots with these names (repeat for multiple)")
|
||||||
@@ -219,8 +188,11 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
|||||||
cmd := &cobra.Command{
|
cmd := &cobra.Command{
|
||||||
Use: "verify <snapshot-id>",
|
Use: "verify <snapshot-id>",
|
||||||
Short: "Verify snapshot integrity",
|
Short: "Verify snapshot integrity",
|
||||||
Long: "Verifies that all blobs referenced in a snapshot exist",
|
Long: "Verifies that all blobs referenced in a snapshot exist.\n\n" +
|
||||||
Args: requireSnapshotIDArg,
|
"The snapshot may be named by its ID or, on a host with no local\n" +
|
||||||
|
"index, by the remote key that 'snapshot list' prints for a\n" +
|
||||||
|
"remote-only snapshot (an unambiguous leading part is enough).",
|
||||||
|
Args: requireSnapshotIDArg,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
snapshotID := args[0]
|
snapshotID := args[0]
|
||||||
|
|
||||||
@@ -232,47 +204,22 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
|||||||
|
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
return RunOperation(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.Options{
|
LogOptions: log.Options{
|
||||||
Verbose: rootFlags.Verbose,
|
Verbose: rootFlags.Verbose,
|
||||||
Debug: rootFlags.Debug,
|
Debug: rootFlags.Debug,
|
||||||
Quiet: rootFlags.Quiet || opts.JSON,
|
Quiet: rootFlags.Quiet || opts.JSON,
|
||||||
},
|
},
|
||||||
Modules: []fx.Option{},
|
}, func(v *vaultik.Vaultik) error {
|
||||||
Invokes: []fx.Option{
|
return v.VerifySnapshotWithOptions(snapshotID, opts)
|
||||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
}, func(err error) {
|
||||||
lc.Append(fx.Hook{
|
if opts.JSON {
|
||||||
OnStart: func(_ context.Context) error {
|
return
|
||||||
go func() {
|
}
|
||||||
err := v.VerifySnapshotWithOptions(snapshotID, opts)
|
|
||||||
if err != nil {
|
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
if !opts.JSON {
|
|
||||||
log.Error("Verification failed", "error", err)
|
|
||||||
ReportErrorf("Verification failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
os.Exit(1)
|
log.Error("Verification failed", "error", err)
|
||||||
}
|
ReportErrorf("Verification failed: %v", err)
|
||||||
}
|
|
||||||
|
|
||||||
err = v.Shutdowner.Shutdown()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
OnStop: func(_ context.Context) error {
|
|
||||||
v.Cancel()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,16 +1,8 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"go.uber.org/fx"
|
|
||||||
"sneak.berlin/go/vaultik/internal/config"
|
|
||||||
"sneak.berlin/go/vaultik/internal/globals"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -25,15 +17,6 @@ type RestoreOptions struct {
|
|||||||
Verify bool // Verify restored files after restore
|
Verify bool // Verify restored files after restore
|
||||||
}
|
}
|
||||||
|
|
||||||
// RestoreApp contains all dependencies needed for restore
|
|
||||||
type RestoreApp struct {
|
|
||||||
Globals *globals.Globals
|
|
||||||
Config *config.Config
|
|
||||||
Storage storage.Storer
|
|
||||||
Vaultik *vaultik.Vaultik
|
|
||||||
Shutdowner fx.Shutdowner
|
|
||||||
}
|
|
||||||
|
|
||||||
// newSnapshotRestoreCommand creates the 'snapshot restore' subcommand
|
// newSnapshotRestoreCommand creates the 'snapshot restore' subcommand
|
||||||
func newSnapshotRestoreCommand() *cobra.Command {
|
func newSnapshotRestoreCommand() *cobra.Command {
|
||||||
opts := &RestoreOptions{}
|
opts := &RestoreOptions{}
|
||||||
@@ -48,6 +31,10 @@ target directory.
|
|||||||
If no paths are specified, all files are restored.
|
If no paths are specified, all files are restored.
|
||||||
If paths are specified, only matching files/directories are restored.
|
If paths are specified, only matching files/directories are restored.
|
||||||
|
|
||||||
|
The snapshot may be named by its ID or, when restoring on a host with no
|
||||||
|
local index, by the remote key that 'snapshot list' prints for a
|
||||||
|
remote-only snapshot (an unambiguous leading part is enough).
|
||||||
|
|
||||||
Requires the VAULTIK_AGE_SECRET_KEY environment variable to be set with
|
Requires the VAULTIK_AGE_SECRET_KEY environment variable to be set with
|
||||||
the age private key.
|
the age private key.
|
||||||
|
|
||||||
@@ -77,7 +64,8 @@ Examples:
|
|||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
// runRestore parses arguments and runs the restore operation through the app framework
|
// runRestore parses arguments and runs the restore operation through the
|
||||||
|
// app framework.
|
||||||
func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
|
func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
|
||||||
snapshotID := args[0]
|
snapshotID := args[0]
|
||||||
|
|
||||||
@@ -86,87 +74,30 @@ func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error {
|
|||||||
opts.Paths = args[restoreMinArgs:]
|
opts.Paths = args[restoreMinArgs:]
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use unified config resolution
|
|
||||||
configPath, err := ResolveConfigPath()
|
configPath, err := ResolveConfigPath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use the app framework like other commands
|
|
||||||
rootFlags := GetRootFlags()
|
rootFlags := GetRootFlags()
|
||||||
|
|
||||||
return RunWithApp(cmd.Context(), AppOptions{
|
return RunOperation(cmd.Context(), AppOptions{
|
||||||
ConfigPath: configPath,
|
ConfigPath: configPath,
|
||||||
LogOptions: log.Options{
|
LogOptions: log.Options{
|
||||||
Verbose: rootFlags.Verbose,
|
Verbose: rootFlags.Verbose,
|
||||||
Debug: rootFlags.Debug,
|
Debug: rootFlags.Debug,
|
||||||
Quiet: rootFlags.Quiet,
|
Quiet: rootFlags.Quiet,
|
||||||
},
|
},
|
||||||
Modules: buildRestoreModules(),
|
}, func(v *vaultik.Vaultik) error {
|
||||||
Invokes: buildRestoreInvokes(snapshotID, opts),
|
return v.Restore(&vaultik.RestoreOptions{
|
||||||
|
SnapshotID: snapshotID,
|
||||||
|
TargetDir: opts.TargetDir,
|
||||||
|
Paths: opts.Paths,
|
||||||
|
Verify: opts.Verify,
|
||||||
|
SkipErrors: rootFlags.SkipErrors,
|
||||||
|
})
|
||||||
|
}, func(err error) {
|
||||||
|
log.Error("Restore operation failed", "error", err)
|
||||||
|
ReportErrorf("Restore failed: %v", err)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildRestoreModules returns the fx.Options for dependency injection in restore
|
|
||||||
func buildRestoreModules() []fx.Option {
|
|
||||||
return []fx.Option{
|
|
||||||
fx.Provide(fx.Annotate(
|
|
||||||
func(g *globals.Globals, cfg *config.Config,
|
|
||||||
storer storage.Storer, v *vaultik.Vaultik, shutdowner fx.Shutdowner) *RestoreApp {
|
|
||||||
return &RestoreApp{
|
|
||||||
Globals: g,
|
|
||||||
Config: cfg,
|
|
||||||
Storage: storer,
|
|
||||||
Vaultik: v,
|
|
||||||
Shutdowner: shutdowner,
|
|
||||||
}
|
|
||||||
},
|
|
||||||
)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildRestoreInvokes returns the fx.Options that wire up the restore lifecycle
|
|
||||||
func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option {
|
|
||||||
return []fx.Option{
|
|
||||||
fx.Invoke(func(app *RestoreApp, lc fx.Lifecycle) {
|
|
||||||
lc.Append(fx.Hook{
|
|
||||||
OnStart: func(_ context.Context) error {
|
|
||||||
// Start the restore operation in a goroutine
|
|
||||||
go func() {
|
|
||||||
// Run the restore operation
|
|
||||||
restoreOpts := &vaultik.RestoreOptions{
|
|
||||||
SnapshotID: snapshotID,
|
|
||||||
TargetDir: opts.TargetDir,
|
|
||||||
Paths: opts.Paths,
|
|
||||||
Verify: opts.Verify,
|
|
||||||
SkipErrors: GetRootFlags().SkipErrors,
|
|
||||||
}
|
|
||||||
|
|
||||||
err := app.Vaultik.Restore(restoreOpts)
|
|
||||||
if err != nil {
|
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
log.Error("Restore operation failed", "error", err)
|
|
||||||
ReportErrorf("Restore failed: %v", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Shutdown the app when restore completes
|
|
||||||
err = app.Shutdowner.Shutdown()
|
|
||||||
if err != nil {
|
|
||||||
log.Error("Failed to shutdown", "error", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
OnStop: func(_ context.Context) error {
|
|
||||||
log.Debug("Stopping restore operation")
|
|
||||||
app.Vaultik.Cancel()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,12 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import "time"
|
|
||||||
|
|
||||||
// SnapshotInfo represents snapshot information for listing
|
|
||||||
//
|
|
||||||
//nolint:tagliatelle // snake_case is the established output format
|
|
||||||
type SnapshotInfo struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
Timestamp time.Time `json:"timestamp"`
|
|
||||||
CompressedSize int64 `json:"compressed_size"`
|
|
||||||
}
|
|
||||||
@@ -1,67 +0,0 @@
|
|||||||
// Package models defines shared value types describing files, chunks,
|
|
||||||
// blobs, and snapshots as they move through the backup pipeline.
|
|
||||||
package models
|
|
||||||
|
|
||||||
import (
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// FileInfo represents a file in the backup system
|
|
||||||
type FileInfo struct {
|
|
||||||
Path string
|
|
||||||
MTime time.Time
|
|
||||||
Size int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// ChunkInfo represents a content-addressed chunk
|
|
||||||
type ChunkInfo struct {
|
|
||||||
Hash string // SHA256 hash
|
|
||||||
Size int64
|
|
||||||
Offset int64 // Offset within source file
|
|
||||||
}
|
|
||||||
|
|
||||||
// ChunkRef represents a reference to a chunk in a blob or file
|
|
||||||
type ChunkRef struct {
|
|
||||||
ChunkHash string
|
|
||||||
Offset int64
|
|
||||||
Length int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// BlobInfo represents an encrypted blob containing multiple chunks
|
|
||||||
type BlobInfo struct {
|
|
||||||
Hash string // SHA256 hash of the blob content (content-addressable)
|
|
||||||
CreatedAt time.Time
|
|
||||||
Size int64
|
|
||||||
ChunkCount int
|
|
||||||
}
|
|
||||||
|
|
||||||
// Snapshot represents a backup snapshot
|
|
||||||
type Snapshot struct {
|
|
||||||
ID string // ISO8601 timestamp
|
|
||||||
Hostname string
|
|
||||||
Version string
|
|
||||||
CreatedAt time.Time
|
|
||||||
FileCount int64
|
|
||||||
ChunkCount int64
|
|
||||||
BlobCount int64
|
|
||||||
TotalSize int64
|
|
||||||
MetadataSize int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// SnapshotMetadata contains the full metadata for a snapshot
|
|
||||||
type SnapshotMetadata struct {
|
|
||||||
Snapshot *Snapshot
|
|
||||||
Files map[string]*FileInfo
|
|
||||||
Chunks map[string]*ChunkInfo
|
|
||||||
Blobs map[string]*BlobInfo
|
|
||||||
FileChunks map[string][]*ChunkRef // path -> chunks
|
|
||||||
BlobChunks map[string][]*ChunkRef // blob hash -> chunks
|
|
||||||
}
|
|
||||||
|
|
||||||
// Chunk represents a data chunk for processing
|
|
||||||
type Chunk struct {
|
|
||||||
Data []byte
|
|
||||||
Hash string
|
|
||||||
Offset int64
|
|
||||||
Length int64
|
|
||||||
}
|
|
||||||
@@ -1,58 +0,0 @@
|
|||||||
package models_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/models"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestModelsCompilation ensures all model types can be instantiated
|
|
||||||
func TestModelsCompilation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// This test primarily serves as a compilation test
|
|
||||||
// to ensure all types are properly defined
|
|
||||||
|
|
||||||
// Test FileInfo
|
|
||||||
fi := &models.FileInfo{
|
|
||||||
Path: "/test/file.txt",
|
|
||||||
MTime: time.Now(),
|
|
||||||
Size: 1024,
|
|
||||||
}
|
|
||||||
if fi.Path != "/test/file.txt" {
|
|
||||||
t.Errorf("FileInfo.Path not set correctly")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test ChunkInfo
|
|
||||||
ci := &models.ChunkInfo{
|
|
||||||
Hash: "abc123",
|
|
||||||
Size: 512,
|
|
||||||
Offset: 0,
|
|
||||||
}
|
|
||||||
if ci.Hash != "abc123" {
|
|
||||||
t.Errorf("ChunkInfo.Hash not set correctly")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test BlobInfo
|
|
||||||
bi := &models.BlobInfo{
|
|
||||||
Hash: "blob123",
|
|
||||||
CreatedAt: time.Now(),
|
|
||||||
Size: 1024,
|
|
||||||
ChunkCount: 2,
|
|
||||||
}
|
|
||||||
if bi.Hash != "blob123" {
|
|
||||||
t.Errorf("BlobInfo.Hash not set correctly")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test Snapshot
|
|
||||||
s := &models.Snapshot{
|
|
||||||
ID: "2024-01-01T00:00:00Z",
|
|
||||||
Hostname: "test-host",
|
|
||||||
Version: "1.0.0",
|
|
||||||
CreatedAt: time.Now(),
|
|
||||||
}
|
|
||||||
if s.ID != "2024-01-01T00:00:00Z" {
|
|
||||||
t.Errorf("Snapshot.ID not set correctly")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+13
-5
@@ -219,11 +219,7 @@ func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) {
|
|||||||
Key: aws.String(fullKey),
|
Key: aws.String(fullKey),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
var (
|
if IsNotFound(err) {
|
||||||
notFound *s3types.NotFound
|
|
||||||
noSuchKey *s3types.NoSuchKey
|
|
||||||
)
|
|
||||||
if errors.As(err, ¬Found) || errors.As(err, &noSuchKey) {
|
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -233,6 +229,18 @@ func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) {
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IsNotFound reports whether err indicates that an object does not exist.
|
||||||
|
// Head and Get requests surface a missing object as different SDK types,
|
||||||
|
// so both are checked here.
|
||||||
|
func IsNotFound(err error) bool {
|
||||||
|
var (
|
||||||
|
notFound *s3types.NotFound
|
||||||
|
noSuchKey *s3types.NoSuchKey
|
||||||
|
)
|
||||||
|
|
||||||
|
return errors.As(err, ¬Found) || errors.As(err, &noSuchKey)
|
||||||
|
}
|
||||||
|
|
||||||
// ObjectInfo contains information about an S3 object.
|
// ObjectInfo contains information about an S3 object.
|
||||||
// It is used by ListObjectsStream to return object metadata
|
// It is used by ListObjectsStream to return object metadata
|
||||||
// along with any errors encountered during listing.
|
// along with any errors encountered during listing.
|
||||||
|
|||||||
@@ -22,8 +22,9 @@ const remoteKeyPrefix = "vaultik|"
|
|||||||
//
|
//
|
||||||
// - the "metadata/<remote-key>/..." subdirectory on the storage
|
// - the "metadata/<remote-key>/..." subdirectory on the storage
|
||||||
// backend so a directory listing of the bucket / file:// dest
|
// backend so a directory listing of the bucket / file:// dest
|
||||||
// doesn't reveal hostnames, configured snapshot names, or backup
|
// doesn't reveal hostnames or configured snapshot names. (The
|
||||||
// timestamps;
|
// backup time is not hidden: the manifest.json.zst inside that
|
||||||
|
// directory carries a plaintext RFC3339 timestamp.)
|
||||||
// - the `snapshot_id` field of the unencrypted manifest.json.zst
|
// - the `snapshot_id` field of the unencrypted manifest.json.zst
|
||||||
// for the same reason;
|
// for the same reason;
|
||||||
// - any code path that needs to translate a known local snapshot ID
|
// - any code path that needs to translate a known local snapshot ID
|
||||||
|
|||||||
@@ -44,7 +44,6 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -669,14 +668,31 @@ func (sm *SnapshotManager) collectCleanupStats(
|
|||||||
|
|
||||||
// vacuumDatabase runs VACUUM on the database to remove deleted data and compact
|
// vacuumDatabase runs VACUUM on the database to remove deleted data and compact
|
||||||
// This is critical for security - ensures no stale/deleted data pages are uploaded
|
// This is critical for security - ensures no stale/deleted data pages are uploaded
|
||||||
|
//
|
||||||
|
// VACUUM runs through the modernc.org/sqlite driver, on a freshly opened
|
||||||
|
// connection with no transaction in flight (VACUUM cannot run inside one).
|
||||||
|
// The database opens in WAL mode, so VACUUM's rewrite lands in the WAL; the
|
||||||
|
// checkpoint on Close flushes it into the main file, which is the file we
|
||||||
|
// then compress and upload.
|
||||||
func (sm *SnapshotManager) vacuumDatabase(ctx context.Context, dbPath string) error {
|
func (sm *SnapshotManager) vacuumDatabase(ctx context.Context, dbPath string) error {
|
||||||
log.Debug("Running VACUUM on database", "path", dbPath)
|
log.Debug("Running VACUUM on database", "path", dbPath)
|
||||||
//nolint:gosec // G204: fixed argv; dbPath is our own temp file path
|
|
||||||
cmd := exec.CommandContext(ctx, "sqlite3", dbPath, "VACUUM;")
|
|
||||||
|
|
||||||
output, err := cmd.CombinedOutput()
|
db, err := database.New(ctx, dbPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("running VACUUM: %w (output: %s)", err, string(output))
|
return fmt.Errorf("opening database for VACUUM: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
cerr := db.Close()
|
||||||
|
if cerr != nil {
|
||||||
|
log.Debug("Failed to close database after VACUUM",
|
||||||
|
"path", dbPath, "error", cerr)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
_, err = db.ExecWithLog(ctx, "VACUUM")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("running VACUUM: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -840,8 +856,10 @@ func (sm *SnapshotManager) generateBlobManifest(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Create manifest. SnapshotID in the unencrypted manifest is the
|
// Create manifest. SnapshotID in the unencrypted manifest is the
|
||||||
// double-SHA256 remote key, not the human ID, so the public bytes
|
// double-SHA256 remote key (see RemoteSnapshotKey), not the human ID,
|
||||||
// don't reveal hostname/snapshot-name/timestamp metadata.
|
// so neither this field nor the directory name reveals the hostname or
|
||||||
|
// snapshot name. Timestamp below is written in the clear, so the backup
|
||||||
|
// time is observable to anyone who can read the manifest.
|
||||||
manifest := &Manifest{
|
manifest := &Manifest{
|
||||||
SnapshotID: RemoteSnapshotKey(snapshotID),
|
SnapshotID: RemoteSnapshotKey(snapshotID),
|
||||||
Timestamp: time.Now().UTC().Format(time.RFC3339),
|
Timestamp: time.Now().UTC().Format(time.RFC3339),
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
package snapshot
|
package snapshot
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"io"
|
"io"
|
||||||
@@ -96,6 +97,97 @@ func verifyCleanedDB(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestVacuumDatabaseRemovesDeletedData proves the export path uploads a
|
||||||
|
// compacted database: after rows carrying a recognizable marker are deleted
|
||||||
|
// and vacuumDatabase runs, no page holding that marker survives in the file
|
||||||
|
// on disk (the file compressFile later reads for upload).
|
||||||
|
func TestVacuumDatabaseRemovesDeletedData(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
dbPath := filepath.Join(tempDir, "snapshot.db")
|
||||||
|
|
||||||
|
db, err := database.New(ctx, dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create database: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A marker distinctive enough that its presence in the raw file can only
|
||||||
|
// come from the rows inserted below.
|
||||||
|
marker := []byte("VACUUM_PROBE_DEADBEEF_DELETED_ROW")
|
||||||
|
payload := bytes.Repeat(marker, 128) // ~4 KiB per row
|
||||||
|
|
||||||
|
_, err = db.Conn().ExecContext(ctx,
|
||||||
|
"CREATE TABLE vacuum_probe (id INTEGER PRIMARY KEY, payload BLOB)")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create probe table: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for range 512 {
|
||||||
|
_, err = db.Conn().ExecContext(ctx,
|
||||||
|
"INSERT INTO vacuum_probe (payload) VALUES (?)", payload)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to insert probe row: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = db.Conn().ExecContext(ctx, "DELETE FROM vacuum_probe")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to delete probe rows: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close so the deletes reach the main file, mirroring the state
|
||||||
|
// prepareExportDB hands to vacuumDatabase.
|
||||||
|
err = db.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to close database: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeInfo, err := fs.Stat(dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to stat database before vacuum: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeBytes, err := afero.ReadFile(fs, dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to read database before vacuum: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !bytes.Contains(beforeBytes, marker) {
|
||||||
|
t.Fatalf("expected deleted-row data to linger before vacuum")
|
||||||
|
}
|
||||||
|
|
||||||
|
sm := &SnapshotManager{fs: fs}
|
||||||
|
|
||||||
|
err = sm.vacuumDatabase(ctx, dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("vacuumDatabase failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
afterBytes, err := afero.ReadFile(fs, dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to read database after vacuum: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if bytes.Contains(afterBytes, marker) {
|
||||||
|
t.Fatalf("deleted-row data survived vacuum in the uploaded file")
|
||||||
|
}
|
||||||
|
|
||||||
|
afterInfo, err := fs.Stat(dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to stat database after vacuum: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if afterInfo.Size() >= beforeInfo.Size() {
|
||||||
|
t.Fatalf("expected vacuum to shrink the file: before=%d after=%d",
|
||||||
|
beforeInfo.Size(), afterInfo.Size())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
|
func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
|
||||||
// Initialize logger
|
// Initialize logger
|
||||||
log.Initialize(log.Config{})
|
log.Initialize(log.Config{})
|
||||||
|
|||||||
@@ -0,0 +1,198 @@
|
|||||||
|
package storage_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"reflect"
|
||||||
|
"sort"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// runStorerConformance is the shared Storer contract. Every backend that
|
||||||
|
// can run in-process is expected to pass it: TestFileStorer runs it against
|
||||||
|
// file://, TestS3Storer against s3://. A new backend inherits this coverage
|
||||||
|
// by passing its own constructor, so the contract is defined once.
|
||||||
|
//
|
||||||
|
// It exercises the public Storer interface: round-trip, stat, list with
|
||||||
|
// prefix filtering, overwrite, delete, delete-of-missing, and not-found on
|
||||||
|
// Get and Stat. Each section takes its own fresh backend instance, so the
|
||||||
|
// order of sections never matters and no section sees another's objects.
|
||||||
|
func runStorerConformance(t *testing.T, newStorer func(*testing.T) storage.Storer) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
conformanceRoundTrip(t, newStorer(t))
|
||||||
|
conformanceOverwrite(t, newStorer(t))
|
||||||
|
conformanceList(t, newStorer(t))
|
||||||
|
conformanceDelete(t, newStorer(t))
|
||||||
|
conformanceNotFound(t, newStorer(t))
|
||||||
|
}
|
||||||
|
|
||||||
|
// conformanceRoundTrip stores a nested key, then reads it back and stats it.
|
||||||
|
func conformanceRoundTrip(t *testing.T, s storage.Storer) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
key := "blobs/aa/bb/object.bin"
|
||||||
|
want := []byte("round-trip payload")
|
||||||
|
|
||||||
|
err := s.Put(ctx, key, bytes.NewReader(want))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := getBytes(t, s, key)
|
||||||
|
if !bytes.Equal(got, want) {
|
||||||
|
t.Errorf("Get returned %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := s.Stat(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Stat: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if info.Key != key {
|
||||||
|
t.Errorf("Stat key = %q, want %q", info.Key, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
if info.Size != int64(len(want)) {
|
||||||
|
t.Errorf("Stat size = %d, want %d", info.Size, len(want))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// conformanceOverwrite checks that a second Put replaces the first.
|
||||||
|
func conformanceOverwrite(t *testing.T, s storage.Storer) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
key := "meta/snapshot.json"
|
||||||
|
|
||||||
|
err := s.Put(ctx, key, bytes.NewReader([]byte("first")))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first Put: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []byte("second and longer payload")
|
||||||
|
|
||||||
|
err = s.Put(ctx, key, bytes.NewReader(want))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second Put: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := getBytes(t, s, key)
|
||||||
|
if !bytes.Equal(got, want) {
|
||||||
|
t.Errorf("after overwrite Get returned %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// conformanceList checks prefix filtering and the empty result for a
|
||||||
|
// prefix that matches nothing.
|
||||||
|
func conformanceList(t *testing.T, s storage.Storer) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
keys := []string{"blobs/aa/one", "blobs/bb/two", "meta/three"}
|
||||||
|
|
||||||
|
for _, k := range keys {
|
||||||
|
err := s.Put(ctx, k, bytes.NewReader([]byte("data")))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put %q: %v", k, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := listSorted(t, s, ""); !reflect.DeepEqual(got, keys) {
|
||||||
|
t.Errorf("List(\"\") = %v, want %v", got, keys)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantBlobs := []string{"blobs/aa/one", "blobs/bb/two"}
|
||||||
|
if got := listSorted(t, s, "blobs/"); !reflect.DeepEqual(got, wantBlobs) {
|
||||||
|
t.Errorf("List(\"blobs/\") = %v, want %v", got, wantBlobs)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := listSorted(t, s, "absent/"); len(got) != 0 {
|
||||||
|
t.Errorf("List(\"absent/\") = %v, want empty", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// conformanceDelete checks that Delete removes an object and that deleting
|
||||||
|
// a missing key is not an error.
|
||||||
|
func conformanceDelete(t *testing.T, s storage.Storer) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
key := "blobs/cc/gone.bin"
|
||||||
|
|
||||||
|
err := s.Put(ctx, key, bytes.NewReader([]byte("temporary")))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = s.Delete(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Delete: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = s.Get(ctx, key)
|
||||||
|
if !errors.Is(err, storage.ErrNotFound) {
|
||||||
|
t.Errorf("Get after Delete error = %v, want ErrNotFound", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = s.Delete(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Delete of missing key = %v, want nil", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// conformanceNotFound checks Get and Stat on an absent key.
|
||||||
|
func conformanceNotFound(t *testing.T, s storage.Storer) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
key := "never/written"
|
||||||
|
|
||||||
|
_, err := s.Get(ctx, key)
|
||||||
|
if !errors.Is(err, storage.ErrNotFound) {
|
||||||
|
t.Errorf("Get error = %v, want ErrNotFound", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = s.Stat(ctx, key)
|
||||||
|
if !errors.Is(err, storage.ErrNotFound) {
|
||||||
|
t.Errorf("Stat error = %v, want ErrNotFound", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// getBytes reads a key fully and closes the reader.
|
||||||
|
func getBytes(t *testing.T, s storage.Storer, key string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
rc, err := s.Get(context.Background(), key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Get %q: %v", key, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = rc.Close() }()
|
||||||
|
|
||||||
|
data, err := io.ReadAll(rc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %q: %v", key, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
// listSorted returns the keys under a prefix in a stable order.
|
||||||
|
func listSorted(t *testing.T, s storage.Storer, prefix string) []string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
keys, err := s.List(context.Background(), prefix)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("List %q: %v", prefix, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Strings(keys)
|
||||||
|
|
||||||
|
return keys
|
||||||
|
}
|
||||||
+79
-54
@@ -46,31 +46,18 @@ func (f *FileStorer) SetFilesystem(fs afero.Fs) {
|
|||||||
// storage base path.
|
// storage base path.
|
||||||
const storageDirPerm = 0o755
|
const storageDirPerm = 0o755
|
||||||
|
|
||||||
|
// tempSuffix marks a partially written object. writeAtomic streams into a
|
||||||
|
// temp file carrying this suffix and only renames it onto the real key once
|
||||||
|
// the whole object is on disk, so an interrupted write can never leave a
|
||||||
|
// truncated object at the key a later run would Stat and trust as a complete
|
||||||
|
// blob. List and ListStream skip these files, so a leftover from an
|
||||||
|
// interrupted write is never listed or trusted as a blob; it is otherwise
|
||||||
|
// harmless and is overwritten when the same key is written again.
|
||||||
|
const tempSuffix = ".partial"
|
||||||
|
|
||||||
// Put stores data at the specified key.
|
// Put stores data at the specified key.
|
||||||
func (f *FileStorer) Put(_ context.Context, key string, data io.Reader) error {
|
func (f *FileStorer) Put(_ context.Context, key string, data io.Reader) error {
|
||||||
path := f.fullPath(key)
|
return f.writeAtomic(key, data, nil)
|
||||||
|
|
||||||
// Create parent directories
|
|
||||||
dir := filepath.Dir(path)
|
|
||||||
|
|
||||||
err := f.fs.MkdirAll(dir, storageDirPerm)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("creating directories: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := f.fs.Create(path)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("creating file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = file.Close() }()
|
|
||||||
|
|
||||||
_, err = io.Copy(file, data)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("writing file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// PutWithProgress stores data with progress reporting.
|
// PutWithProgress stores data with progress reporting.
|
||||||
@@ -78,35 +65,7 @@ func (f *FileStorer) PutWithProgress(
|
|||||||
_ context.Context, key string, data io.Reader,
|
_ context.Context, key string, data io.Reader,
|
||||||
_ int64, progress ProgressCallback,
|
_ int64, progress ProgressCallback,
|
||||||
) error {
|
) error {
|
||||||
path := f.fullPath(key)
|
return f.writeAtomic(key, data, progress)
|
||||||
|
|
||||||
// Create parent directories
|
|
||||||
dir := filepath.Dir(path)
|
|
||||||
|
|
||||||
err := f.fs.MkdirAll(dir, storageDirPerm)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("creating directories: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := f.fs.Create(path)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("creating file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = file.Close() }()
|
|
||||||
|
|
||||||
// Wrap with progress tracking
|
|
||||||
pw := &progressWriter{
|
|
||||||
writer: file,
|
|
||||||
callback: progress,
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.Copy(pw, data)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("writing file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get retrieves data from the specified key.
|
// Get retrieves data from the specified key.
|
||||||
@@ -188,7 +147,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
if !info.IsDir() {
|
if !info.IsDir() && !strings.HasSuffix(info.Name(), tempSuffix) {
|
||||||
// Convert back to key (relative path from basePath)
|
// Convert back to key (relative path from basePath)
|
||||||
relPath, err := filepath.Rel(f.basePath, path)
|
relPath, err := filepath.Rel(f.basePath, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -245,7 +204,7 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
|
|||||||
return nil //nolint:nilerr // continue walking despite errors
|
return nil //nolint:nilerr // continue walking despite errors
|
||||||
}
|
}
|
||||||
|
|
||||||
if !info.IsDir() {
|
if !info.IsDir() && !strings.HasSuffix(info.Name(), tempSuffix) {
|
||||||
relPath, err := filepath.Rel(f.basePath, path)
|
relPath, err := filepath.Rel(f.basePath, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ch <- ObjectInfo{Err: fmt.Errorf("computing relative path: %w", err)}
|
ch <- ObjectInfo{Err: fmt.Errorf("computing relative path: %w", err)}
|
||||||
@@ -275,6 +234,72 @@ func (f *FileStorer) Info() Info {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// writeAtomic streams data into a temp file in the destination directory,
|
||||||
|
// fsyncs it, and renames it onto the final key. The key therefore appears
|
||||||
|
// only once the whole object has been durably written; a failure part-way
|
||||||
|
// leaves a temp file (removed here on the failing path) rather than a
|
||||||
|
// truncated object at the key.
|
||||||
|
func (f *FileStorer) writeAtomic(
|
||||||
|
key string, data io.Reader, progress ProgressCallback,
|
||||||
|
) error {
|
||||||
|
path := f.fullPath(key)
|
||||||
|
dir := filepath.Dir(path)
|
||||||
|
|
||||||
|
err := f.fs.MkdirAll(dir, storageDirPerm)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("creating directories: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tmp, err := afero.TempFile(f.fs, dir, filepath.Base(path)+"-*"+tempSuffix)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("creating temp file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tmpPath := tmp.Name()
|
||||||
|
|
||||||
|
// Remove the temp file unless the rename below claims it. On the success
|
||||||
|
// path renamed is true, so the deferred Close and Remove are harmless
|
||||||
|
// no-ops on a name that no longer exists.
|
||||||
|
renamed := false
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
_ = tmp.Close()
|
||||||
|
|
||||||
|
if !renamed {
|
||||||
|
_ = f.fs.Remove(tmpPath)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
var w io.Writer = tmp
|
||||||
|
if progress != nil {
|
||||||
|
w = &progressWriter{writer: tmp, callback: progress}
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = io.Copy(w, data)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("writing file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = tmp.Sync()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("syncing temp file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = tmp.Close()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("closing temp file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = f.fs.Rename(tmpPath, path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("renaming temp file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
renamed = true
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// fullPath returns the full filesystem path for a key.
|
// fullPath returns the full filesystem path for a key.
|
||||||
func (f *FileStorer) fullPath(key string) string {
|
func (f *FileStorer) fullPath(key string) string {
|
||||||
return filepath.Join(f.basePath, key)
|
return filepath.Join(f.basePath, key)
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
package storage_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// errStreamInterrupted stands in for an upload cut off mid-stream.
|
||||||
|
var errStreamInterrupted = errors.New("connection reset mid-upload")
|
||||||
|
|
||||||
|
// failingReader yields its data once, then fails.
|
||||||
|
type failingReader struct {
|
||||||
|
data []byte
|
||||||
|
done bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *failingReader) Read(p []byte) (int, error) {
|
||||||
|
if r.done {
|
||||||
|
return 0, errStreamInterrupted
|
||||||
|
}
|
||||||
|
|
||||||
|
n := copy(p, r.data)
|
||||||
|
r.done = true
|
||||||
|
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFileStorer_InterruptedWriteLeavesNoTrustedObject checks that a write
|
||||||
|
// cut off mid-stream leaves nothing at the destination key, so a later run
|
||||||
|
// cannot Stat a truncated object and trust it as a complete blob.
|
||||||
|
func TestFileStorer_InterruptedWriteLeavesNoTrustedObject(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
f, err := storage.NewFileStorer(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFileStorer: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
key := "blobs/aa/bb/aabbccddeeff"
|
||||||
|
|
||||||
|
err = f.PutWithProgress(ctx, key, &failingReader{data: []byte("partial")}, 4096, nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected the interrupted write to fail, got nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = f.Stat(ctx, key)
|
||||||
|
if !errors.Is(err, storage.ErrNotFound) {
|
||||||
|
t.Fatalf("expected key absent after interrupted write, got Stat err %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
keys, err := f.List(ctx, "blobs/")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("List: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(keys) != 0 {
|
||||||
|
t.Fatalf("expected no keys listed after interrupted write, got %v", keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFileStorer_ListSkipsPartialFiles checks that a leftover temp file (the
|
||||||
|
// storage layer names them with a ".partial" suffix) is never surfaced as a
|
||||||
|
// key by List or ListStream.
|
||||||
|
func TestFileStorer_ListSkipsPartialFiles(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
base := t.TempDir()
|
||||||
|
|
||||||
|
f, err := storage.NewFileStorer(base)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFileStorer: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
realKey := "blobs/aa/bb/aabbccddeeff"
|
||||||
|
|
||||||
|
err = f.Put(ctx, realKey, strings.NewReader("blob-bytes"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A stray temp file, as an interrupted write would leave behind.
|
||||||
|
leftover := filepath.Join(base, "blobs/aa/bb/aabbccddeeff-123456.partial")
|
||||||
|
|
||||||
|
err = os.WriteFile(leftover, []byte("half"), 0o600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("writing leftover temp file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
keys, err := f.List(ctx, "blobs/")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("List: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(keys) != 1 || keys[0] != realKey {
|
||||||
|
t.Fatalf("List should return only the real key, got %v", keys)
|
||||||
|
}
|
||||||
|
|
||||||
|
var streamed []string
|
||||||
|
|
||||||
|
for obj := range f.ListStream(ctx, "blobs/") {
|
||||||
|
if obj.Err != nil {
|
||||||
|
t.Fatalf("ListStream: %v", obj.Err)
|
||||||
|
}
|
||||||
|
|
||||||
|
streamed = append(streamed, obj.Key)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(streamed) != 1 || streamed[0] != realKey {
|
||||||
|
t.Fatalf("ListStream should return only the real key, got %v", streamed)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package storage_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// newFileStorer builds a file:// backend rooted at a fresh temp directory.
|
||||||
|
//
|
||||||
|
//nolint:ireturn // conformance runs against the Storer interface by design
|
||||||
|
func newFileStorer(t *testing.T) storage.Storer {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
s, err := storage.NewFileStorer(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewFileStorer: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFileStorer runs the shared Storer contract against the file:// backend.
|
||||||
|
func TestFileStorer(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
runStorerConformance(t, newFileStorer)
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
package storage_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The rclone backend is a thin adapter over the rclone library: it turns a
|
||||||
|
// (remote, path) pair into rclone's "remote:path" string, hands it to
|
||||||
|
// rclone, and maps rclone's own results back to the Storer interface. What
|
||||||
|
// can be tested in-process, without a configured remote or network, is that
|
||||||
|
// adapter layer — how the arguments are shaped and how construction errors
|
||||||
|
// are reported. The data-plane operations (Put/Get/List/Delete) are rclone's
|
||||||
|
// own, exercised against a real provider (drive, s3-via-rclone, ...), which
|
||||||
|
// needs a configured remote with credentials and network access and so is
|
||||||
|
// out of reach of a unit test. The shared Storer conformance suite therefore
|
||||||
|
// runs against the in-process file and s3 backends; the rclone backend
|
||||||
|
// inherits that contract once a remote is configured.
|
||||||
|
//
|
||||||
|
// These tests use rclone's ":local:" on-the-fly backend, which addresses the
|
||||||
|
// local filesystem directly without any configured remote, so construction
|
||||||
|
// runs entirely in-process.
|
||||||
|
|
||||||
|
// TestNewRcloneStorerConstruction checks that a valid remote constructs a
|
||||||
|
// backend and that Info() reports the shaped "remote:path" location.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
|
||||||
|
func TestNewRcloneStorerConstruction(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
s, err := storage.NewRcloneStorer(context.Background(), ":local", dir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRcloneStorer: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Info().Location is the "remote:path" string the adapter builds from
|
||||||
|
// its two arguments, so asserting it confirms the argument shaping.
|
||||||
|
want := ":local:" + dir
|
||||||
|
if got := s.Info().Location; got != want {
|
||||||
|
t.Errorf("Info().Location = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNewRcloneStorerUnknownRemote checks that a remote that is not in the
|
||||||
|
// rclone config fails construction with the ErrRemoteNotFound sentinel,
|
||||||
|
// rather than silently returning a backend pointed nowhere.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
|
||||||
|
func TestNewRcloneStorerUnknownRemote(t *testing.T) {
|
||||||
|
_, err := storage.NewRcloneStorer(
|
||||||
|
context.Background(), "vaultik-no-such-remote", "path")
|
||||||
|
if !errors.Is(err, storage.ErrRemoteNotFound) {
|
||||||
|
t.Errorf("NewRcloneStorer error = %v, want ErrRemoteNotFound", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
+16
-1
@@ -38,14 +38,29 @@ func (s *S3Storer) PutWithProgress(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Get retrieves data from the specified key.
|
// Get retrieves data from the specified key.
|
||||||
|
// Returns ErrNotFound if the object does not exist.
|
||||||
func (s *S3Storer) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
func (s *S3Storer) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||||
return s.client.GetObject(ctx, key)
|
rc, err := s.client.GetObject(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
if s3.IsNotFound(err) {
|
||||||
|
return nil, fmt.Errorf("get %q: %w", key, ErrNotFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return rc, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stat returns metadata about an object without retrieving its contents.
|
// Stat returns metadata about an object without retrieving its contents.
|
||||||
|
// Returns ErrNotFound if the object does not exist.
|
||||||
func (s *S3Storer) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
|
func (s *S3Storer) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
|
||||||
info, err := s.client.StatObject(ctx, key)
|
info, err := s.client.StatObject(ctx, key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if s3.IsNotFound(err) {
|
||||||
|
return nil, fmt.Errorf("stat %q: %w", key, ErrNotFound)
|
||||||
|
}
|
||||||
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
package storage_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/johannesboyne/gofakes3"
|
||||||
|
"github.com/johannesboyne/gofakes3/backend/s3mem"
|
||||||
|
|
||||||
|
"sneak.berlin/go/vaultik/internal/s3"
|
||||||
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// s3TestBucket is the bucket created for each in-process S3 server.
|
||||||
|
const s3TestBucket = "test-bucket"
|
||||||
|
|
||||||
|
// newS3Storer builds an s3:// backend backed by a fresh in-process
|
||||||
|
// S3 server. It reuses the same in-memory S3 harness (gofakes3 + s3mem
|
||||||
|
// over httptest) that internal/s3 and the not-found regression test use,
|
||||||
|
// so no new mock or dependency is introduced. Each call gets its own
|
||||||
|
// server, bucket, and client, so the conformance suite's per-section
|
||||||
|
// instances stay isolated.
|
||||||
|
//
|
||||||
|
//nolint:ireturn // conformance runs against the Storer interface by design
|
||||||
|
func newS3Storer(t *testing.T) storage.Storer {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
backend := s3mem.New()
|
||||||
|
|
||||||
|
err := backend.CreateBucket(s3TestBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("create bucket: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
srv := httptest.NewServer(gofakes3.New(backend).Server())
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
|
||||||
|
client, err := s3.NewClient(context.Background(), s3.Config{
|
||||||
|
Endpoint: srv.URL,
|
||||||
|
Bucket: s3TestBucket,
|
||||||
|
AccessKeyID: "test",
|
||||||
|
SecretAccessKey: "test",
|
||||||
|
Region: "us-east-1",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("new client: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return storage.NewS3Storer(client)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestS3Storer runs the shared Storer contract against the s3:// backend,
|
||||||
|
// so it is held to the same round-trip, list, delete, and not-found
|
||||||
|
// behaviour as the file:// backend.
|
||||||
|
func TestS3Storer(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
runStorerConformance(t, newS3Storer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestS3StorerMissingKeyMapsToErrNotFound pins the specific contract that a
|
||||||
|
// missing object surfaces as storage.ErrNotFound rather than the raw AWS SDK
|
||||||
|
// error. Without the mapping, errors.Is(err, storage.ErrNotFound) is false on
|
||||||
|
// s3 and callers would branch differently per backend.
|
||||||
|
func TestS3StorerMissingKeyMapsToErrNotFound(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
storer := newS3Storer(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
_, err := storer.Get(ctx, "does-not-exist")
|
||||||
|
if !errors.Is(err, storage.ErrNotFound) {
|
||||||
|
t.Errorf("Get on missing key: got %v, want ErrNotFound", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = storer.Stat(ctx, "does-not-exist")
|
||||||
|
if !errors.Is(err, storage.ErrNotFound) {
|
||||||
|
t.Errorf("Stat on missing key: got %v, want ErrNotFound", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
package storage_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestParseStorageURLValid checks that each supported scheme parses into
|
||||||
|
// the expected fields, since those fields decide which backend is built.
|
||||||
|
func TestParseStorageURLValid(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const bucket = "mybucket"
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
raw string
|
||||||
|
want *storage.URL
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "file absolute path",
|
||||||
|
raw: "file:///var/backups/vaultik",
|
||||||
|
want: &storage.URL{Scheme: "file", Prefix: "/var/backups/vaultik"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "s3 bucket and prefix, ssl defaults on",
|
||||||
|
raw: "s3://mybucket/backups/host",
|
||||||
|
want: &storage.URL{
|
||||||
|
Scheme: "s3", Bucket: bucket,
|
||||||
|
Prefix: "backups/host", UseSSL: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "s3 bucket only",
|
||||||
|
raw: "s3://mybucket",
|
||||||
|
want: &storage.URL{Scheme: "s3", Bucket: bucket, UseSSL: true},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "s3 with endpoint, region, ssl off",
|
||||||
|
raw: "s3://mybucket?endpoint=minio.example.com®ion=us-west-2&ssl=false",
|
||||||
|
want: &storage.URL{
|
||||||
|
Scheme: "s3", Bucket: bucket,
|
||||||
|
Endpoint: "minio.example.com", Region: "us-west-2", UseSSL: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "rclone remote and path",
|
||||||
|
raw: "rclone://gdrive/backups/host",
|
||||||
|
want: &storage.URL{
|
||||||
|
Scheme: "rclone", RcloneRemote: "gdrive", Prefix: "backups/host",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "rclone remote only",
|
||||||
|
raw: "rclone://gdrive",
|
||||||
|
want: &storage.URL{Scheme: "rclone", RcloneRemote: "gdrive"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
got, err := storage.ParseStorageURL(tc.raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseStorageURL(%q) returned error: %v", tc.raw, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !reflect.DeepEqual(got, tc.want) {
|
||||||
|
t.Errorf("ParseStorageURL(%q) = %+v, want %+v", tc.raw, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestParseStorageURLErrors checks that empty, missing, and unknown-scheme
|
||||||
|
// inputs fail with the documented sentinel errors instead of parsing to a
|
||||||
|
// wrong destination.
|
||||||
|
func TestParseStorageURLErrors(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
raw string
|
||||||
|
wantErr error
|
||||||
|
}{
|
||||||
|
{"empty url", "", storage.ErrEmptyStorageURL},
|
||||||
|
{"file empty path", "file://", storage.ErrEmptyFilePath},
|
||||||
|
{"s3 missing bucket", "s3://", storage.ErrMissingBucket},
|
||||||
|
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket},
|
||||||
|
{"rclone missing remote", "rclone://", storage.ErrMissingRemote},
|
||||||
|
{"unknown scheme", "gs://bucket/x", storage.ErrUnsupportedScheme},
|
||||||
|
{"no scheme", "/local/path", storage.ErrUnsupportedScheme},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
_, err := storage.ParseStorageURL(tc.raw)
|
||||||
|
if !errors.Is(err, tc.wantErr) {
|
||||||
|
t.Errorf("ParseStorageURL(%q) error = %v, want %v",
|
||||||
|
tc.raw, err, tc.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
package vaultik_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/spf13/afero"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
|
"sneak.berlin/go/vaultik/internal/ui"
|
||||||
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestDeepVerifyAcceptsHealthyAndRejectsCorruptBlob backs up a real
|
||||||
|
// snapshot with the on-disk storage backend, runs deep verification on
|
||||||
|
// it, then flips a byte inside one stored blob and runs deep
|
||||||
|
// verification again. A healthy snapshot must pass; a corrupted blob
|
||||||
|
// must fail. The healthy case is the regression guard: deep
|
||||||
|
// verification used to hash the encrypted blob bytes and compare them
|
||||||
|
// to the blob's ID (the double SHA256 of the plaintext), so it reported
|
||||||
|
// every healthy blob as corrupt.
|
||||||
|
func TestDeepVerifyAcceptsHealthyAndRejectsCorruptBlob(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
|
||||||
|
dataDir := filepath.Join(tempDir, "source")
|
||||||
|
storeDir := filepath.Join(tempDir, "remote")
|
||||||
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||||
|
|
||||||
|
chunkSize := int64(64 * 1024)
|
||||||
|
maxBlobSize := int64(512 * 1024)
|
||||||
|
|
||||||
|
// One file large enough to span several chunks within a single blob.
|
||||||
|
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
||||||
|
require.NoError(t, afero.WriteFile(fs,
|
||||||
|
filepath.Join(dataDir, "data.bin"),
|
||||||
|
bytesPattern("deep-", int(chunkSize*3)), 0o644))
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// runFileStorageBackup writes a real snapshot to storeDir and closes
|
||||||
|
// the source index, so verification runs from remote bytes only.
|
||||||
|
cfg, storer, snapshotID := runFileStorageBackup(
|
||||||
|
ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
|
||||||
|
|
||||||
|
newVerifier := func() *vaultik.Vaultik {
|
||||||
|
v := &vaultik.Vaultik{
|
||||||
|
Config: cfg,
|
||||||
|
Storage: storer,
|
||||||
|
Fs: fs,
|
||||||
|
Stdout: io.Discard,
|
||||||
|
Stderr: io.Discard,
|
||||||
|
UI: ui.NewWithColor(io.Discard, false),
|
||||||
|
}
|
||||||
|
v.SetContext(ctx)
|
||||||
|
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t,
|
||||||
|
newVerifier().RunDeepVerify(snapshotID, &vaultik.VerifyOptions{Deep: true}),
|
||||||
|
"deep verify should pass on a healthy snapshot")
|
||||||
|
|
||||||
|
// Flip a byte inside one blob without changing its length, so the
|
||||||
|
// blob-existence and size checks still pass and verification reaches
|
||||||
|
// the blob-content stage.
|
||||||
|
corruptOneBlob(t, fs, filepath.Join(storeDir, "blobs"))
|
||||||
|
|
||||||
|
require.Error(t,
|
||||||
|
newVerifier().RunDeepVerify(snapshotID, &vaultik.VerifyOptions{Deep: true}),
|
||||||
|
"deep verify should fail on a corrupted blob")
|
||||||
|
}
|
||||||
|
|
||||||
|
// corruptOneBlob flips a middle byte of the first blob file found under
|
||||||
|
// blobsDir, leaving the file length unchanged.
|
||||||
|
func corruptOneBlob(t *testing.T, fs afero.Fs, blobsDir string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var blobPath string
|
||||||
|
|
||||||
|
err := afero.Walk(fs, blobsDir,
|
||||||
|
func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if blobPath == "" && !info.IsDir() {
|
||||||
|
blobPath = path
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotEmpty(t, blobPath, "expected at least one blob on disk")
|
||||||
|
|
||||||
|
data, err := afero.ReadFile(fs, blobPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotEmpty(t, data)
|
||||||
|
|
||||||
|
data[len(data)/2] ^= 0xff
|
||||||
|
require.NoError(t, afero.WriteFile(fs, blobPath, data, 0o644))
|
||||||
|
}
|
||||||
@@ -33,8 +33,9 @@ func ubytes(n int64) string {
|
|||||||
var (
|
var (
|
||||||
errMalformedSnapshotID = errors.New(
|
errMalformedSnapshotID = errors.New(
|
||||||
"invalid snapshot ID format: expected hostname_snapshotname_timestamp")
|
"invalid snapshot ID format: expected hostname_snapshotname_timestamp")
|
||||||
errInvalidDuration = errors.New("invalid duration")
|
errInvalidDuration = errors.New("invalid duration")
|
||||||
errUnknownTimeUnit = errors.New("unknown time unit")
|
errUnknownTimeUnit = errors.New("unknown time unit")
|
||||||
|
errNegativeDuration = errors.New("negative durations are not supported")
|
||||||
)
|
)
|
||||||
|
|
||||||
// Time-unit lengths used by parseDuration.
|
// Time-unit lengths used by parseDuration.
|
||||||
@@ -138,8 +139,13 @@ func parseSnapshotName(snapshotID string) string {
|
|||||||
|
|
||||||
// parseDuration parses a duration string with support for human-friendly units:
|
// parseDuration parses a duration string with support for human-friendly units:
|
||||||
// d/day/days, w/week/weeks, mo/month/months, y/year/years, plus standard Go
|
// d/day/days, w/week/weeks, mo/month/months, y/year/years, plus standard Go
|
||||||
// duration units (h, m, s).
|
// duration units. Following Go, m is minutes and mo is months. A bare number,
|
||||||
|
// an unknown unit, and a negative value are all rejected.
|
||||||
func parseDuration(s string) (time.Duration, error) {
|
func parseDuration(s string) (time.Duration, error) {
|
||||||
|
if strings.HasPrefix(strings.TrimSpace(s), "-") {
|
||||||
|
return 0, errNegativeDuration
|
||||||
|
}
|
||||||
|
|
||||||
d, err := time.ParseDuration(s)
|
d, err := time.ParseDuration(s)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return d, nil
|
return d, nil
|
||||||
|
|||||||
@@ -51,13 +51,32 @@ func TestParseDuration(t *testing.T) {
|
|||||||
want time.Duration
|
want time.Duration
|
||||||
err bool
|
err bool
|
||||||
}{
|
}{
|
||||||
{"30d", 30 * 24 * time.Hour, false},
|
// Go units, including the m-is-minutes / mo-is-months distinction
|
||||||
{"4w", 4 * 7 * 24 * time.Hour, false},
|
// that this parser exists to keep straight.
|
||||||
{"6mo", 6 * 30 * 24 * time.Hour, false},
|
{"10ns", 10 * time.Nanosecond, false},
|
||||||
{"1y", 365 * 24 * time.Hour, false},
|
{"10us", 10 * time.Microsecond, false},
|
||||||
{"2w3d", 2*7*24*time.Hour + 3*24*time.Hour, false},
|
{"500ms", 500 * time.Millisecond, false},
|
||||||
{"1h", time.Hour, false},
|
|
||||||
{"30s", 30 * time.Second, false},
|
{"30s", 30 * time.Second, false},
|
||||||
|
{"6m", 6 * time.Minute, false},
|
||||||
|
{"1h", time.Hour, false},
|
||||||
|
// Extended calendar units.
|
||||||
|
{"30d", 30 * 24 * time.Hour, false},
|
||||||
|
{"3days", 3 * 24 * time.Hour, false},
|
||||||
|
{"4w", 4 * 7 * 24 * time.Hour, false},
|
||||||
|
{"2weeks", 2 * 7 * 24 * time.Hour, false},
|
||||||
|
{"6mo", 180 * 24 * time.Hour, false},
|
||||||
|
{"1month", 30 * 24 * time.Hour, false},
|
||||||
|
{"1y", 365 * 24 * time.Hour, false},
|
||||||
|
{"2years", 2 * 365 * 24 * time.Hour, false},
|
||||||
|
// Combined units.
|
||||||
|
{"2w3d", 2*7*24*time.Hour + 3*24*time.Hour, false},
|
||||||
|
{"1y6mo", 365*24*time.Hour + 180*24*time.Hour, false},
|
||||||
|
// Rejected inputs.
|
||||||
|
{"6", 0, true}, // bare number, no unit
|
||||||
|
{"5x", 0, true}, // unknown unit
|
||||||
|
{"-5d", 0, true}, // negative, extended unit
|
||||||
|
{"-5h", 0, true}, // negative, Go unit
|
||||||
|
{"", 0, true}, // empty
|
||||||
{"garbage", 0, true},
|
{"garbage", 0, true},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
package vaultik //nolint:testpackage // exercises unexported count helpers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestTableCountForReportSurfacesReadFailure is the regression guard for
|
||||||
|
// the discarded-error bug: getTableCount for a table its query cannot
|
||||||
|
// resolve must not silently become 0. A count that could not be read is
|
||||||
|
// reported as unknown, which a reader can tell apart from an empty table.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||||
|
func TestTableCountForReportSurfacesReadFailure(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
db, err := database.New(ctx, ":memory:")
|
||||||
|
require.NoError(t, err)
|
||||||
|
t.Cleanup(func() { _ = db.Close() })
|
||||||
|
|
||||||
|
v := &Vaultik{DB: db}
|
||||||
|
v.SetContext(ctx)
|
||||||
|
|
||||||
|
// A table present in the schema reads as a real count.
|
||||||
|
blobs := v.tableCountForReport("blobs")
|
||||||
|
require.NotNil(t, blobs, "an existing table must read as a real count")
|
||||||
|
assert.Equal(t, int64(0), *blobs)
|
||||||
|
|
||||||
|
// A syntactically valid name the sanitizer accepts but whose table
|
||||||
|
// the query cannot resolve is the exact shape #96 describes: a
|
||||||
|
// would-be loud failure that used to be discarded into a 0.
|
||||||
|
_, err = v.getTableCount("snapshots_missing")
|
||||||
|
require.Error(t, err, "a query against a nonexistent table must fail")
|
||||||
|
|
||||||
|
missing := v.tableCountForReport("snapshots_missing")
|
||||||
|
assert.Nil(t, missing, "a failed read is unknown, not a count")
|
||||||
|
|
||||||
|
// The rendered count for a failed read must say unknown, never 0.
|
||||||
|
assert.Equal(t, countUnknown, countText(missing))
|
||||||
|
assert.NotEqual(t, "0", countText(missing))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCountTextDistinguishesEmptyFromUnknown pins the distinction the
|
||||||
|
// output has to preserve: 0 means the table was empty, "unknown" means
|
||||||
|
// the count could not be read.
|
||||||
|
func TestCountTextDistinguishesEmptyFromUnknown(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
zero := int64(0)
|
||||||
|
seven := int64(7)
|
||||||
|
|
||||||
|
assert.Equal(t, "0", countText(&zero))
|
||||||
|
assert.Equal(t, "7", countText(&seven))
|
||||||
|
assert.Equal(t, countUnknown, countText(nil))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCountDiffUnknownWhenEitherSideUnknown checks that a delta computed
|
||||||
|
// from an unreadable count is itself unknown rather than a plausible
|
||||||
|
// number.
|
||||||
|
func TestCountDiffUnknownWhenEitherSideUnknown(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
before := int64(10)
|
||||||
|
after := int64(3)
|
||||||
|
|
||||||
|
require.NotNil(t, countDiff(&before, &after))
|
||||||
|
assert.Equal(t, int64(7), *countDiff(&before, &after))
|
||||||
|
|
||||||
|
assert.Nil(t, countDiff(nil, &after), "unknown before yields unknown delta")
|
||||||
|
assert.Nil(t, countDiff(&before, nil), "unknown after yields unknown delta")
|
||||||
|
assert.Nil(t, countDiff(nil, nil))
|
||||||
|
}
|
||||||
@@ -18,7 +18,6 @@ import (
|
|||||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||||
"sneak.berlin/go/vaultik/internal/database"
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
|
||||||
"sneak.berlin/go/vaultik/internal/types"
|
"sneak.berlin/go/vaultik/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -577,14 +576,20 @@ func (v *Vaultik) handleRestoreVerification(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// downloadSnapshotDB downloads and decrypts the snapshot metadata
|
// downloadSnapshotDB downloads and decrypts the snapshot metadata
|
||||||
// database. The snapshotID is the human ID; we hash it to the remote
|
// database. The identifier is resolved to the snapshot's remote key: a
|
||||||
// key for the storage path.
|
// human ID is hashed, and a remote key (or its abbreviation, as printed
|
||||||
|
// for a remote-only snapshot) is used as-is, so a host with no local
|
||||||
|
// index can restore the snapshots it can only see on the store.
|
||||||
func (v *Vaultik) downloadSnapshotDB(
|
func (v *Vaultik) downloadSnapshotDB(
|
||||||
snapshotID string, identity age.Identity,
|
snapshotID string, identity age.Identity,
|
||||||
) (*database.DB, error) {
|
) (*database.DB, error) {
|
||||||
|
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
// Download encrypted database from storage
|
// Download encrypted database from storage
|
||||||
dbKey := fmt.Sprintf("metadata/%s/db.zst.age",
|
dbKey := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
|
||||||
snapshot.RemoteSnapshotKey(snapshotID))
|
|
||||||
|
|
||||||
reader, err := v.Storage.Get(v.ctx, dbKey)
|
reader, err := v.Storage.Get(v.ctx, dbKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,167 @@
|
|||||||
|
package vaultik_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/spf13/afero"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/config"
|
||||||
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||||
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
|
"sneak.berlin/go/vaultik/internal/ui"
|
||||||
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestRestoreOnAnotherMachine proves the disaster-recovery path: a host
|
||||||
|
// that has only the vaultik binary, the age secret key, and the storage
|
||||||
|
// credentials — no local index, a different hostname, and no
|
||||||
|
// age_recipients configured — can list, restore, and verify a snapshot
|
||||||
|
// straight from the destination store.
|
||||||
|
//
|
||||||
|
// The backup half writes a snapshot with one index and hostname. The
|
||||||
|
// restore half throws that index away entirely: a fresh, empty index and
|
||||||
|
// a config that shares nothing with the original but the storage location
|
||||||
|
// and the secret key. If restore or verify needed the original local
|
||||||
|
// index — or the human snapshot ID that only that index holds — this test
|
||||||
|
// could not run, because the recovery host can know neither.
|
||||||
|
func TestRestoreOnAnotherMachine(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
|
||||||
|
dataDir := filepath.Join(tempDir, "source")
|
||||||
|
storeDir := filepath.Join(tempDir, "remote")
|
||||||
|
restoreDir := filepath.Join(tempDir, "restored")
|
||||||
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||||
|
|
||||||
|
chunkSize := int64(64 * 1024)
|
||||||
|
maxBlobSize := int64(512 * 1024)
|
||||||
|
|
||||||
|
sourceFiles := writeRecoverySourceTree(t, fs, dataDir, chunkSize)
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// Backup host: one index, hostname test-host, age_recipients set.
|
||||||
|
// runFileStorageBackup closes the index before returning, so nothing
|
||||||
|
// below can lean on it.
|
||||||
|
_, storer, originalID := runFileStorageBackup(
|
||||||
|
ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
|
||||||
|
|
||||||
|
// Recovery host: a fresh empty index, a different hostname, and no
|
||||||
|
// age_recipients — only the secret key and the same storage location.
|
||||||
|
recovery, stdout := newRecoveryHost(ctx, t, fs, storer)
|
||||||
|
|
||||||
|
// The recovery index really is empty. This is the assertion that makes
|
||||||
|
// the test a guard against restore quietly depending on the original
|
||||||
|
// index: if it did, an empty index would make restore fail.
|
||||||
|
localSnaps, err := recovery.Repositories.Snapshots.ListRecent(ctx, 100)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Empty(t, localSnaps, "recovery host must start with no local index")
|
||||||
|
|
||||||
|
// List: the snapshot shows up as remote-only, identified by its remote
|
||||||
|
// key, with no recoverable human ID.
|
||||||
|
require.NoError(t, recovery.ListSnapshots(true))
|
||||||
|
|
||||||
|
rows := decodeListJSON(t, stdout.String())
|
||||||
|
require.Len(t, rows, 1)
|
||||||
|
|
||||||
|
remote := rows[0]
|
||||||
|
assert.False(t, remote.LocallyTracked, "snapshot must be remote-only here")
|
||||||
|
assert.Empty(t, remote.ID, "the human ID is unknown to the recovery host")
|
||||||
|
require.Len(t, remote.RemoteKey, 64)
|
||||||
|
assert.Equal(t, snapshot.RemoteSnapshotKey(originalID), remote.RemoteKey,
|
||||||
|
"the listed key is the hashed snapshot ID")
|
||||||
|
|
||||||
|
// Restore driven by the abbreviated identifier the table prints (the
|
||||||
|
// first 12 hex of the remote key), then deep-verify from the store
|
||||||
|
// keyed by the full remote key. Both are what a recovery host can know.
|
||||||
|
require.NoError(t, recovery.Restore(&vaultik.RestoreOptions{
|
||||||
|
SnapshotID: remote.RemoteKey[:12],
|
||||||
|
TargetDir: restoreDir,
|
||||||
|
Verify: true,
|
||||||
|
}))
|
||||||
|
require.NoError(t, recovery.RunDeepVerify(
|
||||||
|
remote.RemoteKey, &vaultik.VerifyOptions{Deep: true}))
|
||||||
|
|
||||||
|
assertRestoredTreeMatches(t, fs, restoreDir, sourceFiles)
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeRecoverySourceTree writes a small source tree spanning several
|
||||||
|
// chunks (so restore reassembles real multi-chunk files) and returns the
|
||||||
|
// content keyed by absolute path.
|
||||||
|
func writeRecoverySourceTree(
|
||||||
|
t *testing.T, fs afero.Fs, dataDir string, chunkSize int64,
|
||||||
|
) map[string][]byte {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
sourceFiles := map[string][]byte{
|
||||||
|
filepath.Join(dataDir, "notes.txt"): []byte("recover me"),
|
||||||
|
filepath.Join(dataDir, "sub", "big.bin"): bytesPattern("big-", int(chunkSize*3)),
|
||||||
|
filepath.Join(dataDir, "sub", "small.bin"): bytesPattern("small-", 128),
|
||||||
|
}
|
||||||
|
|
||||||
|
for path, content := range sourceFiles {
|
||||||
|
require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
|
||||||
|
require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
|
||||||
|
}
|
||||||
|
|
||||||
|
return sourceFiles
|
||||||
|
}
|
||||||
|
|
||||||
|
// newRecoveryHost builds the Vaultik a replacement machine would run: an
|
||||||
|
// empty in-memory index, a hostname different from the backup host, no
|
||||||
|
// age_recipients, and only the secret key plus the shared storer. It
|
||||||
|
// returns the instance and the buffer its stdout is wired to.
|
||||||
|
func newRecoveryHost(
|
||||||
|
ctx context.Context, t *testing.T, fs afero.Fs, storer storage.Storer,
|
||||||
|
) (*vaultik.Vaultik, *bytes.Buffer) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
recoveryDB, err := database.New(ctx, ":memory:")
|
||||||
|
require.NoError(t, err)
|
||||||
|
t.Cleanup(func() { _ = recoveryDB.Close() })
|
||||||
|
|
||||||
|
stdout := &bytes.Buffer{}
|
||||||
|
|
||||||
|
recovery := &vaultik.Vaultik{
|
||||||
|
Config: &config.Config{
|
||||||
|
AgeSecretKey: testAgeSecretKey,
|
||||||
|
Hostname: "recovery-host",
|
||||||
|
},
|
||||||
|
Storage: storer,
|
||||||
|
Fs: fs,
|
||||||
|
Repositories: database.NewRepositories(recoveryDB),
|
||||||
|
DB: recoveryDB,
|
||||||
|
Stdout: stdout,
|
||||||
|
Stderr: io.Discard,
|
||||||
|
UI: ui.NewWithColor(io.Discard, false),
|
||||||
|
}
|
||||||
|
recovery.SetContext(ctx)
|
||||||
|
|
||||||
|
return recovery, stdout
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertRestoredTreeMatches byte-compares every restored file against its
|
||||||
|
// source content.
|
||||||
|
func assertRestoredTreeMatches(
|
||||||
|
t *testing.T, fs afero.Fs, restoreDir string, sourceFiles map[string][]byte,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
for origPath, expected := range sourceFiles {
|
||||||
|
restored := filepath.Join(restoreDir, origPath)
|
||||||
|
got, err := afero.ReadFile(fs, restored)
|
||||||
|
require.NoErrorf(t, err, "restored file missing: %s", restored)
|
||||||
|
require.Truef(t, bytes.Equal(got, expected),
|
||||||
|
"byte mismatch for %s", origPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -669,9 +670,11 @@ func (v *Vaultik) VerifySnapshotWithOptions(
|
|||||||
|
|
||||||
v.printVerifyHeader(snapshotID, opts)
|
v.printVerifyHeader(snapshotID, opts)
|
||||||
|
|
||||||
// Download and parse manifest. The caller supplies a human
|
// Resolve the identifier to the snapshot's remote key and download the
|
||||||
// snapshot ID; we hash it to address remote storage.
|
// manifest. A human ID is hashed; a remote key (or its abbreviation,
|
||||||
manifest, err := v.downloadManifestByKey(snapshot.RemoteSnapshotKey(snapshotID))
|
// as printed for a remote-only snapshot) is used as-is, so a host with
|
||||||
|
// no local index can verify a snapshot it can only see on the store.
|
||||||
|
manifest, err := v.resolveAndDownloadManifest(snapshotID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if opts.JSON {
|
if opts.JSON {
|
||||||
result.Status = verifyStatusFailed
|
result.Status = verifyStatusFailed
|
||||||
@@ -1540,12 +1543,17 @@ func (v *Vaultik) outputRemoveJSON(result *RemoveResult) error {
|
|||||||
return encoder.Encode(result)
|
return encoder.Encode(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
// PruneResult contains statistics about the prune operation
|
// PruneResult contains statistics about the prune operation.
|
||||||
|
// SnapshotsDeleted counts snapshots actually deleted. FilesDeleted,
|
||||||
|
// ChunksDeleted, and BlobsDeleted are derived from before/after row
|
||||||
|
// counts of the local index; each is nil when a count could not be read,
|
||||||
|
// so an unreadable count is reported as unknown rather than silently
|
||||||
|
// as 0.
|
||||||
type PruneResult struct {
|
type PruneResult struct {
|
||||||
SnapshotsDeleted int64
|
SnapshotsDeleted int64
|
||||||
FilesDeleted int64
|
FilesDeleted *int64
|
||||||
ChunksDeleted int64
|
ChunksDeleted *int64
|
||||||
BlobsDeleted int64
|
BlobsDeleted *int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// PruneDatabase removes incomplete snapshots and orphaned files, chunks,
|
// PruneDatabase removes incomplete snapshots and orphaned files, chunks,
|
||||||
@@ -1560,7 +1568,7 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) {
|
|||||||
result := &PruneResult{}
|
result := &PruneResult{}
|
||||||
|
|
||||||
// Snapshot counts before deletion of incompletes.
|
// Snapshot counts before deletion of incompletes.
|
||||||
snapshotCountBefore, _ := v.getTableCount("snapshots")
|
snapshotCountBefore := v.tableCountForReport("snapshots")
|
||||||
|
|
||||||
// First, delete any incomplete snapshots
|
// First, delete any incomplete snapshots
|
||||||
incompleteSnapshots, err := v.Repositories.Snapshots.GetIncompleteSnapshots(v.ctx)
|
incompleteSnapshots, err := v.Repositories.Snapshots.GetIncompleteSnapshots(v.ctx)
|
||||||
@@ -1575,9 +1583,9 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Get counts before cleanup for reporting
|
// Get counts before cleanup for reporting
|
||||||
fileCountBefore, _ := v.getTableCount("files")
|
fileCountBefore := v.tableCountForReport("files")
|
||||||
chunkCountBefore, _ := v.getTableCount("chunks")
|
chunkCountBefore := v.tableCountForReport("chunks")
|
||||||
blobCountBefore, _ := v.getTableCount("blobs")
|
blobCountBefore := v.tableCountForReport("blobs")
|
||||||
|
|
||||||
// Run the cleanup
|
// Run the cleanup
|
||||||
err = v.SnapshotManager.CleanupOrphanedData(v.ctx)
|
err = v.SnapshotManager.CleanupOrphanedData(v.ctx)
|
||||||
@@ -1586,36 +1594,83 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Get counts after cleanup
|
// Get counts after cleanup
|
||||||
fileCountAfter, _ := v.getTableCount("files")
|
fileCountAfter := v.tableCountForReport("files")
|
||||||
chunkCountAfter, _ := v.getTableCount("chunks")
|
chunkCountAfter := v.tableCountForReport("chunks")
|
||||||
blobCountAfter, _ := v.getTableCount("blobs")
|
blobCountAfter := v.tableCountForReport("blobs")
|
||||||
|
|
||||||
result.FilesDeleted = fileCountBefore - fileCountAfter
|
result.FilesDeleted = countDiff(fileCountBefore, fileCountAfter)
|
||||||
result.ChunksDeleted = chunkCountBefore - chunkCountAfter
|
result.ChunksDeleted = countDiff(chunkCountBefore, chunkCountAfter)
|
||||||
result.BlobsDeleted = blobCountBefore - blobCountAfter
|
result.BlobsDeleted = countDiff(blobCountBefore, blobCountAfter)
|
||||||
|
|
||||||
log.Info("Local database prune complete",
|
log.Info("Local database prune complete",
|
||||||
"incomplete_snapshots", result.SnapshotsDeleted,
|
"incomplete_snapshots", result.SnapshotsDeleted,
|
||||||
"orphaned_files", result.FilesDeleted,
|
"orphaned_files", countText(result.FilesDeleted),
|
||||||
"orphaned_chunks", result.ChunksDeleted,
|
"orphaned_chunks", countText(result.ChunksDeleted),
|
||||||
"orphaned_blobs", result.BlobsDeleted,
|
"orphaned_blobs", countText(result.BlobsDeleted),
|
||||||
)
|
)
|
||||||
|
|
||||||
snapshotCountAfter := snapshotCountBefore - result.SnapshotsDeleted
|
// Snapshots remaining after removing the incomplete ones; unknown if
|
||||||
|
// the pre-prune snapshot count could not be read.
|
||||||
|
snapshotsRemain := countDiff(snapshotCountBefore, &result.SnapshotsDeleted)
|
||||||
|
|
||||||
v.UI.Completef("Pruned local index database.")
|
v.UI.Completef("Pruned local index database.")
|
||||||
v.UI.Detailf("Incomplete snapshots: %d removed (%d remain).",
|
v.UI.Detailf("Incomplete snapshots: %s removed (%s remain).",
|
||||||
result.SnapshotsDeleted, snapshotCountAfter)
|
countText(&result.SnapshotsDeleted), countText(snapshotsRemain))
|
||||||
v.UI.Detailf("Orphaned files: %d removed (%d remain).",
|
v.UI.Detailf("Orphaned files: %s removed (%s remain).",
|
||||||
result.FilesDeleted, fileCountAfter)
|
countText(result.FilesDeleted), countText(fileCountAfter))
|
||||||
v.UI.Detailf("Orphaned chunks: %d removed (%d remain).",
|
v.UI.Detailf("Orphaned chunks: %s removed (%s remain).",
|
||||||
result.ChunksDeleted, chunkCountAfter)
|
countText(result.ChunksDeleted), countText(chunkCountAfter))
|
||||||
v.UI.Detailf("Orphaned blobs: %d removed (%d remain).",
|
v.UI.Detailf("Orphaned blobs: %s removed (%s remain).",
|
||||||
result.BlobsDeleted, blobCountAfter)
|
countText(result.BlobsDeleted), countText(blobCountAfter))
|
||||||
|
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// countUnknown is what a count reads as when its query could not be run,
|
||||||
|
// distinct from "0", which means the table really was empty.
|
||||||
|
const countUnknown = "unknown"
|
||||||
|
|
||||||
|
// tableCountForReport returns the row count of a table for the prune
|
||||||
|
// summary, or nil if the count could not be read. A read failure is
|
||||||
|
// logged at warn — visible even under --json, which routes warnings to
|
||||||
|
// stderr — and then rendered as unknown rather than silently becoming 0,
|
||||||
|
// so a broken query is a visible failure instead of a plausible wrong
|
||||||
|
// number.
|
||||||
|
func (v *Vaultik) tableCountForReport(tableName string) *int64 {
|
||||||
|
count, err := v.getTableCount(tableName)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("could not read table row count for prune summary",
|
||||||
|
"table", tableName, "error", err)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return &count
|
||||||
|
}
|
||||||
|
|
||||||
|
// countDiff returns before-after, or nil if either count is unknown so
|
||||||
|
// that an unreadable count does not collapse into a plausible delta.
|
||||||
|
func countDiff(before, after *int64) *int64 {
|
||||||
|
if before == nil || after == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
diff := *before - *after
|
||||||
|
|
||||||
|
return &diff
|
||||||
|
}
|
||||||
|
|
||||||
|
// countText renders a count that may be unknown: nil (the read failed)
|
||||||
|
// becomes "unknown", never "0", so a reader can tell an empty table from
|
||||||
|
// one that could not be queried.
|
||||||
|
func countText(count *int64) string {
|
||||||
|
if count == nil {
|
||||||
|
return countUnknown
|
||||||
|
}
|
||||||
|
|
||||||
|
return strconv.FormatInt(*count, 10)
|
||||||
|
}
|
||||||
|
|
||||||
// validTableNameRe matches table names containing only lowercase
|
// validTableNameRe matches table names containing only lowercase
|
||||||
// alphanumeric characters and underscores.
|
// alphanumeric characters and underscores.
|
||||||
var validTableNameRe = regexp.MustCompile(`^[a-z0-9_]+$`)
|
var validTableNameRe = regexp.MustCompile(`^[a-z0-9_]+$`)
|
||||||
|
|||||||
@@ -0,0 +1,101 @@
|
|||||||
|
package vaultik
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||||
|
)
|
||||||
|
|
||||||
|
// remoteKeyHexLen is the length of a full remote snapshot key: a SHA256
|
||||||
|
// digest rendered as lowercase hex.
|
||||||
|
const remoteKeyHexLen = 64
|
||||||
|
|
||||||
|
// Sentinel errors for resolving a snapshot identifier against the store.
|
||||||
|
var (
|
||||||
|
errSnapshotKeyNotFound = errors.New(
|
||||||
|
"no snapshot on the destination store matches this identifier")
|
||||||
|
errSnapshotKeyAmbiguous = errors.New(
|
||||||
|
"identifier matches more than one snapshot on the destination store")
|
||||||
|
)
|
||||||
|
|
||||||
|
// resolveSnapshotRemoteKey turns a snapshot identifier supplied on the
|
||||||
|
// command line into the remote key that names the snapshot's metadata
|
||||||
|
// directory on the destination store. Every remote path a restore or
|
||||||
|
// verify reads is built from that key.
|
||||||
|
//
|
||||||
|
// Two forms are accepted, matching the two things a host can know:
|
||||||
|
//
|
||||||
|
// - A human snapshot ID (hostname_name_timestamp), which a host holding
|
||||||
|
// the local index has. It is hashed to its remote key; the store is
|
||||||
|
// not consulted.
|
||||||
|
// - A remote key, or the leading part of one, which is all a host with
|
||||||
|
// no local index can know — it is exactly what `snapshot list` prints
|
||||||
|
// for a remote-only snapshot (see formatRemoteOnlyID). It is resolved
|
||||||
|
// against the destination store's metadata listing; an identifier that
|
||||||
|
// matches no snapshot, or more than one, is an error.
|
||||||
|
//
|
||||||
|
// The two are told apart by shape: a remote key is lowercase hex, and a
|
||||||
|
// human snapshot ID never is (it carries a hostname, underscores, and an
|
||||||
|
// RFC3339 timestamp).
|
||||||
|
func (v *Vaultik) resolveSnapshotRemoteKey(identifier string) (string, error) {
|
||||||
|
if !isRemoteKeyOrPrefix(identifier) {
|
||||||
|
return snapshot.RemoteSnapshotKey(identifier), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
keys, err := v.listAllRemoteSnapshotKeys()
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"listing destination store to resolve %q: %w", identifier, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var matches []string
|
||||||
|
|
||||||
|
for _, key := range keys {
|
||||||
|
if strings.HasPrefix(key, identifier) {
|
||||||
|
matches = append(matches, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
switch len(matches) {
|
||||||
|
case 1:
|
||||||
|
return matches[0], nil
|
||||||
|
case 0:
|
||||||
|
return "", fmt.Errorf("%w: %s", errSnapshotKeyNotFound, identifier)
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("%w: %s (%d matches)",
|
||||||
|
errSnapshotKeyAmbiguous, identifier, len(matches))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveAndDownloadManifest resolves a snapshot identifier to its remote
|
||||||
|
// key (see resolveSnapshotRemoteKey) and downloads that snapshot's
|
||||||
|
// manifest.
|
||||||
|
func (v *Vaultik) resolveAndDownloadManifest(
|
||||||
|
identifier string,
|
||||||
|
) (*snapshot.Manifest, error) {
|
||||||
|
remoteKey, err := v.resolveSnapshotRemoteKey(identifier)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return v.downloadManifestByKey(remoteKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
// isRemoteKeyOrPrefix reports whether s is a full remote key or the
|
||||||
|
// leading part of one: 1 to 64 lowercase hex characters. A human snapshot
|
||||||
|
// ID is never all hex, so this shape test is enough to tell the two apart.
|
||||||
|
func isRemoteKeyOrPrefix(s string) bool {
|
||||||
|
if s == "" || len(s) > remoteKeyHexLen {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, r := range s {
|
||||||
|
if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
+37
-23
@@ -138,8 +138,15 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
|||||||
func (v *Vaultik) loadVerificationData(
|
func (v *Vaultik) loadVerificationData(
|
||||||
snapshotID string, opts *VerifyOptions, result *VerifyResult,
|
snapshotID string, opts *VerifyOptions, result *VerifyResult,
|
||||||
) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) {
|
) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) {
|
||||||
// All remote paths use the hashed key derived from the human ID.
|
// Resolve the identifier to the snapshot's remote key. A human ID is
|
||||||
remoteKey := snapshot.RemoteSnapshotKey(snapshotID)
|
// hashed; a remote key (or its abbreviation, as printed for a
|
||||||
|
// remote-only snapshot) is used as-is, so a host with no local index
|
||||||
|
// can verify a snapshot it can only see on the store.
|
||||||
|
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, nil, v.deepVerifyFailure(result, opts,
|
||||||
|
fmt.Sprintf("resolving snapshot identifier: %v", err), err)
|
||||||
|
}
|
||||||
|
|
||||||
// Download manifest. downloadManifestByKey is the single reader for
|
// Download manifest. downloadManifestByKey is the single reader for
|
||||||
// remote manifests; see its doc comment.
|
// remote manifests; see its doc comment.
|
||||||
@@ -186,7 +193,7 @@ func (v *Vaultik) loadVerificationData(
|
|||||||
fmt.Errorf("failed to decrypt database: %w", err))
|
fmt.Errorf("failed to decrypt database: %w", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
dbBlobs, err := v.getBlobsFromDatabase(snapshotID, tdb.DB)
|
dbBlobs, err := v.getBlobsFromDatabase(tdb.DB)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = tdb.Close()
|
_ = tdb.Close()
|
||||||
|
|
||||||
@@ -344,12 +351,8 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error {
|
|||||||
return fmt.Errorf("failed to get decryptor: %w", err)
|
return fmt.Errorf("failed to get decryptor: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Hash the encrypted blob data as it streams through to decryption
|
// Decrypt blob
|
||||||
blobHasher := sha256.New()
|
decryptedReader, err := decryptor.DecryptStream(reader)
|
||||||
teeReader := io.TeeReader(reader, blobHasher)
|
|
||||||
|
|
||||||
// Decrypt blob (reading through teeReader to hash encrypted data)
|
|
||||||
decryptedReader, err := decryptor.DecryptStream(teeReader)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to decrypt: %w", err)
|
return fmt.Errorf("failed to decrypt: %w", err)
|
||||||
}
|
}
|
||||||
@@ -361,12 +364,19 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error {
|
|||||||
}
|
}
|
||||||
defer decompressor.Close()
|
defer decompressor.Close()
|
||||||
|
|
||||||
chunkCount, err := v.verifyBlobChunks(db, blobInfo.Hash, decompressor)
|
// A blob's hash — its remote name — is the double SHA256 of its
|
||||||
|
// decompressed plaintext (see blobgen.Writer.Sum256), not of the
|
||||||
|
// encrypted bytes. Hash the plaintext as chunk verification streams
|
||||||
|
// it, then compare on completion.
|
||||||
|
plaintextHasher := sha256.New()
|
||||||
|
hashedStream := io.TeeReader(decompressor, plaintextHasher)
|
||||||
|
|
||||||
|
chunkCount, err := v.verifyBlobChunks(db, blobInfo.Hash, hashedStream)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = v.verifyBlobFinalIntegrity(decompressor, blobHasher, blobInfo.Hash)
|
err = v.verifyBlobFinalIntegrity(hashedStream, plaintextHasher, blobInfo.Hash)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -470,14 +480,13 @@ func (v *Vaultik) verifyBlobChunks(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// verifyBlobFinalIntegrity checks that no trailing data exists in the
|
// verifyBlobFinalIntegrity checks that no trailing data exists in the
|
||||||
// decompressed stream and that the encrypted blob hash matches the
|
// decompressed stream and that the blob hash matches the expected value.
|
||||||
// expected value.
|
|
||||||
func (v *Vaultik) verifyBlobFinalIntegrity(
|
func (v *Vaultik) verifyBlobFinalIntegrity(
|
||||||
decompressor io.Reader, blobHasher hash.Hash, expectedHash string,
|
plaintext io.Reader, plaintextHasher hash.Hash, expectedHash string,
|
||||||
) error {
|
) error {
|
||||||
// Verify no remaining data in blob - if the chunk list is accurate,
|
// Verify no remaining data in blob - if the chunk list is accurate,
|
||||||
// the blob should be fully consumed.
|
// the blob should be fully consumed.
|
||||||
remaining, err := io.Copy(io.Discard, decompressor)
|
remaining, err := io.Copy(io.Discard, plaintext)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to check for remaining blob data: %w", err)
|
return fmt.Errorf("failed to check for remaining blob data: %w", err)
|
||||||
}
|
}
|
||||||
@@ -486,8 +495,11 @@ func (v *Vaultik) verifyBlobFinalIntegrity(
|
|||||||
return fmt.Errorf("%w: %d bytes", errTrailingBlobData, remaining)
|
return fmt.Errorf("%w: %d bytes", errTrailingBlobData, remaining)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify blob hash matches the encrypted data we downloaded
|
// The blob hash is the double SHA256 of its plaintext content.
|
||||||
calculatedBlobHash := hex.EncodeToString(blobHasher.Sum(nil))
|
firstHash := plaintextHasher.Sum(nil)
|
||||||
|
secondHash := sha256.Sum256(firstHash)
|
||||||
|
calculatedBlobHash := hex.EncodeToString(secondHash[:])
|
||||||
|
|
||||||
if calculatedBlobHash != expectedHash {
|
if calculatedBlobHash != expectedHash {
|
||||||
return fmt.Errorf("%w: calculated %s, expected %s",
|
return fmt.Errorf("%w: calculated %s, expected %s",
|
||||||
errBlobHashMismatch, calculatedBlobHash, expectedHash)
|
errBlobHashMismatch, calculatedBlobHash, expectedHash)
|
||||||
@@ -496,19 +508,21 @@ func (v *Vaultik) verifyBlobFinalIntegrity(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// getBlobsFromDatabase gets all blobs for the snapshot from the database
|
// getBlobsFromDatabase gets all blobs for the snapshot from the database.
|
||||||
func (v *Vaultik) getBlobsFromDatabase(
|
//
|
||||||
snapshotID string, db *sql.DB,
|
// The exported per-snapshot database holds exactly one snapshot's data
|
||||||
) ([]snapshot.BlobInfo, error) {
|
// (see cleanSnapshotDB), so every row in snapshot_blobs belongs to it.
|
||||||
|
// We select them directly rather than filtering by the human snapshot ID,
|
||||||
|
// which a host restoring from the store alone does not have.
|
||||||
|
func (v *Vaultik) getBlobsFromDatabase(db *sql.DB) ([]snapshot.BlobInfo, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT b.blob_hash, b.compressed_size
|
SELECT b.blob_hash, b.compressed_size
|
||||||
FROM snapshot_blobs sb
|
FROM snapshot_blobs sb
|
||||||
JOIN blobs b ON sb.blob_hash = b.blob_hash
|
JOIN blobs b ON sb.blob_hash = b.blob_hash
|
||||||
WHERE sb.snapshot_id = ?
|
|
||||||
ORDER BY b.blob_hash
|
ORDER BY b.blob_hash
|
||||||
`
|
`
|
||||||
|
|
||||||
rows, err := db.QueryContext(v.ctx, query, snapshotID)
|
rows, err := db.QueryContext(v.ctx, query)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to query snapshot blobs: %w", err)
|
return nil, fmt.Errorf("failed to query snapshot blobs: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -114,9 +114,6 @@ main() {
|
|||||||
# from CI. Nothing on the host is ever used as a linter, at any
|
# from CI. Nothing on the host is ever used as a linter, at any
|
||||||
# version, so installing one here would buy nothing.
|
# version, so installing one here would buy nothing.
|
||||||
|
|
||||||
# sqlite3 CLI: the test suite shells out to it (VACUUM).
|
|
||||||
if missing sqlite3; then pkg_install sqlite sqlite3 sqlite sqlite; fi
|
|
||||||
|
|
||||||
# goreleaser, at the version pinned by script/install-goreleaser and
|
# goreleaser, at the version pinned by script/install-goreleaser and
|
||||||
# verified against a hardcoded sha256. Package managers are not used
|
# verified against a hardcoded sha256. Package managers are not used
|
||||||
# for it: they ship whatever version they happen to carry, and the
|
# for it: they ship whatever version they happen to carry, and the
|
||||||
|
|||||||
+16
-1
@@ -56,8 +56,23 @@ main() {
|
|||||||
docker build --output=type=cacheonly \
|
docker build --output=type=cacheonly \
|
||||||
--build-arg CHECK_EPOCH="$epoch" -f Dockerfile.lint .
|
--build-arg CHECK_EPOCH="$epoch" -f Dockerfile.lint .
|
||||||
|
|
||||||
|
# Version, commit and build date are computed here on the host, the
|
||||||
|
# same way script/docker does, and passed into the product build so
|
||||||
|
# the CI-built image reports its real source. The build context
|
||||||
|
# excludes .git (see .dockerignore), so the build cannot derive them
|
||||||
|
# itself; without these it would stamp the Dockerfile's dev/unknown
|
||||||
|
# fallbacks. VERSION comes from script/version, the source of truth
|
||||||
|
# shared with the Makefile.
|
||||||
|
version="$("$ROOT/script/version")"
|
||||||
|
commit="$(git rev-parse HEAD 2>/dev/null || echo unknown)"
|
||||||
|
commit_date="$(git show -s --format=%cs HEAD 2>/dev/null || echo unknown)"
|
||||||
|
|
||||||
epoch="$(date +%s%N)$$"
|
epoch="$(date +%s%N)$$"
|
||||||
docker build --build-arg CHECK_EPOCH="$epoch" .
|
docker build --build-arg CHECK_EPOCH="$epoch" \
|
||||||
|
--build-arg VERSION="$version" \
|
||||||
|
--build-arg COMMIT="$commit" \
|
||||||
|
--build-arg COMMIT_DATE="$commit_date" \
|
||||||
|
.
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
@@ -24,7 +24,24 @@ main() {
|
|||||||
# whether the tree is clean. The Dockerfile now refuses to build
|
# whether the tree is clean. The Dockerfile now refuses to build
|
||||||
# without a non-empty value, so this is required, not optional.
|
# without a non-empty value, so this is required, not optional.
|
||||||
epoch="$(date +%s%N)$$"
|
epoch="$(date +%s%N)$$"
|
||||||
|
|
||||||
|
# Version, commit and build date are computed here on the host,
|
||||||
|
# where .git exists, and passed into the build. The build context
|
||||||
|
# excludes .git (see .dockerignore), so the container cannot derive
|
||||||
|
# them itself -- it used to try and always got "unknown", giving
|
||||||
|
# every image a "commit: unknown" it could not be traced from.
|
||||||
|
# VERSION comes from script/version, the source of truth shared with
|
||||||
|
# the Makefile, so a Docker build reports the same string (tag,
|
||||||
|
# dev-<sha>, or a -dirty variant) that a local build of the same
|
||||||
|
# tree would.
|
||||||
|
version="$("$SCRIPT_DIR/version")"
|
||||||
|
commit="$(git rev-parse HEAD 2>/dev/null || echo unknown)"
|
||||||
|
commit_date="$(git show -s --format=%cs HEAD 2>/dev/null || echo unknown)"
|
||||||
|
|
||||||
docker build --build-arg CHECK_EPOCH="$epoch" \
|
docker build --build-arg CHECK_EPOCH="$epoch" \
|
||||||
|
--build-arg VERSION="$version" \
|
||||||
|
--build-arg COMMIT="$commit" \
|
||||||
|
--build-arg COMMIT_DATE="$commit_date" \
|
||||||
-t "$("$SCRIPT_DIR/projectname")" .
|
-t "$("$SCRIPT_DIR/projectname")" .
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Executable
+161
@@ -0,0 +1,161 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/install-go: install the Go toolchain pinned by go.mod into the
|
||||||
|
# repo-local tool directory, verified against a committed sha256. Our
|
||||||
|
# own extension to scripts-to-rule-them-all. Idempotent: exits at once
|
||||||
|
# when the pinned toolchain is already installed.
|
||||||
|
#
|
||||||
|
# Only .gitea/workflows/release.yml calls this. goreleaser is not a
|
||||||
|
# compiler: it shells out to `go` for the `before:` hook and for every
|
||||||
|
# one of the four cross-compiles, so the release runner needs a Go
|
||||||
|
# toolchain on PATH. check.yml never does -- it builds inside the
|
||||||
|
# digest-pinned Dockerfile images -- so this is the release path's only
|
||||||
|
# host Go, and per REPO_POLICIES.md it must be pinned by hash.
|
||||||
|
# actions/setup-go exposes no checksum input, so Go is installed the way
|
||||||
|
# script/install-goreleaser installs goreleaser: download the exact
|
||||||
|
# archive from go.dev and refuse it unless its sha256 matches the value
|
||||||
|
# committed below.
|
||||||
|
#
|
||||||
|
# The version is go.mod's `go` directive, the single source of truth for
|
||||||
|
# the toolchain. GO_VERSION below MUST equal it, and this script fails
|
||||||
|
# when they disagree -- so bumping Go is one reviewed change touching
|
||||||
|
# go.mod, the checksum here, and the Dockerfile golang digest together.
|
||||||
|
#
|
||||||
|
# Linux only, because that is what the release runner is. A darwin dev
|
||||||
|
# building a snapshot uses their own Go; supporting an OS means adding
|
||||||
|
# its checksums.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
# Go 1.26.1, 2026-09-21. Checksums are the sha256 values go.dev publishes
|
||||||
|
# for each archive at https://go.dev/dl/ (also in its ?mode=json
|
||||||
|
# manifest).
|
||||||
|
GO_VERSION="1.26.1"
|
||||||
|
SHA256_LINUX_AMD64="031f088e5d955bab8657ede27ad4e3bc5b7c1ba281f05f245bcc304f327c987a"
|
||||||
|
SHA256_LINUX_ARM64="a290581cfe4fe28ddd737dde3095f3dbeb7f2e4065cab4eae44dfc53b760c2f7"
|
||||||
|
|
||||||
|
GOROOT_DIR="$ROOT/.tool/go"
|
||||||
|
GOCMD="$GOROOT_DIR/bin/go"
|
||||||
|
|
||||||
|
# The `go` directive in go.mod, e.g. "1.26.1" from `go 1.26.1`.
|
||||||
|
gomod_go_version() {
|
||||||
|
sed -n 's/^go \([0-9][0-9.]*\).*/\1/p' "$ROOT/go.mod" | head -n 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# Print the version of the go at $1 as "1.26.1", or nothing if it is not
|
||||||
|
# usable. `go version` prints "go version go1.26.1 linux/amd64".
|
||||||
|
go_version() {
|
||||||
|
[ -x "$1" ] || return 0
|
||||||
|
"$1" version 2>/dev/null |
|
||||||
|
sed -n 's/^go version go\([0-9][0-9.]*\) .*/\1/p' |
|
||||||
|
head -n 1
|
||||||
|
}
|
||||||
|
|
||||||
|
verify_sha256() {
|
||||||
|
file="$1"
|
||||||
|
want="$2"
|
||||||
|
if command -v sha256sum >/dev/null 2>&1; then
|
||||||
|
got="$(sha256sum "$file" | cut -d' ' -f1)"
|
||||||
|
elif command -v shasum >/dev/null 2>&1; then
|
||||||
|
got="$(shasum -a 256 "$file" | cut -d' ' -f1)"
|
||||||
|
else
|
||||||
|
echo "install-go: no sha256sum or shasum available" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if [ "$got" != "$want" ]; then
|
||||||
|
echo "install-go: checksum mismatch for $file" >&2
|
||||||
|
echo " expected: $want" >&2
|
||||||
|
echo " actual: $got" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# On a Gitea/GitHub Actions runner, put the toolchain on PATH for the
|
||||||
|
# steps that follow by appending to the file named by $GITHUB_PATH. A
|
||||||
|
# no-op off CI, where the caller manages its own PATH.
|
||||||
|
export_ci_path() {
|
||||||
|
[ -n "${GITHUB_PATH:-}" ] || return 0
|
||||||
|
echo "$GOROOT_DIR/bin" >>"$GITHUB_PATH"
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
want="$(gomod_go_version)"
|
||||||
|
if [ "$want" != "$GO_VERSION" ]; then
|
||||||
|
echo "install-go: go.mod says go $want but this script pins" \
|
||||||
|
"$GO_VERSION." >&2
|
||||||
|
echo " Update GO_VERSION and the checksums in this script to" \
|
||||||
|
"match go.mod." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Already installed from a previous run? Then just fix PATH and stop.
|
||||||
|
if [ "$(go_version "$GOCMD")" = "$GO_VERSION" ]; then
|
||||||
|
echo "go $GO_VERSION already installed in .tool/go"
|
||||||
|
export_ci_path
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
os="$(uname -s)"
|
||||||
|
arch="$(uname -m)"
|
||||||
|
case "$os" in
|
||||||
|
Linux) os="linux" ;;
|
||||||
|
*)
|
||||||
|
echo "install-go: unsupported OS $os (release runner is Linux)" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
case "$arch" in
|
||||||
|
x86_64 | amd64)
|
||||||
|
arch="amd64"
|
||||||
|
sum="$SHA256_LINUX_AMD64"
|
||||||
|
;;
|
||||||
|
arm64 | aarch64)
|
||||||
|
arch="arm64"
|
||||||
|
sum="$SHA256_LINUX_ARM64"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "install-go: no pinned checksum for architecture $arch" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
archive="go${GO_VERSION}.${os}-${arch}.tar.gz"
|
||||||
|
url="https://go.dev/dl/${archive}"
|
||||||
|
|
||||||
|
if ! command -v curl >/dev/null 2>&1; then
|
||||||
|
echo "install-go: curl is required" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
dl="$(mktemp -d)"
|
||||||
|
mkdir -p "$ROOT/.tool"
|
||||||
|
stage="$(mktemp -d "$ROOT/.tool/.go-install.XXXXXX")"
|
||||||
|
# shellcheck disable=SC2064 # expand the paths now, not at trap time
|
||||||
|
trap "rm -rf '$dl' '$stage'" EXIT INT TERM
|
||||||
|
|
||||||
|
echo "installing go $GO_VERSION for ${os}-${arch}"
|
||||||
|
curl -fsSL --retry 3 -o "$dl/$archive" "$url"
|
||||||
|
verify_sha256 "$dl/$archive" "$sum"
|
||||||
|
|
||||||
|
# The archive unpacks to a top-level `go/` directory. Extract it into
|
||||||
|
# a staging directory on the same filesystem as the destination, then
|
||||||
|
# rename it into place so a concurrent run never observes a
|
||||||
|
# half-written toolchain.
|
||||||
|
tar -xzf "$dl/$archive" -C "$stage"
|
||||||
|
rm -rf "$GOROOT_DIR"
|
||||||
|
mv "$stage/go" "$GOROOT_DIR"
|
||||||
|
|
||||||
|
installed="$(go_version "$GOCMD")"
|
||||||
|
if [ "$installed" != "$GO_VERSION" ]; then
|
||||||
|
echo "install-go: installed toolchain reports '$installed'," \
|
||||||
|
"expected '$GO_VERSION'" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "go $GO_VERSION installed to .tool/go"
|
||||||
|
export_ci_path
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
# Vaultik test configuration
|
|
||||||
hostname: test-host
|
|
||||||
index_path: /tmp/vaultik-test/index.db
|
|
||||||
source_dirs:
|
|
||||||
- /tmp/vaultik-test/source
|
|
||||||
|
|
||||||
# S3 configuration
|
|
||||||
s3:
|
|
||||||
endpoint: http://localhost:19000 # gofakes3 test endpoint
|
|
||||||
bucket: test-bucket
|
|
||||||
prefix: test-
|
|
||||||
access_key_id: test-key
|
|
||||||
secret_access_key: test-secret
|
|
||||||
region: us-east-1
|
|
||||||
|
|
||||||
# Chunking configuration
|
|
||||||
chunk_size: 65536 # 64KB average chunk size
|
|
||||||
min_chunk_size: 32768 # 32KB minimum
|
|
||||||
max_chunk_size: 131072 # 128KB maximum
|
|
||||||
blob_size: 1048576 # 1MB blobs for testing
|
|
||||||
|
|
||||||
# Compression
|
|
||||||
compression_level: 3
|
|
||||||
|
|
||||||
# Encryption
|
|
||||||
# age_recipients:
|
|
||||||
# - age1qyqszqgpqyqszqgpqyqszqgpqyqszqgpqyqszqgpqyqszqgpqyqs3mw88h
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
age_recipients:
|
|
||||||
- age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj # sneak's long term age key
|
|
||||||
- age1ezrjmfpwsc95svdg0y54mums3zevgzu0x0ecq2f7tp8a05gl0sjq9q9wjg # insecure integration test key
|
|
||||||
source_dirs:
|
|
||||||
- /tmp/vaultik-test-source
|
|
||||||
exclude:
|
|
||||||
- '*.log'
|
|
||||||
- '*.tmp'
|
|
||||||
- '.git'
|
|
||||||
- 'node_modules'
|
|
||||||
s3:
|
|
||||||
endpoint: http://ber1app1.local:3900/
|
|
||||||
bucket: vaultik-integration-test
|
|
||||||
prefix: test-host/
|
|
||||||
access_key_id: GKbc8e6d35fdf50847f155aca5
|
|
||||||
secret_access_key: 217046bee47c050301e3cc13e3cba1a8a943cf5f37f8c7979c349c5254441d18
|
|
||||||
region: us-east-1
|
|
||||||
use_ssl: false
|
|
||||||
part_size: 5242880 # 5MB
|
|
||||||
index_path: /tmp/vaultik-integration-test.sqlite
|
|
||||||
chunk_size: 10MB
|
|
||||||
blob_size_limit: 10GB
|
|
||||||
compression_level: 3
|
|
||||||
hostname: test-host
|
|
||||||
Reference in New Issue
Block a user