1 Commits
Author SHA1 Message Date
sneak 0f2ebbdafa Reject a duration with characters outside its parts (closes #215)
check / check (pull_request) Successful in 4m21s
parseDuration fell back to an unanchored search for number-and-unit
pieces when time.ParseDuration failed, and skipped everything in
between. 1.0y became 0, so `snapshot create --prune --keep-newer-than
1.0y` deleted every snapshot of the backed-up names, the new one
included. 2.1w became one week and 1,5y five years. The fallback now
requires the whole input to be whole-number-and-unit parts with nothing
between them.

Judgement call: a space between number and unit (`30 days`) was
accepted and is now an error, matching Go's own units.

Model: opus-5-5
2026-10-06 02:51:29 +00:00
115 changed files with 2101 additions and 6246 deletions
+10 -19
View File
@@ -17,17 +17,7 @@
# stage that compiles runs `git describe --tags --always` on .git, which # stage that compiles runs `git describe --tags --always` on .git, which
# does not need .git/config; that file can hold a credential, such as a # does not need .git/config; that file can hold a credential, such as a
# password in a remote URL or the token the CI checkout step stores there. # password in a remote URL or the token the CI checkout step stores there.
# Each submodule keeps a config with the same exposure in its git directory .git/config
# under .git/modules/, nested again for a submodule's own submodules, or in
# its own .git directory when it keeps one.
# KNOWN GAP: a submodule whose name has a `config` segment (`config`,
# `deploy/config`, `config/lib`) loses its whole git directory, because
# `**/.git/modules/**/config` also matches that segment's directory
# under .git/modules/. Go's version stamping then fails the build;
# nothing leaks. Name such a submodule without that segment:
# `git submodule add --name`.
**/.git/config
**/.git/modules/**/config
# Agent scratch: one full checkout of the repo per in-flight agent. # Agent scratch: one full checkout of the repo per in-flight agent.
# Anchored because it occurs once where agents run at the repo root. # Anchored because it occurs once where agents run at the repo root.
@@ -51,9 +41,7 @@
**/[iI][dD]_[rR][sS][aA] **/[iI][dD]_[rR][sS][aA]
**/[iI][dD]_[dD][sS][aA] **/[iI][dD]_[dD][sS][aA]
**/[iI][dD]_[eE][cC][dD][sS][aA] **/[iI][dD]_[eE][cC][dD][sS][aA]
**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
**/[iI][dD]_[eE][dD]25519 **/[iI][dD]_[eE][dD]25519
**/[iI][dD]_[eE][dD]25519_[sS][kK]
# Dependencies: restored inside the image, never copied in. # Dependencies: restored inside the image, never copied in.
**/node_modules **/node_modules
@@ -71,9 +59,12 @@
**/.vscode **/.vscode
**/*.sublime-* **/*.sublime-*
# This repo's own host-built artifacts. # This repo's own entries.
/vaultik .gitea
/dist *.md
/.tool LICENSE
/coverage.out vaultik
/coverage.html dist
.tool
coverage.out
coverage.html
-3
View File
@@ -10,6 +10,3 @@ insert_final_newline = true
[Makefile] [Makefile]
indent_style = tab indent_style = tab
[*.go]
indent_style = tab
+12 -7
View File
@@ -1,9 +1,14 @@
name: check name: check
on: [push] on:
push:
branches: [main, next]
pull_request:
branches: [main, next]
jobs: jobs:
check: check:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
# actions/checkout v4.2.2, 2026-02-22 # actions/checkout v4, 2024-09-16
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
- run: script/cibuild - name: Build and check
run: script/cibuild
+5 -6
View File
@@ -16,12 +16,11 @@ jobs:
fetch-depth: 0 fetch-depth: 0
# goreleaser is not a compiler: it shells out to `go` for the # goreleaser is not a compiler: it shells out to `go` for the
# `before:` hook and for every one of the four cross-compiles. # `before:` hook and for every one of the four cross-compiles.
# Without this step the release either fails at the before-hook or, # Nothing else in this repo puts a Go toolchain on the runner --
# worse, ships binaries built by whatever Go the runner happens to # check.yml runs script/cibuild, which does all of its work inside
# carry. check.yml's runner gets the same Go through # the digest-pinned Dockerfile images -- so without this step the
# script/bootstrap, which calls script/install-go, and uses it only # release either fails at the before-hook or, worse, ships binaries
# for `go mod download` and gofmt; it compiles inside the # built by whatever Go the runner happens to carry.
# digest-pinned Dockerfile images.
# #
# actions/setup-go would pin the action by commit sha, but the Go # actions/setup-go would pin the action by commit sha, but the Go
# tarball it downloads at runtime is verified against no value in # tarball it downloads at runtime is verified against no value in
+23 -57
View File
@@ -1,62 +1,28 @@
# Binary
/vaultik
# goreleaser output
/dist/
# Locally installed pinned tools (script/install-goreleaser)
/.tool/
# Test artifacts
*.out
*.test
coverage.html
coverage.out
# IDE
.vscode/
.idea/
*.swp
*.swo
# OS # OS
.DS_Store .DS_Store
Thumbs.db Thumbs.db
# Editors # Local config for development
*.swp
*.swo
*~
*.bak
.idea/
.vscode/
*.sublime-*
# Agent scratch (worktrees of this repo, created and destroyed by
# in-flight tooling). Unanchored: .gitignore patterns already match at
# every depth, so no prefix is wanted here. This is not a .dockerignore
# entry and must not be given a `**/` prefix on the way into one.
.claude/
# Node
node_modules/
# Secrets. Unanchored like every entry above, so each matches at every
# depth. Matching is case-sensitive on Linux, so names use character
# ranges rather than a lowercase form that misses `Server.Key`.
# Environment files. `*.env` covers bare `.env` and the `prod.env`
# convention. Only the templates `example.env` and `sample.env` are
# re-included below. A repository that commits any other template adds
# its own negation after these lines, for example `!.env.example`.
*.[eE][nN][vV]
.[eE][nN][vV].*
.[eE][nN][vV][rR][cC]
!example.env
!sample.env
# Private keys and the bundles carrying them.
*.[pP][eE][mM]
*.[kK][eE][yY]
*.[pP]12
*.[pP][fF][xX]
[iI][dD]_[rR][sS][aA]
[iI][dD]_[dD][sS][aA]
[iI][dD]_[eE][cC][dD][sS][aA]
[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK]
[iI][dD]_[eE][dD]25519
[iI][dD]_[eE][dD]25519_[sS][kK]
# Go build and test output.
*.log
*.out
*.test
coverage.html
# This repo's own host-built artifacts.
/vaultik
/dist/
/.tool/
# Local configs for development; they hold storage credentials.
local-config.yaml local-config.yaml
dev-config.yaml dev-config.yaml
-1
View File
@@ -17,7 +17,6 @@ linters:
disable: disable:
# Genuinely incompatible with project patterns # Genuinely incompatible with project patterns
- exhaustruct # Requires all struct fields - exhaustruct # Requires all struct fields
- exhaustruct_v5 # Requires all struct fields (successor to exhaustruct)
- godot # Requires comments to end with periods - godot # Requires comments to end with periods
- wrapcheck # Too verbose for internal packages - wrapcheck # Too verbose for internal packages
- varnamelen # Short names like db, id are idiomatic Go - varnamelen # Short names like db, id are idiomatic Go
+1 -18
View File
@@ -108,23 +108,6 @@ Version: 2025-06-08
code that touches the affected tables) directly. After 1.0, each schema code that touches the affected tables) directly. After 1.0, each schema
change is a new numbered file in that directory and a released file is change is a new numbered file in that directory and a released file is
never edited; an existing local database is then migrated when vaultik never edited; an existing local database is then migrated when vaultik
is updated. Before 1.0, a local database left on an older schema is is updated. See
deleted and re-created by a full backup. See
[`docs/DATAMODEL.md`](docs/DATAMODEL.md#schema-migrations). [`docs/DATAMODEL.md`](docs/DATAMODEL.md#schema-migrations).
14. Never use `git add -A`. Stage only the files you intentionally
changed.
15. Commit messages carry no attribution or advertising trailers for the
tool that helped write the code, or for its vendor: the owner is the
sole author of code written with a tool.
16. Run the whole test suite with `make test` every time, and read its full
output. Never run `go test`, a single test or a single package, and
never grep the output.
17. Do not stop working on a task until the definition of done given in the
initial instruction is met: all of the work, not part or most of it.
18. For estimates: backing up over 2.5Gbit/s ethernet to an S3 server
backed by 2000MB/sec SSD takes about 4 seconds per gigabyte.
+5 -9
View File
@@ -54,12 +54,10 @@ The database tracks five primary entities and their relationships:
#### File (`database.File`) #### File (`database.File`)
Represents a file, directory, or symlink in the backup system. Stores metadata needed for restoration: Represents a file, directory, or symlink in the backup system. Stores metadata needed for restoration:
- Path, mtime - Path, source_path (for restore path stripping), mtime
- Size, mode, ownership (uid, gid) - Size, mode, ownership (uid, gid)
- Symlink target (if applicable) - Symlink target (if applicable)
It also stores `source_path`, the source directory the scan found it under, made absolute and with symlinks resolved. Restore does not read it.
#### Chunk (`database.Chunk`) #### Chunk (`database.Chunk`)
A content-addressed unit of data. Files are split into variable-size chunks using the FastCDC algorithm: A content-addressed unit of data. Files are split into variable-size chunks using the FastCDC algorithm:
- `ChunkHash`: SHA256 hash of chunk content (primary key) - `ChunkHash`: SHA256 hash of chunk content (primary key)
@@ -84,11 +82,9 @@ The final storage unit uploaded to S3. Contains many compressed and encrypted ch
Blob creation process: Blob creation process:
1. Chunks are accumulated (up to MaxBlobSize, typically 10GB) 1. Chunks are accumulated (up to MaxBlobSize, typically 10GB)
2. As each chunk is added, its uncompressed bytes are fed to a running SHA-256 2. As each chunk is added, its uncompressed bytes are fed to a running SHA-256
3. Concurrently, the same bytes are compressed with zstd, then encrypted with age (recipients configured in config), and written to a temporary file 3. Concurrently, the same bytes are compressed with zstd, then encrypted with age (recipients configured in config), and streamed to storage
4. On finalize, the blob's name is the double SHA-256 of the uncompressed contents — `hex(SHA256(SHA256(...)))` — not a hash of the compressed, encrypted bytes 4. On finalize, the blob's name is the double SHA-256 of the uncompressed contents — `hex(SHA256(SHA256(...)))` — not a hash of the compressed, encrypted bytes
5. The finished file is uploaded to `blobs/{hash[0:2]}/{hash[2:4]}/{hash}` and then deleted 5. Uploaded to `blobs/{hash[0:2]}/{hash[2:4]}/{hash}`
A backup needs free temporary space, because each blob is written whole to a temporary file before it is uploaded (up to about `blob_size_limit`; an rclone destination that cannot stream uploads needs about twice that) and the metadata export writes copies of the local index. Temporary files go to `$TMPDIR` (default `/tmp`); with `TMPDIR` unset, SQLite writes one of those copies to `/var/tmp`.
#### BlobChunk (`database.BlobChunk`) #### BlobChunk (`database.BlobChunk`)
Maps chunks to their position within blobs: Maps chunks to their position within blobs:
@@ -339,10 +335,10 @@ CreateSnapshot(opts)
│ │ │ │
│ └─► Accumulate statistics │ └─► Accumulate statistics
│ │
├─► SnapshotManager.PopulateSnapshotBlobs() // record referenced blobs
│
├─► SnapshotManager.UpdateSnapshotStatsExtended() ├─► SnapshotManager.UpdateSnapshotStatsExtended()
│ │
├─► SnapshotManager.PopulateSnapshotBlobs() // record referenced blobs
│
├─► SnapshotManager.ExportSnapshotMetadata() ├─► SnapshotManager.ExportSnapshotMetadata()
│ │ │ │
│ ├─► Copy database to temp file │ ├─► Copy database to temp file
+44
View File
@@ -0,0 +1,44 @@
# Rules
Read the rules in AGENTS.md and follow them.
# Memory
* Claude is an inanimate tool. The spam that Claude attempts to insert into
commit messages (which it erroneously refers to as "attribution") is not
attribution, as I am the sole author of code created using Claude. It is
corporate advertising for Anthropic and is therefore completely
unacceptable in commit messages.
* NEVER use `git add -A`. Always add only the files you intentionally
changed.
* Tests should always be run before committing code. No commits should be
made that do not pass tests.
* Code should always be formatted before committing. Do not commit
unformatted code.
* Code should always be linted before committing. Do not commit
unlinted code.
* The test suite is fast and local. When running tests, don't run
individual parts of the test suite, always run the whole thing by running
"make test".
* Do not stop working on a task until you have reached the definition of
done provided to you in the initial instruction. Don't do part or most of
the work, do all of the work until the criteria for done are met.
* We do not add migrations before 1.0; schema upgrades can be handled by
deleting the local state file and doing a full backup to re-create it.
* When testing on a 2.5Gbit/s ethernet to an s3 server backed by 2000MB/sec SSD,
estimate about 4 seconds per gigabyte of backup time.
* When running tests, don't run individual tests, or grep the output. run
the entire test suite every time and read the full output.
* When running tests, don't run individual tests, or try to grep the output.
never run "go test". only ever run "make test" to run the full test
suite, and examine the full output.
+80 -64
View File
@@ -1,76 +1,92 @@
# Lint phase. The linter is invoked directly rather than through `make # This file has no lint stage, deliberately.
# lint` or `script/lint`, which are themselves a docker build and would #
# recurse into a daemon that does not exist in a build step. # Linting lives in Dockerfile.lint, built by script/lint, and
# golangci/golangci-lint:v2.14.0, 2026-10-05 # script/cibuild builds both. A lint stage here would have to either
FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint # shell out to `make lint` -- which is now `docker build`, so
WORKDIR /src # docker-in-docker inside a BuildKit step with no daemon -- or call
COPY go.mod go.sum ./ # golangci-lint directly, which would mean a second, independently
RUN go mod download # bumpable digest pin for the linter alongside the one in
COPY . . # Dockerfile.lint. Two pins for one tool is the drift that
# `golangci-lint run` silently ignores an unknown top-level key in # https://git.eeqj.de/sneak/vaultik/issues/78 was filed over. See
# .golangci.yml, such as a misspelt `linters:`; `config verify` fails on it. # https://git.eeqj.de/sneak/vaultik/issues/113 for the ruling.
RUN golangci-lint config verify --config .golangci.yml #
RUN golangci-lint run --config .golangci.yml ./... # Consequence, stated rather than left to be discovered: script/docker
# builds this file only and therefore does not lint. `make fmt-check`
# and `make test` still run here, so what a green build of this file
# means is "formatted, tested, and it compiles" -- the lint verdict
# comes from script/lint or script/cibuild.
# Test phase. -race needs cgo and so a C compiler, which the Debian Go # Build stage
# image ships and the alpine one does not.
# golang:1.26.1 (Debian trixie), 2026-10-05
FROM golang:1.26.1@sha256:cd78d88e00afadbedd272f977d375a6247455f3a4b1178f8ae8bbcb201743a8a AS test
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN go test -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
go test -timeout 90s -race -v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies
# are what make BuildKit build them first, so this stage cannot run
# unless lint and test passed.
# golang:1.26.1-alpine, 2026-03-17 # golang:1.26.1-alpine, 2026-03-17
FROM golang:1.26.1-alpine@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f6c5bd6be49ed82039 AS builder FROM golang:1.26.1-alpine@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f6c5bd6be49ed82039 AS builder
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null # Build tooling: make, plus a C toolchain because `go test -race` needs cgo,
RUN apk add --no-cache git # and git, which derives the version below. The sqlite driver is pure Go
# A tar-stream context keeps the sender's file owners, which git refuses. # (modernc.org/sqlite), so no sqlite library or CLI is required.
RUN git config --system --add safe.directory /src RUN apk add --no-cache make build-base git
WORKDIR /src WORKDIR /src
# Copy go mod files first for better layer caching
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
# Copy source code
COPY . . COPY . .
# The VERSION build arg when one is given, otherwise # Run the format check and the tests.
# `git describe --tags --always` on the .git in the build context. The #
# commit and its date always come from that .git. With .git present, a # CHECK_EPOCH must stay immediately above these RUNs. These layers are
# version that is still empty, dev or unknown, or a commit or date that # keyed on its value, so they are cache-eligible only for a value
# is unknown, fails the build: git is missing or could not read the # already built against this same tree. script/cibuild and script/docker
# checkout, as when .git is a file pointing outside the context. A # each pass a fresh value on every invocation, which is what makes their
# context without .git, such as a source export, stamps "dev" and an # green mean the checks really executed.
# "unknown" commit and date. #
ARG VERSION # The value is expanded into each check command rather than left to a
RUN VERSION="${VERSION:-$(git describe --tags --always || echo dev)}"; \ # bare declaration, so the cache miss does not depend on BuildKit's
commit="$(git rev-parse HEAD || echo unknown)"; \ # unreferenced-ARG handling staying as it is. It also puts the epoch in
commit_date="$(git show -s --format=%cs HEAD || echo unknown)"; \ # the build log, where a reader can see the layer was keyed fresh.
if [ -e .git ]; then \ #
case "$VERSION" in ""|dev|unknown) \ # A build that passes no CHECK_EPOCH, such as a plain `docker build .`,
echo "version is '$VERSION' although .git is present" >&2; \ # keys these layers on the empty string, so rebuilding an unchanged
exit 1 ;; \ # checkout replays them from cache and runs nothing. Only the scripts'
esac; \ # builds mean the checks executed.
if [ "$commit" = unknown ] || [ "$commit_date" = unknown ]; then \ #
echo "commit is '$commit' and its date '$commit_date'" \ # Everything above this line (apk, go.mod, `go mod download`) is
"although .git is present" >&2; \ # deliberately outside the busted range and keeps caching.
exit 1; \ ARG CHECK_EPOCH
fi; \ RUN echo "check epoch: ${CHECK_EPOCH}" && make fmt-check
fi; \ RUN echo "check epoch: ${CHECK_EPOCH}" && make test
globals=sneak.berlin/go/vaultik/internal/globals; \
CGO_ENABLED=0 go build -trimpath \
-ldflags="-s -w -X ${globals}.Version=${VERSION} \
-X ${globals}.Commit=${commit} \
-X ${globals}.CommitDate=${commit_date}" \
-o /vaultik ./cmd/vaultik
# Runtime stage, and the last one: a plain `docker build .` builds this # Version, commit and build date: the build args when given (script/docker
# stage's chain and nothing else. # and script/cibuild pass the ones they compute on the host), otherwise
# derived from the .git in the build context. The version is then `git
# describe --tags --always`: the tag on a tagged commit, tag-N-gHASH after
# one, the short commit when no tag is reachable. A context that carries
# .git and still yields no version fails the build; one without .git, as
# from a source tarball, stamps "dev" and an "unknown" commit and date.
#
# These ARGs sit here, after the checks, rather than at the top of the
# stage: every commit changes their values, and a value change
# invalidates all layers below the ARG. Declared up top they would bust
# `go mod download`; here they only rekey this build layer, which the
# COPY of the sources above already rebuilds on any change anyway.
ARG VERSION
ARG COMMIT
ARG COMMIT_DATE
# Build (pure Go, no CGO required since we use modernc.org/sqlite)
RUN version="${VERSION:-$(git describe --tags --always || echo dev)}"; \
if [ -e .git ] && { [ -z "$version" ] || [ "$version" = dev ] || \
[ "$version" = unknown ]; }; then \
echo "the build context carries .git but yields no version" >&2; \
exit 1; \
fi; \
commit="${COMMIT:-$(git rev-parse HEAD || echo unknown)}"; \
commit_date="${COMMIT_DATE:-$(git show -s --format=%cs HEAD || echo unknown)}"; \
CGO_ENABLED=0 go build -ldflags "-X 'sneak.berlin/go/vaultik/internal/globals.Version=${version}' -X 'sneak.berlin/go/vaultik/internal/globals.Commit=${commit}' -X 'sneak.berlin/go/vaultik/internal/globals.CommitDate=${commit_date}'" -o /vaultik ./cmd/vaultik
# Runtime stage
# alpine:3.21, 2026-02-25 # alpine:3.21, 2026-02-25
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709 FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
+104
View File
@@ -0,0 +1,104 @@
# Lint image.
#
# Every lint run in this repo happens inside this image, invoked through
# script/lint, and linting is a BUILD STEP rather than a container
# command: a successful build of this file IS a clean lint. That shape
# also works where the docker daemon is remote and bind mounts are
# impossible, which `docker run` against a mounted worktree does not.
#
# This FROM line is the single source of truth for the linter version in
# this repo. Nothing else pins golangci-lint: the product Dockerfile has
# no lint stage, deliberately, so there is no second digest to bump and
# no pair of pins that can drift apart. Bump the tag AND the digest here
# and nowhere else.
#
# Note for readers coming from REPO_POLICIES.md: that document still
# describes the older pattern, a lint stage inside the product
# Dockerfile wired up with `COPY --from=lint /src/go.sum /dev/null`.
# That pattern is superseded here by the owner's ruling recorded in
# https://git.eeqj.de/sneak/vaultik/issues/113 -- lint runs in its own
# image, per run, with its own cache and its own lock, which is what
# makes concurrent runs on one host safe. The policy text is org-wide
# and is being amended separately; this file is what this repo does.
#
# golangci/golangci-lint:v2.12.2, 2026-08-10
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240
WORKDIR /src
# Copy the dependency manifests first so the module download layer stays
# cached until they change. Everything above the ARG below is cacheable
# on purpose; a cold module download on every lint would make the inner
# loop unusable and buys nothing, because it is not what the gate is
# asserting.
COPY go.mod go.sum ./
RUN go mod download
COPY . .
# Force the check layers to execute on every invocation.
#
# CHECK_EPOCH must stay immediately above the RUNs below. Those layers
# are keyed on its value, so they are cache-eligible only for a value
# already built against this same tree; script/lint and script/cibuild
# each pass a fresh value on every invocation, which is what makes their
# green mean the linter really ran. Without it, `docker build -f
# Dockerfile.lint .` on an unchanged tree exits 0 in well under a second
# having linted nothing.
#
# The value is expanded into each check command itself rather than left
# to a bare declaration, so the cache miss does not depend on BuildKit's
# unreferenced-ARG handling staying as it is. It also puts the epoch in
# the build log, where a reader can see the layer was keyed fresh.
#
# The guard is what makes a build that omits --build-arg fail instead of
# lie. An unset ARG is an empty string, and an empty string is a
# perfectly stable cache key: without the guard the first such build
# lints and every one after it on an unchanged tree replays this layer,
# executes nothing, and still exits 0. Failed steps are never cached, so
# the guard fails on EVERY invocation rather than once. Do not give
# CHECK_EPOCH a default value; a default would satisfy the guard with a
# constant and restore the hole.
ARG CHECK_EPOCH
RUN [ -n "$CHECK_EPOCH" ] || exit 1
# Validate .golangci.yml before linting with it.
#
# This is not belt-and-braces; it closes a hole that `golangci-lint run`
# leaves wide open. `run` rejects YAML it cannot PARSE, but it silently
# IGNORES an unknown top-level KEY. Renaming `linters:` to `linterz:` --
# one character -- discards `default: all`, the whole disable list and
# every threshold, leaves only golangci-lint's small default linter set
# running, and exits 0 reporting `0 issues.` on a tree the real config
# fails. Demonstrated on this repo at this pin, recorded on
# https://git.eeqj.de/sneak/vaultik/pulls/114: with a planted
# over-length line, `script/lint` exits 1 naming the `revive` finding
# with `linters:` and exits 0 with `linterz:`. A set-but-ineffective
# config quietly falling back to defaults is precisely the false-green
# class this gate exists to eliminate, so it must not sit in the gate's
# own configuration.
#
# `config verify` catches it, and it does so OFFLINE at this pinned
# version -- verified, not assumed. Under `docker run --network none`
# against the pinned digest it exits 0 on this repo's config and exits 3
# on the `linterz:` variant with `additional properties 'linterz' not
# allowed`. An earlier revision of this file asserted the opposite, that
# the schema is fetched over live HTTPS from an unpinned URL, and used
# that to justify omitting this line. That claim was false at v2.12.2;
# the schema is embedded. If a future bump reintroduces a network fetch
# the failure is loud and this comment is where to record it.
#
# It is keyed on CHECK_EPOCH, like the lint run below, so it executes on
# every invocation. Content-addressing alone would arguably be enough --
# .golangci.yml arrives through `COPY . .`, so a cache hit here implies
# a byte-identical config was validated when the layer really ran. That
# argument is exactly the one that would also excuse caching the lint
# layer, and this repo has ruled it insufficient: a cached check layer
# checks nothing, and the cost of being wrong is silent. Forcing it costs
# milliseconds and puts the epoch in the log, where a reader can see that
# this validation ran rather than being replayed.
RUN echo "check epoch: ${CHECK_EPOCH}" && \
golangci-lint config verify --config .golangci.yml
RUN echo "check epoch: ${CHECK_EPOCH}" && \
golangci-lint run --config .golangci.yml ./...
+11 -15
View File
@@ -1,8 +1,5 @@
.PHONY: all bootstrap setup check test lint lint-fix fmt fmt-check build clean deps test-coverage local install release release-snapshot docker hooks .PHONY: all bootstrap setup check test lint lint-fix fmt fmt-check build clean deps test-coverage local install release release-snapshot docker hooks
# Where script/bootstrap installs Go when the host has none.
export PATH := $(PATH):$(CURDIR)/.tool/go/bin
# Version number, derived from git by script/version (`git describe # Version number, derived from git by script/version (`git describe
# --tags --always --dirty`). This used to be a hardcoded # --tags --always --dirty`). This used to be a hardcoded
# constant, which meant every local build claimed to be a release that # constant, which meant every local build claimed to be a release that
@@ -43,10 +40,9 @@ setup:
check: check:
@script/check @script/check
# Run tests only, by building the test phase of the Dockerfile. This # Run tests only. This runs the ENTIRE suite -- there is no separate
# runs the ENTIRE suite -- there is no separate integration target and # integration target and no build-tagged subset held back. In
# no build-tagged subset held back. In particular # particular internal/vaultik/integration_test.go, which does full
# internal/vaultik/integration_test.go, which does full
# chunk -> pack -> encrypt -> upload -> restore round-trips, runs here. # chunk -> pack -> encrypt -> upload -> restore round-trips, runs here.
# A `test-integration` target used to exist and was removed: no file in # A `test-integration` target used to exist and was removed: no file in
# the repo carried a build tag, so `-tags=integration` selected nothing # the repo carried a build tag, so `-tags=integration` selected nothing
@@ -91,17 +87,17 @@ clean:
go clean go clean
# Install dependencies. The linter is deliberately not installed here: # Install dependencies. The linter is deliberately not installed here:
# script/lint lints by building the lint phase of the Dockerfile, whose # script/lint lints by building Dockerfile.lint, whose FROM line is the
# FROM line is the single source of truth for the linter version. A # single source of truth for the linter version. A second, separately
# second, separately pinned copy on PATH could drift from it and make a # pinned copy on PATH could drift from it and make a local `make lint`
# local `make lint` disagree with CI. # disagree with CI.
deps: deps:
go mod download go mod download
# Run tests with coverage, on the host. -count=1 because without it an # Run tests with coverage. -count=1 for the same reason script/test
# unchanged package is served from Go's test result cache, and a # uses it: without it an unchanged package is served from Go's test
# coverage profile assembled from cached results describes a run that # result cache, and a coverage profile assembled from cached results
# did not happen. # describes a run that did not happen.
test-coverage: test-coverage:
go test -v -count=1 -coverprofile=coverage.out ./... go test -v -count=1 -coverprofile=coverage.out ./...
go tool cover -html=coverage.out -o coverage.html go tool cover -html=coverage.out -o coverage.html
+136 -119
View File
@@ -178,7 +178,7 @@ vaultik version
* `--verbose`, `-v`: Enable verbose output (on stderr — see below) * `--verbose`, `-v`: Enable verbose output (on stderr — see below)
* `--debug`: Enable debug output (on stderr — see below) * `--debug`: Enable debug output (on stderr — see below)
* `--quiet`, `-q`: Suppress non-error output (also suppresses startup banner) * `--quiet`, `-q`: Suppress non-error output (also suppresses startup banner)
* `--skip-errors`: Skip files that cannot be read when creating a snapshot, or that cannot be restored when restoring, instead of aborting. Packing and storage errors while creating a snapshot (which would leave a chunk recorded but not stored) still abort the run. * `--skip-errors`: Skip files that cannot be read when creating a snapshot, or that cannot be restored when restoring, instead of aborting. Packing and storage errors (which would leave a chunk recorded but not stored) still abort the run.
### locking ### locking
@@ -200,10 +200,8 @@ local index or the destination store. `config`, `database delete`,
### stdout and stderr ### stdout and stderr
Log output — everything from `--verbose` and `--debug`, and every Log output — everything from `--verbose` and `--debug`, and every
warning and error the logger emits — goes to **stderr**, and so does the warning and error the logger emits — goes to **stderr**. stdout carries
startup banner. stdout carries the output you asked for: tables, the the output you asked for: tables, and the documents produced by `--json`.
documents produced by `--json`, `config get` values, and completion
scripts.
This means `vaultik snapshot list --verbose > out.txt` captures the This means `vaultik snapshot list --verbose > out.txt` captures the
listing and leaves the diagnostics on your terminal. To capture both, listing and leaves the diagnostics on your terminal. To capture both,
@@ -352,9 +350,8 @@ may hold snapshots this host doesn't know about), which is what
prune` invocation to run as a follow-up. Local row cleanup (files, prune` invocation to run as a follow-up. Local row cleanup (files,
chunks, blobs the snapshot was the last referrer for) runs chunks, blobs the snapshot was the last referrer for) runs
automatically. If the destination store is unreachable, the local-DB automatically. If the destination store is unreachable, the local-DB
removal still completes and a warning is emitted; run `vaultik snapshot removal still completes and a warning is emitted; rerun `vaultik prune`
remove <snapshot-id>` again once the store is reachable to remove the once the store is reachable to finish remote cleanup. To wipe everything
snapshot's metadata from it (`vaultik prune` does not). To wipe everything
on the destination in one go, use `vaultik remote nuke --force`. on the destination in one go, use `vaultik remote nuke --force`.
* `--local-only`: Skip remote cleanup; only touch the local index * `--local-only`: Skip remote cleanup; only touch the local index
* `--dry-run`: Show what would be deleted without deleting * `--dry-run`: Show what would be deleted without deleting
@@ -390,13 +387,7 @@ recipients, and local database statistics.
**`remote info`**: Show storage backend type and location plus detailed **`remote info`**: Show storage backend type and location plus detailed
remote storage inventory: per-snapshot metadata sizes, blob counts, and remote storage inventory: per-snapshot metadata sizes, blob counts, and
orphaned blob detection. A name under `metadata/` that is not a remote orphaned blob detection.
key is skipped with a warning and is not printed. If a listed
`manifest.json.zst` cannot be read, or sits under a skipped name, the
orphaned blob figures are reported as unknown; `--json` gives them as
`null`, lists the remote key of each unreadable manifest in
`unreadable_manifests` and counts the manifests under skipped names in
`skipped_manifest_count`.
* `--json`: Output as JSON * `--json`: Output as JSON
**`remote nuke`**: Delete every snapshot's metadata and every blob from the **`remote nuke`**: Delete every snapshot's metadata and every blob from the
@@ -428,16 +419,6 @@ a local or mounted filesystem. Useful for testing or backing up to a NAS.
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud **Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
providers. Requires rclone to be configured separately (`rclone config`). providers. Requires rclone to be configured separately (`rclone config`).
An upload cut off part-way leaves nothing under the object's name on S3, which
shows an object only once its upload has completed, and on the local filesystem
backend, which writes a temporary file and renames it into place. Rclone
remotes with a server-side move (local and sftp among them) are written under a
temporary name ending in `.partial` and moved into place. Rclone remotes without
one are written in place: where such a remote shows a file while it is still
being written, a killed upload can leave a truncated object under its name,
which a later backup takes for complete. A leftover `.partial` file is ignored
and can be deleted.
Legacy S3 configuration via `s3.*` fields (endpoint, bucket, prefix, etc.) is Legacy S3 configuration via `s3.*` fields (endpoint, bucket, prefix, etc.) is
still supported for backward compatibility. `storage_url` takes precedence if still supported for backward compatibility. `storage_url` takes precedence if
both are set. both are set.
@@ -549,14 +530,14 @@ complete annotated example also lives in
| Field | Default | Description | | Field | Default | Description |
|-------|---------|-------------| |-------|---------|-------------|
| `age_recipients` | (required by `snapshot create`) | Age public keys for encryption. Other commands run without one, so a machine that only restores can leave it empty | | `age_recipients` | (required) | Age public keys for encryption |
| `age_secret_key` | (unset) | Age private key for decryption (`snapshot restore`, `snapshot verify --deep`). Setting it in the config file places the private key on the backed-up host, defeating the public-key-only design (see "why" above). Prefer the `VAULTIK_AGE_SECRET_KEY` environment variable, supplied only on the machine you restore from. | | `age_secret_key` | (unset) | Age private key for decryption (`snapshot restore`, `snapshot verify --deep`). Setting it in the config file places the private key on the backed-up host, defeating the public-key-only design (see "why" above). Prefer the `VAULTIK_AGE_SECRET_KEY` environment variable, supplied only on the machine you restore from. |
| `snapshots` | (required) | Named snapshot definitions with paths and excludes | | `snapshots` | (required) | Named snapshot definitions with paths and excludes |
| `storage_url` | | Storage backend URL (`s3://`, `file://`, `rclone://`) | | `storage_url` | | Storage backend URL (`s3://`, `file://`, `rclone://`) |
| `s3.*` | | Legacy S3 configuration (endpoint, bucket, credentials) | | `s3.*` | | Legacy S3 configuration (endpoint, bucket, credentials) |
| `exclude` | | Global exclude patterns (applied to all snapshots) | | `exclude` | | Global exclude patterns (applied to all snapshots) |
| `chunk_size` | `10MB` | Average chunk size for content-defined chunking | | `chunk_size` | `10MB` | Average chunk size for content-defined chunking |
| `blob_size_limit` | `10GB` | Maximum blob size before splitting. Must be at least four times `chunk_size` (the largest chunk the chunker can emit), otherwise a single-chunk blob could exceed the limit. A backup needs free temporary space, because each blob is written whole to a temporary file before it is uploaded (up to about `blob_size_limit`; an rclone destination that cannot stream uploads needs about twice that) and the metadata export writes copies of the local index. Temporary files go to `$TMPDIR` (default `/tmp`); with `TMPDIR` unset, SQLite writes one of those copies to `/var/tmp` | | `blob_size_limit` | `10GB` | Maximum blob size before splitting. Must be at least four times `chunk_size` (the largest chunk the chunker can emit), otherwise a single-chunk blob could exceed the limit |
| `compression_level` | `3` | zstd compression level (1-19) | | `compression_level` | `3` | zstd compression level (1-19) |
| `hostname` | system hostname | Hostname used in snapshot IDs | | `hostname` | system hostname | Hostname used in snapshot IDs |
| `index_path` | platform data dir | Local SQLite index path | | `index_path` | platform data dir | Local SQLite index path |
@@ -664,8 +645,8 @@ Work planned after 1.0. Loosely ordered by priority.
## output style ## output style
Every command's user-facing output is governed by `internal/ui`, in one Every command's user-facing output is governed by `internal/ui`, in one
of two ways. Color is enabled when the stream written to is a TTY and of two ways. Color is enabled when stdout is a TTY and the `NO_COLOR`
the `NO_COLOR` environment variable is unset (https://no-color.org/). environment variable is unset (https://no-color.org/).
* **Status, progress, warnings, and errors** go through the `internal/ui` * **Status, progress, warnings, and errors** go through the `internal/ui`
message methods below: marker-prefixed, colored on a TTY, and — except message methods below: marker-prefixed, colored on a TTY, and — except
@@ -675,26 +656,24 @@ the `NO_COLOR` environment variable is unset (https://no-color.org/).
`config init`, `config set`, and `database delete`. `config init`, `config set`, and `database delete`.
* **The data a command exists to produce** is written plain, with no * **The data a command exists to produce** is written plain, with no
marker and no color, because a marker would corrupt a table or a marker and no color, because a marker would corrupt a table or a
parsed document. This covers the `version`, `info`, `remote info` and parsed document. This covers the `version`, `info`, and `remote info`
`snapshot verify` reports, the `snapshot list` table, `config get` reports, the `snapshot list` table, `config get` values, and every
values, and every `--json` document. `--quiet` silences the human `--json` document. `--quiet` silences the human reports and tables
reports and tables (`version`, `info`, `remote info`, `snapshot (`version`, `info`, `remote info`, `snapshot list`) but never the
verify`, `snapshot list`) but never the machine-consumed `config get` machine-consumed `config get` value or the `--json` documents, which a
value or the `--json` documents, which a script depends on. The script depends on. The `database delete` confirmation prompt is also
`database delete` confirmation prompt is also written this way and written this way and always shown: it is an interactive exchange the
always shown: it is an interactive exchange the operator must see. operator must see.
`internal/ui` writes to stdout; it is the output the user asked for. The `internal/ui` writes to stdout; it is the output the user asked for.
exceptions are the startup banner and the error a failed command ends Structured log records are a different thing and go through
with, which go to stderr. Structured log records are a different thing `internal/log`, which writes to stderr (see "stdout and stderr" above).
and go through `internal/log`, which writes to stderr (see "stdout and
stderr" above).
Message classes: Message classes:
| Class | Marker | Alignment | Use for | | Class | Marker | Alignment | Use for |
|-------|--------|-----------|---------| |-------|--------|-----------|---------|
| Banner | none | column 0 | The startup line printed once per invocation, on stderr | | Banner | none | column 0 | The startup line printed once per invocation |
| Begin | `》` (white) | column 0 | An operation is about to start (present-continuous verb) | | Begin | `》` (white) | column 0 | An operation is about to start (present-continuous verb) |
| Complete | `》` (green) | column 0 | An operation just finished (past-tense verb) | | Complete | `》` (green) | column 0 | An operation just finished (past-tense verb) |
| Info | `》` (white) | column 0 | Neutral status update | | Info | `》` (white) | column 0 | Neutral status update |
@@ -749,11 +728,12 @@ regardless of color setting (emoji are not color).
## requirements ## requirements
* Go 1.26 or later * Go 1.26 or later
* Docker, with a reachable daemon, to test, lint, check, or commit: * Docker, with a reachable daemon, to lint, check, or commit:
`script/test` and `script/lint` build the `test` and `lint` phases of `script/lint` lints by building `Dockerfile.lint`, which runs the
the `Dockerfile`, and `make check` and the pre-commit hook both run digest-pinned `golangci-lint` image as a build step, and `make check`
them. A `golangci-lint` installed on `PATH` is not a substitute and is and the pre-commit hook both run it. A `golangci-lint` installed on
never used on a host, whatever its version. `PATH` is not a substitute and is never used on a host, whatever its
version.
* S3-compatible object storage (or local filesystem, or rclone remote) * S3-compatible object storage (or local filesystem, or rclone remote)
## development workflow ## development workflow
@@ -784,20 +764,17 @@ standard: normalized scripts in `script/` are the entrypoints for the
development workflow, and the Makefile targets are thin shims that call development workflow, and the Makefile targets are thin shims that call
them. We provide: them. We provide:
* `script/bootstrap` — install all development dependencies (Go, Go * `script/bootstrap` — install all development dependencies (go, Go
module download). A host without Go gets the `go.mod` version through module download). It deliberately does not install `golangci-lint`;
`script/install-go`, in `.tool/go`, which `script/bootstrap` itself, see `script/lint` below.
the `Makefile`, `script/fmt`, `script/fmt-check`, `script/precommit`
and `script/release` add to their `PATH`. It
deliberately does not install `golangci-lint`; see `script/lint`
below.
* `script/setup` — make a fresh clone ready for development: runs * `script/setup` — make a fresh clone ready for development: runs
`script/bootstrap`, then `script/install-precommit` `script/bootstrap`, then `script/install-precommit`
* `script/projectname` — print the project name (used for the Docker * `script/projectname` — print the project name (used for the Docker
image tag) image tag)
* `script/version` — print the version string to bake into the binary. * `script/version` — print the version string to bake into the binary.
The `Makefile`'s `LDFLAGS` call this. See [releasing](#releasing) for The `Makefile`'s `LDFLAGS` call this, and `script/docker` and
the rules. `script/cibuild` pass its output to the image build. See
[releasing](#releasing) for the rules.
* `script/install-goreleaser` — install the pinned `goreleaser` into * `script/install-goreleaser` — install the pinned `goreleaser` into
`.tool/bin` from a sha256-verified release archive. Idempotent, and `.tool/bin` from a sha256-verified release archive. Idempotent, and
called by `script/bootstrap`; the release workflow calls it directly called by `script/bootstrap`; the release workflow calls it directly
@@ -805,60 +782,102 @@ them. We provide:
`script/bootstrap` insists on. `script/bootstrap` insists on.
* `script/install-go` — install the Go toolchain named by `go.mod`'s * `script/install-go` — install the Go toolchain named by `go.mod`'s
`go` directive into `.tool/go` from a sha256-verified `go.dev` `go` directive into `.tool/go` from a sha256-verified `go.dev`
archive, for Linux or macOS on amd64 or arm64. Idempotent. On a CI archive, and put it on `PATH`. Idempotent. Called only by the release
runner it also puts `.tool/go/bin` on `PATH` for the steps that workflow, which needs a host Go for `goreleaser` to shell out to;
follow. Called by the release workflow, which needs a host Go for nothing else on the release runner does. `actions/setup-go` is not
`goreleaser` to shell out to, and by `script/bootstrap` on a host used because it verifies the downloaded toolchain against no value in
without Go. `actions/setup-go` is not used because it verifies the this repo. Bumping Go edits `go.mod`, the checksum in this script, and
downloaded toolchain against no value in this repo. Bumping Go edits the `Dockerfile` `golang` digest together.
`go.mod`, the checksums in this script, and the two `golang` digests
in the `Dockerfile` together.
* `script/release` — cross-compile and publish the release artifacts * `script/release` — cross-compile and publish the release artifacts
with the pinned `goreleaser`. Refuses a `goreleaser` on `PATH` whose with the pinned `goreleaser`. Refuses a `goreleaser` on `PATH` whose
version is not the pinned one, because a different version would build version is not the pinned one, on the same reasoning as `script/lint`.
a different release from the same tag.
* `script/release-snapshot` — the same build with no publishing and no * `script/release-snapshot` — the same build with no publishing and no
tagging, into `./dist` tagging, into `./dist`
* `script/test` — run the test suite by building the `test` phase of * `script/test` — run the test suite (verbose rerun on failure). This
the `Dockerfile` (verbose rerun on failure). This runs *everything*: runs *everything*: there is no separate integration target and no
there is no separate integration target and no build-tagged subset build-tagged subset held back, so the full round-trip tests in
held back, so the full round-trip tests in `internal/vaultik/integration_test.go` run on every invocation. It
`internal/vaultik/integration_test.go` run on every invocation. The passes `-count=1`, which disables Go's test result cache. That is
90s `-timeout` is a hang backstop rather than a performance budget; it deliberate and it is not free: on this repo's suite it costs about 11
applies per test binary, to test execution only. seconds on every repeat run (measured, back to back: 0.4s cached
* `script/lint` — lint by building the `lint` phase of the versus 11.6s with `-count=1`). That is the price of the run meaning
`Dockerfile`, which runs `golangci-lint config verify` and then anything, because without it an unchanged package prints
`golangci-lint run --config .golangci.yml ./...` as build steps in `ok <pkg> (cached)`, which is indistinguishable from a package that
the digest-pinned `golangci-lint` image. Nothing lints on the host. really ran, so the whole suite can report a full set of `ok` lines in
That `FROM` line is the only pin of the linter version, and it changes under half a second having executed nothing. The `-timeout` is a hang
in the same commit as a re-vendored `.golangci.yml`. backstop rather than a performance budget — it applies per test binary
* `script/lint-fix` — apply the linter's autofixes (rewrites files), to test execution only, not to compilation — and is set well above the
using the same pinned image, parsed out of the `lint` phase's `FROM` slowest package's measured runtime. Its 120s value deliberately
line. It cannot be a build step, because fixes have to land in the diverges from the 30s `REPO_POLICIES.md` mandates; the reasoning is in
worktree, so it bind-mounts the tree into a `docker run` and therefore the comment in the script, and issue #101 proposes amending the policy
needs a *local* daemon. It is a developer convenience and never a text.
gate: no gate reads its exit status. Run `make lint` afterwards to * `script/lint` — lint by building `Dockerfile.lint`, which runs
find out whether the tree is clean. `golangci-lint run --config .golangci.yml ./...` as a build step
* `script/fmt` — format all code (writes) inside the digest-pinned `golangci-lint` image, so a successful build
* `script/fmt-check` — check formatting (read-only). It runs `gofmt` on *is* a clean lint. Nothing lints on the host, at any version, ever;
the host, over every Go file outside `.tool`. the script requires Docker and fails loudly rather than falling back
* `script/check` — run `script/test`, `script/lint`, and to a `golangci-lint` on `PATH`. That `FROM` line is the single source
`script/fmt-check`. of truth for the linter version — bump it there and nowhere else.
* `script/docker` — build the Docker image tagged via
`script/projectname`, stamped with the version
`git describe --tags --always --dirty` gives on the host, or `unknown`
outside a git checkout. The image's build stage depends on the `lint`
and `test` phases, so this lints and tests too.
* `script/cibuild` — CI entrypoint: runs `script/bootstrap`,
`script/check`, and then the same image build as `script/docker`.
`.gitea/workflows/check.yml` runs it on every push.
Every `docker build` in these scripts passes `--no-cache`, because on It takes no arguments, because a build step has no command line to
an unchanged tree a cached check layer is replayed without running and pass flags to, and it passes a fresh `--build-arg CHECK_EPOCH` on
the build still exits 0. A plain `docker build .` is therefore no every invocation so the lint layer cannot be replayed from cache (see
evidence that the checks ran. The cost is that `script/cibuild` runs `script/cibuild` below for what that mechanism defends against). To
the `lint` and `test` phases twice: once in `script/check` and again watch the linter execute, run it as
in the image build. `BUILDKIT_PROGRESS=plain script/lint` and check that the lint layer
says `RUN … golangci-lint` rather than `CACHED`.
One container per run means one lint cache and one `golangci-lint`
lock per run, both private to it and discarded with it, so concurrent
runs on one host cannot contaminate or block each other.
* `script/lint-fix` — apply the linter's autofixes (rewrites files),
using the same pinned image, parsed out of `Dockerfile.lint`. It
cannot be a build step, because fixes have to land in the worktree, so
it bind-mounts the tree into a `docker run` and therefore needs a
*local* daemon. It is a developer convenience and never a gate: no
gate reads its exit status. Run `make lint` afterwards to find out
whether the tree is clean.
* `script/fmt` — format all code (writes)
* `script/fmt-check` — check formatting (read-only)
* `script/check` — run `script/test`, `script/lint`, and
`script/fmt-check`. This is authoritative *because* `script/lint`
builds `Dockerfile.lint`: a local `make check` and CI cannot disagree
about lint findings.
* `script/docker` — build the Docker image tagged via
`script/projectname`. Passes a fresh `--build-arg CHECK_EPOCH` for the
same reason `script/cibuild` does, so a local image build cannot be
green on checks it replayed from cache. It builds the *product* image
only, and the product `Dockerfile` has no lint stage, so it does not
lint: a green here means formatted, tested, and it compiles.
* `script/cibuild` — CI entrypoint, and the full gate. Two builds, in
order: `Dockerfile.lint` (the linter, as a build step) and then
`Dockerfile` (`make fmt-check` and `make test` in its builder stage,
then the product image). Either failing fails the script. It runs the
checks in the same containers CI does, from a clean copy of the tree,
so it also catches anything that depends on host state.
`.gitea/workflows/check.yml` runs it on every push to `main` and
`next` and on every pull request against either.
It passes a fresh `--build-arg CHECK_EPOCH` to each build, unique per
invocation, which both files declare immediately above their check
`RUN`s and expand into each check command. Those layers are keyed on
that value, so a new value re-runs them even on a byte-identical tree,
and a green from this script means the checks executed. Dependency and
module layers sit above the `ARG` and still cache, so a build is not
cold.
A `docker build -f Dockerfile.lint .` that supplies no `CHECK_EPOCH`
fails rather than lying. An unset `ARG` is an empty string and an
empty string is a stable cache key, so without a guard such a build
would serve the lint layer from cache, execute nothing, and still exit
0. `Dockerfile.lint` therefore asserts the value is non-empty before
running anything, and because failed steps are never cached that
assertion fires on every invocation rather than once. The product
`Dockerfile` has no such guard, because a plain `docker build .` must
succeed: without `CHECK_EPOCH`, rebuilding an unchanged checkout
replays its check layers from cache. Use `script/lint`,
`script/docker` or `script/cibuild`, which pass the arg, when the
checks must run.
* `script/precommit` — pre-commit gate: `go mod tidy` + `go fmt` (must * `script/precommit` — pre-commit gate: `go mod tidy` + `go fmt` (must
not change files), then `script/check` not change files), then `script/check`
* `script/install-precommit` — install the git pre-commit hook that * `script/install-precommit` — install the git pre-commit hook that
@@ -870,8 +889,8 @@ them. We provide:
The version a binary reports comes from git, not from a constant in a The version a binary reports comes from git, not from a constant in a
file. It is `git describe --tags --always --dirty`, which file. It is `git describe --tags --always --dirty`, which
`script/version` runs for the `Makefile`, and `script/docker` and `script/version` runs for the `Makefile`, `script/docker` and
`script/cibuild` run themselves: `script/cibuild`:
* `HEAD` is exactly on a tag → that tag, such as `v1.0.0`. * `HEAD` is exactly on a tag → that tag, such as `v1.0.0`.
* a commit after a tag → `<tag>-<N>-g<short sha>`. * a commit after a tag → `<tag>-<N>-g<short sha>`.
@@ -882,17 +901,15 @@ file. It is `git describe --tags --always --dirty`, which
A `docker build .` of a clone, with no build arguments, runs the same A `docker build .` of a clone, with no build arguments, runs the same
`git describe` (without `--dirty`) on the `.git` in its build context, `git describe` (without `--dirty`) on the `.git` in its build context,
so it stamps the same value for a clean commit; the build fails if the so it stamps the same value for a clean commit; the build fails if the
context carries `.git` and no version, commit or commit date comes out. context carries `.git` and no version comes out. A binary built without
A binary built without git metadata reports `dev`.
git metadata reports `dev`, or `unknown` when `script/docker` or
`script/cibuild` built it outside a git checkout.
`goreleaser` stamps a release binary with the tag minus its leading `goreleaser` stamps a release binary with the tag minus its leading
`v`, so the tag `v1.0.0` produces `vaultik 1.0.0`, matching the archive `v`, so the tag `v1.0.0` produces `vaultik 1.0.0`, matching the archive
name `vaultik_1.0.0_linux_amd64.tar.gz`. `goreleaser --snapshot` stamps name `vaultik_1.0.0_linux_amd64.tar.gz`. `goreleaser --snapshot` stamps
`dev-<12 chars of the commit sha>` rather than inventing the next patch `dev-<12 chars of the commit sha>` rather than inventing the next patch
number. `vaultik version` calls a build a development build when its number. `vaultik version` calls a build a development build when its
version is `dev`, `unknown`, `dev-<sha>`, the short commit sha or version is `dev`, `dev-<sha>`, the short commit sha or
`<tag>-<N>-g<short sha>`, with or without `-dirty`; only a plain tag, `<tag>-<N>-g<short sha>`, with or without `-dirty`; only a plain tag,
such as `v1.0.0` or `1.0.0`, is a release. If `script/version` cannot such as `v1.0.0` or `1.0.0`, is a release. If `script/version` cannot
be run at all, `make` stops with an error instead of building an be run at all, `make` stops with an error instead of building an
@@ -926,14 +943,14 @@ It is passed to `goreleaser` as `GITEA_TOKEN`. The runner's automatic
token is deliberately not used: it is not guaranteed to carry release token is deliberately not used: it is not guaranteed to carry release
write access. write access.
The Go toolchain that compiles the released binaries is installed by The Go toolchain that compiles the released binaries comes from an
`script/install-go`, which downloads the version named by `go.mod` `actions/setup-go` step pinned by commit sha, reading its version from
(currently `1.26.1`, the same version the `Dockerfile` builder stage `go.mod` (currently `1.26.1`, the same version the `Dockerfile` builder
pins by digest) and refuses the archive unless its sha256 matches the stage pins by digest). `goreleaser` shells out to `go` for every
value committed in the script. `goreleaser` shells out to `go` for every
cross-compile, so without that step the release would either fail cross-compile, so without that step the release would either fail
outright or ship binaries built by whatever unpinned toolchain the outright or ship binaries built by whatever unpinned toolchain the
runner happened to carry. runner happened to carry — the one unpinned thing in an otherwise
hash-pinned release path.
To rehearse the whole build without publishing or tagging anything: To rehearse the whole build without publishing or tagging anything:
+84 -355
View File
@@ -1,6 +1,6 @@
--- ---
title: Repository Policies title: Repository Policies
last_modified: 2026-10-04 last_modified: 2026-07-06
--- ---
This document covers repository structure, tooling, and workflow standards. Code This document covers repository structure, tooling, and workflow standards. Code
@@ -60,28 +60,17 @@ style conventions are in separate documents:
prerequisite since nvm requires bash. yarn is then pinned via prerequisite since nvm requires bash. yarn is then pinned via
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts"; `corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
always exact versions. `script/cibuild` runs the CI build: it changes to the always exact versions. `script/cibuild` runs the CI build: it changes to the
repo root, runs `script/bootstrap`, runs `script/check`, and builds the image repo root and runs `docker build .`; the Gitea workflow calls it. Four further
with the version; the Gitea workflow calls it. **`script/cibuild` runs scripts are our own extensions to the standard: `script/check` runs
`script/bootstrap` first**, because the workflow checks out the repo and runs `script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
nothing else, while `script/fmt-check` runs the formatter on the host: on a what the git pre-commit hook runs, and it calls `script/check`;
pristine checkout with nothing installed the run dies there, after the `script/install-precommit` installs the git pre-commit hook (the `make hooks`
containerised gates have passed. **The bootstrap alone is not enough**: target shims to it); and `script/projectname` (literally that filename) simply
`script/bootstrap` installs node and yarn under nvm and leaves neither on the outputs the project's name. Scripts that need the name call
`PATH` of the shell that called it, so a bare `yarn` still exits 127. The host `script/projectname` — e.g. `script/docker` assembles its image tag from it —
entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore so those scripts stay byte-identical across all repos. Repo-type-specific
source nvm for the pinned node version before invoking it, exactly as pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
`script/bootstrap`'s own install step does. A runner carrying nothing but `script/precommit`, not in the hook itself. Model scripts are at
docker and git then gets through `script/check`. Four further scripts are our
own extensions to the standard: `script/check` runs `script/test`,
`script/lint` and `script/fmt-check`; `script/precommit` is what the git
pre-commit hook runs, and it calls `script/check`; `script/install-precommit`
installs the git pre-commit hook (the `make hooks` target shims to it); and
`script/projectname` (literally that filename) simply outputs the project's
name. Scripts that need the name call `script/projectname` — e.g.
`script/docker` assembles its image tag from it — so those scripts stay
byte-identical across all repos. Repo-type-specific pre-commit extras (e.g.
`go mod tidy` verification in Go repos) belong in `script/precommit`, not in
the hook itself. Model scripts are at
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
must document the provided scripts in an **Entrypoints** section (see the must document the provided scripts in an **Entrypoints** section (see the
README requirements below). README requirements below).
@@ -100,198 +89,87 @@ style conventions are in separate documents:
contributor should be able to understand the entire development workflow by contributor should be able to understand the entire development workflow by
reading the Makefile. reading the Makefile.
- Every repo should have a `Dockerfile`, and it carries the repo's gates: a - Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
`lint` phase and a `test` phase, with the final stage depending on both so the as a build step so the build fails if the branch is not green. For non-server
image cannot be built unless they pass. For non-server repos the final stage repos, the Dockerfile should bring up a development environment and run
brings up a development environment; for server repos it is the runtime image. `make check`. For server repos, `make check` should run as an early build
The gate phases and the build stage start from their pinned base images and stage before the final image is assembled. Dockerfiles install development
install what those images lack either inline, as the canonical Go `Dockerfile` prerequisites by running `script/bootstrap` rather than duplicating installs
below does for `git`, or by running `script/bootstrap`, as the `prompts` inline; COPY `script/` and the dependency manifests (`package.json` +
repo's own `Dockerfile` does for its yarn packages. The development `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
environment stage installs development prerequisites by running layer stays cached until dependencies change.
`script/bootstrap` rather than duplicating its installs inline. A stage that
runs `script/bootstrap` COPYs `script/` and the dependency manifests
(`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it.
- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is - **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
no separate lint file. `script/lint` and `script/test` each build one phase repos use a multistage build where linting runs in an independent stage based
and nothing else: on the `golangci/golangci-lint` image (pinned by hash). This stage runs
`make fmt-check` and `make lint` before the full build begins. The build stage
then declares an explicit dependency on the lint stage via
`COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
linting before proceeding to compilation and tests. This ensures lint failures
surface in seconds rather than minutes, without blocking on dependency
download or compilation in the build stage.
```sh The standard pattern for a Go repo Dockerfile is:
docker build --no-cache --target lint -t "$(script/projectname)-lint" .
docker build --no-cache --target test -t "$(script/projectname)-test" .
```
**A stage that is not the last one in the file is built only when the final
stage's chain depends on it, or when `--target` names it.** That is why the
two gates are always invoked by name here, and why the final stage carries a
`COPY --from=` of a harmless file from each of them: without that edge a
plain `docker build .` builds the last stage alone and exits 0 having linted
and tested nothing.
**Every `docker build` in `script/` is tagged**, here and in
`script/cibuild` and `script/docker`. An untagged build leaves a dangling
image behind on every invocation, on every developer host and every CI
runner; a tagged one replaces the previous image.
Inside a phase the tool is invoked directly — `golangci-lint`, `go test`,
`eslint`, `prettier` — never through `make lint` or `script/test`, which are
themselves a `docker build` and would recurse into a daemon that does not
exist in a build step. Formatting is the exception and stays on the host:
`script/fmt` writes the working tree, and `script/fmt-check` is its
read-only twin.
**No lint verdict may come from a host invocation of the linter.** On a
shared host golangci-lint reads a result cache keyed on file content rather
than location, so a second checkout of the same content is served the first
one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs
exit non-zero with `parallel golangci-lint is running` — a status a caller
cannot tell from real findings. Both have produced wrong verdicts in this
org, in both directions. A container has its own cache, its own `TMPDIR` and
a digest-pinned binary, so neither is reachable.
- **Any build that runs checks is built with `--no-cache`.** Docker invalidates
a `COPY` layer only when the copied content changes, so on an unchanged tree
the check `RUN` is served from cache, nothing executes, and the build still
exits 0. Every `docker build` in `script/` therefore passes `--no-cache`:
`script/lint`, `script/test`, `script/cibuild` and `script/docker` are the
four, and there is no fifth — `script/check` runs the two gate phases and
`script/fmt-check`, and builds no image of its own. A bare `docker build .` is
not evidence that anything ran: a sub-second build reporting success is a
cache hit, not a result. Never invalidate by pruning — `docker builder prune`
and friends destroy a build cache shared with every other build on the host.
When a check is added or changed, prove it works by planting a defect it must
catch and watching the run fail on it, then revert the defect. A green run
alone shows neither that the check ran nor that it covers what it should.
- **The gate phases are separate stages, and the build stage depends on both.**
The lint phase is based on the `golangci/golangci-lint` image (pinned by
hash), so lint failures surface in seconds rather than after a full compile,
and the test phase is based on the Debian Go image. The canonical Go repo
`Dockerfile`:
```dockerfile ```dockerfile
# Lint phase # Lint stage — fast feedback on formatting and lint issues
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD # golangci/golangci-lint:v2.x.x, YYYY-MM-DD
FROM golangci/golangci-lint@sha256:... AS lint FROM golangci/golangci-lint@sha256:... AS lint
WORKDIR /src WORKDIR /src
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
COPY . . COPY . .
RUN golangci-lint run --config .golangci.yml ./... RUN make fmt-check
RUN make lint
# Test phase. -race needs cgo and so a C compiler, which the Debian Go # Build stage
# image ships and the alpine one does not.
# golang:1.x, YYYY-MM-DD
FROM golang@sha256:... AS test
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN go test -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
go test -timeout 90s -race -v ./...; exit 1; }
# Build stage. Nothing is wanted from either phase above; the copies
# are what make BuildKit build them first, so this stage cannot run
# unless lint and test passed.
# golang:1.x-alpine, YYYY-MM-DD # golang:1.x-alpine, YYYY-MM-DD
FROM golang@sha256:... AS builder FROM golang@sha256:... AS builder
COPY --from=lint /src/go.sum /dev/null
COPY --from=test /src/go.sum /dev/null
RUN apk add --no-cache git
# A tar-stream context keeps the sender's file owners, which git refuses.
RUN git config --system --add safe.directory /src
WORKDIR /src WORKDIR /src
# Force BuildKit to run the lint stage before proceeding
COPY --from=lint /src/go.sum /dev/null
COPY go.mod go.sum ./ COPY go.mod go.sum ./
RUN go mod download RUN go mod download
COPY . . COPY . .
RUN make test
# The VERSION build arg when one is given, otherwise ARG VERSION=dev
# `git describe --tags --always` on the .git in the build context. With RUN CGO_ENABLED=0 go build -trimpath \
# .git present, a version that is still empty, dev or unknown fails the -ldflags="-s -w -X main.Version=${VERSION}" \
# build: git is missing or could not read the checkout. -o /app ./cmd/app/
ARG VERSION
RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \
if [ -e .git ]; then \
case "$VERSION" in ""|dev|unknown) \
echo "version is '$VERSION' although .git is present" >&2; \
exit 1 ;; \
esac; \
fi; \
CGO_ENABLED=0 go build -trimpath \
-ldflags="-s -w -X main.Version=${VERSION}" \
-o /app ./cmd/app/
# Runtime stage, and the last one # Runtime stage
FROM alpine@sha256:... FROM alpine@sha256:...
COPY --from=builder /app /usr/local/bin/app COPY --from=builder /app /usr/local/bin/app
ENTRYPOINT ["app"] ENTRYPOINT ["app"]
``` ```
Key points: Key points:
- The lint phase uses the `golangci/golangci-lint` image directly (it has - The lint stage uses the `golangci/golangci-lint` image directly (it
both Go and the linter), so nothing needs installing. includes both Go and the linter), so there is no need to install the
- `COPY --from=<phase> /src/go.sum /dev/null` is a no-op copy whose only linter separately.
purpose is the ordering edge. BuildKit runs stages in parallel by default, - `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
and a stage nothing depends on is not built at all, so without these two a stage dependency. BuildKit runs stages in parallel by default; without
lines a red gate would not fail the build. this line, the build stage would not wait for lint to finish and a lint
- Keep the runtime stage last, and if you add a stage after it, give it the failure might not fail the overall build.
same two copies. A plain `docker build .` builds the last stage's chain
and nothing else.
- If the project uses `//go:embed` directives that reference build artifacts - If the project uses `//go:embed` directives that reference build artifacts
(e.g. a web frontend compiled in a separate stage), the lint phase must (e.g. a web frontend compiled in a separate stage), the lint stage must
create placeholder files so the embed directives resolve. Example: create placeholder files so the embed directives resolve. Example:
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
- If the project requires CGO or system libraries for linting, install them The lint stage should not depend on the actual build output — it exists to
in the lint phase. The `golangci/golangci-lint` image is Debian-based and fail fast.
has no `apk`, so install with `apt-get` under the Debian package name - If the project requires CGO or system libraries for linting (e.g.
(`libvips-dev`, where alpine says `vips-dev`), and delete the package `vips-dev`), install them in the lint stage with `apk add`.
lists in the same `RUN`, so the layer does not keep them: - The build stage runs `make test` after compilation setup. Tests run in the
build stage, not the lint stage, because they may require compiled
```dockerfile artifacts or heavier dependencies.
RUN apt-get update \
&& apt-get install -y --no-install-recommends libvips-dev \
&& rm -rf /var/lib/apt/lists/*
```
- `.dockerignore` lets `.git` into the build context. It keeps out every git
`config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the
repository's own, each submodule's under `.git/modules/`, and that of a
submodule keeping its own `.git` directory. `git describe` does not need
them, and each can hold a credential: a password in a remote URL, or the
token the CI checkout step stores there. A submodule whose name has a
`config` segment (`config`, `deploy/config`, `config/lib`) loses its whole
git directory to `**/.git/modules/**/config`, and Go's version stamping
then fails the build: give it a name without that segment
(`git submodule add --name`). The stage that compiles has `git` (the
Debian Go image has it; an alpine one needs `apk add --no-cache git`) and
takes the version from the `VERSION` build argument when one is given,
otherwise from `git describe --tags --always`. That gives the tag on a
tagged commit; on a later commit, the tag, the number of commits since it
and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no
tag is reachable. The stage that compiles also marks its working directory
safe for git (`git config --system --add safe.directory /src`): a context
sent as a tar stream keeps the sender's file owners, and git refuses a
checkout owned by another user, so the version would come out empty.
`ARG VERSION` has no default, and the build fails if the context carries
`.git` and the version still comes out empty, `dev` or `unknown`. A plain
`docker build .` with no build arguments must succeed; a Dockerfile that
refuses an empty build argument drops that refusal and keeps the argument.
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
runs `script/cibuild` on push, and checks out the repo as its only other step. runs `script/cibuild` (which runs `docker build .`) on push. Since the
That script bootstraps, runs the gate phases, and then builds the image, so a Dockerfile already runs `make check`, a successful build implies all checks
successful run means every check passed; a bare `docker build .` does not pass.
carry the same guarantee, because its gate phases may come from the cache. The
image build is uncached and so runs the gate phases a second time. That is the
price of the rule above, and it is worth paying: the image that ships is built
from a run of its own gates rather than from a cache entry. A separate
workflow limited to `main` by a `branches` list under `on: push` cannot be
checked by review: to try a change to it, add the feature branch to that list
and push, then remove the branch from the list again before merging. Keep any
job in it that publishes behind `if: github.ref_name == 'main'`, so the run
from the feature branch publishes nothing.
- Use platform-standard formatters: `black` for Python, `prettier` for - Use platform-standard formatters: `black` for Python, `prettier` for
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
@@ -311,21 +189,14 @@ style conventions are in separate documents:
module under test to verify it compiles/parses. There is no excuse for module under test to verify it compiles/parses. There is no excuse for
`make test` to be a no-op. `make test` to be a no-op.
- `make test` must complete in under 60 seconds. That is the hard cap, and a - `make test` must complete in under 20 seconds. Add a 30-second timeout in the
suite that exceeds it fails. Under 20 seconds is the target. A suite between Makefile.
20 and 60 seconds is still green, but the overage must be filed as an
improvement bug against that repo. Add a 90-second timeout to the test
invocation (`go test -timeout 90s`). The backstop deliberately sits above the
hard cap so that it catches a genuinely hung test rather than a merely slow
one.
- **The test command should use the conditional verbose rerun pattern.** Run - **`make test` should use the conditional verbose rerun pattern.** Run tests
tests without `-v` (verbose) first. If tests fail, automatically rerun with without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
`-v` to show full output. This keeps CI logs and `docker build` output clean show full output. This keeps CI logs and `docker build` output clean on
on success (just package/suite summaries) while providing full diagnostic success (just package/suite summaries) while providing full diagnostic detail
detail on failure (every test case, every assertion). The command lives in the on failure (every test case, every assertion). The general shell pattern:
`test` phase of the `Dockerfile`, since `script/test` builds that phase; the
Makefile form below is the same pattern for any repo-local invocation:
```makefile ```makefile
test: test:
@@ -338,26 +209,11 @@ style conventions are in separate documents:
```makefile ```makefile
test: test:
@go test -count=1 -timeout 90s -race -cover ./... || \ @go test -timeout 30s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \ { echo "--- Rerunning with -v for details ---"; \
go test -count=1 -timeout 90s -race -v ./...; exit 1; } go test -timeout 30s -race -v ./...; exit 1; }
``` ```
`-count=1` is required on both invocations: it defeats Go's test _result_
cache, so neither run can report a stored pass in place of running the
tests. It leaves the build cache alone, so it costs the runtime of the suite
and no recompilation.
That cache is Go's own, separate from Docker's layer cache. Go stores a
passing result in its cache directory (`GOCACHE`), and when the same tests
run again on unchanged code it prints that result, marked `(cached)`,
without running them. That matters on a developer's machine, where this
target runs and the directory lasts from one run to the next. The `test`
phase of the `Dockerfile` needs no `-count=1`: its base image holds no
result for this repo's tests and nothing before its `go test` step runs a
test, so there is nothing to replay. `--no-cache` (above) is what makes that
step run on an unchanged tree.
Python example: Python example:
```makefile ```makefile
@@ -383,84 +239,10 @@ style conventions are in separate documents:
must be in `.gitignore`. No exceptions. must be in `.gitignore`. No exceptions.
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), - `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
language build artifacts, and `node_modules/`. Fetch the standard `.gitignore` Fetch the standard `.gitignore` from
from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
setting up a new repo. These patterns are written to `.gitignore`'s own a new repo.
semantics, in which an unanchored pattern already matches at every depth; they
are not a `.dockerignore` and must not be transplanted into one unmodified.
- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns
across unmodified leaves secrets in the build context.** Docker matches with
`moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so
`*` does not cross `/` and a pattern without a leading `**/` is anchored at
the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key`
therefore excludes only the copies at the repository root, while `config/.env`
and `certs/server.key` still reach the context and can land in an image layer
— which is more dangerous than a short file with no secret patterns at all,
because it reads as solved and stops anyone looking. Give every
depth-independent pattern the `**/` prefix and leave only genuinely
root-anchored entries unprefixed: `.claude`, and the repo's own host-built
binary, written `/myapp` and never `**/myapp`, which would also match
`cmd/myapp/` and delete the package directory from the context. Matching is
case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so
secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`,
and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern
also catches something the build needs, re-include it with a negation
(`!docs/example.env`); deleting the pattern reopens the exposure for every
other file it covers. Fetch the standard `.dockerignore` from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend
it with the repo's own artifacts.
- **In-repo agent scratch belongs in both files, written to each file's own
semantics.** `.claude/` holds one worktree per in-flight agent — an entire
additional checkout of the repo — so under `COPY . .` the build context
inflates by a multiple of the repo and another session's unreviewed work can
be copied into an image layer. In `.gitignore` the entry is `.claude/`,
unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/`
prefix, because the prefixed form would also delete any nested directory of
that name from the build. Anchoring carries a known gap that the canonical
`.dockerignore` states in its own comment, since consuming repos receive the
file and not the tracker: the directory is created in the agent's working
directory, so a repo running agents in subdirectories still ships
`services/api/.claude/` and must add its own anchored entry there.
- **A plain `docker build .` of a clone stamps the version that
`git describe --tags --always` gives**, derived from the `.git` in the build
context as the canonical `Dockerfile` above shows. Without its failure check,
a missing `git` or an unreadable checkout would leave `-X main.Version=` empty
and the build would still exit 0. `script/docker` and `script/cibuild` pass
the version they compute on the host; it takes precedence. They do this
byte-identically across repos:
```sh
# Own line: a failing command substitution inside an argument does not
# trip `set -e`, so the inline form degrades to an empty constant.
version="$(git describe --tags --always --dirty 2>/dev/null || true)"
[ -n "$version" ] || version="unknown"
docker build --no-cache \
--build-arg VERSION="$version" \
-t "$(script/projectname)" .
```
`--always` makes an untagged repo yield an abbreviated commit hash rather
than failing, and the `[ -n "$version" ]` line is the single place the
fallback is applied — a live check that fires on a build from an export with
no `.git` and on a repository with no commits yet. Do not fold it into the
substitution as `|| echo unknown`, which makes the guard unreachable. The
Dockerfile's side is `ARG VERSION` in the stage that compiles, declared
there because `ARG` is stage-scoped; passing `VERSION` to a repo whose
Dockerfile declares no such `ARG` is ignored and costs nothing, which is why
the scripts stay byte-identical. One consequence for CI: the standard
checkout action clones shallow and fetches no tags, so a repo that embeds a
tag-derived version must set `fetch-depth: 0` on its checkout step.
- **Verify `.dockerignore` by enumerating the image, not by reading the
patterns.** Plant files at the root _and_ at least two directories deep, build
a probe image that does `COPY . .`, and list what actually landed
(`docker run --rm --entrypoint find IMAGE /app`). The `transferring context`
size is not a substitute: a nested secret is a few bytes, and BuildKit
transfers only the delta from the previous build.
- **No build artifacts in version control.** Code-derived data (compiled - **No build artifacts in version control.** Code-derived data (compiled
bundles, minified output, generated assets) must never be committed to the bundles, minified output, generated assets) must never be committed to the
@@ -476,56 +258,9 @@ style conventions are in separate documents:
- Make all changes on a feature branch. You can do whatever you want on a - Make all changes on a feature branch. You can do whatever you want on a
feature branch. feature branch.
- `.golangci.yml` is standardized. The vendored copy in a consuming repo must - `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
_NEVER_ be modified by an agent: fetch it from manually by the user. Fetch from
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
byte-identical, so that no repo can quietly loosen its own linting. Linter
configuration changes are made to the canonical copy in the `prompts` repo and
reach consuming repos by re-vendoring; an agent may open a PR against
canonical, which only the user merges. One list is exempt from byte-identity,
because it cannot be written once for every repo: the `deny` list of the
`test-support` depguard rule, where a repo names its own test-support packages
by full import path. A repo adds entries there and changes nothing else, and a
re-vendor carries its entries forward. The canonical golangci-lint version is
v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base
image
(`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`,
which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go`
directive must not name a newer Go minor version than the one golangci-lint
was built with, or golangci-lint refuses to lint it: this release lints
`go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo
installs golangci-lint on the host. A repo sets the lint phase digest to the
one named here and re-vendors `.golangci.yml` in the same commit, whichever of
the two prompted the change: the canonical copy can name linters that an older
golangci-lint rejects, and a newer golangci-lint can add linters that
`default: all` switches on until the canonical copy disables them.
- **`script/bootstrap` installs a pinned tool by comparing versions, never by
testing presence.** An `if ! command -v <tool>; then install; fi` guard tests
`PATH` only, so on an already-provisioned machine the pin is inert and a
version bump is a silent no-op — while the Dockerfile, installing into a clean
image, gets the pinned version, so a local `make check` and `make docker` can
disagree about what the tool even is. The canonical form:
- compares the installed version against the pin over the **whole** version
token; a parser that stops at the first `-` reports `2.12.2` for a host
running `2.12.2-rc1` and skips the install;
- treats absent, non-zero, empty or unrecognised `--version` output as a
mismatch, so the failure direction is a redundant install and never a
skipped one;
- after installing, re-resolves the binary the way callers do — `hash -r`,
then through `PATH`, not through the directory the installer wrote to —
and fails naming the resolved path, since an install that a shadowing
binary hides succeeds while changing nothing any caller sees;
- is actually called, and prints the version on both success paths: a
function defined and never invoked has the same exit status and the same
empty output as one that worked.
Keep it POSIX sh: no arrays, no `[[`, no `grep -P`.
A Go tool a repo needs on the host is installed with `go install` pinned to
a commit hash (`go install <package>@<commit hash>`). It is never tracked as
a `go.mod` tool dependency or through a `tools.go` file, either of which
pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`.
- When pinning images or packages by hash, add a comment above the reference - When pinning images or packages by hash, add a comment above the reference
with the version and date (YYYY-MM-DD). with the version and date (YYYY-MM-DD).
@@ -639,14 +374,12 @@ style conventions are in separate documents:
settings. settings.
- Avoid putting files in the repo root unless necessary. Root should contain - Avoid putting files in the repo root unless necessary. Root should contain
only project-level config files (`README.md`, `AGENTS.md`, `Makefile`, only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
`Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
and language-specific config). Everything else goes in a subdirectory. language-specific config). Everything else goes in a subdirectory. Canonical
Canonical subdirectory names: subdirectory names:
- `bin/` — executable scripts and tools - `bin/` — executable scripts and tools
- `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose - `cmd/` — Go command entrypoints
body is a single call into `internal/` or `pkg/`, no project logic in
`cmd/`
- `configs/` — configuration templates and examples - `configs/` — configuration templates and examples
- `deploy/` — deployment manifests (k8s, compose, terraform) - `deploy/` — deployment manifests (k8s, compose, terraform)
- `docs/` — documentation and markdown (README.md stays in root) - `docs/` — documentation and markdown (README.md stays in root)
@@ -673,7 +406,3 @@ style conventions are in separate documents:
- Go: `go.mod`, `go.sum`, `.golangci.yml` - Go: `go.mod`, `go.sum`, `.golangci.yml`
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore` - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
- Python: `pyproject.toml` - Python: `pyproject.toml`
- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It
is never committed under a file or directory named after one agent tool, such
as `CLAUDE.md` or `.claude/`, and never split into separate memory files.
+2 -271
View File
@@ -22,283 +22,14 @@ the tag exists and is exercised; what is left is merging `next` to
# Completed Steps # Completed Steps
- 2026-10-07: Made a symlink whose target cannot be read stop the backup
([issue #269](https://git.eeqj.de/sneak/vaultik/issues/269)). It was
left out of the snapshot with only a debug log line, even without
`--skip-errors`. It now aborts the run, or with `--skip-errors` is
skipped with the `Failed to access` error line that any other entry
the scan cannot read gets.
- 2026-10-07: Stopped a killed rclone upload from leaving a truncated
object under its key
([issue #266](https://git.eeqj.de/sneak/vaultik/issues/266)). The
rclone backend wrote each object straight to its key, so killing an
upload to a local or sftp remote left a truncated object there; the
next backup found the key with `Stat`, skipped the upload and recorded
a snapshot that could not be restored. On a remote with a server-side
move, an object is now written under a temporary name ending in
`.partial` and moved into place, and listings skip such names. Remotes
without one are still written in place.
- 2026-10-07: Made an interrupted command exit 130 and say so
([issue #267](https://git.eeqj.de/sneak/vaultik/issues/267)). Ctrl-C
or SIGTERM during `snapshot create`, `snapshot restore` or `snapshot
verify` exited 0 with no error line (`snapshot verify --json` exited
1, also without one), so a `--cron` run that never finished looked
like a success. A command stopped by either signal now exits 130 and
prints `interrupted before the command finished` on stderr, under
`--cron` and `--json` too.
- 2026-10-07: Corrected documentation, help text and comments that were
false about the code
([issue #233](https://git.eeqj.de/sneak/vaultik/issues/233)). A blob
is not streamed to storage. The README, `ARCHITECTURE.md` and
`config.example.yml` now say a backup needs free temporary space,
because each blob is written whole to a temporary file before it is
uploaded (up to about `blob_size_limit`; an rclone destination that
cannot stream uploads needs about twice that) and the metadata export
writes copies of the local index. Temporary files go to `$TMPDIR`
(default `/tmp`); with `TMPDIR` unset, SQLite writes one of those
copies to `/var/tmp`. Also corrected: the snapshot ID format, what restore
reads and how incomplete snapshots are removed in `docs/DATAMODEL.md`,
what `source_path` holds, the `index_path` and config file defaults,
what `snapshot remove` cleans up, how the release gets its Go
toolchain, and the `script/release` and `script/fmt-check` comments.
- 2026-10-07: Cut the time the `internal/vaultik` and `internal/database`
tests take ([issue #235](https://git.eeqj.de/sneak/vaultik/issues/235)).
Most of the `internal/vaultik` time went to 24 tests that ran one at a
time only because they call `log.Initialize`; they now call it before
`t.Parallel()`, as the package's other tests do. `TestLargeDatasets`
committed each of its 1,500 inserts on its own and now makes them in
one transaction. `TestDedupOnlySnapshotRestores` gives its second
backup its own snapshot name instead of sleeping past the one-second
timestamp in the snapshot ID.
- 2026-10-07: Made two messages say only what is true
([issue #240](https://git.eeqj.de/sneak/vaultik/issues/240)). A config
file that others can read was warned about as containing S3
credentials even when it set none, as a `file://` config does. The
warning now says the file may contain S3 credentials only when
`s3.access_key_id` or `s3.secret_access_key` is set, since either may
come from a `${...}` reference rather than the file, and otherwise
says the file is readable by others. `snapshot purge` against a
destination store it could not list gave an error with
`listing remote snapshots:` in it twice; the prefix now appears once.
- 2026-10-07: Made `s3.part_size` set the multipart upload part size
([issue #232](https://git.eeqj.de/sneak/vaultik/issues/232)). It was
loaded and defaulted but never passed to the S3 client, whose uploader
used a fixed 10MiB part. It now reaches the uploader for `storage_url`
and for the `s3.*` fields, and a part size S3 refuses, below 5MiB or
above 5GiB, `0` included, fails at config load. A blob too large for
S3's limit of 10,000 parts at the configured size is uploaded in larger
parts. The docs gave the default as `5MB`, which the config file reads
as 5,000,000 bytes, below the minimum; they now say `5MiB`.
- 2026-10-07: Made per-name retention work when the hostname contains `_`
([issue #230](https://git.eeqj.de/sneak/vaultik/issues/230)). A
snapshot ID is `hostname_name_timestamp`, and the name was read as
everything between the first and the last `_`, so with
`hostname: my_host` the name `home` came out as `host_home`.
`snapshot purge --keep-latest --snapshot home` then printed "No
snapshots to delete", and `snapshot create --prune` purged nothing
without a message. The name is now read using the hostname the
`snapshots` table stores with each snapshot, cut at its first `.` as it
is in the ID.
- 2026-10-07: Made `remote info` stop reporting a snapshot's blobs as
orphaned when its manifest cannot be read, and stop printing raw
names from under `metadata/`
([issue #228](https://git.eeqj.de/sneak/vaultik/issues/228)). A
manifest it failed to read was skipped, so that snapshot's blobs were
counted as orphaned and the report advised running `vaultik prune`.
The orphan figures are now unknown in that case, with no prune
advice, and `--json` gives them as `null` with the unreadable remote
keys in `unreadable_manifests`. A name under `metadata/` that is not
64 lowercase hex characters is now skipped with a warning instead of
being printed, control characters included. A manifest under a
skipped name is then not read either, so it also leaves the orphan
figures unknown, and `--json` counts such manifests in
`skipped_manifest_count`. A directory with no manifest in it, as left
by an interrupted backup, leaves the figures known.
- 2026-10-07: Made `config set` keep a string that looks like a number
([issue #229](https://git.eeqj.de/sneak/vaultik/issues/229)). It wrote
every value unquoted, and `config.Load` reads the file through untyped
YAML, so an access key `00112233` loaded as `38043` and a hostname `007`
as `7`. A value for a string setting in `config.Config` is now tagged as
a YAML string, which the file quotes wherever YAML would read a number or
a boolean; other settings are still written unquoted.
- 2026-10-07: Made a backup notice a file rewritten with its size
unchanged and a new mtime in the same second as the one in the index
([issue #226](https://git.eeqj.de/sneak/vaultik/issues/226)). The
`files` table held mtime in whole seconds and the scanner compared
whole seconds, so every later snapshot kept the old content. A new
`mtime_nsec` column holds the nanoseconds within the second that
`mtime` holds, and the scanner compares the full mtime. A local index
created before the change lacks the column and is rebuilt with
`vaultik database delete` and a full backup.
- 2026-10-07: Made taking the process-wide lock atomic
([issue #227](https://git.eeqj.de/sneak/vaultik/issues/227)). The lock
read `vaultik.pid`, checked whether that PID was alive and then wrote
its own, so two writers started together could both pass the check and
both run. It is now an `flock` on `vaultik.pid`, held until the run
ends; the kernel drops it when the process exits, so a crash leaves no
lock behind. A clean exit now empties the file instead of deleting it,
because deleting it would let two later runs each lock a different
file.
- 2026-10-06: Made `snapshot remove --json` write only its document to
stdout when the destination store cannot be reached
([issue #251](https://git.eeqj.de/sneak/vaultik/issues/251)). Its
warning that the snapshot's metadata was left on the destination store
went to stdout ahead of the document, breaking `| jq` on a command that
exited 0. Under `--json` the warning now reaches stderr only, through
the logger. The warning, the README and the command's help said
`vaultik prune` would finish the cleanup, but `prune` never removes
snapshot metadata; they now say to run `vaultik snapshot remove` for the
snapshot again once the destination store is reachable.
- 2026-10-06: Made the backup summary and the `snapshots` row count each
file, byte and upload once
([issue #225](https://git.eeqj.de/sneak/vaultik/issues/225)). The
scanner added a file's bytes again for each new chunk and counted a
file as unchanged for each chunk already stored, so a first backup
reported twice its size and "backed up" could go negative. Upload
figures came from the progress reporter, which `--cron` turns off, and
`blob_count` counted earlier paths' blobs again for each later path.
The scanner now counts uploads itself; `blob_size`,
`blob_uncompressed_size` and `compression_ratio` describe the blobs
the snapshot references, and `docs/DATAMODEL.md` now says
`chunk_count` and `blob_count` count what the run added.
- 2026-10-06: Made command output follow the README's stdout and stderr
rules ([issue #224](https://git.eeqj.de/sneak/vaultik/issues/224)). The
startup banner went to stdout, so a `completion` script or a
`config get` value started with it; the banner now goes to stderr. A
failing `remote info`, `prune` or `snapshot remove` under `--json`
printed nothing on either stream, and now reports its error on stderr.
`snapshot verify --quiet` printed its whole report; it now prints
none, and a failure still reaches stderr with the same exit status.
- 2026-10-06: Made a backup without `--cron` of a snapshot with two or
more `paths` complete instead of panicking with `close of closed
channel` ([issue #253](https://git.eeqj.de/sneak/vaultik/issues/253)).
`Scan` runs once per path and started and stopped the progress
reporter each time, and a second stop panics. The reporter is now
started and stopped once per snapshot, around the scans of all its
paths.
- 2026-10-06: Made a restore path argument select only that path and
what is beneath it
([issue #223](https://git.eeqj.de/sneak/vaultik/issues/223)). The
lookup matched with SQL `LIKE`, so `/home/u/doc` also restored
`doc2`, `DOC` and `doc.txt.bak`, and a `_` or `%` in the path acted
as a wildcard. A backup used the same lookup to load the known files
of each configured path, so files of a longer sibling path were
counted as deleted. `FileRepository.ListUnderPath`, which replaces
`ListByPrefix`, returns the file at the path and every file whose
path starts with the path plus `/`, compared case-sensitively.
- 2026-10-06: Made restore return an error instead of panicking on a
malformed snapshot database
([issue #231](https://git.eeqj.de/sneak/vaultik/issues/231)). A chunk
hash shorter than 16 characters crashed the error message naming it,
and `--verify` dereferenced a missing `chunks` row and allocated
whatever chunk size the database gave. Those messages now go through
`shortHash`, a missing row is an error, and `--verify` rejects a
negative size and hashes each chunk as a stream.
- 2026-10-06: Made `s3://bucket/prefix` and `s3://bucket/prefix/` the same
destination ([issue #222](https://git.eeqj.de/sneak/vaultik/issues/222)).
The S3 client put the prefix directly in front of each key, so a prefix
without a trailing slash stored `prefixblobs/...`. A non-empty prefix is
now joined to every key with one `/`, giving the README's
`<bucket>/<prefix>/blobs/...` layout. The `s3.prefix` config setting goes
through the same client and gets the same join.
- 2026-10-06: Made restore apply owners, modes and times in an order
that keeps them
([issue #219](https://git.eeqj.de/sneak/vaultik/issues/219)). A
directory got its stored mode and mtime before its contents were
written, so a read-only directory came back without its files and a
non-empty one carried the time of the restore. Directories are now
created owner-only and get their stored owner, mode and mtime once the
restore loop is done, each before its parent. A file's mode is applied
after its chown, which on Linux clears setuid and setgid, and a
symlink gets its stored owner (as root) and mtime on the link itself.
- 2026-10-06: Made `go.mod` what `go mod tidy` writes, so the
pre-commit hook no longer stops every commit
([issue #246](https://git.eeqj.de/sneak/vaultik/issues/246)). A test
in `internal/cli` imports `github.com/spf13/pflag` directly, but
`go.mod` still marked it `// indirect`, and `script/precommit` fails
whenever the tidy changes `go.mod`. It is now in the direct `require`
block.
- 2026-10-06: Made the README's steps for restoring on another machine
work ([issue #221](https://git.eeqj.de/sneak/vaultik/issues/221)).
`config init` wrote a placeholder recipient that `config.Load`
rejects, so every command on the new machine failed before reaching
the store. The file now has an empty `age_recipients` list,
`config.Load` accepts an empty list, and `snapshot create` refuses to
run without a recipient.
- 2026-10-06: Made `snapshot restore --skip-errors` skip the files that
need a blob it cannot download
([issue #218](https://git.eeqj.de/sneak/vaultik/issues/218)). A missing
or damaged blob ended the restore even with the flag, after restoring
whichever files happened to come first. Every file that needs such a
blob is now reported as failed, the rest are restored, and the command
still exits non-zero. Without the flag the blob error still aborts.
- 2026-10-06: Re-vendored the canonical files from `sneak/prompts` at
`dd4027b` ([issue #213](https://git.eeqj.de/sneak/vaultik/issues/213)).
Linting and testing are now the `lint` and `test` phases of the
`Dockerfile`, and the build stage depends on both. `Dockerfile.lint`,
`CHECK_EPOCH` and the tests that checked them are gone; every
`docker build` in `script/` passes `--no-cache` instead. golangci-lint
is v2.14.0, with its new findings fixed in the code, and the rules in
`CLAUDE.md` now live in `AGENTS.md`.
- 2026-10-06: Made the local index actually run in WAL mode with a busy
timeout ([issue #217](https://git.eeqj.de/sneak/vaultik/issues/217)).
The connection settings were written in a form the SQLite driver
ignores, so the index ran without either and `snapshot list` or `info`
during a backup could make the backup's next write fail with
`database is locked`. They are now `_pragma=` parameters, and the
metadata export copies the open index with `VACUUM INTO`, because a
copy of the file alone misses rows still in the `-wal` file.
- 2026-10-06: Made a backup record the real uid and gid of files,
directories and symlinks
([issue #216](https://git.eeqj.de/sneak/vaultik/issues/216)). The
scanner asked the stat result for `Uid()` and `Gid()` methods, which
`*syscall.Stat_t` does not have, so every entry was stored as `0:0`
and a restore as root gave every file to root. It now reads the
`Uid` and `Gid` fields of `*syscall.Stat_t`.
- 2026-10-06: Made a `file://` destination whose directory is missing
count as one that cannot be listed
([issue #220](https://git.eeqj.de/sneak/vaultik/issues/220)). The file
backend listed a missing directory as an empty store, so with the
quickstart's USB stick unplugged `snapshot list` reported every local
snapshot as missing from the store, `snapshot remove` claimed to have
removed metadata it never reached, and `prune` dropped every local
snapshot record. Listing a missing destination directory is now an
error; a first backup still creates the directory.
- 2026-10-06: Made `--older-than` and `--keep-newer-than` reject a - 2026-10-06: Made `--older-than` and `--keep-newer-than` reject a
duration with characters outside its number-and-unit parts duration with characters outside its number-and-unit parts
([issue #215](https://git.eeqj.de/sneak/vaultik/issues/215)). The ([issue #215](https://git.eeqj.de/sneak/vaultik/issues/215)). The
parser picked out the parts it recognised and skipped the rest, so parser picked out the parts it recognised and skipped the rest, so
`1.0y` became zero and `--prune --keep-newer-than 1.0y` deleted every `1.0y` became zero and `--prune --keep-newer-than 1.0y` deleted every
snapshot of the backed-up names, the new one included. `1.0y`, snapshot of the backed-up names, the new one included. `1.0y`,
`1,5y`, `30 days`, `x7d` and a bare `0` are now errors; decimals still `1,5y`, `30 days` and `x7d` are now errors; decimals still work in Go
work in Go units such as `1.5h`. units such as `1.5h`.
- 2026-10-06: Made a backup re-chunk a known file that lists a chunk no - 2026-10-06: Made a backup re-chunk a known file that lists a chunk no
uploaded blob holds uploaded blob holds
+44 -84
View File
@@ -12,44 +12,48 @@ import (
// #75). The failure it protects against is silent: the image still // #75). The failure it protects against is silent: the image still
// builds and runs, but `vaultik version` inside it reports "commit: // builds and runs, but `vaultik version` inside it reports "commit:
// unknown" or a version of "dev", so an operator cannot tell which // unknown" or a version of "dev", so an operator cannot tell which
// source produced a given backup. The build takes the version as a // source produced a given backup. The build takes the values as build
// build arg, which script/docker computes on the host, and otherwise // args, which script/docker computes on the host, and otherwise derives
// derives it from the .git in its context; the commit and its date // them from the .git in its context.
// always come from that .git.
// //
// These are parses of the committed files, because shelling out to // These are parses of the committed files, for the same reason the lint
// docker would nest a build inside `make test`. That `vaultik version` // guards next door are: shelling out to docker would nest a build
// in the built image really prints the host's version is verified by // inside `make test`. That `vaultik version` in the built image really
// hand. // prints the host's version is verified by hand and recorded on the
// pull request.
// The files under guard, relative to the repository root. // dockerScript is script/docker, relative to the repository root.
const ( const dockerScript = "script/docker"
productDockerfile = "Dockerfile"
dockerScript = "script/docker"
)
// TestProductDockerfileTakesVersionAsBuildArg fails unless the build // versionArgs are the ldflag targets the build stamps and, matching
// declares ARG VERSION, with no default, and stamps it into the binary // them, the build args the host must supply. The names line up so the
// whenever it is given, ahead of the value derived in the container. // same list checks both files.
func TestProductDockerfileTakesVersionAsBuildArg(t *testing.T) { func versionArgs() []string {
return []string{"VERSION", "COMMIT", "COMMIT_DATE"}
}
// TestProductDockerfileTakesVersionAsBuildArgs fails unless the build
// declares each version arg, with no default, and stamps it into the
// binary whenever it is given, ahead of the value derived in the
// container.
func TestProductDockerfileTakesVersionAsBuildArgs(t *testing.T) {
t.Parallel() t.Parallel()
found := instructions(t, productDockerfile) found := instructions(t, productDockerfile)
require.Contains(t, found, "ARG VERSION", for _, arg := range versionArgs() {
"%s must declare `ARG VERSION`, with no default, so the host can"+ require.Contains(t, found, "ARG "+arg,
" pass it in", productDockerfile) "%s must declare `ARG %s`, with no default, so the host can"+
" pass it in", productDockerfile, arg)
assertLdflagReferences(t, found, "VERSION") assertLdflagReferences(t, found, arg)
}
} }
// TestProductDockerfileDerivesVersionFromGit fails unless a build given // TestProductDockerfileDerivesVersionFromGit fails unless a build given
// no VERSION, such as a plain `docker build .` of a clone, takes it from // no VERSION, such as a plain `docker build .` of a clone, takes it from
// `git describe` of the .git in its context, stamps "dev" when the // `git describe` of the .git in its context, and fails rather than
// context has no .git, and fails rather than stamp "dev" when that .git // stamp "dev" when that .git yields no version.
// yields no version. The commit and its date come from the same .git,
// and the build fails rather than stamp them "unknown" when it is
// present.
func TestProductDockerfileDerivesVersionFromGit(t *testing.T) { func TestProductDockerfileDerivesVersionFromGit(t *testing.T) {
t.Parallel() t.Parallel()
@@ -58,31 +62,31 @@ func TestProductDockerfileDerivesVersionFromGit(t *testing.T) {
buildAt := indexContaining(found, "go build") buildAt := indexContaining(found, "go build")
require.GreaterOrEqual(t, buildAt, 0, "%s must build", productDockerfile) require.GreaterOrEqual(t, buildAt, 0, "%s must build", productDockerfile)
assert.Contains(t, found[buildAt], "git describe --tags --always || echo dev", assert.Contains(t, found[buildAt], "git describe --tags --always",
"%s must derive the version from git when no VERSION is given,"+ "%s must derive the version from git when no VERSION is given",
" and stamp dev when the context has no .git", productDockerfile) productDockerfile)
assert.Contains(t, found[buildAt], "[ -e .git ]", assert.Contains(t, found[buildAt], "[ -e .git ]",
"%s must fail when the context carries .git but yields no version", "%s must fail when the context carries .git but yields no version",
productDockerfile) productDockerfile)
assert.Contains(t, found[buildAt], "git rev-parse HEAD",
"%s must stamp the commit from git", productDockerfile)
assert.Contains(t, found[buildAt], "git show -s --format=%cs HEAD",
"%s must stamp the commit date from git", productDockerfile)
assert.Contains(t, found[buildAt],
`[ "$commit" = unknown ] || [ "$commit_date" = unknown ]`,
"%s must fail when the context carries .git but yields no commit"+
" or date", productDockerfile)
} }
// TestDockerScriptComputesVersionOnTheHost fails unless script/docker // TestDockerScriptComputesVersionOnTheHost fails unless script/docker
// passes the version it derives where .git exists as a build arg. // derives each value where .git exists and passes it as a build arg,
// with VERSION coming from script/version so a Docker build reports the
// same string a local build of the same tree would.
func TestDockerScriptComputesVersionOnTheHost(t *testing.T) { func TestDockerScriptComputesVersionOnTheHost(t *testing.T) {
t.Parallel() t.Parallel()
script := readRepoFile(t, dockerScript) script := readRepoFile(t, dockerScript)
assert.Contains(t, script, "--build-arg VERSION=", for _, arg := range versionArgs() {
"%s must pass --build-arg VERSION to the build", dockerScript) assert.Contains(t, script, "--build-arg "+arg+"=",
"%s must pass --build-arg %s to the build", dockerScript, arg)
}
assert.Contains(t, script, "/version",
"%s must take VERSION from script/version, as the Makefile does",
dockerScript)
} }
// assertLdflagReferences fails unless the build instruction uses the // assertLdflagReferences fails unless the build instruction uses the
@@ -103,47 +107,3 @@ func assertLdflagReferences(t *testing.T, found []string, arg string) {
"the go build in %s must use ${%s:-...}, or the arg is passed and"+ "the go build in %s must use ${%s:-...}, or the arg is passed and"+
" discarded", productDockerfile, arg) " discarded", productDockerfile, arg)
} }
// instructions returns the Dockerfile's instructions, one per element,
// with comments and blank lines dropped and continuation lines joined,
// so a multi-line RUN is one string.
func instructions(t *testing.T, name string) []string {
t.Helper()
var (
out []string
continued string
isContinued bool
)
for line := range strings.SplitSeq(readRepoFile(t, name), "\n") {
trimmed := strings.TrimSpace(line)
if !isContinued && (trimmed == "" || strings.HasPrefix(trimmed, "#")) {
continue
}
isContinued = strings.HasSuffix(trimmed, `\`)
continued += strings.TrimSuffix(trimmed, `\`)
if isContinued {
continue
}
out = append(out, strings.Join(strings.Fields(continued), " "))
continued = ""
}
return out
}
// indexContaining returns the position of the first instruction
// containing want; -1 if there is none.
func indexContaining(found []string, want string) int {
for i, instruction := range found {
if strings.Contains(instruction, want) {
return i
}
}
return -1
}
+368
View File
@@ -0,0 +1,368 @@
package main_test
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// This file guards the shape of the lint gate. Every property asserted
// here is one whose loss is SILENT: the build still exits 0, the gate
// still looks green, and nothing was linted or tested.
//
// The gate is a build step. script/lint builds Dockerfile.lint, which
// runs golangci-lint as a RUN instruction, so a successful build is a
// clean lint. BuildKit will happily replay that RUN from cache on an
// unchanged tree in well under a second, which is why the check layers
// are keyed on a CHECK_EPOCH build arg that the calling script
// regenerates per invocation, and why an empty value is a hard error
// rather than a stable cache key.
//
// These are parses rather than invocations. Shelling out to docker from
// the test suite would nest a build inside `make test`, which itself
// runs inside a build in CI. The one property a parse cannot establish
// -- that a real finding actually fails the build -- is verified by
// hand against a deliberately broken tree, recorded on the pull
// request.
//
// One property is deliberately NOT tested here: that no script runs the
// linter on the host. script/lint is the only lint entry point, and it
// runs golangci-lint only inside the container; keeping it that way is a
// review matter, not something a test in this file establishes.
// The files under guard, relative to the repository root.
const (
lintDockerfile = "Dockerfile.lint"
productDockerfile = "Dockerfile"
lintScript = "script/lint"
cibuildScript = "script/cibuild"
)
// linterBinary is the linter's command name, used to locate the
// config-verify and lint steps in Dockerfile.lint.
const linterBinary = "golangci-lint"
// checkEpochARG is the declaration, with no default value. A default
// would satisfy the non-empty guard with a constant, and a constant is
// a stable cache key: the checks would be replayed from cache forever
// after the first build.
const checkEpochARG = "ARG CHECK_EPOCH"
// checkEpochGuard is what turns a build that omits --build-arg into a
// loud failure instead of a quiet green. Failed steps are never cached,
// so it fires on every such invocation rather than once.
const checkEpochGuard = `RUN [ -n "$CHECK_EPOCH" ] || exit 1`
// freshEpoch is the epoch computation the calling scripts must use, as
// a bare assignment on its own line. Inline in an argument, a failing
// `date` would not abort under `set -eu`; CHECK_EPOCH would become the
// empty string, and the guard above would be the only thing standing
// between that and a permanently cached green. `$$` is required because
// `date +%s` is second-granular and busybox silently drops `%N`, so
// without the pid two concurrent runs in one second can collide.
const freshEpoch = `epoch="$(date +%s%N)$$"`
// TestLintDockerfilePinsTheLinterByDigest fails if the lint image stops
// being pinned. An unpinned tag makes the gate's verdict depend on
// whatever the registry currently serves under that name.
func TestLintDockerfilePinsTheLinterByDigest(t *testing.T) {
t.Parallel()
from := ""
for _, instruction := range instructions(t, lintDockerfile) {
if strings.HasPrefix(instruction, "FROM ") {
from = instruction
break
}
}
require.NotEmpty(t, from, "%s declares no FROM", lintDockerfile)
assert.Contains(t, from, "golangci/golangci-lint",
"the lint image must be the golangci-lint image")
assert.Contains(t, from, "@sha256:",
"the lint image must be pinned by digest, not by tag alone")
}
// TestLintDockerfileCannotBeCachedGreen pins the whole cache-busting
// mechanism in the file that lints: the declaration with no default,
// the non-empty guard, and the value expanded into the lint command
// itself rather than merely declared.
func TestLintDockerfileCannotBeCachedGreen(t *testing.T) {
t.Parallel()
found := instructions(t, lintDockerfile)
argAt := indexOf(found, checkEpochARG)
require.GreaterOrEqual(t, argAt, 0,
"%s must declare `%s` with no default value",
lintDockerfile, checkEpochARG)
assert.GreaterOrEqual(t, indexOf(found, checkEpochGuard), argAt,
"%s must guard against an empty CHECK_EPOCH with `%s`",
lintDockerfile, checkEpochGuard)
assertEpochExpandedInto(t, found[argAt:], "golangci-lint run")
// Dependency layers must stay above the ARG, or every lint run
// re-downloads the module cache and the inner loop becomes
// unusable.
download := indexOf(found, "RUN go mod download")
require.GreaterOrEqual(t, download, 0,
"%s must download modules in their own layer", lintDockerfile)
assert.Less(t, download, argAt,
"`%s` must come after `go mod download` so dependency layers"+
" still cache", checkEpochARG)
}
// TestLintDockerfileVerifiesTheLinterConfig guards the validation of
// .golangci.yml itself. `golangci-lint run` rejects a config it cannot
// parse but silently IGNORES an unknown top-level key, so renaming
// `linters:` to `linterz:` discards `default: all` and every threshold
// and still exits 0 reporting no issues. `config verify` is what turns
// that into a failure, and it has to run BEFORE the lint, or the lint
// spends a minute reporting a verdict from a config already known to be
// wrong.
func TestLintDockerfileVerifiesTheLinterConfig(t *testing.T) {
t.Parallel()
found := instructions(t, lintDockerfile)
verify := linterBinary + " config verify"
verifyAt := indexContaining(found, verify)
require.GreaterOrEqual(t, verifyAt, 0,
"%s must run `%s --config .golangci.yml`: without it a typo'd"+
" top-level key in .golangci.yml is silently ignored and the"+
" gate passes with only the default linter set", lintDockerfile,
verify)
runAt := indexContaining(found, linterBinary+" run")
require.GreaterOrEqual(t, runAt, 0, "%s must lint", lintDockerfile)
assert.Less(t, verifyAt, runAt,
"%s must verify the config before linting with it", lintDockerfile)
// Keyed on the epoch like every other check layer, so it executes
// per invocation rather than being replayed. A cached validation
// validates nothing.
assertEpochExpandedInto(t, found, verify)
}
// TestProductDockerfileKeysChecksOnTheEpoch holds the same line for the
// checks that remain in the product image build, but without the guard:
// a plain `docker build .` with no build arguments must succeed.
func TestProductDockerfileKeysChecksOnTheEpoch(t *testing.T) {
t.Parallel()
found := instructions(t, productDockerfile)
argAt := indexOf(found, checkEpochARG)
require.GreaterOrEqual(t, argAt, 0,
"%s must declare `%s`", productDockerfile, checkEpochARG)
assert.Equal(t, -1, indexOf(found, checkEpochGuard),
"%s must not refuse an empty CHECK_EPOCH: a plain `docker build .`"+
" must succeed", productDockerfile)
assertEpochExpandedInto(t, found[argAt:], "make fmt-check")
assertEpochExpandedInto(t, found[argAt:], "make test")
}
// TestProductDockerfileDoesNotLint records the split deliberately: the
// linter lives in Dockerfile.lint and nowhere else, so there is exactly
// one digest pinning it. A lint stage reintroduced here would either be
// docker-in-docker (`make lint` is now `docker build`) or a second,
// independently bumpable pin.
func TestProductDockerfileDoesNotLint(t *testing.T) {
t.Parallel()
contents := readRepoFile(t, productDockerfile)
for _, forbidden := range []string{"golangci", "make lint"} {
assert.NotContains(t, instructionText(contents), forbidden,
"%s must not lint: the linter is pinned once, in %s",
productDockerfile, lintDockerfile)
}
}
// TestLintScriptBuildsTheLintDockerfileWithAFreshEpoch is the other
// half of the mechanism. The Dockerfile's guard only rejects an EMPTY
// epoch; a constant non-empty one would satisfy it and still be served
// from cache forever.
func TestLintScriptBuildsTheLintDockerfileWithAFreshEpoch(t *testing.T) {
t.Parallel()
script := readRepoFile(t, lintScript)
assertBareEpochAssignment(t, script, lintScript)
assert.Contains(t, script, `--build-arg CHECK_EPOCH="$epoch"`,
"%s must pass the fresh epoch to the build", lintScript)
assert.Contains(t, script, lintDockerfile,
"%s must build %s", lintScript, lintDockerfile)
}
// TestCibuildBuildsBothDockerfilesWithFreshEpochs guards the CI gate:
// dropping either build silently removes a whole class of check from
// CI while leaving it green.
func TestCibuildBuildsBothDockerfilesWithFreshEpochs(t *testing.T) {
t.Parallel()
script := readRepoFile(t, cibuildScript)
assertBareEpochAssignment(t, script, cibuildScript)
assert.Equal(t, 2, strings.Count(script, freshEpoch),
"%s must compute a fresh epoch for each of its two builds",
cibuildScript)
assert.Equal(t, 2,
strings.Count(script, `--build-arg CHECK_EPOCH="$epoch"`),
"%s must pass a fresh epoch to both builds", cibuildScript)
assert.Contains(t, script, "-f Dockerfile.lint",
"%s must build %s", cibuildScript, lintDockerfile)
}
// assertEpochExpandedInto fails unless some instruction runs the named
// command with the epoch expanded into it. Expansion, not mere
// declaration: an ARG that no instruction references is not guaranteed
// to key the layer, and the expansion also puts the value in the build
// log where a reader can see the layer was keyed fresh.
func assertEpochExpandedInto(t *testing.T, found []string, command string) {
t.Helper()
for _, instruction := range found {
if !strings.HasPrefix(instruction, "RUN ") {
continue
}
if strings.Contains(instruction, command) &&
strings.Contains(instruction, "${CHECK_EPOCH}") {
return
}
}
assert.Fail(t, "no epoch-keyed layer runs the command",
"`%s` must run in a layer that expands ${CHECK_EPOCH}, or it"+
" will be replayed from cache without executing", command)
}
// assertBareEpochAssignment fails unless the script computes the epoch
// as a bare assignment on its own line.
func assertBareEpochAssignment(t *testing.T, script, name string) {
t.Helper()
for line := range strings.SplitSeq(script, "\n") {
if strings.TrimSpace(line) == freshEpoch {
return
}
}
assert.Fail(t, "no bare epoch assignment",
"%s must compute `%s` as a bare assignment on its own line, so"+
" `set -e` catches a failing date instead of quietly"+
" building with an empty epoch", name, freshEpoch)
}
// instructions returns the Dockerfile's instructions, one per element,
// with comments and blank lines dropped and continuation lines joined,
// so a multi-line RUN is one string.
func instructions(t *testing.T, name string) []string {
t.Helper()
return strings.Split(instructionText(readRepoFile(t, name)), "\n")
}
// instructionText is instructions' parse, before splitting: it is also
// what a "must not contain" assertion should look at, so that a word
// appearing only in a comment is not mistaken for behaviour.
func instructionText(contents string) string {
var (
out []string
continued string
isContinued bool
)
for line := range strings.SplitSeq(contents, "\n") {
trimmed := strings.TrimSpace(line)
if !isContinued && (trimmed == "" || strings.HasPrefix(trimmed, "#")) {
continue
}
isContinued = strings.HasSuffix(trimmed, `\`)
continued += strings.TrimSuffix(trimmed, `\`)
if isContinued {
continue
}
out = append(out, strings.Join(strings.Fields(continued), " "))
continued = ""
}
return strings.Join(out, "\n")
}
// indexOf returns the position of the first instruction equal to, or
// beginning with, want; -1 if there is none. An `ARG NAME=default`
// counts as beginning with `ARG NAME`, so a declared arg is found
// whether or not it carries a default.
func indexOf(found []string, want string) int {
for i, instruction := range found {
if instruction == want ||
strings.HasPrefix(instruction, want+" ") ||
strings.HasPrefix(instruction, want+"=") {
return i
}
}
return -1
}
// indexContaining returns the position of the first instruction
// containing want; -1 if there is none.
func indexContaining(found []string, want string) int {
for i, instruction := range found {
if strings.Contains(instruction, want) {
return i
}
}
return -1
}
// readRepoFile reads a file by its path relative to the repository
// root.
func readRepoFile(t *testing.T, name string) string {
t.Helper()
//nolint:gosec // G304: the path is a constant relative to this repo
contents, err := os.ReadFile(filepath.Join(repoRoot(t), name))
require.NoError(t, err)
return string(contents)
}
// repoRoot returns the repository root. The test binary runs with its
// package directory as the working directory, so the root is found by
// walking up until the module file appears.
func repoRoot(t *testing.T) string {
t.Helper()
dir, err := os.Getwd()
require.NoError(t, err)
for {
_, err = os.Stat(filepath.Join(dir, "go.mod"))
if err == nil {
return dir
}
parent := filepath.Dir(dir)
require.NotEqual(t, dir, parent,
"walked to the filesystem root without finding a go.mod")
dir = parent
}
}
+2 -38
View File
@@ -1,8 +1,6 @@
package main_test package main_test
import ( import (
"os"
"path/filepath"
"regexp" "regexp"
"slices" "slices"
"strings" "strings"
@@ -86,48 +84,14 @@ func TestBuildTargetBuildsTheBinary(t *testing.T) {
"`make build` must depend on the rule that builds the binary") "`make build` must depend on the rule that builds the binary")
} }
// readMakefile returns the contents of the repository's Makefile. // readMakefile returns the contents of the repository's Makefile. The
// root is located by the shared walk in lintdocker_test.go.
func readMakefile(t *testing.T) string { func readMakefile(t *testing.T) string {
t.Helper() t.Helper()
return readRepoFile(t, "Makefile") return readRepoFile(t, "Makefile")
} }
// readRepoFile reads a file by its path relative to the repository
// root.
func readRepoFile(t *testing.T, name string) string {
t.Helper()
//nolint:gosec // G304: the path is a constant relative to this repo
contents, err := os.ReadFile(filepath.Join(repoRoot(t), name))
require.NoError(t, err)
return string(contents)
}
// repoRoot returns the repository root. The test binary runs with its
// package directory as the working directory, so the root is found by
// walking up until the module file appears.
func repoRoot(t *testing.T) string {
t.Helper()
dir, err := os.Getwd()
require.NoError(t, err)
for {
_, err = os.Stat(filepath.Join(dir, "go.mod"))
if err == nil {
return dir
}
parent := filepath.Dir(dir)
require.NotEqual(t, dir, parent,
"walked to the filesystem root without finding a go.mod")
dir = parent
}
}
// phonyTargets returns every name declared phony, across all .PHONY // phonyTargets returns every name declared phony, across all .PHONY
// lines. // lines.
func phonyTargets(makefile string) []string { func phonyTargets(makefile string) []string {
+7 -17
View File
@@ -3,8 +3,7 @@
# Copy this file and uncomment/modify the values you need # Copy this file and uncomment/modify the values you need
# Age recipient public keys for encryption # Age recipient public keys for encryption
# Backups are encrypted to these public keys. snapshot create needs at least # This is REQUIRED - backups are encrypted to these public keys
# one; listing, verifying and restoring do not
# Generate with: age-keygen | grep "public key" # Generate with: age-keygen | grep "public key"
age_recipients: age_recipients:
- age1cj2k2addawy294f6k2gr2mf9gps9r3syplryxca3nvxj3daqm96qfp84tz - age1cj2k2addawy294f6k2gr2mf9gps9r3syplryxca3nvxj3daqm96qfp84tz
@@ -287,18 +286,15 @@ storage_url: "rclone://myremote/path/to/backups"
# #use_ssl: true # #use_ssl: true
# #
# # Part size for multipart uploads # # Part size for multipart uploads
# # Minimum 5MiB, maximum 5GiB; affects memory usage during upload # # Minimum 5MB, affects memory usage during upload
# # A blob too large for 10,000 parts of this size gets larger parts # # Supports: 5MB, 10M, 100MiB, etc.
# # Supports: 10MB, 16MiB, 100MiB, etc. (5MB is below the minimum) # # Default: 5MB
# # Default: 5MiB # #part_size: 5MB
# #part_size: 5MiB
# Path to local SQLite index database # Path to local SQLite index database
# This database tracks file state for incremental backups # This database tracks file state for incremental backups
# Default: the platform data directory, e.g. # Default: /var/lib/vaultik/index.sqlite
# macOS: ~/Library/Application Support/vaultik/index.sqlite #index_path: /var/lib/vaultik/index.sqlite
# Linux: ~/.local/share/vaultik/index.sqlite
#index_path: /path/to/index.sqlite
# Average chunk size for content-defined chunking # Average chunk size for content-defined chunking
# Smaller chunks = better deduplication but more metadata # Smaller chunks = better deduplication but more metadata
@@ -313,12 +309,6 @@ storage_url: "rclone://myremote/path/to/backups"
# Chunking uses no secret (the FastCDC parameters are fixed and public). At a # Chunking uses no secret (the FastCDC parameters are fixed and public). At a
# large limit a blob holds hundreds of chunks, so individual chunk lengths are # large limit a blob holds hundreds of chunks, so individual chunk lengths are
# not visible in its size; lowering the limit toward chunk_size exposes them. # not visible in its size; lowering the limit toward chunk_size exposes them.
# A backup needs free temporary space, because each blob is written whole to
# a temporary file before it is uploaded (up to about blob_size_limit; an
# rclone destination that cannot stream uploads needs about twice that) and
# the metadata export writes copies of the local index. Temporary files go to
# $TMPDIR (default /tmp); with TMPDIR unset, SQLite writes one of those copies
# to /var/tmp.
# Supports: 1GB, 10G, 500MB, 1GiB, etc. # Supports: 1GB, 10G, 500MB, 1GiB, etc.
# Default: 10GB # Default: 10GB
#blob_size_limit: 10GB #blob_size_limit: 10GB
+11 -11
View File
@@ -36,8 +36,7 @@ Stores metadata about files in the filesystem being backed up.
**Columns:** **Columns:**
- `id` (TEXT PRIMARY KEY) - UUID for the file record - `id` (TEXT PRIMARY KEY) - UUID for the file record
- `path` (TEXT NOT NULL UNIQUE) - Absolute file path - `path` (TEXT NOT NULL UNIQUE) - Absolute file path
- `mtime` (INTEGER NOT NULL) - Modification time, whole seconds since the Unix epoch - `mtime` (INTEGER NOT NULL) - Modification time as Unix timestamp
- `mtime_nsec` (INTEGER NOT NULL) - Nanoseconds within that second, 0 to 999999999
- `size` (INTEGER NOT NULL) - File size in bytes - `size` (INTEGER NOT NULL) - File size in bytes
- `mode` (INTEGER NOT NULL) - Unix file permissions and type - `mode` (INTEGER NOT NULL) - Unix file permissions and type
- `uid` (INTEGER NOT NULL) - User ID of file owner - `uid` (INTEGER NOT NULL) - User ID of file owner
@@ -111,17 +110,17 @@ Maps chunks to the blobs that contain them.
Tracks backup snapshots. Tracks backup snapshots.
**Columns:** **Columns:**
- `id` (TEXT PRIMARY KEY) - Snapshot ID (format: `hostname_name_timestamp`, e.g. `server1_home_2025-06-01T12:00:00Z`: the hostname up to its first `.`, the snapshot name, and an RFC 3339 UTC timestamp) - `id` (TEXT PRIMARY KEY) - Snapshot ID (format: hostname-YYYYMMDD-HHMMSSZ)
- `hostname` (TEXT) - Hostname where backup was created - `hostname` (TEXT) - Hostname where backup was created
- `vaultik_version` (TEXT) - Version of Vaultik used - `vaultik_version` (TEXT) - Version of Vaultik used
- `vaultik_git_revision` (TEXT) - Git revision of Vaultik used - `vaultik_git_revision` (TEXT) - Git revision of Vaultik used
- `started_at` (INTEGER) - Start timestamp - `started_at` (INTEGER) - Start timestamp
- `completed_at` (INTEGER) - Completion timestamp (NULL if in progress) - `completed_at` (INTEGER) - Completion timestamp (NULL if in progress)
- `file_count` (INTEGER) - Number of files in snapshot - `file_count` (INTEGER) - Number of files in snapshot
- `chunk_count` (INTEGER) - Number of chunks this snapshot stored that were not stored before - `chunk_count` (INTEGER) - Number of unique chunks
- `blob_count` (INTEGER) - Number of blobs this snapshot created - `blob_count` (INTEGER) - Number of blobs referenced
- `total_size` (INTEGER) - Total size of all files - `total_size` (INTEGER) - Total size of all files
- `blob_size` (INTEGER) - Total compressed size of all referenced blobs - `blob_size` (INTEGER) - Total size of all blobs (compressed)
- `blob_uncompressed_size` (INTEGER) - Total uncompressed size of all referenced blobs - `blob_uncompressed_size` (INTEGER) - Total uncompressed size of all referenced blobs
- `compression_ratio` (REAL) - Compression ratio achieved - `compression_ratio` (REAL) - Compression ratio achieved
- `compression_level` (INTEGER) - Compression level used for this snapshot - `compression_level` (INTEGER) - Compression level used for this snapshot
@@ -218,8 +217,8 @@ The `{remote-key}` directory name is a one-way hash of the human snapshot ID, so
### 4. Restore Process ### 4. Restore Process
The restore process doesn't use the local database. Instead: The restore process doesn't use the local database. Instead:
1. Downloads and decrypts the snapshot's metadata database (`db.zst.age`) from S3 1. Downloads snapshot metadata from S3
2. Downloads the blobs holding the chunks of the files being restored, found through that database's `blob_chunks` table; the manifest is not read 2. Downloads required blobs based on manifest
3. Reconstructs files from decrypted and decompressed chunks 3. Reconstructs files from decrypted and decompressed chunks
### 5. Pruning ### 5. Pruning
@@ -232,8 +231,9 @@ The restore process doesn't use the local database. Instead:
Before each backup: Before each backup:
1. Query incomplete snapshots (where `completed_at IS NULL`) 1. Query incomplete snapshots (where `completed_at IS NULL`)
2. Delete each one and all its associations, without checking S3 for its metadata 2. Check if metadata exists in S3
3. Clean up orphaned files, chunks, and blobs 3. If no metadata, delete snapshot and all associations
4. Clean up orphaned files, chunks, and blobs
## Repository Pattern ## Repository Pattern
@@ -280,7 +280,7 @@ This ensures consistency, especially important for operations like:
3. **Batch Operations**: Where possible, operations are batched within transactions 3. **Batch Operations**: Where possible, operations are batched within transactions
4. **Write-Ahead Logging**: The local index runs in SQLite WAL mode with a 10-second busy timeout, so a read-only command such as `snapshot list` can read it while a backup writes to it. Committed rows can sit in the `-wal` file beside the index until a checkpoint, so the metadata export copies the index through SQLite (`VACUUM INTO`), not as a file 4. **Write-Ahead Logging**: SQLite WAL mode is enabled for better concurrency
## Data Integrity ## Data Integrity
+2 -2
View File
@@ -20,11 +20,9 @@ require (
github.com/rclone/rclone v1.72.1 github.com/rclone/rclone v1.72.1
github.com/spf13/afero v1.15.0 github.com/spf13/afero v1.15.0
github.com/spf13/cobra v1.10.1 github.com/spf13/cobra v1.10.1
github.com/spf13/pflag v1.0.10
github.com/stretchr/testify v1.11.1 github.com/stretchr/testify v1.11.1
go.uber.org/fx v1.24.0 go.uber.org/fx v1.24.0
golang.org/x/sync v0.18.0 golang.org/x/sync v0.18.0
golang.org/x/sys v0.38.0
golang.org/x/term v0.37.0 golang.org/x/term v0.37.0
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.38.0 modernc.org/sqlite v1.38.0
@@ -227,6 +225,7 @@ require (
github.com/smarty/assertions v1.16.0 // indirect github.com/smarty/assertions v1.16.0 // indirect
github.com/sony/gobreaker v1.0.0 // indirect github.com/sony/gobreaker v1.0.0 // indirect
github.com/spacemonkeygo/monkit/v3 v3.0.25-0.20251022131615-eb24eb109368 // indirect github.com/spacemonkeygo/monkit/v3 v3.0.25-0.20251022131615-eb24eb109368 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 // indirect github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 // indirect
github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect
github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/match v1.1.1 // indirect
@@ -264,6 +263,7 @@ require (
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
golang.org/x/net v0.47.0 // indirect golang.org/x/net v0.47.0 // indirect
golang.org/x/oauth2 v0.33.0 // indirect golang.org/x/oauth2 v0.33.0 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/text v0.31.0 // indirect golang.org/x/text v0.31.0 // indirect
golang.org/x/time v0.14.0 // indirect golang.org/x/time v0.14.0 // indirect
golang.org/x/tools v0.38.0 // indirect golang.org/x/tools v0.38.0 // indirect
+28 -48
View File
@@ -200,16 +200,12 @@ func RunApp(ctx context.Context, app *fx.App) error {
} }
// errReported marks a failure the operation has already shown the user // errReported marks a failure the operation has already shown the user
// (or, under `snapshot verify --json`, put in its document). Entry // (and deliberately withheld under --json). Entry turns it into a
// turns it into a non-zero exit status without printing anything // non-zero exit status without printing anything further, so the error
// further, so the error line is not doubled. It flows up from // line is not doubled. It flows up from RunOperation through cobra to
// RunOperation through cobra to Entry. // Entry.
var errReported = errors.New("operation failed") var errReported = errors.New("operation failed")
// errInterrupted marks an operation that SIGINT or SIGTERM stopped
// before it finished. Entry shows it and returns exitCodeInterrupted.
var errInterrupted = errors.New("interrupted before the command finished")
// RunOperation runs op against the Vaultik instance inside the fx app // RunOperation runs op against the Vaultik instance inside the fx app
// and turns a failure into a returned error rather than an os.Exit from // and turns a failure into a returned error rather than an os.Exit from
// within the goroutine. An os.Exit there skipped main's deferred // within the goroutine. An os.Exit there skipped main's deferred
@@ -224,21 +220,17 @@ var errInterrupted = errors.New("interrupted before the command finished")
// interrupt OnStop cancels op and waits for the goroutine to return, so // interrupt OnStop cancels op and waits for the goroutine to return, so
// op's cleanup (removing decrypted scratch files) runs before the // op's cleanup (removing decrypted scratch files) runs before the
// process exits; the wait is bounded by shutdownTimeout. report is // process exits; the wait is bounded by shutdownTimeout. report is
// called with a failure so the caller can show it to the user before // called with a non-canceled failure so the caller can log it (and
// it becomes errReported. // suppress it under --json) before it becomes errReported. A context
// // cancellation is the interrupt path, not a failure: it is neither
// The run counts as interrupted unless op returned, without an // reported nor counted as one.
// interrupt having cancelled it, before RunWithApp returned. An
// interrupted op is not reported, whatever it returned; RunOperation
// returns errInterrupted instead.
func RunOperation( func RunOperation(
ctx context.Context, opts AppOptions, ctx context.Context, opts AppOptions,
op func(v *vaultik.Vaultik) error, report func(err error), op func(v *vaultik.Vaultik) error, report func(err error),
) error { ) error {
var ( var (
mu sync.Mutex mu sync.Mutex
finished bool // op returned before any interrupt cancelled it failed bool
failed bool // op finished with an error
) )
opts.Invokes = append(opts.Invokes, opts.Invokes = append(opts.Invokes,
@@ -249,21 +241,11 @@ func RunOperation(
OnStart: func(_ context.Context) error { OnStart: func(_ context.Context) error {
stop = v.StartOperation(func() { stop = v.StartOperation(func() {
err := op(v) err := op(v)
if err != nil && !errors.Is(err, context.Canceled) {
// Only stop, called from OnStop below, cancels the report(err)
// Vaultik context, so a live context means no
// interrupt cancelled op. Check the context, not
// err: an interrupted op need not return
// context.Canceled (`snapshot verify --json`
// returns a verification failure).
if v.Context().Err() == nil {
if err != nil {
report(err)
}
mu.Lock() mu.Lock()
finished = true failed = true
failed = err != nil
mu.Unlock() mu.Unlock()
} }
@@ -296,34 +278,28 @@ func RunOperation(
return err return err
} }
// RunWithApp returns only after the app was asked to stop, either by // The goroutine sets failed before triggering the shutdown that lets
// an interrupt or by the goroutine's Shutdown call. When op finished // RunWithApp return, so the write is in place by the time we read it.
// without being cancelled, the goroutine set finished before that
// call. So if finished is unset here, an interrupt stopped the app,
// and op either returned after it was cancelled or is still running
// because the shutdown timed out.
mu.Lock() mu.Lock()
defer mu.Unlock() defer mu.Unlock()
switch { if failed {
case !finished:
return errInterrupted
case failed:
return errReported return errReported
default:
return nil
} }
return nil
} }
// runVaultikApp runs the standard single-operation command lifecycle // runVaultikApp runs the standard single-operation command lifecycle
// shared by the snapshot list/purge/remove and remote nuke subcommands: // shared by the snapshot list/purge/remove and remote nuke subcommands:
// resolve the config, then run op against the Vaultik instance through // resolve the config, then run op against the Vaultik instance through
// RunOperation, reporting a failure prefixed with failMsg on stderr. mode // RunOperation, reporting a failure prefixed with failMsg (suppressed
// says whether the command takes the PID lock. jsonOutput marks a command // while suppressErrors is true, e.g. under --json). mode says whether the
// whose stdout is a JSON document: it quiets the UI but, unlike Quiet, // command takes the PID lock. jsonOutput marks a command whose stdout is a
// leaves the stderr log level alone. // JSON document: it quiets the UI but, unlike Quiet, leaves the stderr log
// level alone.
func runVaultikApp( func runVaultikApp(
cmd *cobra.Command, mode lockMode, jsonOutput bool, cmd *cobra.Command, mode lockMode, jsonOutput, suppressErrors bool,
failMsg string, op func(v *vaultik.Vaultik) error, failMsg string, op func(v *vaultik.Vaultik) error,
) error { ) error {
configPath, err := ResolveConfigPath() configPath, err := ResolveConfigPath()
@@ -343,6 +319,10 @@ func runVaultikApp(
}, },
Mode: mode, Mode: mode,
}, op, func(err error) { }, op, func(err error) {
if suppressErrors {
return
}
log.Error(failMsg, "error", err) log.Error(failMsg, "error", err)
ReportErrorf("%s: %v", failMsg, err) ReportErrorf("%s: %v", failMsg, err)
}) })
+5 -63
View File
@@ -7,14 +7,11 @@ import (
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"reflect"
"strconv" "strconv"
"strings" "strings"
"unicode/utf8"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/ui" "sneak.berlin/go/vaultik/internal/ui"
) )
@@ -34,9 +31,6 @@ const configDirMode = 0o755
// yaml.Marshal's 4-space default. // yaml.Marshal's 4-space default.
const configYAMLIndent = 2 const configYAMLIndent = 2
// yamlStringTag is YAML's tag for a string scalar.
const yamlStringTag = "!!str"
var ( var (
errConfigExists = errors.New("config file already exists") errConfigExists = errors.New("config file already exists")
errEmptyConfig = errors.New("empty config file") errEmptyConfig = errors.New("empty config file")
@@ -51,19 +45,16 @@ const defaultConfigTemplate = `# vaultik configuration
# ─── REQUIRED ──────────────────────────────────────────────────────────────── # ─── REQUIRED ────────────────────────────────────────────────────────────────
# Age recipient public keys for encryption. snapshot create needs at least # Age recipient public keys for encryption.
# one; listing, verifying and restoring do not, so a machine that only
# restores can leave this empty.
# Backups are encrypted to ALL listed recipients; any one of the corresponding # Backups are encrypted to ALL listed recipients; any one of the corresponding
# private keys can decrypt. Adding a recipient later does not re-encrypt data # private keys can decrypt. Adding a recipient later does not re-encrypt data
# already stored: deduplicated chunks and existing blobs stay encrypted to the # already stored: deduplicated chunks and existing blobs stay encrypted to the
# earlier recipients, so a newly added key cannot restore them on its own (see # earlier recipients, so a newly added key cannot restore them on its own (see
# docs/REPOSTRUCTURE.md, Accepted Risks). Generate a keypair and add its # docs/REPOSTRUCTURE.md, Accepted Risks). Generate a keypair with:
# public key with:
# age-keygen -o vaultik_backup_private_key.txt # age-keygen -o vaultik_backup_private_key.txt
# grep 'public key' vaultik_backup_private_key.txt # grep 'public key' vaultik_backup_private_key.txt
# vaultik config set age_recipients.0 age1... age_recipients:
age_recipients: [] - age1REPLACE_WITH_YOUR_PUBLIC_KEY
# Named snapshots. Each snapshot backs up one or more paths and can have its # Named snapshots. Each snapshot backs up one or more paths and can have its
# own exclude patterns in addition to the global excludes below. # own exclude patterns in addition to the global excludes below.
@@ -205,7 +196,7 @@ storage_url: ""
# access_key_id: YOUR_ACCESS_KEY # access_key_id: YOUR_ACCESS_KEY
# secret_access_key: YOUR_SECRET_KEY # secret_access_key: YOUR_SECRET_KEY
# # region: us-east-1 # Default: us-east-1 # # region: us-east-1 # Default: us-east-1
# # part_size: 5MiB # Upload part size, 5MiB to 5GiB. Default: 5MiB # # part_size: 5MB # Multipart upload part size. Default: 5MB
# # For the s3:// form, disable TLS with ?ssl=false in the URL, not use_ssl. # # For the s3:// form, disable TLS with ?ssl=false in the URL, not use_ssl.
# ─── OPTIONAL ──────────────────────────────────────────────────────────────── # ─── OPTIONAL ────────────────────────────────────────────────────────────────
@@ -589,58 +580,9 @@ func yamlPathSet(root *yaml.Node, keys []string, value string) error {
} }
} }
// config.Load reads the file through untyped YAML, which turns an
// unquoted 00112233 into the number 38043 and 1e5 into 100000. Tagging
// a string setting as a string makes the encoder quote such a value.
// Other settings stay unquoted, so compression_level 9 is a number.
// The encoder refuses to write a value that is not valid UTF-8 as a
// string. Left untagged, such a value is written as base64 !!binary and
// loads back unchanged.
if configKeyIsString(keys) && utf8.ValidString(value) {
node.Tag = yamlStringTag
}
return nil return nil
} }
// configKeyIsString reports whether the dotted key names a string in
// config.Config, following the fields' yaml tags, as s3.access_key_id and
// snapshots.home.exclude.0 do.
func configKeyIsString(keys []string) bool {
typ := reflect.TypeFor[config.Config]()
for _, key := range keys {
switch {
case typ.Kind() == reflect.Map || typ.Kind() == reflect.Slice:
// The key is a snapshot name or a list index.
typ = typ.Elem()
case typ.Kind() == reflect.Struct:
field, ok := yamlField(typ, key)
if !ok {
return false
}
typ = field.Type
default:
return false
}
}
return typ.Kind() == reflect.String
}
// yamlField returns the field of struct type typ whose yaml tag names key.
func yamlField(typ reflect.Type, key string) (reflect.StructField, bool) {
for field := range typ.Fields() {
name, _, _ := strings.Cut(field.Tag.Get("yaml"), ",")
if name == key {
return field, true
}
}
return reflect.StructField{}, false
}
// yamlSetInMapping resolves (creating if needed) the value node for key // yamlSetInMapping resolves (creating if needed) the value node for key
// within a mapping node, setting it to value when it is the final path // within a mapping node, setting it to value when it is the final path
// element, and returns the node to descend into. // element, and returns the node to descend into.
+2 -146
View File
@@ -4,7 +4,6 @@ import (
"bytes" "bytes"
"os" "os"
"path/filepath" "path/filepath"
"strconv"
"strings" "strings"
"testing" "testing"
@@ -25,10 +24,8 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
t.Fatalf("default config template is not valid YAML: %v", err) t.Fatalf("default config template is not valid YAML: %v", err)
} }
// A placeholder recipient would fail config.Load, so the template if len(cfg.AgeRecipients) != 1 {
// leaves the list empty. t.Errorf("expected 1 placeholder age recipient, got %d", len(cfg.AgeRecipients))
if len(cfg.AgeRecipients) != 0 {
t.Errorf("expected no age recipients, got %d", len(cfg.AgeRecipients))
} }
home, ok := cfg.Snapshots["home"] home, ok := cfg.Snapshots["home"]
@@ -58,147 +55,6 @@ func TestDefaultConfigTemplateParses(t *testing.T) {
} }
} }
// TestConfigSetRecipientOnFreshConfig follows the README quickstart: on the
// file `config init` writes, `config set age_recipients.0` and
// `config set storage_url` give a config that loads with that recipient.
func TestConfigSetRecipientOnFreshConfig(t *testing.T) {
t.Parallel()
const recipient = "age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode)
if err != nil {
t.Fatalf("write config: %v", err)
}
out := ui.NewWithColor(&bytes.Buffer{}, false)
err = writeConfigSet(out, path, "age_recipients.0", recipient)
if err != nil {
t.Fatalf("config set age_recipients.0: %v", err)
}
err = writeConfigSet(out, path, "storage_url", "file:///mnt/backups")
if err != nil {
t.Fatalf("config set storage_url: %v", err)
}
cfg, err := config.Load(path)
if err != nil {
t.Fatalf("config.Load: %v", err)
}
if len(cfg.AgeRecipients) != 1 || cfg.AgeRecipients[0] != recipient {
t.Errorf("age_recipients = %v, want [%s]", cfg.AgeRecipients, recipient)
}
}
// TestConfigSetStringLooksLikeNumber sets string settings to values that
// YAML reads as numbers or booleans when they are unquoted, and checks that
// config.Load returns each one unchanged.
func TestConfigSetStringLooksLikeNumber(t *testing.T) {
t.Parallel()
tests := []struct {
key string
value string
field func(cfg *config.Config) string
}{
{"s3.access_key_id", "00112233",
func(cfg *config.Config) string { return cfg.S3.AccessKeyID }},
{"s3.secret_access_key", "12345678901234567890123456789012",
func(cfg *config.Config) string { return cfg.S3.SecretAccessKey }},
{"hostname", "007",
func(cfg *config.Config) string { return cfg.Hostname }},
{"s3.prefix", "1e5",
func(cfg *config.Config) string { return cfg.S3.Prefix }},
{"s3.bucket", "true",
func(cfg *config.Config) string { return cfg.S3.Bucket }},
{"s3.region", "FALSE",
func(cfg *config.Config) string { return cfg.S3.Region }},
{"snapshots.home.exclude.0", "1.10",
func(cfg *config.Config) string { return cfg.Snapshots["home"].Exclude[0] }},
}
for _, tt := range tests {
t.Run(tt.key+"="+tt.value, func(t *testing.T) {
t.Parallel()
cfg := loadAfterConfigSet(t, tt.key, tt.value)
got := tt.field(cfg)
if got != tt.value {
t.Errorf("%s = %q after config set %q", tt.key, got, tt.value)
}
})
}
}
// TestConfigSetNonUTF8Path checks that config set still accepts a value that
// is not valid UTF-8, such as a path with a Latin-1 file name, and that
// config.Load returns it unchanged.
func TestConfigSetNonUTF8Path(t *testing.T) {
t.Parallel()
const dir = "/srv/caf\xe9"
cfg := loadAfterConfigSet(t, "snapshots.home.paths.0", dir)
got := cfg.Snapshots["home"].Paths[0]
if got != dir {
t.Errorf("snapshots.home.paths.0 = %q, want %q", got, dir)
}
}
// TestConfigSetNumberStaysNumber checks that a number set for an integer
// setting is still read as a number, not as a quoted string.
func TestConfigSetNumberStaysNumber(t *testing.T) {
t.Parallel()
const level = 9
cfg := loadAfterConfigSet(t, "compression_level", strconv.Itoa(level))
if cfg.CompressionLevel != level {
t.Errorf("compression_level = %d, want %d", cfg.CompressionLevel, level)
}
}
// loadAfterConfigSet writes the file `config init` writes, sets storage_url
// to a local directory so that the file passes validation, applies
// `config set key value` and returns what config.Load reads back.
func loadAfterConfigSet(t *testing.T, key, value string) *config.Config {
t.Helper()
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(defaultConfigTemplate), configFileMode)
if err != nil {
t.Fatalf("write config: %v", err)
}
out := ui.NewWithColor(&bytes.Buffer{}, false)
err = writeConfigSet(out, path, "storage_url", "file:///mnt/backups")
if err != nil {
t.Fatalf("config set storage_url: %v", err)
}
err = writeConfigSet(out, path, key, value)
if err != nil {
t.Fatalf("config set %s: %v", key, err)
}
cfg, err := config.Load(path)
if err != nil {
t.Fatalf("config.Load: %v", err)
}
return cfg
}
const testYAML = `# top comment const testYAML = `# top comment
compression_level: 3 compression_level: 3
age_recipients: age_recipients:
+13 -25
View File
@@ -15,25 +15,17 @@ import (
// the startup banner. // the startup banner.
const shortCommitLen = 12 const shortCommitLen = 12
// exitCodeInterrupted is the exit status of a command that SIGINT or
// SIGTERM stopped. It is 128 plus SIGINT's number, 2, which is what a
// shell reports for a command stopped by Ctrl-C.
const exitCodeInterrupted = 130
// Entry is the main entry point for the CLI application. // Entry is the main entry point for the CLI application.
// It prints the startup banner to stderr (unless a banner-suppressing // It prints the startup banner to stdout (unless a banner-suppressing
// flag is present in os.Args — see bannerSuppressedInArgs), executes the // flag is present in os.Args — see bannerSuppressedInArgs), executes the
// root cobra command, and routes any returned error through the // root cobra command, and routes any returned error through the
// ui.Writer so the user sees a properly formatted "🛑 ERROR:" line. // ui.Writer so the user sees a properly formatted "🛑 ERROR:" line.
// The banner goes to stderr because stdout carries only the output the
// user asked for, such as a completion script or a `config get` value.
// //
// It returns the process exit code (0 on success, 130 when interrupted, // It returns the process exit code (0 on success, 1 on error) rather
// 1 on any other error) rather than calling os.Exit, so that main's // than calling os.Exit, so that main's deferred profile writers run
// deferred profile writers run before the process ends. See run in // before the process ends. See run in cmd/vaultik/main.go.
// cmd/vaultik/main.go.
func Entry() int { func Entry() int {
emitStartupBanner(os.Args[1:], os.Stderr) emitStartupBanner(os.Args[1:], os.Stdout)
rootCmd := NewRootCommand() rootCmd := NewRootCommand()
rootCmd.SilenceErrors = true rootCmd.SilenceErrors = true
@@ -41,20 +33,14 @@ func Entry() int {
err := rootCmd.Execute() err := rootCmd.Execute()
if err != nil { if err != nil {
// An operation that ran inside the fx app has already reported // An operation that ran inside the fx app has already reported
// its own failure (`snapshot verify --json` puts it in the // its own failure (and suppressed it under --json); errReported
// document instead); errReported says so. Printing it again // says so. Printing it again here would double the error line.
// here would double the error line.
// Every other error — bad arguments, a config that would not // Every other error — bad arguments, a config that would not
// load, an interrupt — reaches Entry unreported, so it is shown // load — reaches Entry unreported, so it is shown here.
// here.
if !errors.Is(err, errReported) { if !errors.Is(err, errReported) {
ReportErrorf("%s", err.Error()) ReportErrorf("%s", err.Error())
} }
if errors.Is(err, errInterrupted) {
return exitCodeInterrupted
}
return 1 return 1
} }
@@ -63,8 +49,9 @@ func Entry() int {
// emitStartupBanner writes the startup banner to w unless args (the // emitStartupBanner writes the startup banner to w unless args (the
// argument vector with the program name already stripped) contains a // argument vector with the program name already stripped) contains a
// flag that suppresses it. Split out of Entry so that the decision is // flag that suppresses it. Split out of Entry so that the decision — the
// reachable from a test without running the whole CLI. // only thing standing between a --json invocation and a parseable
// stdout — is reachable from a test without running the whole CLI.
func emitStartupBanner(args []string, w io.Writer) { func emitStartupBanner(args []string, w io.Writer) {
if bannerSuppressedInArgs(args) { if bannerSuppressedInArgs(args) {
return return
@@ -99,7 +86,8 @@ func ReportErrorf(format string, args ...any) {
// --json is a subcommand flag rather than a persistent one, but so is // --json is a subcommand flag rather than a persistent one, but so is
// --cron (it exists only on `snapshot create`), so this adds no new // --cron (it exists only on `snapshot create`), so this adds no new
// class of imprecision. The only cost of a false positive is a missing // class of imprecision. The only cost of a false positive is a missing
// decorative banner. // decorative banner; the cost of a false negative is a corrupt document
// on stdout, so the scan errs deliberately in that direction.
func bannerSuppressedInArgs(args []string) bool { func bannerSuppressedInArgs(args []string) bool {
for _, a := range args { for _, a := range args {
if a == "--" { if a == "--" {
+49 -25
View File
@@ -36,14 +36,23 @@ const (
// strips it before scanning, so it has to be present. // strips it before scanning, so it has to be present.
programName = "vaultik" programName = "vaultik"
// someSnapshotID only fills the positional argument; no test needs // someSnapshotID is any snapshot identifier: these tests never run
// the snapshot to exist. // the command, so it only has to occupy the positional argument.
someSnapshotID = "host_2026-01-01T00:00:00Z" someSnapshotID = "host_2026-01-01T00:00:00Z"
) )
// placeholderJSONDocument stands in for whatever document a --json
// command writes to stdout. `snapshot list --json` with no snapshots
// prints exactly this; the other --json commands print an object rather
// than an array, but this test is not about their shape. It is about
// what is on stdout *before* them, which is the same for all of them
// because Entry prints the banner before cobra has parsed anything and
// therefore before it can know which command is running.
const placeholderJSONDocument = "[]\n"
// jsonArgumentVectors are the argument vectors of every --json // jsonArgumentVectors are the argument vectors of every --json
// invocation the CLI accepts, with the program name stripped exactly as // invocation the CLI accepts, with the program name stripped exactly as
// Entry strips it. Each one must suppress the banner. // Entry strips it. Each one must leave stdout untouched by the banner.
// //
//nolint:gochecknoglobals // read-only test fixture shared by two tests //nolint:gochecknoglobals // read-only test fixture shared by two tests
var jsonArgumentVectors = map[string][]string{ var jsonArgumentVectors = map[string][]string{
@@ -65,23 +74,39 @@ var jsonArgumentVectors = map[string][]string{
}, },
} }
// TestJSONInvocationSuppressesBanner checks that every --json // TestJSONInvocationStdoutIsExactlyOneDocument is the CLI-layer
// invocation suppresses the startup banner, as the README says --json // regression guard for issue #106: `vaultik snapshot list --json | jq`
// does along with --quiet and --cron. The scan is over the raw argument // must work with no other flags.
// vector, so each position and spelling of --json is listed. //
func TestJSONInvocationSuppressesBanner(t *testing.T) { // internal/vaultik's TestListSnapshots_JSONStdoutIsOnlyTheDocument
// guards the same contract one layer down, but it calls the library
// function directly and so cannot see Entry, which is where the
// contamination was: the startup banner is written to stdout before
// cobra parses anything, and the suppression scan did not know about
// --json. The two banner lines and the blank line landed ahead of the
// document and `jq` refused the result.
//
// The document is a constant here because this test is about the
// argument vectors, one per --json command; the one that runs a real
// command end to end is TestEntryJSONStdoutIsExactlyOneDocument below.
func TestJSONInvocationStdoutIsExactlyOneDocument(t *testing.T) {
t.Parallel() t.Parallel()
for name, argv := range jsonArgumentVectors { for name, argv := range jsonArgumentVectors {
t.Run(name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
t.Parallel() t.Parallel()
var banner bytes.Buffer var stdout bytes.Buffer
emitStartupBanner(argv, &banner) emitStartupBanner(argv, &stdout)
assert.Empty(t, banner.String(), require.Empty(t, stdout.String(),
"--json suppresses the banner") "nothing may reach stdout ahead of a --json document")
_, err := stdout.WriteString(placeholderJSONDocument)
require.NoError(t, err)
requireExactlyOneJSONDocument(t, stdout.String())
}) })
} }
} }
@@ -102,11 +127,11 @@ func TestBannerStillPrintedWithoutSuppressingFlag(t *testing.T) {
t.Run(name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
t.Parallel() t.Parallel()
var banner bytes.Buffer var stdout bytes.Buffer
emitStartupBanner(argv, &banner) emitStartupBanner(argv, &stdout)
assert.Contains(t, banner.String(), "starting up at", assert.Contains(t, stdout.String(), "starting up at",
"the banner belongs on invocations that did not opt out") "the banner belongs on invocations that did not opt out")
}) })
} }
@@ -147,9 +172,9 @@ func TestBannerSuppressedInArgs(t *testing.T) {
// hermeticConfig is a complete, valid config that needs no network and // hermeticConfig is a complete, valid config that needs no network and
// no credentials: file:// storage is exempt from the S3 credential // no credentials: file:// storage is exempt from the S3 credential
// checks. A test that lists the destination must create its directory // checks, and FileStorer over a directory that does not exist lists
// first, because listing a directory that does not exist is an error. // zero objects without erroring. Chunk, blob and compression settings
// Chunk, blob and compression settings are filled in by config.Load. // are filled in by config.Load.
const hermeticConfig = `age_recipients: const hermeticConfig = `age_recipients:
- age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj - age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj
snapshots: snapshots:
@@ -171,25 +196,22 @@ hostname: test-host
// `snapshot list` is the command chosen because it is the only --json // `snapshot list` is the command chosen because it is the only --json
// command that reaches its document without a populated destination // command that reaches its document without a populated destination
// store: it reads the local index, streams `metadata/` (empty here), // store: it reads the local index, streams `metadata/` (empty here),
// and treats an empty destination directory as an empty list rather // and treats a barren destination as an empty list rather than a
// than a failure. // failure.
// //
// Not parallel: it replaces os.Args, os.Stdout and the xdg globals. // Not parallel: it replaces os.Args, os.Stdout and the xdg globals.
func TestEntryJSONStdoutIsExactlyOneDocument(t *testing.T) { func TestEntryJSONStdoutIsExactlyOneDocument(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
configPath := filepath.Join(dir, "config.yml") configPath := filepath.Join(dir, "config.yml")
storeDir := filepath.Join(dir, "store")
contents := fmt.Sprintf(hermeticConfig, contents := fmt.Sprintf(hermeticConfig,
filepath.Join(dir, "source"), filepath.Join(dir, "source"),
storeDir, filepath.Join(dir, "store"),
filepath.Join(dir, "index.sqlite")) filepath.Join(dir, "index.sqlite"))
require.NoError(t, require.NoError(t,
os.WriteFile(configPath, []byte(contents), configFileMode)) os.WriteFile(configPath, []byte(contents), configFileMode))
require.NoError(t, os.Mkdir(storeDir, 0o750))
// The PID lock lives under xdg.DataHome, which xdg resolves at // The PID lock lives under xdg.DataHome, which xdg resolves at
// package init; point it at the temp dir so the test neither // package init; point it at the temp dir so the test neither
// touches nor collides with the real one. // touches nor collides with the real one.
@@ -222,7 +244,9 @@ func TestEntryJSONStdoutIsExactlyOneDocument(t *testing.T) {
// captureProcessStdout redirects the process's own stdout to a pipe for // captureProcessStdout redirects the process's own stdout to a pipe for
// the duration of fn and returns what was written to it. The redirection // the duration of fn and returns what was written to it. The redirection
// has to be at the file-descriptor level rather than through an injected // has to be at the file-descriptor level rather than through an injected
// writer, because the commands Entry runs reach os.Stdout directly. // writer, because the banner and the JSON encoder reach os.Stdout
// independently and the point of the test is that both land in the same
// place.
// //
// Not parallel-safe: os.Stdout is process-global. // Not parallel-safe: os.Stdout is process-global.
func captureProcessStdout(t *testing.T, fn func()) string { func captureProcessStdout(t *testing.T, fn func()) string {
-203
View File
@@ -1,203 +0,0 @@
package cli //nolint:testpackage // shares runEntry and the argument constants
import (
"fmt"
"net/http"
"net/http/httptest"
"os"
"os/signal"
"path/filepath"
"strings"
"testing"
"time"
"github.com/adrg/xdg"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// stalledStoreConfig is hermeticConfig with an s3:// destination store
// in place of the file:// one. The server behind it accepts any
// credentials.
const stalledStoreConfig = `age_recipients:
- age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj
snapshots:
test:
paths:
- %s
storage_url: s3://bucket?endpoint=%s&ssl=false
s3:
access_key_id: key
secret_access_key: secret
index_path: %s
hostname: test-host
`
// interruptRepeat is how often interruptOnFirstRequest sends SIGINT.
const interruptRepeat = 50 * time.Millisecond
// TestEntryInterruptedRun sends SIGINT to the test process while a
// command waits on the destination store, and checks that Entry returns
// 130 and prints one line on stderr saying the run was interrupted. The
// store is a local HTTP server that holds every request open, so the
// command is always mid-operation when the signal arrives. The two
// cases cover --cron and --json, which silence other output.
//
// Not parallel: it signals the process and replaces os.Args, os.Stdout,
// os.Stderr and the xdg globals.
//
//nolint:paralleltest // signals the process and replaces process globals
func TestEntryInterruptedRun(t *testing.T) {
for _, testCase := range []struct {
name string
args []string
}{
{
name: "snapshot create --cron",
args: []string{cmdSnapshot, cmdCreate, "--cron"},
},
{
name: "snapshot verify --json",
args: []string{cmdSnapshot, cmdVerify, someSnapshotID, flagJSON},
},
} {
t.Run(testCase.name, func(t *testing.T) {
endpoint, requestArrived := startStalledStore(t)
configPath := writeStalledStoreConfig(t, endpoint)
interruptOnFirstRequest(t, requestArrived)
code, _, stderr := runEntry(t,
append([]string{flagConfig, configPath}, testCase.args...)...)
assert.Equal(t, 130, code)
assert.Equal(t, 1,
strings.Count(stderr, errInterrupted.Error()), stderr)
})
}
}
// interruptOnFirstRequest sends SIGINT to the test process every
// interruptRepeat, from the first request to the destination store until
// the test ends. One signal is not enough: the command can reach the
// store before fx has started catching signals. The test catches SIGINT
// too, so that a signal fx is not catching does not kill the test
// binary.
func interruptOnFirstRequest(t *testing.T, requestArrived <-chan struct{}) {
t.Helper()
self, err := os.FindProcess(os.Getpid())
require.NoError(t, err)
caught := make(chan os.Signal, 1)
signal.Notify(caught, os.Interrupt)
testEnded := make(chan struct{})
senderDone := make(chan struct{})
// Stop catching SIGINT only after the sender has returned. The sender
// waits for each SIGINT it sends to arrive on caught; one still on
// its way after signal.Stop would kill the test binary.
t.Cleanup(func() {
close(testEnded)
<-senderDone
signal.Stop(caught)
})
go func() {
defer close(senderDone)
select {
case <-requestArrived:
case <-testEnded:
return
}
ticker := time.NewTicker(interruptRepeat)
defer ticker.Stop()
for {
// Empty caught, so that the receive below waits for this
// SIGINT rather than an earlier one.
select {
case <-caught:
default:
}
sendErr := self.Signal(os.Interrupt)
if sendErr != nil {
t.Errorf("sending SIGINT: %v", sendErr)
return
}
<-caught
select {
case <-testEnded:
return
case <-ticker.C:
}
}
}()
}
// startStalledStore starts an HTTP server that never answers: each
// request is held until the client gives up on it or the test ends.
// It returns the server's host:port and a channel that receives a value
// when the first request arrives.
func startStalledStore(t *testing.T) (string, <-chan struct{}) {
t.Helper()
requestArrived := make(chan struct{}, 1)
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(
func(_ http.ResponseWriter, r *http.Request) {
select {
case requestArrived <- struct{}{}:
default:
}
select {
case <-r.Context().Done():
case <-release:
}
}))
// Cleanups run last-registered first, so release lets any held
// request return before Close waits for it.
t.Cleanup(server.Close)
t.Cleanup(func() { close(release) })
return server.Listener.Addr().String(), requestArrived
}
// writeStalledStoreConfig writes a config whose destination store is the
// server at endpoint and whose snapshot source holds one small file, so
// that `snapshot create` has a blob to upload. Returns the config path.
func writeStalledStoreConfig(t *testing.T, endpoint string) string {
t.Helper()
dir := t.TempDir()
configPath := filepath.Join(dir, "config.yml")
sourceDir := filepath.Join(dir, "source")
require.NoError(t, os.Mkdir(sourceDir, 0o750))
require.NoError(t, os.WriteFile(filepath.Join(sourceDir, "file.txt"),
[]byte("contents"), 0o600))
contents := fmt.Sprintf(stalledStoreConfig,
sourceDir, endpoint, filepath.Join(dir, "index.sqlite"))
require.NoError(t,
os.WriteFile(configPath, []byte(contents), configFileMode))
// The PID lock lives under xdg.DataHome, which xdg resolves at
// package init; point it at the temp dir so the test neither
// touches nor collides with the real one.
t.Setenv("XDG_DATA_HOME", filepath.Join(dir, "data"))
xdg.Reload()
t.Cleanup(xdg.Reload)
return configPath
}
+5 -10
View File
@@ -98,30 +98,25 @@ func TestEntryPruneJSONStdoutIsExactlyOneDocument(t *testing.T) {
} }
} }
// writeHermeticPruneConfig builds a config over a temp directory with an // writeHermeticPruneConfig builds a config over a temp directory and, if
// empty destination directory and, if seedStale is set, creates the // seedStale is set, creates the index database up front with one
// index database up front with one snapshot record that has no // snapshot record that has no counterpart on the destination store.
// counterpart on the destination store. Returns the config path. // Returns the config path.
func writeHermeticPruneConfig(t *testing.T, seedStale bool) string { func writeHermeticPruneConfig(t *testing.T, seedStale bool) string {
t.Helper() t.Helper()
dir := t.TempDir() dir := t.TempDir()
configPath := filepath.Join(dir, "config.yml") configPath := filepath.Join(dir, "config.yml")
indexPath := filepath.Join(dir, "index.sqlite") indexPath := filepath.Join(dir, "index.sqlite")
storeDir := filepath.Join(dir, "store")
contents := fmt.Sprintf(hermeticConfig, contents := fmt.Sprintf(hermeticConfig,
filepath.Join(dir, "source"), filepath.Join(dir, "source"),
storeDir, filepath.Join(dir, "store"),
indexPath) indexPath)
require.NoError(t, require.NoError(t,
os.WriteFile(configPath, []byte(contents), configFileMode)) os.WriteFile(configPath, []byte(contents), configFileMode))
// prune fails on a destination directory that does not exist, so the
// empty store is created here rather than left to a first backup.
require.NoError(t, os.Mkdir(storeDir, 0o750))
// The PID lock lives under xdg.DataHome, which xdg resolves at // The PID lock lives under xdg.DataHome, which xdg resolves at
// package init; point it at the temp dir so the test neither // package init; point it at the temp dir so the test neither
// touches nor collides with the real one. // touches nor collides with the real one.
+3 -3
View File
@@ -15,8 +15,8 @@ import (
// run, so a failing command must come back with a non-zero code rather // run, so a failing command must come back with a non-zero code rather
// than ending the process here. // than ending the process here.
// //
// Stdout and stderr are captured only to keep the banner and command // Stdout is captured only to keep the banner and command output off the
// output off the test log; the assertion is on the returned code. // test log; the assertion is on the returned code.
// //
//nolint:paralleltest // replaces os.Args and rootFlags //nolint:paralleltest // replaces os.Args and rootFlags
func TestEntryReturnsStatusCode(t *testing.T) { func TestEntryReturnsStatusCode(t *testing.T) {
@@ -50,7 +50,7 @@ func TestEntryReturnsStatusCode(t *testing.T) {
var code int var code int
_, _ = captureProcessStdoutAndStderr(t, func() { code = Entry() }) _ = captureProcessStdout(t, func() { code = Entry() })
assert.Equal(t, testCase.want, code) assert.Equal(t, testCase.want, code)
}) })
-200
View File
@@ -1,200 +0,0 @@
package cli //nolint:testpackage // shares hermeticConfig and the capture helpers
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"os"
"path/filepath"
"strings"
"testing"
"github.com/adrg/xdg"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/database"
)
// TestEntryCompletionStdoutIsTheScript runs `vaultik completion bash`,
// whose stdout the README tells the user to source. The script has to
// start on the first line.
//
//nolint:paralleltest // replaces os.Args, os.Stdout and os.Stderr
func TestEntryCompletionStdoutIsTheScript(t *testing.T) {
code, stdout, _ := runEntry(t, "completion", "bash")
require.Equal(t, 0, code)
firstLine, _, _ := strings.Cut(stdout, "\n")
assert.True(t, strings.HasPrefix(firstLine, "# bash completion"),
"the first line of stdout must be the script's, got %q", firstLine)
}
// TestEntryConfigGetStdoutIsTheValue runs `vaultik config get`, whose
// stdout a script reads as the value and nothing else.
//
//nolint:paralleltest // replaces os.Args, os.Stdout and os.Stderr
func TestEntryConfigGetStdoutIsTheValue(t *testing.T) {
configPath := filepath.Join(t.TempDir(), "config.yml")
require.NoError(t, os.WriteFile(configPath,
[]byte("hostname: test-host\n"), configFileMode))
code, stdout, _ := runEntry(t,
flagConfig, configPath, "config", "get", "hostname")
require.Equal(t, 0, code)
assert.Equal(t, "test-host\n", stdout)
}
// TestEntryJSONFailureIsReportedOnStderr runs each --json command that
// writes no document when it fails, against a destination it cannot
// use. The error must reach stderr, and stdout must stay empty.
//
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
func TestEntryJSONFailureIsReportedOnStderr(t *testing.T) {
for _, testCase := range []struct {
name string
args []string
wantOnStderr string
}{
{
name: "remote info",
args: []string{cmdRemote, cmdInfo, flagJSON},
wantOnStderr: "Failed to get remote info",
},
{
name: "prune",
args: []string{cmdPrune, flagJSON},
wantOnStderr: "Prune failed",
},
{
name: "snapshot remove",
args: []string{cmdSnapshot, cmdRemove, someSnapshotID, flagJSON},
wantOnStderr: "Failed to remove snapshot",
},
} {
t.Run(testCase.name, func(t *testing.T) {
configPath := writeUnusableDestinationConfig(t)
code, stdout, stderr := runEntry(t,
append([]string{flagConfig, configPath}, testCase.args...)...)
assert.Equal(t, 1, code)
assert.Empty(t, stdout,
"a failed --json command has no document to write")
assert.Contains(t, stderr, testCase.wantOnStderr,
"the failure must be reported on stderr")
})
}
}
// TestEntrySnapshotRemoveJSONWarningIsOnStderr runs `snapshot remove
// --json` on a snapshot in the local index, against a destination
// directory that does not exist. The command removes the snapshot from
// the local index and still exits 0. Its stdout must hold the document
// alone, with the warning about the destination store on stderr: the
// command to run again once it is reachable, and the snapshot's ID in
// the record's snapshot_id field.
//
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
func TestEntrySnapshotRemoveJSONWarningIsOnStderr(t *testing.T) {
configPath, indexPath := writeMissingDestinationConfig(t)
seedStaleSnapshotRecord(t, indexPath)
code, stdout, stderr := runEntry(t, flagConfig, configPath,
cmdSnapshot, cmdRemove, stalePruneSnapshotID, flagJSON)
require.Equal(t, 0, code)
requireExactlyOneJSONDocument(t, stdout)
// stderr is a pipe here, so the logger writes one JSON record a line.
var warning map[string]any
for line := range strings.Lines(stderr) {
if strings.Contains(line,
"Could not remove snapshot metadata from remote storage") {
require.NoError(t, json.Unmarshal([]byte(line), &warning))
}
}
require.NotNil(t, warning, "the warning must reach stderr")
assert.Contains(t, warning[slog.MessageKey],
"run 'vaultik snapshot remove' with the snapshot's ID again")
assert.Equal(t, stalePruneSnapshotID, warning["snapshot_id"])
}
// writeMissingDestinationConfig builds a config whose destination
// directory does not exist. Returns the config path and the path of
// its local index, which is not created here.
func writeMissingDestinationConfig(t *testing.T) (string, string) {
t.Helper()
dir := t.TempDir()
configPath := filepath.Join(dir, "config.yml")
indexPath := filepath.Join(dir, "index.sqlite")
contents := fmt.Sprintf(hermeticConfig,
filepath.Join(dir, "source"),
filepath.Join(dir, "missing-store"),
indexPath)
require.NoError(t,
os.WriteFile(configPath, []byte(contents), configFileMode))
// The PID lock lives under xdg.DataHome, which xdg resolves at
// package init; point it at the temp dir so the test neither
// touches nor collides with the real one.
t.Setenv("XDG_DATA_HOME", filepath.Join(dir, "data"))
xdg.Reload()
t.Cleanup(xdg.Reload)
return configPath, indexPath
}
// writeUnusableDestinationConfig builds a config whose destination
// directory does not exist, which fails `remote info`, and whose local
// index is bound to another destination, which fails `prune` and
// `snapshot remove` (a missing destination alone only makes `snapshot
// remove` warn). Returns the config path.
func writeUnusableDestinationConfig(t *testing.T) string {
t.Helper()
configPath, indexPath := writeMissingDestinationConfig(t)
ctx := context.Background()
db, err := database.New(ctx, indexPath)
require.NoError(t, err)
defer func() { require.NoError(t, db.Close()) }()
require.NoError(t, database.NewRepositories(db).LocalMeta.Set(ctx,
database.LocalMetaKeyStorageURL, "file://"+t.TempDir()))
return configPath
}
// runEntry runs Entry with args after the program name and returns its
// exit code and what it wrote to stdout and stderr.
//
// Not parallel-safe: it replaces os.Args, os.Stdout and os.Stderr.
func runEntry(t *testing.T, args ...string) (int, string, string) {
t.Helper()
previousArgs := os.Args
t.Cleanup(func() {
os.Args = previousArgs
rootFlags = RootFlags{}
})
os.Args = append([]string{programName}, args...)
var code int
stdout, stderr := captureProcessStdoutAndStderr(t,
func() { code = Entry() })
return code, stdout, stderr
}
+7 -5
View File
@@ -22,11 +22,9 @@ scans every snapshot manifest in the destination store, builds the
set of still-referenced blob hashes, and deletes any blob not in that set of still-referenced blob hashes, and deletes any blob not in that
set. set.
Snapshot create --prune runs the same cleanup automatically; this Snapshot create --prune and snapshot remove run the same cleanup
command is the manual entry point for the same work (e.g. after a automatically; this command is the manual entry point for the same
crashed backup or to reclaim storage). Snapshot remove leaves blobs in work (e.g. after a crashed backup or to reclaim storage).`,
place; run this command afterwards to delete the ones no longer
referenced.`,
Args: cobra.NoArgs, Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error { RunE: func(cmd *cobra.Command, _ []string) error {
// Use unified config resolution // Use unified config resolution
@@ -50,6 +48,10 @@ referenced.`,
}, func(v *vaultik.Vaultik) error { }, func(v *vaultik.Vaultik) error {
return v.Prune(opts) return v.Prune(opts)
}, func(err error) { }, func(err error) {
if opts.JSON {
return
}
log.Error("Prune operation failed", "error", err) log.Error("Prune operation failed", "error", err)
ReportErrorf("Prune failed: %v", err) ReportErrorf("Prune failed: %v", err)
}) })
+5 -1
View File
@@ -45,7 +45,7 @@ This is destructive and irreversible. Requires --force.`,
return errNukeNeedsForce return errNukeNeedsForce
} }
return runVaultikApp(cmd, mutating, false, "Remote nuke failed", return runVaultikApp(cmd, mutating, false, false, "Remote nuke failed",
func(v *vaultik.Vaultik) error { func(v *vaultik.Vaultik) error {
return v.NukeRemote(true) return v.NukeRemote(true)
}) })
@@ -92,6 +92,10 @@ func newRemoteInfoCommand() *cobra.Command {
}, func(v *vaultik.Vaultik) error { }, func(v *vaultik.Vaultik) error {
return v.RemoteInfo(jsonOutput) return v.RemoteInfo(jsonOutput)
}, func(err error) { }, func(err error) {
if jsonOutput {
return
}
log.Error("Failed to get remote info", "error", err) log.Error("Failed to get remote info", "error", err)
ReportErrorf("Failed to get remote info: %v", err) ReportErrorf("Failed to get remote info: %v", err)
}) })
+1 -2
View File
@@ -60,8 +60,7 @@ on the source system.`,
cmd.PersistentFlags().BoolVar(&rootFlags.SkipErrors, "skip-errors", false, cmd.PersistentFlags().BoolVar(&rootFlags.SkipErrors, "skip-errors", false,
"Skip files that cannot be read when creating a snapshot, or "+ "Skip files that cannot be read when creating a snapshot, or "+
"that cannot be restored when restoring, instead of aborting "+ "that cannot be restored when restoring, instead of aborting "+
"(packing and storage errors while creating a snapshot still "+ "(packing and storage errors still abort)")
"abort)")
// Add subcommands // Add subcommands
cmd.AddCommand( cmd.AddCommand(
+7 -9
View File
@@ -66,9 +66,8 @@ func newSnapshotCreateCommand() *cobra.Command {
If snapshot names are provided, only those snapshots are created. If snapshot names are provided, only those snapshots are created.
If no names are provided, all configured snapshots are created. If no names are provided, all configured snapshots are created.
The config is read from the path given by --config or VAULTIK_CONFIG; Config is located at /etc/vaultik/config.yml by default, but can be overridden by
otherwise from the platform config directory (~/.config/vaultik/config.yml specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
on Linux), then /etc/vaultik/config.yml.`,
Args: cobra.ArbitraryArgs, Args: cobra.ArbitraryArgs,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
// Pass snapshot names from args // Pass snapshot names from args
@@ -127,7 +126,7 @@ func newSnapshotListCommand() *cobra.Command {
Long: "Lists all snapshots with their ID, timestamp, and compressed size", Long: "Lists all snapshots with their ID, timestamp, and compressed size",
Args: cobra.NoArgs, Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error { RunE: func(cmd *cobra.Command, _ []string) error {
return runVaultikApp(cmd, readOnly, false, return runVaultikApp(cmd, readOnly, false, false,
"Failed to list snapshots", "Failed to list snapshots",
func(v *vaultik.Vaultik) error { func(v *vaultik.Vaultik) error {
return v.ListSnapshots(jsonOutput) return v.ListSnapshots(jsonOutput)
@@ -163,7 +162,7 @@ restrict the operation to specific snapshot names.`,
return errPurgeCriteriaBoth return errPurgeCriteriaBoth
} }
return runVaultikApp(cmd, mutating, false, return runVaultikApp(cmd, mutating, false, false,
"Failed to purge snapshots", "Failed to purge snapshots",
func(v *vaultik.Vaultik) error { func(v *vaultik.Vaultik) error {
return v.PurgeSnapshotsWithOptions(opts) return v.PurgeSnapshotsWithOptions(opts)
@@ -259,15 +258,14 @@ Use --local-only to skip the remote half (e.g. when you want to forget a
snapshot locally without touching the destination store). snapshot locally without touching the destination store).
If the remote is unreachable, the local-database removal still completes If the remote is unreachable, the local-database removal still completes
and a warning is emitted; run 'vaultik snapshot remove <snapshot-id>' again and a warning is emitted; rerun 'vaultik prune' once the destination store
once the destination store is reachable to remove the snapshot's metadata is reachable to finish remote cleanup.
from it ('vaultik prune' does not).
To wipe the entire destination store and start over, use 'vaultik remote To wipe the entire destination store and start over, use 'vaultik remote
nuke --force' — it is the single supported entry point for that.`, nuke --force' — it is the single supported entry point for that.`,
Args: requireSnapshotIDArg, Args: requireSnapshotIDArg,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runVaultikApp(cmd, mutating, opts.JSON, return runVaultikApp(cmd, mutating, opts.JSON, opts.JSON,
"Failed to remove snapshot", "Failed to remove snapshot",
func(v *vaultik.Vaultik) error { func(v *vaultik.Vaultik) error {
_, err := v.RemoveSnapshot(args[0], opts) _, err := v.RemoveSnapshot(args[0], opts)
+16 -30
View File
@@ -33,19 +33,18 @@ const secretKeyPrefix = "AGE-SECRET-KEY-"
const ( const (
defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB defaultBlobSizeLimit = Size(10 * 1024 * 1024 * 1024) // 10GB
defaultChunkSize = Size(10 * 1024 * 1024) // 10MB defaultChunkSize = Size(10 * 1024 * 1024) // 10MB
defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB defaultS3PartSize = Size(5 * 1024 * 1024) // 5MB
defaultCompressionLevel = 3 defaultCompressionLevel = 3
minChunkSize = 1024 * 1024 // 1MB minChunkSize = 1024 * 1024 // 1MB
minCompressionLevel = 1 minCompressionLevel = 1
maxCompressionLevel = 19 maxCompressionLevel = 19
// S3 accepts a multipart upload part from 5MiB to 5GiB.
minS3PartSize = 5 * 1024 * 1024
maxS3PartSize = 5 * 1024 * 1024 * 1024
) )
// Sentinel validation errors. // Sentinel validation errors.
var ( var (
errNoConfigPath = errors.New("config path not provided") errNoConfigPath = errors.New("config path not provided")
errNoAgeRecipients = errors.New(
"at least one age_recipient is required (generate with: age-keygen)")
errRecipientIsSecretKey = errors.New( errRecipientIsSecretKey = errors.New(
"an age secret key was given where a public key (age1...) belongs") "an age secret key was given where a public key (age1...) belongs")
errRecipientNotX25519 = errors.New( errRecipientNotX25519 = errors.New(
@@ -58,7 +57,6 @@ var (
"blob_size_limit must be at least the largest chunk the chunker can " + "blob_size_limit must be at least the largest chunk the chunker can " +
"emit (chunk_size times the FastCDC size spread)") "emit (chunk_size times the FastCDC size spread)")
errBadCompression = errors.New("compression_level must be between 1 and 19") errBadCompression = errors.New("compression_level must be between 1 and 19")
errBadS3PartSize = errors.New("s3.part_size must be between 5MiB and 5GiB")
errBadStorageScheme = errors.New( errBadStorageScheme = errors.New(
"storage_url must start with s3://, file://, or rclone://") "storage_url must start with s3://, file://, or rclone://")
errStorageNotConfigured = errors.New( errStorageNotConfigured = errors.New(
@@ -248,7 +246,6 @@ func Load(path string) (*Config, error) {
ChunkSize: defaultChunkSize, ChunkSize: defaultChunkSize,
IndexPath: filepath.Join(xdg.DataHome, appName, "index.sqlite"), IndexPath: filepath.Join(xdg.DataHome, appName, "index.sqlite"),
CompressionLevel: defaultCompressionLevel, CompressionLevel: defaultCompressionLevel,
S3: S3Config{PartSize: defaultS3PartSize},
} }
// Convert smartconfig data to YAML then unmarshal // Convert smartconfig data to YAML then unmarshal
@@ -299,13 +296,17 @@ func Load(path string) (*Config, error) {
cfg.S3.Region = "us-east-1" cfg.S3.Region = "us-east-1"
} }
if cfg.S3.PartSize == 0 {
cfg.S3.PartSize = defaultS3PartSize
}
// Check config file permissions (warn if world or group readable) // Check config file permissions (warn if world or group readable)
//nolint:gosec // G703: config path is operator-supplied by design //nolint:gosec // G703: config path is operator-supplied by design
info, statErr := os.Stat(path) info, statErr := os.Stat(path)
if statErr == nil { if statErr == nil {
mode := info.Mode().Perm() mode := info.Mode().Perm()
if mode&0044 != 0 { // group or world readable if mode&0044 != 0 { // group or world readable
log.Warn(cfg.readableByOthersWarning(), log.Warn("Config file has insecure permissions (contains S3 credentials)",
"path", path, "path", path,
"mode", fmt.Sprintf("%04o", mode), "mode", fmt.Sprintf("%04o", mode),
"recommendation", "chmod 600 "+path) "recommendation", "chmod 600 "+path)
@@ -322,10 +323,9 @@ func Load(path string) (*Config, error) {
// Validate checks if the configuration is valid and complete. // Validate checks if the configuration is valid and complete.
// It ensures all required fields are present and have valid values: // It ensures all required fields are present and have valid values:
// - Every age recipient must parse as an X25519 age1... public key (so a // - At least one age recipient must be specified, and every recipient must
// bad entry fails at load, not mid-backup); errors name the position, // parse as an X25519 age1... public key (so a bad entry fails at load, not
// never the value. An empty list is accepted, because only snapshot // mid-backup); errors name the position, never the value
// create needs a recipient and it checks for one itself
// - At least one snapshot must be configured with at least one path // - At least one snapshot must be configured with at least one path
// - Storage must be configured (either storage_url or s3.* fields) // - Storage must be configured (either storage_url or s3.* fields)
// - Chunk size must be at least 1MB // - Chunk size must be at least 1MB
@@ -333,10 +333,13 @@ func Load(path string) (*Config, error) {
// (chunk_size times chunker.ChunkSizeSpread), so a single-chunk blob never // (chunk_size times chunker.ChunkSizeSpread), so a single-chunk blob never
// exceeds the configured limit // exceeds the configured limit
// - Compression level must be between 1 and 19 // - Compression level must be between 1 and 19
// - S3 part size must be between 5MiB and 5GiB, the part sizes S3 accepts
// //
// Returns an error describing the first validation failure encountered. // Returns an error describing the first validation failure encountered.
func (c *Config) Validate() error { func (c *Config) Validate() error {
if len(c.AgeRecipients) == 0 {
return errNoAgeRecipients
}
for i, recipient := range c.AgeRecipients { for i, recipient := range c.AgeRecipients {
err := validateAgeRecipient(recipient) err := validateAgeRecipient(recipient)
if err != nil { if err != nil {
@@ -378,11 +381,6 @@ func (c *Config) Validate() error {
return errBadCompression return errBadCompression
} }
if c.S3.PartSize.Int64() < minS3PartSize ||
c.S3.PartSize.Int64() > maxS3PartSize {
return errBadS3PartSize
}
return nil return nil
} }
@@ -418,18 +416,6 @@ func (c *Config) setAgeSecretKey() {
} }
} }
// readableByOthersWarning is the warning Load logs when others can read
// the config file. It says "may contain" because the S3 credentials are
// seen only after smartconfig has replaced any ${...} reference in the
// file with its value, so a set credential need not be in the file.
func (c *Config) readableByOthersWarning() string {
if c.S3.AccessKeyID != "" || c.S3.SecretAccessKey != "" {
return "Config file is readable by others and may contain S3 credentials"
}
return "Config file is readable by others"
}
// validateStorage validates storage configuration. // validateStorage validates storage configuration.
// If StorageURL is set, it takes precedence. S3 URLs require credentials. // If StorageURL is set, it takes precedence. S3 URLs require credentials.
// File URLs don't require any S3 configuration. // File URLs don't require any S3 configuration.
+2 -247
View File
@@ -8,7 +8,6 @@ import (
"testing" "testing"
"sneak.berlin/go/vaultik/internal/chunker" "sneak.berlin/go/vaultik/internal/chunker"
"sneak.berlin/go/vaultik/internal/log"
) )
const ( const (
@@ -167,7 +166,6 @@ func TestValidateBlobSizeLimit(t *testing.T) {
ChunkSize: chunkSize, ChunkSize: chunkSize,
BlobSizeLimit: blobLimit, BlobSizeLimit: blobLimit,
CompressionLevel: 3, CompressionLevel: 3,
S3: S3Config{PartSize: defaultS3PartSize},
} }
} }
@@ -223,124 +221,9 @@ func TestValidateBlobSizeLimit(t *testing.T) {
} }
} }
// TestValidateS3PartSize checks that s3.part_size is held to the part sizes
// S3 accepts, 5MiB to 5GiB, by changing only the part size of the test
// config. "5MB" in the config file is 5,000,000 bytes, below the minimum.
func TestValidateS3PartSize(t *testing.T) {
t.Parallel()
base, err := Load(os.Getenv("VAULTIK_CONFIG"))
if err != nil {
t.Fatalf("Failed to load config: %v", err)
}
tests := []struct {
name string
partSize Size
wantErr bool
}{
{
name: "5MB is rejected",
partSize: 5_000_000,
wantErr: true,
},
{
name: "one byte below 5MiB is rejected",
partSize: minS3PartSize - 1,
wantErr: true,
},
{
name: "5MiB is accepted",
partSize: minS3PartSize,
wantErr: false,
},
{
name: "5GiB is accepted",
partSize: maxS3PartSize,
wantErr: false,
},
{
name: "one byte above 5GiB is rejected",
partSize: maxS3PartSize + 1,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
cfg := *base
cfg.S3.PartSize = tt.partSize
err := cfg.Validate()
if tt.wantErr {
if !errors.Is(err, errBadS3PartSize) {
t.Fatalf("Validate() error = %v, want errBadS3PartSize", err)
}
return
}
if err != nil {
t.Fatalf("Validate() unexpected error: %v", err)
}
})
}
}
// TestLoadS3PartSize checks that a config file without s3.part_size loads
// with the 5MiB default, and that an explicit 0 fails at load like any other
// part size S3 refuses.
func TestLoadS3PartSize(t *testing.T) {
t.Parallel()
const withoutPartSize = "snapshots:\n" +
" test:\n" +
" paths: [/tmp/vaultik-test-source]\n" +
"storage_url: file:///tmp/vaultik-test-storage\n"
writeConfig := func(t *testing.T, text string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "config.yml")
err := os.WriteFile(path, []byte(text), 0o600)
if err != nil {
t.Fatalf("write config: %v", err)
}
return path
}
t.Run("absent loads as 5MiB", func(t *testing.T) {
t.Parallel()
cfg, err := Load(writeConfig(t, withoutPartSize))
if err != nil {
t.Fatalf("Load() unexpected error: %v", err)
}
if cfg.S3.PartSize != defaultS3PartSize {
t.Errorf("s3.part_size = %d, want %d",
cfg.S3.PartSize, defaultS3PartSize)
}
})
t.Run("0 is rejected", func(t *testing.T) {
t.Parallel()
_, err := Load(writeConfig(t, withoutPartSize+"s3:\n part_size: 0\n"))
if !errors.Is(err, errBadS3PartSize) {
t.Fatalf("Load() error = %v, want errBadS3PartSize", err)
}
})
}
// TestValidateAgeRecipients checks that recipients are parsed at config load // TestValidateAgeRecipients checks that recipients are parsed at config load
// (a bad entry fails immediately, not mid-backup) and that no invalid entry — // (a bad entry fails immediately, not mid-backup) and that no invalid entry —
// least of all a pasted secret key — is echoed in the error. An empty list // least of all a pasted secret key — is echoed in the error.
// loads, because only snapshot create needs a recipient.
func TestValidateAgeRecipients(t *testing.T) { func TestValidateAgeRecipients(t *testing.T) {
t.Parallel() t.Parallel()
@@ -352,7 +235,6 @@ func TestValidateAgeRecipients(t *testing.T) {
ChunkSize: Size(10 * 1024 * 1024), ChunkSize: Size(10 * 1024 * 1024),
BlobSizeLimit: Size(10 * 1024 * 1024 * 1024), BlobSizeLimit: Size(10 * 1024 * 1024 * 1024),
CompressionLevel: 3, CompressionLevel: 3,
S3: S3Config{PartSize: defaultS3PartSize},
} }
} }
@@ -362,12 +244,7 @@ func TestValidateAgeRecipients(t *testing.T) {
wantErr bool wantErr bool
}{ }{
{ {
name: "no recipients is accepted", name: "config init placeholder is rejected",
recipients: nil,
wantErr: false,
},
{
name: "placeholder recipient is rejected",
recipients: []string{"age1REPLACE_WITH_YOUR_PUBLIC_KEY"}, recipients: []string{"age1REPLACE_WITH_YOUR_PUBLIC_KEY"},
wantErr: true, wantErr: true,
}, },
@@ -460,125 +337,3 @@ func TestAgeSecretKeySourceName(t *testing.T) {
}) })
} }
} }
// loadReadableConfig writes configYAML to a file that others can read,
// loads it, and returns what the logger wrote to stderr meanwhile. The
// logger writes to the os.Stderr it finds when it is initialized, so
// os.Stderr is pointed at a file first. Not parallel-safe: os.Stderr and
// the logger are process-global.
func loadReadableConfig(t *testing.T, configYAML string) string {
t.Helper()
dir := t.TempDir()
configPath := filepath.Join(dir, "config.yml")
stderrPath := filepath.Join(dir, "stderr")
err := os.WriteFile(configPath, []byte(configYAML), 0o600)
if err != nil {
t.Fatalf("writing config: %v", err)
}
//nolint:gosec // G302: the test needs a config file others can read
err = os.Chmod(configPath, 0o644)
if err != nil {
t.Fatalf("chmod config: %v", err)
}
stderrFile, err := os.Create(stderrPath) //nolint:gosec // G304: test temp path
if err != nil {
t.Fatalf("creating stderr file: %v", err)
}
previous := os.Stderr
os.Stderr = stderrFile
log.Initialize(log.Config{})
_, loadErr := Load(configPath)
os.Stderr = previous
log.Initialize(log.Config{})
_ = stderrFile.Close()
if loadErr != nil {
t.Fatalf("Load() error = %v", loadErr)
}
captured, err := os.ReadFile(stderrPath) //nolint:gosec // G304: test temp path
if err != nil {
t.Fatalf("reading stderr file: %v", err)
}
return string(captured)
}
// TestLoadWarnsReadableConfigWithoutS3Credentials checks that a config
// file others can read, holding no S3 credentials, is warned about
// without a claim that it holds them.
//
//nolint:paralleltest // loadReadableConfig replaces os.Stderr
func TestLoadWarnsReadableConfigWithoutS3Credentials(t *testing.T) {
stderr := loadReadableConfig(t, `
storage_url: file:///var/backups/vaultik
snapshots:
home:
paths:
- /home
`)
if !strings.Contains(stderr, "Config file is readable by others") {
t.Errorf("expected a warning that the file is readable by others, got %q",
stderr)
}
if strings.Contains(stderr, "S3 credentials") {
t.Errorf("warning names S3 credentials the file does not set: %q", stderr)
}
}
// TestLoadWarnsReadableConfigWithS3Credentials checks that a config file
// others can read and that sets S3 credentials, as values or as ${ENV:...}
// references, is warned about as one that may contain them.
//
//nolint:paralleltest // loadReadableConfig replaces os.Stderr
func TestLoadWarnsReadableConfigWithS3Credentials(t *testing.T) {
t.Setenv("VAULTIK_TEST_ACCESS_KEY_ID", "test-access-key")
t.Setenv("VAULTIK_TEST_SECRET_ACCESS_KEY", "test-secret-key")
configs := map[string]string{
"values": `
storage_url: s3://bucket/prefix?endpoint=s3.example.com
s3:
access_key_id: test-access-key
secret_access_key: test-secret-key
snapshots:
home:
paths:
- /home
`,
"references": `
storage_url: s3://bucket/prefix?endpoint=s3.example.com
s3:
access_key_id: ${ENV:VAULTIK_TEST_ACCESS_KEY_ID}
secret_access_key: ${ENV:VAULTIK_TEST_SECRET_ACCESS_KEY}
snapshots:
home:
paths:
- /home
`,
}
for name, configYAML := range configs {
t.Run(name, func(t *testing.T) {
stderr := loadReadableConfig(t, configYAML)
if !strings.Contains(stderr,
"Config file is readable by others and may contain S3 credentials") {
t.Errorf("expected a warning naming the S3 credentials, got %q",
stderr)
}
})
}
}
+2
View File
@@ -16,6 +16,8 @@ var (
// Size represents a byte size that can be specified in configuration files. // Size represents a byte size that can be specified in configuration files.
// It can unmarshal from both numeric values (interpreted as bytes) and // It can unmarshal from both numeric values (interpreted as bytes) and
// human-readable strings like "10MB", "2.5GB", or "1TB". // human-readable strings like "10MB", "2.5GB", or "1TB".
//
//nolint:recvcheck // UnmarshalYAML requires a pointer; String/Int64 are value reads
type Size int64 type Size int64
// UnmarshalYAML implements yaml.Unmarshaler for Size, allowing it to be // UnmarshalYAML implements yaml.Unmarshaler for Size, allowing it to be
+1 -1
View File
@@ -220,7 +220,7 @@ func (r *ChunkFileRepository) CreateBatch(
cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length) cf.ChunkHash.String(), cf.FileID.String(), cf.FileOffset, cf.Length)
} }
query += querySb183.String() query += querySb183.String() //nolint:gosec // G202: appends "?" placeholders only
query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING" query += " ON CONFLICT(chunk_hash, file_id) DO NOTHING"
+1 -1
View File
@@ -98,7 +98,7 @@ func (r *ChunkRepository) GetByHashes(
args[i] = hash args[i] = hash
} }
query += querySb75.String() query += querySb75.String() //nolint:gosec // G202: appends "?" placeholders only
query += ") ORDER BY chunk_hash" query += ") ORDER BY chunk_hash"
+44 -39
View File
@@ -42,10 +42,6 @@ var schemaFS embed.FS
// table itself. It is applied before the normal migration loop. // table itself. It is applied before the normal migration loop.
const bootstrapVersion = 0 const bootstrapVersion = 0
// busyTimeoutMs is how long a connection to the index waits for another
// connection's lock before failing with "database is locked".
const busyTimeoutMs = 10000
// DB represents the Vaultik local index database connection. // DB represents the Vaultik local index database connection.
// It uses SQLite to track file metadata, content-defined chunks, and blob associations. // It uses SQLite to track file metadata, content-defined chunks, and blob associations.
// The database enables incremental backups by detecting changed files and // The database enables incremental backups by detecting changed files and
@@ -98,27 +94,10 @@ func ParseMigrationVersion(filename string) (int, error) {
return version, nil return version, nil
} }
// indexDSN returns the driver DSN that opens the database at path. The
// driver runs each _pragma parameter on every connection it opens and drops
// any parameter it does not know without an error, so a setting written in
// another form silently does nothing. In WAL mode one connection can write
// while others read; the busy timeout makes a connection wait for a lock
// instead of failing at once.
func indexDSN(path string) string {
return fmt.Sprintf(
"%s?_pragma=busy_timeout(%d)&_pragma=journal_mode(WAL)"+
"&_pragma=synchronous(NORMAL)&_pragma=foreign_keys(1)",
path, busyTimeoutMs)
}
// New creates a new database connection at the specified path. // New creates a new database connection at the specified path.
// It creates the schema if needed. Every connection runs in WAL mode with // It creates the schema if needed and configures SQLite with WAL mode for
// a busy timeout and foreign keys on (see indexDSN), so a read-only command // better concurrency. SQLite handles crash recovery automatically when
// can read the index while a backup writes to it. Committed rows can sit in // opening a database with journal/WAL files present.
// the -wal file beside the database until a checkpoint, so a copy of the
// database file alone may miss them.
// SQLite handles crash recovery automatically when opening a database with
// journal/WAL files present.
// The path parameter can be a file path for persistent storage or ":memory:" // The path parameter can be a file path for persistent storage or ":memory:"
// for an in-memory database (useful for testing). // for an in-memory database (useful for testing).
func New(ctx context.Context, path string) (*DB, error) { func New(ctx context.Context, path string) (*DB, error) {
@@ -131,7 +110,11 @@ func New(ctx context.Context, path string) (*DB, error) {
// First attempt with standard WAL mode // First attempt with standard WAL mode
log.Debug("Attempting to open database with WAL mode", "path", path) log.Debug("Attempting to open database with WAL mode", "path", path)
conn, err := sql.Open("sqlite", indexDSN(path)) conn, err := sql.Open(
"sqlite",
path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000"+
"&_locking_mode=NORMAL&_foreign_keys=ON",
)
if err == nil { if err == nil {
configureConnPool(conn) configureConnPool(conn)
@@ -151,8 +134,8 @@ func New(ctx context.Context, path string) (*DB, error) {
_ = conn.Close() _ = conn.Close()
} }
// If the first attempt failed, try once more // If first attempt failed, try with TRUNCATE mode to clear any locks
return retryOpen(ctx, path) return openWithRecovery(ctx, path)
} }
// configureConnPool serializes all database access through one connection. // configureConnPool serializes all database access through one connection.
@@ -164,12 +147,18 @@ func configureConnPool(conn *sql.DB) {
conn.SetMaxIdleConns(1) conn.SetMaxIdleConns(1)
} }
// finishOpen wraps the connection and applies any pending migrations. On // finishOpen enables foreign keys, wraps the connection, and applies any
// migration failure the connection is closed. // pending migrations. On migration failure the connection is closed.
func finishOpen(ctx context.Context, conn *sql.DB, path string) (*DB, error) { func finishOpen(ctx context.Context, conn *sql.DB, path string) (*DB, error) {
// Enable foreign keys explicitly
_, err := conn.ExecContext(ctx, "PRAGMA foreign_keys = ON")
if err != nil {
log.Warn("Failed to enable foreign keys", "path", path, "error", err)
}
db := &DB{conn: conn, path: path} db := &DB{conn: conn, path: path}
err := applyMigrations(ctx, conn) err = applyMigrations(ctx, conn)
if err != nil { if err != nil {
_ = conn.Close() _ = conn.Close()
@@ -179,15 +168,21 @@ func finishOpen(ctx context.Context, conn *sql.DB, path string) (*DB, error) {
return db, nil return db, nil
} }
// retryOpen makes a second attempt to open the database, with the same // openWithRecovery retries opening the database in TRUNCATE journal mode to
// settings, after the first attempt failed, for example because another // clear stale locks, then switches back to WAL mode.
// process held a lock for longer than the busy timeout. func openWithRecovery(ctx context.Context, path string) (*DB, error) {
func retryOpen(ctx context.Context, path string) (*DB, error) { log.Info(
log.Info("Database appears locked, retrying open", "path", path) "Database appears locked, attempting recovery with TRUNCATE mode",
"path", path,
)
conn, err := sql.Open("sqlite", indexDSN(path)) conn, err := sql.Open(
"sqlite",
path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000"+
"&_foreign_keys=ON",
)
if err != nil { if err != nil {
return nil, fmt.Errorf("opening database on retry: %w", err) return nil, fmt.Errorf("opening database in recovery mode: %w", err)
} }
configureConnPool(conn) configureConnPool(conn)
@@ -195,18 +190,28 @@ func retryOpen(ctx context.Context, path string) (*DB, error) {
err = conn.PingContext(ctx) err = conn.PingContext(ctx)
if err != nil { if err != nil {
log.Debug( log.Debug(
"Failed to ping database on retry, closing", "Failed to ping database in recovery mode, closing",
"path", path, "error", err, "path", path, "error", err,
) )
_ = conn.Close() _ = conn.Close()
return nil, fmt.Errorf( return nil, fmt.Errorf(
"database still locked on retry: %w", "database still locked after recovery attempt: %w",
err, err,
) )
} }
log.Debug("Database opened in TRUNCATE mode", "path", path)
// Switch back to WAL mode
log.Debug("Switching database back to WAL mode", "path", path)
_, err = conn.ExecContext(ctx, "PRAGMA journal_mode=WAL")
if err != nil {
log.Warn("Failed to switch back to WAL mode", "path", path, "error", err)
}
db, err := finishOpen(ctx, conn, path) db, err := finishOpen(ctx, conn, path)
if err != nil { if err != nil {
return nil, err return nil, err
-83
View File
@@ -120,89 +120,6 @@ func TestDatabaseConcurrentAccess(t *testing.T) {
} }
} }
// TestNewSetsJournalModeAndBusyTimeout checks that the connection settings
// New passes reach SQLite. The driver drops a setting it does not recognise
// without an error, so only reading the value back shows it took effect.
func TestNewSetsJournalModeAndBusyTimeout(t *testing.T) {
t.Parallel()
ctx := context.Background()
db, err := New(ctx, filepath.Join(t.TempDir(), "index.db"))
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
var journalMode string
err = db.conn.QueryRowContext(ctx, "PRAGMA journal_mode").Scan(&journalMode)
if err != nil {
t.Fatalf("reading journal_mode: %v", err)
}
if journalMode != "wal" {
t.Errorf("journal_mode = %q, want %q", journalMode, "wal")
}
var busyTimeout int
err = db.conn.QueryRowContext(ctx, "PRAGMA busy_timeout").Scan(&busyTimeout)
if err != nil {
t.Fatalf("reading busy_timeout: %v", err)
}
if busyTimeout != busyTimeoutMs {
t.Errorf("busy_timeout = %d, want %d", busyTimeout, busyTimeoutMs)
}
}
// TestNewWriteSucceedsWhileAnotherHandleReads opens the same index twice,
// as a read-only command does while a backup runs, and checks that a write
// on one handle commits while the other is in the middle of a read.
func TestNewWriteSucceedsWhileAnotherHandleReads(t *testing.T) {
t.Parallel()
ctx := context.Background()
dbPath := filepath.Join(t.TempDir(), "index.db")
reader, err := New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to open reading handle: %v", err)
}
defer func() { _ = reader.Close() }()
writer, err := New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to open writing handle: %v", err)
}
defer func() { _ = writer.Close() }()
// The read lock taken by the SELECT is held until the transaction ends.
readTx, err := reader.BeginTx(ctx, nil)
if err != nil {
t.Fatalf("beginning read transaction: %v", err)
}
defer func() { _ = readTx.Rollback() }()
var count int
err = readTx.QueryRowContext(ctx, "SELECT COUNT(*) FROM chunks").Scan(&count)
if err != nil {
t.Fatalf("reading chunks: %v", err)
}
_, err = writer.ExecWithLog(ctx,
"INSERT INTO chunks (chunk_hash, size) VALUES (?, ?)", "hash", 1024)
if err != nil {
t.Fatalf("write while another handle reads: %v", err)
}
}
func TestParseMigrationVersion(t *testing.T) { func TestParseMigrationVersion(t *testing.T) {
t.Parallel() t.Parallel()
+1 -1
View File
@@ -253,7 +253,7 @@ func (r *FileChunkRepository) CreateBatch(
args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String()) args = append(args, fc.FileID.String(), fc.Idx, fc.ChunkHash.String())
} }
query += querySb211.String() query += querySb211.String() //nolint:gosec // G202: appends "?" placeholders only
query += " ON CONFLICT(file_id, idx) DO NOTHING" query += " ON CONFLICT(file_id, idx) DO NOTHING"
+26 -40
View File
@@ -33,13 +33,11 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
} }
query := ` query := `
INSERT INTO files INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target)
(id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(path) DO UPDATE SET ON CONFLICT(path) DO UPDATE SET
source_path = excluded.source_path, source_path = excluded.source_path,
mtime = excluded.mtime, mtime = excluded.mtime,
mtime_nsec = excluded.mtime_nsec,
size = excluded.size, size = excluded.size,
mode = excluded.mode, mode = excluded.mode,
uid = excluded.uid, uid = excluded.uid,
@@ -56,19 +54,16 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
if tx != nil { if tx != nil {
LogSQL("Execute", query, LogSQL("Execute", query,
file.ID.String(), file.Path.String(), file.SourcePath.String(), file.ID.String(), file.Path.String(), file.SourcePath.String(),
file.MTime.Unix(), file.MTime.Nanosecond(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
file.Size, file.Mode, file.UID, file.GID,
file.LinkTarget.String()) file.LinkTarget.String())
err = tx.QueryRowContext(ctx, query, err = tx.QueryRowContext(ctx, query,
file.ID.String(), file.Path.String(), file.SourcePath.String(), file.ID.String(), file.Path.String(), file.SourcePath.String(),
file.MTime.Unix(), file.MTime.Nanosecond(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
file.Size, file.Mode, file.UID, file.GID,
file.LinkTarget.String()).Scan(&idStr) file.LinkTarget.String()).Scan(&idStr)
} else { } else {
err = r.db.QueryRowWithLog(ctx, query, err = r.db.QueryRowWithLog(ctx, query,
file.ID.String(), file.Path.String(), file.SourcePath.String(), file.ID.String(), file.Path.String(), file.SourcePath.String(),
file.MTime.Unix(), file.MTime.Nanosecond(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
file.Size, file.Mode, file.UID, file.GID,
file.LinkTarget.String()).Scan(&idStr) file.LinkTarget.String()).Scan(&idStr)
} }
@@ -89,7 +84,7 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
// in the index. // in the index.
func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, error) { func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, error) {
query := ` query := `
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files FROM files
WHERE path = ? WHERE path = ?
` `
@@ -109,7 +104,7 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err
// GetByID retrieves a file by its UUID // GetByID retrieves a file by its UUID
func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, error) { func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, error) {
query := ` query := `
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files FROM files
WHERE id = ? WHERE id = ?
` `
@@ -132,7 +127,7 @@ func (r *FileRepository) GetByPathTx(
ctx context.Context, tx *sql.Tx, path string, ctx context.Context, tx *sql.Tx, path string,
) (*File, error) { ) (*File, error) {
query := ` query := `
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files FROM files
WHERE path = ? WHERE path = ?
` `
@@ -163,14 +158,13 @@ func (r *FileRepository) ListModifiedSince(
ctx context.Context, since time.Time, ctx context.Context, since time.Time,
) ([]*File, error) { ) ([]*File, error) {
query := ` query := `
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files FROM files
WHERE (mtime, mtime_nsec) >= (?, ?) WHERE mtime >= ?
ORDER BY path ORDER BY path
` `
rows, err := r.db.conn.QueryContext(ctx, query, rows, err := r.db.conn.QueryContext(ctx, query, since.Unix())
since.Unix(), since.Nanosecond())
if err != nil { if err != nil {
return nil, fmt.Errorf("querying files: %w", err) return nil, fmt.Errorf("querying files: %w", err)
} }
@@ -234,24 +228,19 @@ func (r *FileRepository) DeleteByID(
return nil return nil
} }
// ListUnderPath returns the file at path and every file beneath it, // ListByPrefix returns all files whose path starts with prefix, ordered by
// ordered by path. Paths are compared case-sensitively, and a trailing // path.
// slash on path is ignored, so "/" lists every file. func (r *FileRepository) ListByPrefix(
func (r *FileRepository) ListUnderPath( ctx context.Context, prefix string,
ctx context.Context, path string,
) ([]*File, error) { ) ([]*File, error) {
path = strings.TrimRight(path, "/")
dirPrefix := path + "/"
// LIKE would ignore ASCII case and treat _ and % in path as wildcards.
query := ` query := `
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files FROM files
WHERE path = ? OR substr(path, 1, length(?)) = ? WHERE path LIKE ? || '%'
ORDER BY path ORDER BY path
` `
rows, err := r.db.conn.QueryContext(ctx, query, path, dirPrefix, dirPrefix) rows, err := r.db.conn.QueryContext(ctx, query, prefix)
if err != nil { if err != nil {
return nil, fmt.Errorf("querying files: %w", err) return nil, fmt.Errorf("querying files: %w", err)
} }
@@ -330,7 +319,7 @@ func (r *FileRepository) ListIDsWithChunksNotInUploadedBlobs(
// ListAll returns all files in the database // ListAll returns all files in the database
func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) { func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
query := ` query := `
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files FROM files
ORDER BY path ORDER BY path
` `
@@ -371,7 +360,7 @@ func (r *FileRepository) CreateBatch(
} }
// Each files row binds this many SQL variables. // Each files row binds this many SQL variables.
const fileCols = 10 const fileCols = 9
// Batch at 100 rows to be safe with SQLite's variable limit. // Batch at 100 rows to be safe with SQLite's variable limit.
const batchSize = 100 const batchSize = 100
@@ -382,7 +371,7 @@ func (r *FileRepository) CreateBatch(
batch := files[i:end] batch := files[i:end]
query := `INSERT INTO files query := `INSERT INTO files
(id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target) (id, path, source_path, mtime, size, mode, uid, gid, link_target)
VALUES ` VALUES `
args := make([]any, 0, len(batch)*fileCols) args := make([]any, 0, len(batch)*fileCols)
@@ -394,21 +383,19 @@ func (r *FileRepository) CreateBatch(
querySb325.WriteString(", ") querySb325.WriteString(", ")
} }
querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)")
args = append(args, args = append(args,
f.ID.String(), f.Path.String(), f.SourcePath.String(), f.ID.String(), f.Path.String(), f.SourcePath.String(),
f.MTime.Unix(), f.MTime.Nanosecond(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID,
f.Size, f.Mode, f.UID, f.GID,
f.LinkTarget.String()) f.LinkTarget.String())
} }
query += querySb325.String() query += querySb325.String() //nolint:gosec // G202: appends "?" placeholders only
query += ` ON CONFLICT(path) DO UPDATE SET query += ` ON CONFLICT(path) DO UPDATE SET
source_path = excluded.source_path, source_path = excluded.source_path,
mtime = excluded.mtime, mtime = excluded.mtime,
mtime_nsec = excluded.mtime_nsec,
size = excluded.size, size = excluded.size,
mode = excluded.mode, mode = excluded.mode,
uid = excluded.uid, uid = excluded.uid,
@@ -468,7 +455,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) {
var ( var (
file File file File
idStr, pathStr, sourcePathStr string idStr, pathStr, sourcePathStr string
mtimeUnix, mtimeNsec int64 mtimeUnix int64
linkTarget sql.NullString linkTarget sql.NullString
) )
@@ -477,7 +464,6 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) {
&pathStr, &pathStr,
&sourcePathStr, &sourcePathStr,
&mtimeUnix, &mtimeUnix,
&mtimeNsec,
&file.Size, &file.Size,
&file.Mode, &file.Mode,
&file.UID, &file.UID,
@@ -496,7 +482,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) {
file.Path = types.FilePath(pathStr) file.Path = types.FilePath(pathStr)
file.SourcePath = types.SourcePath(sourcePathStr) file.SourcePath = types.SourcePath(sourcePathStr)
file.MTime = time.Unix(mtimeUnix, mtimeNsec).UTC() file.MTime = time.Unix(mtimeUnix, 0).UTC()
if linkTarget.Valid { if linkTarget.Valid {
file.LinkTarget = types.FilePath(linkTarget.String) file.LinkTarget = types.FilePath(linkTarget.String)
} }
-187
View File
@@ -5,12 +5,10 @@ import (
"database/sql" "database/sql"
"errors" "errors"
"os" "os"
"slices"
"testing" "testing"
"time" "time"
"sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
) )
// errTestRollback is the sentinel returned from transaction bodies to // errTestRollback is the sentinel returned from transaction bodies to
@@ -136,82 +134,6 @@ func TestFileRepositoryListDelete(t *testing.T) {
} }
} }
func TestFileRepositoryListUnderPath(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := database.NewFileRepository(db)
const (
docDir = "/home/u/doc"
docFile = "/home/u/doc/a.txt"
)
// In path order, so the root case can expect all of them as listed.
paths := []string{
"/home/u/50%/x.txt",
"/home/u/50percent/y.txt",
"/home/u/DOC/c.txt",
"/home/u/a_b/x.txt",
"/home/u/axb/y.txt",
docDir,
"/home/u/doc.txt.bak",
docFile,
"/home/u/doc/sub/b.txt",
"/home/u/doc2/b.txt",
}
for _, path := range paths {
err := repo.Create(ctx, nil, &database.File{
Path: types.FilePath(path),
MTime: time.Now().Truncate(time.Second),
Mode: 0644,
})
if err != nil {
t.Fatalf("failed to create %s: %v", path, err)
}
}
docTree := []string{docDir, docFile, "/home/u/doc/sub/b.txt"}
tests := []struct {
name string
path string
want []string
}{
{"directory", docDir, docTree},
{"directory with trailing slash", docDir + "/", docTree},
{"directory differing only in case", "/home/u/DOC",
[]string{"/home/u/DOC/c.txt"}},
{"file", docFile, []string{docFile}},
{"underscore is literal", "/home/u/a_b",
[]string{"/home/u/a_b/x.txt"}},
{"percent is literal", "/home/u/50%",
[]string{"/home/u/50%/x.txt"}},
{"root", "/", paths},
}
for _, tt := range tests {
files, err := repo.ListUnderPath(ctx, tt.path)
if err != nil {
t.Fatalf("%s: failed to list files: %v", tt.name, err)
}
got := make([]string, 0, len(files))
for _, f := range files {
got = append(got, f.Path.String())
}
if !slices.Equal(got, tt.want) {
t.Errorf("%s: listing %q got %q, want %q",
tt.name, tt.path, got, tt.want)
}
}
}
func TestFileRepositorySymlink(t *testing.T) { func TestFileRepositorySymlink(t *testing.T) {
t.Parallel() t.Parallel()
@@ -252,115 +174,6 @@ func TestFileRepositorySymlink(t *testing.T) {
} }
} }
// An mtime after 2262 or before 1678 does not fit in int64 nanoseconds
// since the epoch, and must still come back from the database unchanged.
func TestFileRepositoryMTimeOutsideInt64NanosecondRange(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := database.NewFileRepository(db)
mtimes := []time.Time{
time.Date(2300, time.January, 1, 0, 0, 0, 123456789, time.UTC),
time.Date(1601, time.January, 1, 0, 0, 0, 987654321, time.UTC),
}
for _, mtime := range mtimes {
created := &database.File{
Path: types.FilePath("/created-" + mtime.Format(time.RFC3339Nano)),
MTime: mtime,
}
err := repo.Create(ctx, nil, created)
if err != nil {
t.Fatalf("failed to create file: %v", err)
}
batched := &database.File{
ID: types.NewFileID(),
Path: types.FilePath("/batched-" + mtime.Format(time.RFC3339Nano)),
MTime: mtime,
}
err = repo.CreateBatch(ctx, nil, []*database.File{batched})
if err != nil {
t.Fatalf("failed to batch create file: %v", err)
}
for _, path := range []types.FilePath{created.Path, batched.Path} {
retrieved, err := repo.GetByPath(ctx, path.String())
if err != nil {
t.Fatalf("failed to get file: %v", err)
}
if !retrieved.MTime.Equal(mtime) {
t.Errorf("%s: mtime got %v, want %v",
path, retrieved.MTime, mtime)
}
}
}
}
// A file already in the index and rewritten within the same second must get
// its new nanoseconds stored, through both Create and CreateBatch.
func TestFileRepositoryUpsertMTimeInSameSecond(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := database.NewFileRepository(db)
indexed := time.Date(2026, time.October, 7, 12, 0, 0, 100000000, time.UTC)
rewritten := indexed.Add(800 * time.Millisecond)
tests := []struct {
name string
upsert func(file *database.File) error
}{
{"Create", func(file *database.File) error {
return repo.Create(ctx, nil, file)
}},
{"CreateBatch", func(file *database.File) error {
return repo.CreateBatch(ctx, nil, []*database.File{file})
}},
}
for _, tt := range tests {
file := &database.File{
ID: types.NewFileID(),
Path: types.FilePath("/" + tt.name),
MTime: indexed,
}
err := tt.upsert(file)
if err != nil {
t.Fatalf("%s: failed to create file: %v", tt.name, err)
}
file.MTime = rewritten
err = tt.upsert(file)
if err != nil {
t.Fatalf("%s: failed to update file: %v", tt.name, err)
}
retrieved, err := repo.GetByPath(ctx, file.Path.String())
if err != nil {
t.Fatalf("%s: failed to get file: %v", tt.name, err)
}
if !retrieved.MTime.Equal(rewritten) {
t.Errorf("%s: mtime got %v, want %v",
tt.name, retrieved.MTime, rewritten)
}
}
}
func TestFileRepositoryTransaction(t *testing.T) { func TestFileRepositoryTransaction(t *testing.T) {
t.Parallel() t.Parallel()
+4 -3
View File
@@ -14,7 +14,8 @@ type File struct {
ID types.FileID // UUID primary key ID types.FileID // UUID primary key
Path types.FilePath // Absolute path of the file Path types.FilePath // Absolute path of the file
// SourcePath is the source directory this file came from. // SourcePath is the source directory this file came from (used for
// restore path stripping).
SourcePath types.SourcePath SourcePath types.SourcePath
MTime time.Time MTime time.Time
Size int64 Size int64
@@ -98,8 +99,8 @@ type Snapshot struct {
StartedAt time.Time StartedAt time.Time
CompletedAt *time.Time // nil if still in progress CompletedAt *time.Time // nil if still in progress
FileCount int64 FileCount int64
ChunkCount int64 // Chunks this snapshot stored that were not stored before ChunkCount int64
BlobCount int64 // Blobs this snapshot created BlobCount int64
TotalSize int64 // Total size of all referenced files TotalSize int64 // Total size of all referenced files
// BlobSize is the total size of all referenced blobs (compressed and // BlobSize is the total size of all referenced blobs (compressed and
@@ -824,7 +824,7 @@ func TestTransactionIsolation(t *testing.T) {
} }
// Verify the file was not created (transaction rolled back) // Verify the file was not created (transaction rolled back)
files, err := repos.Files.ListUnderPath(ctx, "/tx-test.txt") files, err := repos.Files.ListByPrefix(ctx, "/tx-test")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -916,7 +916,7 @@ func TestConcurrentOrphanedCleanup(t *testing.T) {
} }
// Verify correct files were deleted // Verify correct files were deleted
files, err := repos.Files.ListAll(ctx) files, err := repos.Files.ListByPrefix(ctx, "/concurrent-")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
+1 -1
View File
@@ -147,7 +147,7 @@ func TestOrphanedFileCleanupDebug(t *testing.T) {
t.Logf("Files count after cleanup: %d", count) t.Logf("Files count after cleanup: %d", count)
// List remaining files // List remaining files
files, err := repos.Files.ListUnderPath(ctx, "/") files, err := repos.Files.ListByPrefix(ctx, "/")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
+27 -34
View File
@@ -3,7 +3,6 @@ package database
import ( import (
"context" "context"
"database/sql"
"fmt" "fmt"
"strings" "strings"
"testing" "testing"
@@ -368,7 +367,7 @@ func verifyBlobNullUploadTS(
} }
// createLargeDatasetFiles creates fileCount files and adds every other // createLargeDatasetFiles creates fileCount files and adds every other
// one to the snapshot, in one transaction as a backup writes them. // one to the snapshot.
func createLargeDatasetFiles( func createLargeDatasetFiles(
t *testing.T, t *testing.T,
repos *Repositories, repos *Repositories,
@@ -377,38 +376,31 @@ func createLargeDatasetFiles(
) { ) {
t.Helper() t.Helper()
ctx := context.Background()
start := time.Now() start := time.Now()
err := repos.WithTx(context.Background(), for i := range fileCount {
func(ctx context.Context, tx *sql.Tx) error { file := &File{
for i := range fileCount { Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)),
file := &File{ MTime: time.Now(),
Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)), Size: int64(i * 1024),
MTime: time.Now(), Mode: 0644,
Size: int64(i * 1024), UID: uint32(1000 + (i % 10)),
Mode: 0644, GID: uint32(1000 + (i % 10)),
UID: uint32(1000 + (i % 10)), }
GID: uint32(1000 + (i % 10)),
}
err := repos.Files.Create(ctx, tx, file) err := repos.Files.Create(ctx, nil, file)
if err != nil { if err != nil {
return fmt.Errorf("creating file %d: %w", i, err) t.Fatalf("failed to create file %d: %v", i, err)
} }
// Add half to snapshot // Add half to snapshot
if i%2 == 0 { if i%2 == 0 {
err = repos.Snapshots.AddFileByID(ctx, tx, snapshotID, file.ID) err = repos.Snapshots.AddFileByID(ctx, nil, snapshotID, file.ID)
if err != nil { if err != nil {
return err t.Fatal(err)
}
}
} }
}
return nil
})
if err != nil {
t.Fatal(err)
} }
t.Logf("Created %d files in %v", fileCount, time.Since(start)) t.Logf("Created %d files in %v", fileCount, time.Since(start))
@@ -450,12 +442,12 @@ func TestLargeDatasets(t *testing.T) {
createLargeDatasetFiles(t, repos, snapshot.ID.String(), fileCount) createLargeDatasetFiles(t, repos, snapshot.ID.String(), fileCount)
}) })
// Test ListUnderPath performance // Test ListByPrefix performance
//nolint:paralleltest // phases share one database and are order-dependent //nolint:paralleltest // phases share one database and are order-dependent
t.Run("list under path performance", func(t *testing.T) { t.Run("list by prefix performance", func(t *testing.T) {
start := time.Now() start := time.Now()
files, err := repos.Files.ListUnderPath(ctx, "/large/") files, err := repos.Files.ListByPrefix(ctx, "/large/")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -480,7 +472,7 @@ func TestLargeDatasets(t *testing.T) {
t.Logf("Cleaned up orphaned files in %v", time.Since(start)) t.Logf("Cleaned up orphaned files in %v", time.Since(start))
// Verify correct number remain // Verify correct number remain
files, err := repos.Files.ListUnderPath(ctx, "/large/") files, err := repos.Files.ListByPrefix(ctx, "/large/")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -614,7 +606,8 @@ func TestTimezoneHandling(t *testing.T) {
t.Skip("timezone not available") t.Skip("timezone not available")
} }
nyTime := time.Now().In(loc) // Use Truncate to remove sub-second precision since we store as Unix timestamps
nyTime := time.Now().In(loc).Truncate(time.Second)
file := &File{ file := &File{
Path: "/timezone-test.txt", Path: "/timezone-test.txt",
MTime: nyTime, MTime: nyTime,
+2 -3
View File
@@ -5,9 +5,8 @@
CREATE TABLE IF NOT EXISTS files ( CREATE TABLE IF NOT EXISTS files (
id TEXT PRIMARY KEY, -- UUID id TEXT PRIMARY KEY, -- UUID
path TEXT NOT NULL UNIQUE, path TEXT NOT NULL UNIQUE,
source_path TEXT NOT NULL DEFAULT '', -- The source directory this file came from source_path TEXT NOT NULL DEFAULT '', -- The source directory this file came from (for restore path stripping)
mtime INTEGER NOT NULL, -- whole seconds since the Unix epoch mtime INTEGER NOT NULL,
mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second, 0 to 999999999
size INTEGER NOT NULL, size INTEGER NOT NULL,
mode INTEGER NOT NULL, mode INTEGER NOT NULL,
uid INTEGER NOT NULL, uid INTEGER NOT NULL,
+4 -29
View File
@@ -127,7 +127,6 @@ func (r *SnapshotRepository) UpdateExtendedStats(
snapshotID string, snapshotID string,
blobUncompressedSize int64, blobUncompressedSize int64,
compressionLevel int, compressionLevel int,
uploadBytes int64,
uploadDurationMs int64, uploadDurationMs int64,
) error { ) error {
compressionRatio, err := r.extendedCompressionRatio( compressionRatio, err := r.extendedCompressionRatio(
@@ -142,7 +141,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(
SET blob_uncompressed_size = ?, SET blob_uncompressed_size = ?,
compression_ratio = ?, compression_ratio = ?,
compression_level = ?, compression_level = ?,
upload_bytes = ?, upload_bytes = blob_size,
upload_duration_ms = ? upload_duration_ms = ?
WHERE id = ? WHERE id = ?
` `
@@ -150,11 +149,11 @@ func (r *SnapshotRepository) UpdateExtendedStats(
if tx != nil { if tx != nil {
_, err = tx.ExecContext(ctx, query, _, err = tx.ExecContext(ctx, query,
blobUncompressedSize, compressionRatio, compressionLevel, blobUncompressedSize, compressionRatio, compressionLevel,
uploadBytes, uploadDurationMs, snapshotID) uploadDurationMs, snapshotID)
} else { } else {
_, err = r.db.ExecWithLog(ctx, query, _, err = r.db.ExecWithLog(ctx, query,
blobUncompressedSize, compressionRatio, compressionLevel, blobUncompressedSize, compressionRatio, compressionLevel,
uploadBytes, uploadDurationMs, snapshotID) uploadDurationMs, snapshotID)
} }
if err != nil { if err != nil {
@@ -396,7 +395,7 @@ func (r *SnapshotRepository) AddFilesByIDBatch(
args = append(args, snapshotID, fileID.String()) args = append(args, snapshotID, fileID.String())
} }
query += querySb312.String() query += querySb312.String() //nolint:gosec // G202: appends "?" placeholders only
var err error var err error
if tx != nil { if tx != nil {
@@ -544,30 +543,6 @@ func (r *SnapshotRepository) GetSnapshotTotalCompressedSize(
return totalSize, nil return totalSize, nil
} }
// GetSnapshotBlobSizes returns the total compressed and uncompressed sizes
// of all blobs referenced by a snapshot.
func (r *SnapshotRepository) GetSnapshotBlobSizes(
ctx context.Context, snapshotID string,
) (int64, int64, error) {
query := `
SELECT COALESCE(SUM(b.compressed_size), 0),
COALESCE(SUM(b.uncompressed_size), 0)
FROM snapshot_blobs sb
JOIN blobs b ON sb.blob_hash = b.blob_hash
WHERE sb.snapshot_id = ?
`
var compressed, uncompressed int64
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(
&compressed, &uncompressed)
if err != nil {
return 0, 0, fmt.Errorf("querying snapshot blob sizes: %w", err)
}
return compressed, uncompressed, nil
}
// GetSnapshotUncompressedChunkSize returns the sum of plaintext sizes of all unique // GetSnapshotUncompressedChunkSize returns the sum of plaintext sizes of all unique
// chunks referenced by a snapshot (via snapshot_files → file_chunks → chunks). // chunks referenced by a snapshot (via snapshot_files → file_chunks → chunks).
func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize( func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(
-59
View File
@@ -145,65 +145,6 @@ func TestSnapshotRepositoryUpdateCounts(t *testing.T) {
} }
} }
// GetSnapshotBlobSizes totals the blobs the snapshot references, and only
// those.
func TestSnapshotRepositoryGetSnapshotBlobSizes(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repos := database.NewRepositories(db)
snapshot := &database.Snapshot{
ID: "2024-01-03T12:00:00Z",
Hostname: testHostname,
VaultikVersion: testVersion,
StartedAt: time.Now().Truncate(time.Second),
}
err := repos.Snapshots.Create(ctx, nil, snapshot)
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
blobs := []*database.Blob{
{Hash: "referenced-1", CompressedSize: 10, UncompressedSize: 100},
{Hash: "referenced-2", CompressedSize: 20, UncompressedSize: 200},
{Hash: "unreferenced", CompressedSize: 40, UncompressedSize: 400},
}
for _, blob := range blobs {
blob.ID = types.NewBlobID()
blob.CreatedTS = time.Now().Truncate(time.Second)
err = repos.Blobs.Create(ctx, nil, blob)
if err != nil {
t.Fatalf("failed to create blob %s: %v", blob.Hash, err)
}
}
for _, blob := range blobs[:2] {
err = repos.Snapshots.AddBlob(ctx, nil, snapshot.ID.String(),
blob.ID, blob.Hash)
if err != nil {
t.Fatalf("failed to add blob %s to snapshot: %v", blob.Hash, err)
}
}
compressed, uncompressed, err := repos.Snapshots.GetSnapshotBlobSizes(
ctx, snapshot.ID.String())
if err != nil {
t.Fatalf("failed to get snapshot blob sizes: %v", err)
}
if compressed != 30 || uncompressed != 300 {
t.Errorf("blob sizes: got %d and %d, want 30 and 300",
compressed, uncompressed)
}
}
func TestSnapshotRepositoryListRecent(t *testing.T) { func TestSnapshotRepositoryListRecent(t *testing.T) {
t.Parallel() t.Parallel()
+16
View File
@@ -158,3 +158,19 @@ type UploadStats struct {
MinDurationMs int64 MinDurationMs int64
MaxDurationMs int64 MaxDurationMs int64
} }
// GetCountBySnapshot returns the count of uploads for a specific snapshot
func (r *UploadRepository) GetCountBySnapshot(
ctx context.Context, snapshotID string,
) (int64, error) {
query := `SELECT COUNT(*) FROM uploads WHERE snapshot_id = ?`
var count int64
err := r.conn.QueryRowContext(ctx, query, snapshotID).Scan(&count)
if err != nil {
return 0, err
}
return count, nil
}
+4 -11
View File
@@ -18,19 +18,14 @@ var Appname = "vaultik" //nolint:gochecknoglobals // set via -ldflags at build t
// deliberately not a number. // deliberately not a number.
const DevVersion = "dev" const DevVersion = "dev"
// Unknown is what Commit and CommitDate hold when the build did not
// stamp them, and the version script/docker and script/cibuild stamp
// when the host has no git checkout.
const Unknown = "unknown"
// Version is the application version, populated from main(). // Version is the application version, populated from main().
var Version = DevVersion //nolint:gochecknoglobals // set via -ldflags at build time var Version = DevVersion //nolint:gochecknoglobals // set via -ldflags at build time
// Commit is the git commit hash, populated from main(). // Commit is the git commit hash, populated from main().
var Commit = Unknown //nolint:gochecknoglobals // set via -ldflags at build time var Commit = "unknown" //nolint:gochecknoglobals // set via -ldflags at build time
// CommitDate is the ISO-8601 date of the commit, populated from main(). // CommitDate is the ISO-8601 date of the commit, populated from main().
var CommitDate = Unknown //nolint:gochecknoglobals // set via -ldflags at build time var CommitDate = "unknown" //nolint:gochecknoglobals // set via -ldflags at build time
// Author identifies the upstream author of vaultik. // Author identifies the upstream author of vaultik.
const Author = "Jeffrey Paul <sneak@sneak.berlin>" const Author = "Jeffrey Paul <sneak@sneak.berlin>"
@@ -77,11 +72,9 @@ func New() (*Globals, error) {
// safe reading of "we could not establish that this is a release" is // safe reading of "we could not establish that this is a release" is
// that it is not one. The Makefile refuses to build at all in that // that it is not one. The Makefile refuses to build at all in that
// case; this is the second line of defence, for a binary linked by // case; this is the second line of defence, for a binary linked by
// something other than the Makefile. Unknown counts for the same // something other than the Makefile.
// reason.
func IsDevVersion(v string) bool { func IsDevVersion(v string) bool {
if v == "" || v == Unknown || v == DevVersion || if v == "" || v == DevVersion || strings.HasPrefix(v, DevVersion+"-") ||
strings.HasPrefix(v, DevVersion+"-") ||
strings.HasSuffix(v, "-dirty") { strings.HasSuffix(v, "-dirty") {
return true return true
} }
-3
View File
@@ -74,9 +74,6 @@ func TestIsDevVersion(t *testing.T) {
// as one. The Makefile refuses to build when script/version // as one. The Makefile refuses to build when script/version
// yields nothing; this covers a binary linked some other way. // yields nothing; this covers a binary linked some other way.
{"", true}, {"", true},
// What script/docker and script/cibuild stamp when the host
// has no git checkout.
{"unknown", true},
} }
for _, tc := range cases { for _, tc := range cases {
+49 -64
View File
@@ -10,18 +10,15 @@ import (
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings" "strings"
"syscall"
"golang.org/x/sys/unix"
) )
// ErrAlreadyRunning indicates another vaultik instance is running. // ErrAlreadyRunning indicates another vaultik instance is running.
var ErrAlreadyRunning = errors.New("another vaultik instance is already running") var ErrAlreadyRunning = errors.New("another vaultik instance is already running")
// Lock represents an acquired PID lock: an flock(2) on the PID file, // Lock represents an acquired PID lock.
// held while the file stays open. The kernel drops it when the process
// exits, however it exits, so a crashed run never leaves the lock held.
type Lock struct { type Lock struct {
file *os.File path string
} }
const ( const (
@@ -32,9 +29,10 @@ const (
) )
// Acquire attempts to acquire a PID lock in the specified directory. // Acquire attempts to acquire a PID lock in the specified directory.
// If another process holds the lock, it returns ErrAlreadyRunning with // If the lock file exists and the process is still running, it returns
// that process's PID. On success, it writes the current PID to the lock // ErrAlreadyRunning with details about the existing process.
// file and returns a Lock that must be released with Release(). // On success, it writes the current PID to the lock file and returns
// a Lock that must be released with Release().
func Acquire(lockDir string) (*Lock, error) { func Acquire(lockDir string) (*Lock, error) {
// Ensure lock directory exists // Ensure lock directory exists
err := os.MkdirAll(lockDir, lockDirPerm) err := os.MkdirAll(lockDir, lockDirPerm)
@@ -44,82 +42,56 @@ func Acquire(lockDir string) (*Lock, error) {
lockPath := filepath.Join(lockDir, "vaultik.pid") lockPath := filepath.Join(lockDir, "vaultik.pid")
// No O_TRUNC: the file may hold the PID of the process that has the // Check for existing lock
// lock, which the error below reports. existingPID, err := readPIDFile(lockPath)
file, err := os.OpenFile( //nolint:gosec // G304: path is our own lock file if err == nil {
lockPath, os.O_RDWR|os.O_CREATE, pidFilePerm) // Lock file exists, check if process is running
if err != nil { if isProcessRunning(existingPID) {
return nil, fmt.Errorf("opening PID file: %w", err) return nil, fmt.Errorf("%w (PID %d)", ErrAlreadyRunning, existingPID)
}
err = unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB)
if err != nil {
_ = file.Close()
if errors.Is(err, unix.EWOULDBLOCK) {
return nil, alreadyRunningError(lockPath)
} }
// Process is not running, stale lock file - we can take over
return nil, fmt.Errorf("locking PID file: %w", err)
} }
err = writePID(file) // Write our PID
pid := os.Getpid()
err = os.WriteFile(lockPath, []byte(strconv.Itoa(pid)), pidFilePerm)
if err != nil { if err != nil {
_ = file.Close() return nil, fmt.Errorf("writing PID file: %w", err)
return nil, err
} }
return &Lock{file: file}, nil return &Lock{path: lockPath}, nil
} }
// Release empties the PID file and closes it, which drops the lock. // Release removes the PID lock file.
// It is safe to call Release multiple times. // It is safe to call Release multiple times.
func (l *Lock) Release() error { func (l *Lock) Release() error {
if l == nil || l.file == nil { if l == nil || l.path == "" {
return nil return nil
} }
file := l.file // Verify we still own the lock (our PID is in the file)
l.file = nil existingPID, err := readPIDFile(l.path)
// Do not remove the file here. A process that opened it a moment
// earlier could then lock the removed file while another creates and
// locks a new one, and both would run.
truncateErr := file.Truncate(0)
closeErr := file.Close()
return errors.Join(truncateErr, closeErr)
}
// writePID replaces the contents of the locked PID file with the current
// PID.
func writePID(file *os.File) error {
err := file.Truncate(0)
if err != nil { if err != nil {
return fmt.Errorf("truncating PID file: %w", err) // File already gone or unreadable - that's fine
return nil //nolint:nilerr // unreadable lock file means nothing to release
} }
_, err = file.WriteAt([]byte(strconv.Itoa(os.Getpid())), 0) if existingPID != os.Getpid() {
if err != nil { // Someone else wrote to our lock file - don't remove it
return fmt.Errorf("writing PID file: %w", err) return nil
} }
err = os.Remove(l.path)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("removing PID file: %w", err)
}
l.path = "" // Prevent double-release
return nil return nil
} }
// alreadyRunningError reports that another process holds the lock,
// naming its PID when the file holds one. The holder writes its PID just
// after it locks, so the file can briefly be empty.
func alreadyRunningError(lockPath string) error {
pid, err := readPIDFile(lockPath)
if err != nil {
return ErrAlreadyRunning
}
return fmt.Errorf("%w (PID %d)", ErrAlreadyRunning, pid)
}
// readPIDFile reads and parses the PID from a lock file. // readPIDFile reads and parses the PID from a lock file.
func readPIDFile(path string) (int, error) { func readPIDFile(path string) (int, error) {
data, err := os.ReadFile(path) //nolint:gosec // G304: path is our own lock file data, err := os.ReadFile(path) //nolint:gosec // G304: path is our own lock file
@@ -134,3 +106,16 @@ func readPIDFile(path string) (int, error) {
return pid, nil return pid, nil
} }
// isProcessRunning checks if a process with the given PID is running.
func isProcessRunning(pid int) bool {
process, err := os.FindProcess(pid)
if err != nil {
return false
}
// On Unix, FindProcess always succeeds. We need to send signal 0 to check.
err = process.Signal(syscall.Signal(0))
return err == nil
}
+3 -63
View File
@@ -4,7 +4,6 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
"sync"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -34,10 +33,9 @@ func TestAcquireAndRelease(t *testing.T) {
err = lock.Release() err = lock.Release()
require.NoError(t, err) require.NoError(t, err)
// Verify PID file is empty // Verify PID file is gone
data, err = os.ReadFile(pidPath) //nolint:gosec // G304: test's own temp file _, err = os.Stat(pidPath)
require.NoError(t, err) assert.True(t, os.IsNotExist(err))
assert.Empty(t, data)
} }
func TestAcquireBlocksSecondInstance(t *testing.T) { func TestAcquireBlocksSecondInstance(t *testing.T) {
@@ -57,64 +55,6 @@ func TestAcquireBlocksSecondInstance(t *testing.T) {
lock2, err := pidlock.Acquire(tmpDir) lock2, err := pidlock.Acquire(tmpDir)
require.ErrorIs(t, err, pidlock.ErrAlreadyRunning) require.ErrorIs(t, err, pidlock.ErrAlreadyRunning)
assert.Nil(t, lock2) assert.Nil(t, lock2)
// Once the first lock is released, the next Acquire succeeds
require.NoError(t, lock1.Release())
lock3, err := pidlock.Acquire(tmpDir)
require.NoError(t, err)
require.NoError(t, lock3.Release())
}
// TestConcurrentAcquireAdmitsOne starts many Acquire calls at the same
// moment, as two cron entries firing together would, and checks that
// exactly one of them gets the lock.
func TestConcurrentAcquireAdmitsOne(t *testing.T) {
t.Parallel()
const callers = 50
tmpDir := t.TempDir()
start := make(chan struct{})
var (
mu sync.Mutex
acquired []*pidlock.Lock
failures []error
wg sync.WaitGroup
)
for range callers {
wg.Go(func() {
<-start
lock, err := pidlock.Acquire(tmpDir)
mu.Lock()
defer mu.Unlock()
if err != nil {
failures = append(failures, err)
return
}
acquired = append(acquired, lock)
})
}
close(start)
wg.Wait()
for _, lock := range acquired {
require.NoError(t, lock.Release())
}
assert.Len(t, acquired, 1, "exactly one caller should hold the lock")
for _, err := range failures {
require.ErrorIs(t, err, pidlock.ErrAlreadyRunning)
}
} }
func TestAcquireWithStaleLock(t *testing.T) { func TestAcquireWithStaleLock(t *testing.T) {
+6 -34
View File
@@ -6,7 +6,6 @@ import (
"context" "context"
"errors" "errors"
"io" "io"
"strings"
"sync/atomic" "sync/atomic"
"github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws"
@@ -27,14 +26,10 @@ type Client struct {
bucket string bucket string
prefix string prefix string
endpoint string endpoint string
partSize int64
} }
// Config contains S3 client configuration. // Config contains S3 client configuration.
// All fields are required except Prefix, which defaults to an empty string, // All fields are required except Prefix, which defaults to an empty string.
// and PartSize, where zero means the SDK default of 5 MiB.
// A non-empty Prefix is joined to every key with one "/", whether or not
// it ends with one.
// The Endpoint field should include the protocol (http:// or https://). // The Endpoint field should include the protocol (http:// or https://).
type Config struct { type Config struct {
Endpoint string Endpoint string
@@ -43,9 +38,6 @@ type Config struct {
AccessKeyID string AccessKeyID string
SecretAccessKey string SecretAccessKey string
Region string Region string
// PartSize is the size in bytes of each part of a multipart upload.
// An upload too large for S3's limit of 10,000 parts gets larger parts.
PartSize int64
} }
// nopLogger is a logger that discards all output. // nopLogger is a logger that discards all output.
@@ -83,19 +75,11 @@ func NewClient(ctx context.Context, cfg Config) (*Client, error) {
s3Client := s3.NewFromConfig(awsCfg, s3Opts) s3Client := s3.NewFromConfig(awsCfg, s3Opts)
// Every method below builds a key as prefix + key, so the prefix
// must carry its own trailing "/".
prefix := strings.TrimRight(cfg.Prefix, "/")
if prefix != "" {
prefix += "/"
}
return &Client{ return &Client{
s3Client: s3Client, s3Client: s3Client,
bucket: cfg.Bucket, bucket: cfg.Bucket,
prefix: prefix, prefix: cfg.Prefix,
endpoint: cfg.Endpoint, endpoint: cfg.Endpoint,
partSize: cfg.PartSize,
}, nil }, nil
} }
@@ -129,9 +113,12 @@ func (c *Client) PutObjectWithProgress(
) error { ) error {
fullKey := c.prefix + key fullKey := c.prefix + key
// uploadPartSize is 10MB for better progress granularity.
const uploadPartSize = 10 * 1024 * 1024
// Create an uploader with the S3 client // Create an uploader with the S3 client
uploader := manager.NewUploader(c.s3Client, func(u *manager.Uploader) { uploader := manager.NewUploader(c.s3Client, func(u *manager.Uploader) {
u.PartSize = uploadPartSize(c.partSize, size) u.PartSize = uploadPartSize
}) })
// Create a progress reader that tracks upload progress // Create a progress reader that tracks upload progress
@@ -152,21 +139,6 @@ func (c *Client) PutObjectWithProgress(
return err return err
} }
// uploadPartSize returns the part size for an upload of size bytes: the
// configured part size (the SDK default when zero), raised where needed so
// the upload fits in S3's limit of 10,000 parts. The uploader cannot raise
// it itself, because it cannot seek the progress reader to learn its size.
func uploadPartSize(configured, size int64) int64 {
if configured == 0 {
configured = manager.DefaultUploadPartSize
}
maxParts := int64(manager.MaxUploadParts)
smallestThatFits := (size + maxParts - 1) / maxParts // rounded up
return max(configured, smallestThatFits)
}
// GetObject downloads an object from S3 with the specified key. // GetObject downloads an object from S3 with the specified key.
// The key is automatically prefixed with the configured prefix. // The key is automatically prefixed with the configured prefix.
// Returns a ReadCloser containing the object data. The caller must // Returns a ReadCloser containing the object data. The caller must
-55
View File
@@ -1,55 +0,0 @@
package s3
import "testing"
// TestUploadPartSize checks that an upload too large for 10,000 parts of the
// configured size gets parts just large enough to fit in 10,000.
func TestUploadPartSize(t *testing.T) {
t.Parallel()
const mib = 1024 * 1024
tests := []struct {
name string
configured int64
size int64
want int64
}{
{
name: "an upload that fits keeps the configured size",
configured: 5 * mib,
size: 10 * 1024 * mib,
want: 5 * mib,
},
{
name: "exactly 10,000 parts keeps the configured size",
configured: 6 * mib,
size: 10_000 * 6 * mib,
want: 6 * mib,
},
{
name: "one byte more than 10,000 parts adds a byte to each",
configured: 6 * mib,
size: 10_000*6*mib + 1,
want: 6*mib + 1,
},
{
name: "zero means the SDK default of 5MiB",
configured: 0,
size: 1,
want: 5 * mib,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got := uploadPartSize(tt.configured, tt.size)
if got != tt.want {
t.Errorf("uploadPartSize(%d, %d) = %d, want %d",
tt.configured, tt.size, got, tt.want)
}
})
}
}
-1
View File
@@ -28,7 +28,6 @@ func provideClient(lc fx.Lifecycle, cfg *config.Config) (*Client, error) {
AccessKeyID: cfg.S3.AccessKeyID, AccessKeyID: cfg.S3.AccessKeyID,
SecretAccessKey: cfg.S3.SecretAccessKey, SecretAccessKey: cfg.S3.SecretAccessKey,
Region: cfg.S3.Region, Region: cfg.S3.Region,
PartSize: cfg.S3.PartSize.Int64(),
}) })
if err != nil { if err != nil {
return nil, err return nil, err
+1 -1
View File
@@ -71,7 +71,7 @@ func verifyBackupFiles(
) { ) {
t.Helper() t.Helper()
files, err := repos.Files.ListAll(ctx) files, err := repos.Files.ListByPrefix(ctx, "")
if err != nil { if err != nil {
t.Fatalf("Failed to list files: %v", err) t.Fatalf("Failed to list files: %v", err)
} }
+8 -16
View File
@@ -1,48 +1,40 @@
//nolint:testpackage // exercises the unexported copyDatabase helper //nolint:testpackage // exercises the unexported copyFile helper
package snapshot package snapshot
import ( import (
"context"
"os" "os"
"path/filepath" "path/filepath"
"syscall" "syscall"
"testing" "testing"
"github.com/spf13/afero" "github.com/spf13/afero"
"sneak.berlin/go/vaultik/internal/database"
) )
// TestCopyDatabaseExportCopyMode verifies that the exported snapshot // TestCopyFileExportCopyMode verifies that the exported snapshot database
// database copy is created owner-only (0600), even under a lenient 022 // copy is created owner-only (0600), even under a lenient 022 umask that
// umask that would otherwise leave a fresh file world-readable. // would otherwise leave a fresh file world-readable.
// //
//nolint:paralleltest // syscall.Umask is process-global; parallel tests would clash //nolint:paralleltest // syscall.Umask is process-global; parallel tests would clash
func TestCopyDatabaseExportCopyMode(t *testing.T) { func TestCopyFileExportCopyMode(t *testing.T) {
restore := syscall.Umask(0o022) restore := syscall.Umask(0o022)
defer syscall.Umask(restore) defer syscall.Umask(restore)
ctx := context.Background()
dir := t.TempDir() dir := t.TempDir()
src := filepath.Join(dir, "index.sqlite") src := filepath.Join(dir, "index.sqlite")
db, err := database.New(ctx, src) err := os.WriteFile(src, []byte("index data"), 0o600)
if err != nil { if err != nil {
t.Fatalf("creating source index: %v", err) t.Fatalf("creating source index: %v", err)
} }
err = db.Close()
if err != nil {
t.Fatalf("closing source index: %v", err)
}
dst := filepath.Join(dir, "snapshot.db") dst := filepath.Join(dir, "snapshot.db")
sm := &SnapshotManager{fs: afero.NewOsFs()} sm := &SnapshotManager{fs: afero.NewOsFs()}
err = sm.copyDatabase(ctx, src, dst) err = sm.copyFile(src, dst)
if err != nil { if err != nil {
t.Fatalf("copyDatabase: %v", err) t.Fatalf("copyFile: %v", err)
} }
info, err := os.Stat(dst) info, err := os.Stat(dst)
+4
View File
@@ -66,6 +66,7 @@ type ProgressStats struct {
BlobsCreated atomic.Int64 BlobsCreated atomic.Int64
BlobsUploaded atomic.Int64 BlobsUploaded atomic.Int64
BytesUploaded atomic.Int64 BytesUploaded atomic.Int64
UploadDurationMs atomic.Int64 // Total milliseconds spent uploading
CurrentFile atomic.Value // stores string CurrentFile atomic.Value // stores string
TotalSize atomic.Int64 // Total size to process (set after scan phase) TotalSize atomic.Int64 // Total size to process (set after scan phase)
TotalFiles atomic.Int64 // Total files to process in phase 2 TotalFiles atomic.Int64 // Total files to process in phase 2
@@ -230,6 +231,9 @@ func (pr *ProgressReporter) ReportUploadComplete(
// Clear current upload // Clear current upload
pr.stats.CurrentUpload.Store((*UploadInfo)(nil)) pr.stats.CurrentUpload.Store((*UploadInfo)(nil))
// Add to total upload duration
pr.stats.UploadDurationMs.Add(duration.Milliseconds())
// Calculate speed // Calculate speed
if duration < time.Millisecond { if duration < time.Millisecond {
duration = time.Millisecond duration = time.Millisecond
+127 -90
View File
@@ -11,7 +11,6 @@ import (
"runtime" "runtime"
"strings" "strings"
"sync" "sync"
"syscall"
"time" "time"
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
@@ -58,8 +57,8 @@ type Scanner struct {
compressionLevel int compressionLevel int
ageRecipient string ageRecipient string
snapshotID string // Current snapshot being processed snapshotID string // Current snapshot being processed
// currentSourcePath is the source directory being scanned, stored with // currentSourcePath is the source directory being scanned (used for
// each file record. // restore path stripping).
currentSourcePath string currentSourcePath string
exclude []string // Glob patterns for files/directories to exclude exclude []string // Glob patterns for files/directories to exclude
compiledExclude []compiledPattern // Compiled glob patterns compiledExclude []compiledPattern // Compiled glob patterns
@@ -92,6 +91,9 @@ type Scanner struct {
// Mutex for coordinating blob creation // Mutex for coordinating blob creation
packerMu sync.Mutex // Blocks chunk production during blob creation packerMu sync.Mutex // Blocks chunk production during blob creation
// Context for cancellation
scanCtx context.Context //nolint:containedctx // set per-Scan for packer callbacks
} }
// Periodic status output intervals and thresholds for the scan and // Periodic status output intervals and thresholds for the scan and
@@ -131,23 +133,18 @@ type ScannerConfig struct {
SkipErrors bool SkipErrors bool
} }
// ScanResult contains the results of a scan operation. Files and bytes // ScanResult contains the results of a scan operation
// are counted per file: BytesScanned is the size of the new and changed
// files, BytesSkipped that of the unchanged ones.
type ScanResult struct { type ScanResult struct {
FilesScanned int FilesScanned int
FilesSkipped int FilesSkipped int
FilesDeleted int FilesDeleted int
BytesScanned int64 BytesScanned int64
BytesSkipped int64 BytesSkipped int64
BytesDeleted int64 BytesDeleted int64
ChunksCreated int ChunksCreated int
BlobsCreated int BlobsCreated int
BlobsUploaded int StartTime time.Time
BytesUploaded int64 EndTime time.Time
UploadDuration time.Duration
StartTime time.Time
EndTime time.Time
} }
// NewScanner creates a new scanner instance // NewScanner creates a new scanner instance
@@ -211,7 +208,9 @@ func (s *Scanner) Scan(
ctx context.Context, path string, snapshotID string, ctx context.Context, path string, snapshotID string,
) (*ScanResult, error) { ) (*ScanResult, error) {
s.snapshotID = snapshotID s.snapshotID = snapshotID
// Store source path for file records (used during restore)
s.currentSourcePath = path s.currentSourcePath = path
s.scanCtx = ctx
result := &ScanResult{ result := &ScanResult{
StartTime: time.Now().UTC(), StartTime: time.Now().UTC(),
} }
@@ -219,13 +218,17 @@ func (s *Scanner) Scan(
// Set blob handler for concurrent upload // Set blob handler for concurrent upload
if s.storage != nil { if s.storage != nil {
log.Debug("Setting blob handler for storage uploads") log.Debug("Setting blob handler for storage uploads")
s.packer.SetBlobHandler(func(blobWithReader *blob.WithReader) error { s.packer.SetBlobHandler(s.handleBlobReady)
return s.handleBlobReady(ctx, blobWithReader, result)
})
} else { } else {
log.Debug("No storage configured, blobs will not be uploaded") log.Debug("No storage configured, blobs will not be uploaded")
} }
// Start progress reporting if enabled
if s.progress != nil {
s.progress.Start()
defer s.progress.Stop()
}
// Phase 0: Repair any state left by an interrupted previous run, then // Phase 0: Repair any state left by an interrupted previous run, then
// load known files and chunks from the database into memory for fast // load known files and chunks from the database into memory for fast
// lookup. // lookup.
@@ -290,14 +293,13 @@ func (s *Scanner) Scan(
log.Info("Phase 2/3: Skipping (no files need processing, metadata-only snapshot)") log.Info("Phase 2/3: Skipping (no files need processing, metadata-only snapshot)")
} }
result.EndTime = time.Now().UTC() // Finalize result with blob statistics
s.finalizeScanResult(ctx, result)
return result, nil return result, nil
} }
// GetProgress returns the progress reporter for this scanner, or nil when // GetProgress returns the progress reporter for this scanner
// progress is off. Scan neither starts nor stops it: the caller does,
// once for all the paths it scans, because a second Stop panics.
func (s *Scanner) GetProgress() *ProgressReporter { func (s *Scanner) GetProgress() *ProgressReporter {
return s.progress return s.progress
} }
@@ -432,16 +434,35 @@ func (s *Scanner) summarizeScanPhase(
s.ui.Completef("%s.", msg) s.ui.Completef("%s.", msg)
} }
// loadKnownFiles loads the known files at and beneath path from the // finalizeScanResult populates final blob statistics in the scan result
// database into a map for fast lookup. Every loaded file the scan does // by querying the packer and database for blob/upload counts
// not find is counted as deleted. This avoids per-file database queries func (s *Scanner) finalizeScanResult(ctx context.Context, result *ScanResult) {
// during the scan phase. blobs := s.packer.GetFinishedBlobs()
result.BlobsCreated += len(blobs)
// Query database for actual blob count created during this snapshot
// The database is authoritative, especially for concurrent blob uploads
// We count uploads rather than all snapshot_blobs to get only NEW blobs
if s.snapshotID != "" {
uploadCount, err := s.repos.Uploads.GetCountBySnapshot(ctx, s.snapshotID)
if err != nil {
log.Warn("Failed to query upload count from database", "error", err)
} else {
result.BlobsCreated = int(uploadCount)
}
}
result.EndTime = time.Now().UTC()
}
// loadKnownFiles loads all known files from the database into a map for fast lookup
// This avoids per-file database queries during the scan phase
func (s *Scanner) loadKnownFiles( func (s *Scanner) loadKnownFiles(
ctx context.Context, path string, ctx context.Context, path string,
) (map[string]*database.File, error) { ) (map[string]*database.File, error) {
files, err := s.repos.Files.ListUnderPath(ctx, path) files, err := s.repos.Files.ListByPrefix(ctx, path)
if err != nil { if err != nil {
return nil, fmt.Errorf("listing files under %s: %w", path, err) return nil, fmt.Errorf("listing files by prefix: %w", err)
} }
result := make(map[string]*database.File, len(files)) result := make(map[string]*database.File, len(files))
@@ -889,10 +910,9 @@ func (s *Scanner) scanPhase(
} }
// Handle symlinks and directories // Handle symlinks and directories
handled, err := s.recordSpecialEntry( if handled := s.recordSpecialEntry(
filePath, info, existingFiles, collector, result) filePath, info, existingFiles, collector, result); handled {
if handled { return nil
return err
} }
// Skip other non-regular files (devices, sockets, etc.) // Skip other non-regular files (devices, sockets, etc.)
@@ -934,25 +954,22 @@ func (s *Scanner) scanPhase(
} }
// recordSpecialEntry records symlinks and directories (which have no // recordSpecialEntry records symlinks and directories (which have no
// data to chunk) and reports whether it handled the entry. For a symlink // data to chunk) and reports whether it handled the entry.
// whose target cannot be read it returns handleWalkError's result.
func (s *Scanner) recordSpecialEntry( func (s *Scanner) recordSpecialEntry(
filePath string, info os.FileInfo, filePath string, info os.FileInfo,
existingFiles map[string]struct{}, existingFiles map[string]struct{},
collector *scanCollector, result *ScanResult, collector *scanCollector, result *ScanResult,
) (bool, error) { ) bool {
// Handle symlinks // Handle symlinks
if info.Mode()&os.ModeSymlink != 0 { if info.Mode()&os.ModeSymlink != 0 {
file, err := s.buildSymlinkEntry(filePath, info) file := s.buildSymlinkEntry(filePath, info)
if err != nil { if file != nil {
return true, s.handleWalkError(filePath, err) existingFiles[filePath] = struct{}{}
collector.addToProcess(filePath, info, file)
s.updateScanEntryStats(result, true, info)
} }
existingFiles[filePath] = struct{}{} return true
collector.addToProcess(filePath, info, file)
s.updateScanEntryStats(result, true, info)
return true, nil
} }
// Handle directories (record for permission/ownership preservation // Handle directories (record for permission/ownership preservation
@@ -962,10 +979,10 @@ func (s *Scanner) recordSpecialEntry(
existingFiles[filePath] = struct{}{} existingFiles[filePath] = struct{}{}
collector.addToProcess(filePath, info, file) collector.addToProcess(filePath, info, file)
return true, nil return true
} }
return false, nil return false
} }
// handleWalkError deals with a filesystem error surfaced by the walk: // handleWalkError deals with a filesystem error surfaced by the walk:
@@ -1115,18 +1132,22 @@ func (s *Scanner) printScanProgressLine(
} }
// buildSymlinkEntry creates a File record for a symlink. // buildSymlinkEntry creates a File record for a symlink.
func (s *Scanner) buildSymlinkEntry( // Returns nil if the link target cannot be read.
path string, info os.FileInfo, func (s *Scanner) buildSymlinkEntry(path string, info os.FileInfo) *database.File {
) (*database.File, error) {
target, err := os.Readlink(path) target, err := os.Readlink(path)
if err != nil { if err != nil {
return nil, err log.Debug("Cannot read symlink target", "path", path, "error", err)
return nil
} }
var uid, gid uint32 var uid, gid uint32
if stat, ok := info.Sys().(*syscall.Stat_t); ok { if stat, ok := info.Sys().(interface {
uid = stat.Uid Uid() uint32
gid = stat.Gid Gid() uint32
}); ok {
uid = stat.Uid()
gid = stat.Gid()
} }
return &database.File{ return &database.File{
@@ -1139,15 +1160,18 @@ func (s *Scanner) buildSymlinkEntry(
UID: uid, UID: uid,
GID: gid, GID: gid,
LinkTarget: types.FilePath(target), LinkTarget: types.FilePath(target),
}, nil }
} }
// buildDirectoryEntry creates a File record for a directory. // buildDirectoryEntry creates a File record for a directory.
func (s *Scanner) buildDirectoryEntry(path string, info os.FileInfo) *database.File { func (s *Scanner) buildDirectoryEntry(path string, info os.FileInfo) *database.File {
var uid, gid uint32 var uid, gid uint32
if stat, ok := info.Sys().(*syscall.Stat_t); ok { if stat, ok := info.Sys().(interface {
uid = stat.Uid Uid() uint32
gid = stat.Gid Gid() uint32
}); ok {
uid = stat.Uid()
gid = stat.Gid()
} }
return &database.File{ return &database.File{
@@ -1180,10 +1204,16 @@ func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess)
func (s *Scanner) checkFileInMemory( func (s *Scanner) checkFileInMemory(
path string, info os.FileInfo, knownFiles map[string]*database.File, path string, info os.FileInfo, knownFiles map[string]*database.File,
) (*database.File, bool) { ) (*database.File, bool) {
// Get file stats
stat, ok := info.Sys().(interface {
Uid() uint32
Gid() uint32
})
var uid, gid uint32 var uid, gid uint32
if stat, ok := info.Sys().(*syscall.Stat_t); ok { if ok {
uid = stat.Uid uid = stat.Uid()
gid = stat.Gid gid = stat.Gid()
} }
// Check against in-memory map first to get existing ID if available // Check against in-memory map first to get existing ID if available
@@ -1200,8 +1230,9 @@ func (s *Scanner) checkFileInMemory(
} }
file := &database.File{ file := &database.File{
ID: fileID, ID: fileID,
Path: types.FilePath(path), Path: types.FilePath(path),
// Store source directory for restore path stripping
SourcePath: types.SourcePath(s.currentSourcePath), SourcePath: types.SourcePath(s.currentSourcePath),
MTime: info.ModTime(), MTime: info.ModTime(),
Size: info.Size(), Size: info.Size(),
@@ -1222,7 +1253,7 @@ func (s *Scanner) checkFileInMemory(
// Check if file has changed // Check if file has changed
if existingFile.Size != file.Size || if existingFile.Size != file.Size ||
!existingFile.MTime.Equal(file.MTime) || existingFile.MTime.Unix() != file.MTime.Unix() ||
existingFile.Mode != file.Mode || existingFile.Mode != file.Mode ||
existingFile.UID != file.UID || existingFile.UID != file.UID ||
existingFile.GID != file.GID { existingFile.GID != file.GID {
@@ -1357,7 +1388,8 @@ func (s *Scanner) processFileWithErrorHandling(
// record a file whose chunk is in no blob and cannot be restored, so // record a file whose chunk is in no blob and cannot be restored, so
// abort the run even under --skip-errors. Only open and read errors // abort the run even under --skip-errors. Only open and read errors
// are skipped below. // are skipped below.
if _, ok := errors.AsType[*packerError](err); ok { var pErr *packerError
if errors.As(err, &pErr) {
return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err) return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err)
} }
// Handle files that were deleted between scan and process phases // Handle files that were deleted between scan and process phases
@@ -1493,24 +1525,24 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
} }
// handleBlobReady is called by the packer when a blob is finalized // handleBlobReady is called by the packer when a blob is finalized
func (s *Scanner) handleBlobReady( func (s *Scanner) handleBlobReady(blobWithReader *blob.WithReader) error {
ctx context.Context, blobWithReader *blob.WithReader, result *ScanResult,
) error {
startTime := time.Now().UTC() startTime := time.Now().UTC()
finishedBlob := blobWithReader.FinishedBlob finishedBlob := blobWithReader.FinishedBlob
result.BlobsCreated++
if s.progress != nil { if s.progress != nil {
s.progress.ReportUploadStart(finishedBlob.Hash, finishedBlob.Compressed) s.progress.ReportUploadStart(finishedBlob.Hash, finishedBlob.Compressed)
s.progress.GetStats().BlobsCreated.Add(1) s.progress.GetStats().BlobsCreated.Add(1)
} }
ctx := s.scanCtx
if ctx == nil {
ctx = context.Background()
}
blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobPath := fmt.Sprintf("blobs/%s/%s/%s",
finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash) finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash)
blobExists, err := s.uploadBlobIfNeeded( blobExists, err := s.uploadBlobIfNeeded(ctx, blobPath, blobWithReader, startTime)
ctx, blobPath, blobWithReader, startTime, result)
if err != nil { if err != nil {
s.cleanupBlobTempFile(blobWithReader) s.cleanupBlobTempFile(blobWithReader)
@@ -1545,7 +1577,6 @@ func (s *Scanner) uploadBlobIfNeeded(
blobPath string, blobPath string,
blobWithReader *blob.WithReader, blobWithReader *blob.WithReader,
startTime time.Time, startTime time.Time,
result *ScanResult,
) (bool, error) { ) (bool, error) {
finishedBlob := blobWithReader.FinishedBlob finishedBlob := blobWithReader.FinishedBlob
@@ -1581,10 +1612,6 @@ func (s *Scanner) uploadBlobIfNeeded(
uploadDuration := time.Since(startTime) uploadDuration := time.Since(startTime)
uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds() uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds()
result.BlobsUploaded++
result.BytesUploaded += finishedBlob.Compressed
result.UploadDuration += uploadDuration
s.ui.Completef("Uploaded blob %s (%s) in %s at %s.", s.ui.Completef("Uploaded blob %s (%s) in %s at %s.",
s.ui.Hex(finishedBlob.Hash), s.ui.Hex(finishedBlob.Hash),
s.ui.Size(finishedBlob.Compressed), s.ui.Size(finishedBlob.Compressed),
@@ -1800,9 +1827,9 @@ func (s *Scanner) processFileStreaming(
size: chunk.Size, size: chunk.Size,
}) })
if !chunkExists { s.updateChunkStats(chunkExists, chunk.Size, result)
s.updateChunkStats(chunk.Size, result)
if !chunkExists {
err := s.addChunkToPacker(ctx, chunk) err := s.addChunkToPacker(ctx, chunk)
if err != nil { if err != nil {
// Mark as a packer error so --skip-errors cannot swallow it: // Mark as a packer error so --skip-errors cannot swallow it:
@@ -1830,16 +1857,26 @@ func (s *Scanner) processFileStreaming(
return nil return nil
} }
// updateChunkStats counts a chunk that was not already stored. The scan // updateChunkStats updates scan result and progress stats for a processed chunk
// result's file counts, BytesScanned and BytesSkipped are not touched func (s *Scanner) updateChunkStats(
// here: the scan phase counts each file once. chunkExists bool, chunkSize int64, result *ScanResult,
func (s *Scanner) updateChunkStats(chunkSize int64, result *ScanResult) { ) {
result.ChunksCreated++ if chunkExists {
result.FilesSkipped++
if s.progress != nil { result.BytesSkipped += chunkSize
s.progress.GetStats().ChunksCreated.Add(1) if s.progress != nil {
s.progress.GetStats().BytesProcessed.Add(chunkSize) s.progress.GetStats().BytesSkipped.Add(chunkSize)
s.progress.UpdateChunkingActivity() }
} else {
result.ChunksCreated++
result.BytesScanned += chunkSize
if s.progress != nil {
s.progress.GetStats().ChunksCreated.Add(1)
s.progress.GetStats().BytesProcessed.Add(chunkSize)
s.progress.UpdateChunkingActivity()
}
} }
} }
+1 -78
View File
@@ -71,7 +71,7 @@ func verifySimpleScanDatabase(
t.Helper() t.Helper()
// Verify files in database - includes regular files and directories // Verify files in database - includes regular files and directories
files, err := repos.Files.ListUnderPath(ctx, "/source") files, err := repos.Files.ListByPrefix(ctx, "/source")
if err != nil { if err != nil {
t.Fatalf("failed to list files: %v", err) t.Fatalf("failed to list files: %v", err)
} }
@@ -307,80 +307,3 @@ func TestScannerLargeFile(t *testing.T) {
} }
} }
} }
// TestScannerRecordsOwnership backs up real files on disk and checks that
// the uid and gid of a file, a directory and a symlink are recorded.
// When the tests run as root both sides are 0, so only a run as another
// user can catch ownership recorded as 0.
func TestScannerRecordsOwnership(t *testing.T) {
t.Parallel()
sourceDir := t.TempDir()
filePath := filepath.Join(sourceDir, "file.txt")
dirPath := filepath.Join(sourceDir, "subdir")
linkPath := filepath.Join(sourceDir, "link")
err := os.WriteFile(filePath, []byte("owned"), 0o600)
if err != nil {
t.Fatal(err)
}
err = os.Mkdir(dirPath, 0o700)
if err != nil {
t.Fatal(err)
}
err = os.Symlink("file.txt", linkPath)
if err != nil {
t.Fatal(err)
}
db, err := database.NewTestDB()
if err != nil {
t.Fatalf("failed to create test database: %v", err)
}
defer func() {
err := db.Close()
if err != nil {
t.Errorf("failed to close database: %v", err)
}
}()
repos := database.NewRepositories(db)
scanner := snapshot.NewScanner(snapshot.ScannerConfig{
FS: afero.NewOsFs(),
ChunkSize: int64(1024 * 16),
Repositories: repos,
MaxBlobSize: int64(1024 * 1024),
CompressionLevel: 3,
AgeRecipients: []string{testAgePublicKey},
})
ctx := context.Background()
snapshotID := "test-snapshot-ownership"
createTestSnapshotRecord(ctx, t, repos, snapshotID)
_, err = scanner.Scan(ctx, sourceDir, snapshotID)
if err != nil {
t.Fatalf("scan failed: %v", err)
}
for _, path := range []string{filePath, dirPath, linkPath} {
file, err := repos.Files.GetByPath(ctx, path)
if err != nil {
t.Fatalf("failed to get %s: %v", path, err)
}
if file == nil {
t.Fatalf("%s was not recorded", path)
}
if int(file.UID) != os.Getuid() || int(file.GID) != os.Getgid() {
t.Errorf("%s recorded as uid %d gid %d, want uid %d gid %d",
path, file.UID, file.GID, os.Getuid(), os.Getgid())
}
}
}
+7 -95
View File
@@ -3,7 +3,6 @@ package snapshot_test
import ( import (
"context" "context"
"errors" "errors"
"io"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -14,7 +13,6 @@ import (
"github.com/spf13/afero" "github.com/spf13/afero"
"sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/snapshot" "sneak.berlin/go/vaultik/internal/snapshot"
"sneak.berlin/go/vaultik/internal/ui"
) )
// errSimTempFail is the one-time temp-file creation failure blobTempFailFs // errSimTempFail is the one-time temp-file creation failure blobTempFailFs
@@ -82,30 +80,6 @@ func (f *readFailFs) Open(name string) (afero.File, error) {
return file, nil return file, nil
} }
// linkRemovedAfterLstatFs is the real filesystem, except that the symlink at
// target is removed right after the walk lstats it, as happens when a link is
// deleted during a backup. The scanner's readlink of it then fails.
type linkRemovedAfterLstatFs struct {
afero.OsFs
t *testing.T
target string
}
func (f *linkRemovedAfterLstatFs) LstatIfPossible(
name string,
) (os.FileInfo, bool, error) {
info, lstatCalled, err := f.OsFs.LstatIfPossible(name)
if err == nil && name == f.target {
rmErr := os.Remove(name)
if rmErr != nil {
f.t.Errorf("removing %s: %v", name, rmErr)
}
}
return info, lstatCalled, err
}
// writeSkipErrorTestFile writes one file into fs with a fixed mtime. // writeSkipErrorTestFile writes one file into fs with a fixed mtime.
func writeSkipErrorTestFile(t *testing.T, fs afero.Fs, path, content string) { func writeSkipErrorTestFile(t *testing.T, fs afero.Fs, path, content string) {
t.Helper() t.Helper()
@@ -128,11 +102,10 @@ func writeSkipErrorTestFile(t *testing.T, fs afero.Fs, path, content string) {
} }
} }
// runSkipErrorScan scans source on fs with the given skip-errors setting, // runSkipErrorScan scans /source on fs with the given skip-errors setting and
// printing user-facing messages to uiw (nil discards them), and returns the // returns the repositories (for inspection) and the scan error.
// repositories (for inspection) and the scan error.
func runSkipErrorScan( func runSkipErrorScan(
t *testing.T, fs afero.Fs, source string, skipErrors bool, uiw *ui.Writer, t *testing.T, fs afero.Fs, skipErrors bool,
) (*database.Repositories, error) { ) (*database.Repositories, error) {
t.Helper() t.Helper()
@@ -157,7 +130,6 @@ func runSkipErrorScan(
MaxBlobSize: int64(1024 * 1024), MaxBlobSize: int64(1024 * 1024),
CompressionLevel: 3, CompressionLevel: 3,
AgeRecipients: []string{testAgePublicKey}, AgeRecipients: []string{testAgePublicKey},
UI: uiw,
SkipErrors: skipErrors, SkipErrors: skipErrors,
}) })
@@ -165,7 +137,7 @@ func runSkipErrorScan(
snapshotID := "test-snapshot-skip-errors" snapshotID := "test-snapshot-skip-errors"
createTestSnapshotRecord(ctx, t, repos, snapshotID) createTestSnapshotRecord(ctx, t, repos, snapshotID)
_, err = scanner.Scan(ctx, source, snapshotID) _, err = scanner.Scan(ctx, "/source", snapshotID)
return repos, err return repos, err
} }
@@ -185,7 +157,7 @@ func TestScannerPackingFailureAbortsUnderSkipErrors(t *testing.T) {
writeSkipErrorTestFile(t, fs, "/source/file1.txt", "first file content") writeSkipErrorTestFile(t, fs, "/source/file1.txt", "first file content")
writeSkipErrorTestFile(t, fs, "/source/file2.txt", "second file content") writeSkipErrorTestFile(t, fs, "/source/file2.txt", "second file content")
repos, err := runSkipErrorScan(t, fs, "/source", true, nil) repos, err := runSkipErrorScan(t, fs, true)
if err == nil { if err == nil {
t.Fatal("expected scan to abort on the packer error, got nil") t.Fatal("expected scan to abort on the packer error, got nil")
} }
@@ -212,7 +184,7 @@ func TestScannerReadErrorAbortsWithoutSkipErrors(t *testing.T) {
fs := &readFailFs{Fs: afero.NewMemMapFs(), target: target} fs := &readFailFs{Fs: afero.NewMemMapFs(), target: target}
writeSkipErrorTestFile(t, fs, target, "content that cannot be read") writeSkipErrorTestFile(t, fs, target, "content that cannot be read")
_, err := runSkipErrorScan(t, fs, "/source", false, nil) _, err := runSkipErrorScan(t, fs, false)
if err == nil { if err == nil {
t.Fatal("expected scan to fail on the read error, got nil") t.Fatal("expected scan to fail on the read error, got nil")
} }
@@ -228,7 +200,7 @@ func TestScannerReadErrorSkippedWithSkipErrors(t *testing.T) {
fs := &readFailFs{Fs: afero.NewMemMapFs(), target: target} fs := &readFailFs{Fs: afero.NewMemMapFs(), target: target}
writeSkipErrorTestFile(t, fs, target, "content that cannot be read") writeSkipErrorTestFile(t, fs, target, "content that cannot be read")
repos, err := runSkipErrorScan(t, fs, "/source", true, nil) repos, err := runSkipErrorScan(t, fs, true)
if err != nil { if err != nil {
t.Fatalf("expected scan to complete with --skip-errors, got %v", err) t.Fatalf("expected scan to complete with --skip-errors, got %v", err)
} }
@@ -242,63 +214,3 @@ func TestScannerReadErrorSkippedWithSkipErrors(t *testing.T) {
t.Fatalf("expected unreadable file skipped, got %d chunks", len(chunks)) t.Fatalf("expected unreadable file skipped, got %d chunks", len(chunks))
} }
} }
// writeSymlinkSource creates a source directory on disk holding one symlink
// and returns the directory and the symlink's path.
func writeSymlinkSource(t *testing.T) (string, string) {
t.Helper()
sourceDir := t.TempDir()
linkPath := filepath.Join(sourceDir, "link")
err := os.Symlink("target.txt", linkPath)
if err != nil {
t.Fatalf("creating symlink: %v", err)
}
return sourceDir, linkPath
}
// TestScannerUnreadableSymlinkAbortsWithoutSkipErrors checks that a symlink
// whose target cannot be read aborts the run when --skip-errors is not set.
func TestScannerUnreadableSymlinkAbortsWithoutSkipErrors(t *testing.T) {
t.Parallel()
sourceDir, linkPath := writeSymlinkSource(t)
fs := &linkRemovedAfterLstatFs{t: t, target: linkPath}
_, err := runSkipErrorScan(t, fs, sourceDir, false, nil)
if !errors.Is(err, os.ErrNotExist) {
t.Fatalf("expected scan to fail on the removed symlink, got %v", err)
}
}
// TestScannerUnreadableSymlinkSkippedWithSkipErrors checks that a symlink
// whose target cannot be read is skipped with an error line, and the run
// completes, when --skip-errors is set.
func TestScannerUnreadableSymlinkSkippedWithSkipErrors(t *testing.T) {
t.Parallel()
sourceDir, linkPath := writeSymlinkSource(t)
fs := &linkRemovedAfterLstatFs{t: t, target: linkPath}
uiw := ui.NewWithColor(io.Discard, false)
repos, err := runSkipErrorScan(t, fs, sourceDir, true, uiw)
if err != nil {
t.Fatalf("expected scan to complete with --skip-errors, got %v", err)
}
if uiw.ErrorCount() != 1 {
t.Fatalf("expected one error line for the symlink, got %d",
uiw.ErrorCount())
}
file, err := repos.Files.GetByPath(context.Background(), linkPath)
if err != nil {
t.Fatalf("getting %s: %v", linkPath, err)
}
if file != nil {
t.Fatalf("expected %s not to be recorded", linkPath)
}
}
+66 -47
View File
@@ -105,21 +105,17 @@ func (sm *SnapshotManager) CreateSnapshot(
return sm.CreateSnapshotWithName(ctx, hostname, "", version, gitRevision) return sm.CreateSnapshotWithName(ctx, hostname, "", version, gitRevision)
} }
// ShortHostname returns hostname up to its first dot. A snapshot ID starts
// with this form, while the snapshots table stores the full hostname.
func ShortHostname(hostname string) string {
short, _, _ := strings.Cut(hostname, ".")
return short
}
// CreateSnapshotWithName creates a new snapshot record with an optional // CreateSnapshotWithName creates a new snapshot record with an optional
// snapshot name. The snapshot ID format is: hostname_name_timestamp or // snapshot name. The snapshot ID format is: hostname_name_timestamp or
// hostname_timestamp if name is empty. // hostname_timestamp if name is empty.
func (sm *SnapshotManager) CreateSnapshotWithName( func (sm *SnapshotManager) CreateSnapshotWithName(
ctx context.Context, hostname, name, version, gitRevision string, ctx context.Context, hostname, name, version, gitRevision string,
) (string, error) { ) (string, error) {
shortHostname := ShortHostname(hostname) // Use short hostname (strip domain if present)
shortHostname := hostname
if before, _, ok := strings.Cut(hostname, "."); ok {
shortHostname = before
}
// Build snapshot ID with optional name // Build snapshot ID with optional name
timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z") timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z")
@@ -158,6 +154,26 @@ func (sm *SnapshotManager) CreateSnapshotWithName(
return snapshotID, nil return snapshotID, nil
} }
// UpdateSnapshotStats updates the statistics for a snapshot during backup
func (sm *SnapshotManager) UpdateSnapshotStats(
ctx context.Context, snapshotID string, stats BackupStats,
) error {
err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID,
int64(stats.FilesScanned),
int64(stats.ChunksCreated),
int64(stats.BlobsCreated),
stats.BytesScanned,
stats.BytesUploaded,
)
})
if err != nil {
return fmt.Errorf("updating snapshot stats: %w", err)
}
return nil
}
// UpdateSnapshotStatsExtended updates snapshot statistics with extended metrics. // UpdateSnapshotStatsExtended updates snapshot statistics with extended metrics.
// This includes compression level, uncompressed blob size, and upload duration. // This includes compression level, uncompressed blob size, and upload duration.
func (sm *SnapshotManager) UpdateSnapshotStatsExtended( func (sm *SnapshotManager) UpdateSnapshotStatsExtended(
@@ -169,8 +185,8 @@ func (sm *SnapshotManager) UpdateSnapshotStatsExtended(
int64(stats.FilesScanned), int64(stats.FilesScanned),
int64(stats.ChunksCreated), int64(stats.ChunksCreated),
int64(stats.BlobsCreated), int64(stats.BlobsCreated),
stats.TotalSize, stats.BytesScanned,
stats.BlobSize, stats.BytesUploaded,
) )
if err != nil { if err != nil {
return err return err
@@ -180,7 +196,6 @@ func (sm *SnapshotManager) UpdateSnapshotStatsExtended(
return sm.repos.Snapshots.UpdateExtendedStats(ctx, tx, snapshotID, return sm.repos.Snapshots.UpdateExtendedStats(ctx, tx, snapshotID,
stats.BlobUncompressedSize, stats.BlobUncompressedSize,
stats.CompressionLevel, stats.CompressionLevel,
stats.BytesUploaded,
stats.UploadDurationMs, stats.UploadDurationMs,
) )
}) })
@@ -371,12 +386,12 @@ func (sm *SnapshotManager) prepareExportDB(
ctx context.Context, dbPath, snapshotID, tempDir string, ctx context.Context, dbPath, snapshotID, tempDir string,
) ([]byte, string, error) { ) ([]byte, string, error) {
// Step 1: Copy database to temp file // Step 1: Copy database to temp file
// The main database is still open here, so it is copied through SQLite // The main database should be closed at this point
tempDBPath := filepath.Join(tempDir, "snapshot.db") tempDBPath := filepath.Join(tempDir, "snapshot.db")
log.Debug("Copying database to temporary location", log.Debug("Copying database to temporary location",
"source", dbPath, "destination", tempDBPath) "source", dbPath, "destination", tempDBPath)
err := sm.copyDatabase(ctx, dbPath, tempDBPath) err := sm.copyFile(dbPath, tempDBPath)
if err != nil { if err != nil {
return nil, "", fmt.Errorf("copying database: %w", err) return nil, "", fmt.Errorf("copying database: %w", err)
} }
@@ -633,10 +648,9 @@ func (sm *SnapshotManager) collectCleanupStats(
// //
// VACUUM runs through the modernc.org/sqlite driver, on a freshly opened // VACUUM runs through the modernc.org/sqlite driver, on a freshly opened
// connection with no transaction in flight (VACUUM cannot run inside one). // connection with no transaction in flight (VACUUM cannot run inside one).
// database.New opens the file in WAL mode, so VACUUM's rewrite lands in the // The database opens in WAL mode, so VACUUM's rewrite lands in the WAL; the
// -wal file. This is the only connection to the file, so closing it // checkpoint on Close flushes it into the main file, which is the file we
// checkpoints the rewrite into the main file and removes the -wal file; the // then compress and upload.
// main file is the one compressFile then compresses and uploads.
func (sm *SnapshotManager) vacuumDatabase(ctx context.Context, dbPath string) error { func (sm *SnapshotManager) vacuumDatabase(ctx context.Context, dbPath string) error {
log.Debug("Running VACUUM on database", "path", dbPath) log.Debug("Running VACUUM on database", "path", dbPath)
@@ -730,15 +744,26 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
// user; it holds the same private index data as the local index file. // user; it holds the same private index data as the local index file.
const exportCopyPerm = 0o600 const exportCopyPerm = 0o600
// copyDatabase copies the database at src to dst with VACUUM INTO. It reads // copyFile copies a file from src to dst. The destination is the exported
// through SQLite, so the copy holds rows committed to src that are still in // snapshot database, so it is created owner-only rather than with the
// its -wal file, which a copy of the file alone would miss. The destination // umask-dependent default.
// is the exported snapshot database, so it is created empty and owner-only func (sm *SnapshotManager) copyFile(src, dst string) error {
// first rather than with the umask-dependent default: VACUUM INTO writes log.Debug("Opening source file for copy", "path", src)
// into an existing empty file and keeps its mode.
func (sm *SnapshotManager) copyDatabase( sourceFile, err := sm.fs.Open(src)
ctx context.Context, src, dst string, if err != nil {
) error { return err
}
defer func() {
log.Debug("Closing source file", "path", src)
err := sourceFile.Close()
if err != nil {
log.Debug("Failed to close source file", "path", src, "error", err)
}
}()
log.Debug("Creating destination file", "path", dst) log.Debug("Creating destination file", "path", dst)
destFile, err := sm.fs.OpenFile( destFile, err := sm.fs.OpenFile(
@@ -748,28 +773,23 @@ func (sm *SnapshotManager) copyDatabase(
return err return err
} }
err = destFile.Close() defer func() {
log.Debug("Closing destination file", "path", dst)
err := destFile.Close()
if err != nil {
log.Debug("Failed to close destination file", "path", dst, "error", err)
}
}()
log.Debug("Copying file data")
n, err := io.Copy(destFile, sourceFile)
if err != nil { if err != nil {
return err return err
} }
db, err := database.New(ctx, src) log.Debug("File copy complete", "bytes_copied", n)
if err != nil {
return fmt.Errorf("opening database to copy: %w", err)
}
defer func() {
cerr := db.Close()
if cerr != nil {
log.Debug("Failed to close database after copy",
"path", src, "error", cerr)
}
}()
_, err = db.ExecWithLog(ctx, "VACUUM INTO ?", dst)
if err != nil {
return fmt.Errorf("running VACUUM INTO: %w", err)
}
return nil return nil
} }
@@ -875,7 +895,7 @@ func (sm *SnapshotManager) getFileSize(path string) int64 {
// BackupStats contains statistics from a backup operation // BackupStats contains statistics from a backup operation
type BackupStats struct { type BackupStats struct {
FilesScanned int FilesScanned int
TotalSize int64 // Total size of all files examined BytesScanned int64
ChunksCreated int ChunksCreated int
BlobsCreated int BlobsCreated int
BytesUploaded int64 BytesUploaded int64
@@ -885,7 +905,6 @@ type BackupStats struct {
type ExtendedBackupStats struct { type ExtendedBackupStats struct {
BackupStats BackupStats
BlobSize int64 // Total compressed size of all referenced blobs
BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs
CompressionLevel int // Compression level used for this snapshot CompressionLevel int // Compression level used for this snapshot
UploadDurationMs int64 // Total milliseconds spent uploading to S3 UploadDurationMs int64 // Total milliseconds spent uploading to S3
-71
View File
@@ -188,77 +188,6 @@ func TestVacuumDatabaseRemovesDeletedData(t *testing.T) {
} }
} }
// TestPrepareExportDBKeepsRowsCommittedToOpenIndex exports from an index
// that is still open, as a backup does. A row committed there can still be
// in the index's -wal file, and the export must hold it all the same.
func TestPrepareExportDBKeepsRowsCommittedToOpenIndex(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
fs := afero.NewOsFs()
dbPath := filepath.Join(t.TempDir(), "index.sqlite")
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
repos := database.NewRepositories(db)
snapshot := &database.Snapshot{ID: "open-index-snapshot", Hostname: "test-host"}
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
return repos.Snapshots.Create(ctx, tx, snapshot)
})
if err != nil {
t.Fatalf("failed to create snapshot: %v", err)
}
sm := &SnapshotManager{
config: &config.Config{
CompressionLevel: 3,
AgeRecipients: []string{testAgeRecipient},
},
fs: fs,
}
_, tempDBPath, err := sm.prepareExportDB(
ctx, dbPath, snapshot.ID.String(), t.TempDir())
if err != nil {
t.Fatalf("prepareExportDB failed: %v", err)
}
// Only the main database file is compressed and uploaded, so open a
// copy of that file alone.
uploadedPath := filepath.Join(t.TempDir(), "uploaded.db")
err = copyFile(fs, tempDBPath, uploadedPath)
if err != nil {
t.Fatalf("failed to copy exported database: %v", err)
}
exported, err := database.OpenReadOnly(ctx, uploadedPath)
if err != nil {
t.Fatalf("failed to open exported database: %v", err)
}
defer func() { _ = exported.Close() }()
got, err := database.NewRepositories(exported).Snapshots.GetByID(
ctx, snapshot.ID.String())
if err != nil {
t.Fatalf("failed to read snapshot from export: %v", err)
}
if got == nil {
t.Fatal("exported database is missing the snapshot row")
}
}
func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
// Initialize logger // Initialize logger
log.Initialize(log.Config{}) log.Initialize(log.Config{})
+2 -3
View File
@@ -14,9 +14,8 @@ import (
// runStorerConformance is the shared Storer contract. Every backend that // runStorerConformance is the shared Storer contract. Every backend that
// can run in-process is expected to pass it: TestFileStorer runs it against // can run in-process is expected to pass it: TestFileStorer runs it against
// file://, TestS3Storer against s3://, TestRcloneStorer against rclone's // file://, TestS3Storer against s3://. A new backend inherits this coverage
// local backend. A new backend inherits this coverage by passing its own // by passing its own constructor, so the contract is defined once.
// constructor, so the contract is defined once.
// //
// It exercises the public Storer interface: round-trip, stat, list with // It exercises the public Storer interface: round-trip, stat, list with
// prefix filtering, overwrite, delete, delete-of-missing, and not-found on // prefix filtering, overwrite, delete, delete-of-missing, and not-found on
+11 -27
View File
@@ -22,11 +22,11 @@ type FileStorer struct {
// //
// Construction is intentionally cheap and does not touch the filesystem. // Construction is intentionally cheap and does not touch the filesystem.
// The basePath is recorded; the directory is created lazily on first // The basePath is recorded; the directory is created lazily on first
// write. Write operations (Put/PutWithProgress) call MkdirAll for the // write. Reads (Get/Stat/List) tolerate a missing basePath — a missing
// or unmounted destination during `snapshot list` should NOT block the
// command, it should degrade to "no remote snapshots reachable" with a
// warning. Write operations (Put/PutWithProgress) call MkdirAll for the
// per-blob parent directory, which also covers basePath on first use. // per-blob parent directory, which also covers basePath on first use.
// Get and Stat report a key under a missing basePath as ErrNotFound.
// List and ListStream fail on a missing basePath, because listing it as
// an empty store would make `prune` drop every local snapshot record.
// //
// Uses the real OS filesystem by default; call SetFilesystem to // Uses the real OS filesystem by default; call SetFilesystem to
// override for testing. // override for testing.
@@ -50,10 +50,9 @@ const storageDirPerm = 0o755
// temp file carrying this suffix and only renames it onto the real key once // temp file carrying this suffix and only renames it onto the real key once
// the whole object is on disk, so an interrupted write can never leave a // the whole object is on disk, so an interrupted write can never leave a
// truncated object at the key a later run would Stat and trust as a complete // truncated object at the key a later run would Stat and trust as a complete
// blob. The rclone backend's upload does the same on remotes with a // blob. List and ListStream skip these files, so a leftover from an
// server-side move. List and ListStream skip these files, so a leftover from // interrupted write is never listed or trusted as a blob; it is otherwise
// an interrupted write is never listed or trusted as a blob; it is otherwise // harmless and is overwritten when the same key is written again.
// harmless.
const tempSuffix = ".partial" const tempSuffix = ".partial"
// Put stores data at the specified key. // Put stores data at the specified key.
@@ -120,20 +119,13 @@ func (f *FileStorer) Delete(_ context.Context, key string) error {
return nil return nil
} }
// List returns all keys with the given prefix. It fails when the // List returns all keys with the given prefix.
// destination directory is missing; a missing prefix under it is an
// empty listing.
func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) { func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error) {
var keys []string var keys []string
_, err := f.fs.Stat(f.basePath)
if err != nil {
return nil, fmt.Errorf("checking destination directory: %w", err)
}
basePath := f.fullPath(prefix) basePath := f.fullPath(prefix)
// Check if the prefix exists // Check if base path exists
exists, err := afero.Exists(f.fs, basePath) exists, err := afero.Exists(f.fs, basePath)
if err != nil { if err != nil {
return nil, fmt.Errorf("checking path: %w", err) return nil, fmt.Errorf("checking path: %w", err)
@@ -175,24 +167,16 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
return keys, nil return keys, nil
} }
// ListStream returns a channel of ObjectInfo for large result sets. Like // ListStream returns a channel of ObjectInfo for large result sets.
// List, it sends an error when the destination directory is missing.
func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan ObjectInfo { func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan ObjectInfo {
ch := make(chan ObjectInfo) ch := make(chan ObjectInfo)
go func() { go func() {
defer close(ch) defer close(ch)
_, err := f.fs.Stat(f.basePath)
if err != nil {
ch <- ObjectInfo{Err: fmt.Errorf("checking destination directory: %w", err)}
return
}
basePath := f.fullPath(prefix) basePath := f.fullPath(prefix)
// Check if the prefix exists // Check if base path exists
exists, err := afero.Exists(f.fs, basePath) exists, err := afero.Exists(f.fs, basePath)
if err != nil { if err != nil {
ch <- ObjectInfo{Err: fmt.Errorf("checking path: %w", err)} ch <- ObjectInfo{Err: fmt.Errorf("checking path: %w", err)}
+2 -5
View File
@@ -14,9 +14,6 @@ import (
// errStreamInterrupted stands in for an upload cut off mid-stream. // errStreamInterrupted stands in for an upload cut off mid-stream.
var errStreamInterrupted = errors.New("connection reset mid-upload") var errStreamInterrupted = errors.New("connection reset mid-upload")
// testBlobKey is a key laid out as a blob's key is.
const testBlobKey = "blobs/aa/bb/aabbccddeeff"
// failingReader yields its data once, then fails. // failingReader yields its data once, then fails.
type failingReader struct { type failingReader struct {
data []byte data []byte
@@ -46,7 +43,7 @@ func TestFileStorer_InterruptedWriteLeavesNoTrustedObject(t *testing.T) {
} }
ctx := context.Background() ctx := context.Background()
key := testBlobKey key := "blobs/aa/bb/aabbccddeeff"
err = f.PutWithProgress(ctx, key, &failingReader{data: []byte("partial")}, 4096, nil) err = f.PutWithProgress(ctx, key, &failingReader{data: []byte("partial")}, 4096, nil)
if err == nil { if err == nil {
@@ -82,7 +79,7 @@ func TestFileStorer_ListSkipsPartialFiles(t *testing.T) {
} }
ctx := context.Background() ctx := context.Background()
realKey := testBlobKey realKey := "blobs/aa/bb/aabbccddeeff"
err = f.Put(ctx, realKey, strings.NewReader("blob-bytes")) err = f.Put(ctx, realKey, strings.NewReader("blob-bytes"))
if err != nil { if err != nil {
-35
View File
@@ -1,10 +1,6 @@
package storage_test package storage_test
import ( import (
"context"
"errors"
"io/fs"
"path/filepath"
"testing" "testing"
"sneak.berlin/go/vaultik/internal/storage" "sneak.berlin/go/vaultik/internal/storage"
@@ -29,34 +25,3 @@ func TestFileStorer(t *testing.T) {
t.Parallel() t.Parallel()
runStorerConformance(t, newFileStorer) runStorerConformance(t, newFileStorer)
} }
// TestFileStorerListMissingDestination checks that List and ListStream
// fail when the destination directory does not exist, instead of
// reporting an empty store.
func TestFileStorerListMissingDestination(t *testing.T) {
t.Parallel()
ctx := context.Background()
s, err := storage.NewFileStorer(filepath.Join(t.TempDir(), "unmounted"))
if err != nil {
t.Fatalf("NewFileStorer: %v", err)
}
keys, err := s.List(ctx, "metadata/")
if !errors.Is(err, fs.ErrNotExist) {
t.Errorf("List = %v, %v; want a not-exist error", keys, err)
}
var streamErr error
for object := range s.ListStream(ctx, "metadata/") {
if object.Err != nil {
streamErr = object.Err
}
}
if !errors.Is(streamErr, fs.ErrNotExist) {
t.Errorf("ListStream error = %v, want a not-exist error", streamErr)
}
}
-2
View File
@@ -99,7 +99,6 @@ func storerFromParsedS3URL(parsed *URL, cfg *config.Config) (Storer, error) {
AccessKeyID: cfg.S3.AccessKeyID, AccessKeyID: cfg.S3.AccessKeyID,
SecretAccessKey: cfg.S3.SecretAccessKey, SecretAccessKey: cfg.S3.SecretAccessKey,
Region: region, Region: region,
PartSize: cfg.S3.PartSize.Int64(),
}) })
if err != nil { if err != nil {
return nil, fmt.Errorf("creating S3 client: %w", err) return nil, fmt.Errorf("creating S3 client: %w", err)
@@ -135,7 +134,6 @@ func storerFromLegacyS3Config(cfg *config.Config) (Storer, error) {
AccessKeyID: cfg.S3.AccessKeyID, AccessKeyID: cfg.S3.AccessKeyID,
SecretAccessKey: cfg.S3.SecretAccessKey, SecretAccessKey: cfg.S3.SecretAccessKey,
Region: region, Region: region,
PartSize: cfg.S3.PartSize.Int64(),
}) })
if err != nil { if err != nil {
return nil, fmt.Errorf("creating S3 client: %w", err) return nil, fmt.Errorf("creating S3 client: %w", err)
+15 -64
View File
@@ -3,7 +3,6 @@ package storage
import ( import (
"bytes" "bytes"
"context" "context"
"crypto/rand"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@@ -69,7 +68,14 @@ func (r *RcloneStorer) Put(ctx context.Context, key string, data io.Reader) erro
return fmt.Errorf("reading data: %w", err) return fmt.Errorf("reading data: %w", err)
} }
return r.upload(ctx, key, bytes.NewReader(buf)) // Upload the object
_, err = operations.Rcat(ctx, r.fsys, key,
io.NopCloser(bytes.NewReader(buf)), time.Now(), nil)
if err != nil {
return fmt.Errorf("uploading object: %w", err)
}
return nil
} }
// PutWithProgress stores data with progress reporting. // PutWithProgress stores data with progress reporting.
@@ -83,7 +89,13 @@ func (r *RcloneStorer) PutWithProgress(
callback: progress, callback: progress,
} }
return r.upload(ctx, key, pr) // Upload the object
_, err := operations.Rcat(ctx, r.fsys, key, io.NopCloser(pr), time.Now(), nil)
if err != nil {
return fmt.Errorf("uploading object: %w", err)
}
return nil
} }
// Get retrieves data from the specified key. // Get retrieves data from the specified key.
@@ -161,10 +173,6 @@ func (r *RcloneStorer) List(ctx context.Context, prefix string) ([]string, error
err := operations.ListFn(ctx, r.fsys, func(obj fs.Object) { err := operations.ListFn(ctx, r.fsys, func(obj fs.Object) {
key := obj.Remote() key := obj.Remote()
if strings.HasSuffix(key, tempSuffix) {
return
}
if prefix == "" || strings.HasPrefix(key, prefix) { if prefix == "" || strings.HasPrefix(key, prefix) {
keys = append(keys, key) keys = append(keys, key)
} }
@@ -194,10 +202,6 @@ func (r *RcloneStorer) ListStream(
} }
key := obj.Remote() key := obj.Remote()
if strings.HasSuffix(key, tempSuffix) {
return
}
if prefix == "" || strings.HasPrefix(key, prefix) { if prefix == "" || strings.HasPrefix(key, prefix) {
ch <- ObjectInfo{ ch <- ObjectInfo{
Key: key, Key: key,
@@ -226,59 +230,6 @@ func (r *RcloneStorer) Info() Info {
} }
} }
// upload writes data to key. Where the remote has a server-side move, it
// writes under a temporary name ending in tempSuffix and moves the object
// onto key once it is complete, so a killed upload cannot leave a truncated
// object at key; a remote without one is written in place. List and
// ListStream skip a temporary object left behind.
//
// rclone's own copy does this only where the remote also sets
// PartialUploads. That flag is not checked here: hdfs, for one, shows a
// file while it is written without setting it.
func (r *RcloneStorer) upload(ctx context.Context, key string, data io.Reader) error {
if r.fsys.Features().Move == nil {
_, err := operations.Rcat(ctx, r.fsys, key, io.NopCloser(data), time.Now(), nil)
if err != nil {
return fmt.Errorf("uploading object: %w", err)
}
return nil
}
tempKey := key + "-" + rand.Text() + tempSuffix
obj, err := operations.Rcat(ctx, r.fsys, tempKey, io.NopCloser(data), time.Now(), nil)
if err != nil {
// Rcat returns the object it wrote when the written data fails its check.
if obj != nil {
_ = obj.Remove(ctx)
}
return fmt.Errorf("uploading object: %w", err)
}
// On drive, dropbox, onedrive and others the remote's own move does not
// replace an object already at key. operations.Move removes the object
// it is given first, and copies where the remote refuses the move.
existing, err := r.fsys.NewObject(ctx, key)
if errors.Is(err, fs.ErrorObjectNotFound) {
existing = nil
} else if err != nil {
_ = obj.Remove(ctx)
return fmt.Errorf("looking up existing object: %w", err)
}
_, err = operations.Move(ctx, r.fsys, existing, key, obj)
if err != nil {
_ = obj.Remove(ctx)
return fmt.Errorf("moving object into place: %w", err)
}
return nil
}
// progressReader wraps an io.Reader to track read progress. // progressReader wraps an io.Reader to track read progress.
type progressReader struct { type progressReader struct {
reader io.Reader reader io.Reader
+14 -307
View File
@@ -1,47 +1,28 @@
package storage_test package storage_test
import ( import (
"bytes"
"context" "context"
"errors" "errors"
"io"
"os"
"path/filepath"
"strings"
"testing" "testing"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config/configmap"
"sneak.berlin/go/vaultik/internal/storage" "sneak.berlin/go/vaultik/internal/storage"
) )
// The rclone backend is a thin adapter over the rclone library: it turns a
// (remote, path) pair into rclone's "remote:path" string, hands it to
// rclone, and maps rclone's own results back to the Storer interface. What
// can be tested in-process, without a configured remote or network, is that
// adapter layer — how the arguments are shaped and how construction errors
// are reported. The data-plane operations (Put/Get/List/Delete) are rclone's
// own, exercised against a real provider (drive, s3-via-rclone, ...), which
// needs a configured remote with credentials and network access and so is
// out of reach of a unit test. The shared Storer conformance suite therefore
// runs against the in-process file and s3 backends; the rclone backend
// inherits that contract once a remote is configured.
//
// These tests use rclone's ":local:" on-the-fly backend, which addresses the // These tests use rclone's ":local:" on-the-fly backend, which addresses the
// local filesystem directly without any configured remote, so they run // local filesystem directly without any configured remote, so construction
// entirely in-process. A remote that needs credentials and network access // runs entirely in-process.
// (drive, s3 via rclone, ...) is out of reach of a unit test.
// newRcloneStorer builds an rclone backend on rclone's local backend,
// rooted at a fresh temp directory.
//
//nolint:ireturn // conformance runs against the Storer interface by design
func newRcloneStorer(t *testing.T) storage.Storer {
t.Helper()
s, err := storage.NewRcloneStorer(context.Background(), ":local", t.TempDir())
if err != nil {
t.Fatalf("NewRcloneStorer: %v", err)
}
return s
}
// TestRcloneStorer runs the shared Storer contract against the rclone
// backend.
//
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
func TestRcloneStorer(t *testing.T) {
runStorerConformance(t, newRcloneStorer)
}
// TestNewRcloneStorerConstruction checks that a valid remote constructs a // TestNewRcloneStorerConstruction checks that a valid remote constructs a
// backend and that Info() reports the shaped "remote:path" location. // backend and that Info() reports the shaped "remote:path" location.
@@ -63,280 +44,6 @@ func TestNewRcloneStorerConstruction(t *testing.T) {
} }
} }
// unbufferedUploadContext returns a context in which rclone neither reads
// ahead of the write nor holds a small upload in memory. Without it, a
// progress callback runs before anything is written to the remote.
func unbufferedUploadContext() context.Context {
ctx, ci := fs.AddConfig(context.Background())
ci.BufferSize = 0
ci.StreamingUploadCutoff = 0
return ctx
}
// objectAtKeyDuringUpload uploads data to testBlobKey and reports whether
// an object was at the key before the upload finished. It fails the test
// unless the key then reads back as the uploaded data.
func objectAtKeyDuringUpload(
ctx context.Context, t *testing.T, s *storage.RcloneStorer,
) bool {
t.Helper()
data := bytes.Repeat([]byte("blob-bytes"), 1000)
seen := false
err := s.PutWithProgress(ctx, testBlobKey, bytes.NewReader(data),
int64(len(data)), func(int64) error {
_, statErr := s.Stat(ctx, testBlobKey)
if statErr == nil {
seen = true
}
return nil
})
if err != nil {
t.Fatalf("PutWithProgress: %v", err)
}
got := readObject(ctx, t, s, testBlobKey)
if !bytes.Equal(got, data) {
t.Errorf("object read back as %d bytes, want the %d uploaded",
len(got), len(data))
}
return seen
}
// readObject returns the contents of the object at key.
func readObject(
ctx context.Context, t *testing.T, s *storage.RcloneStorer, key string,
) []byte {
t.Helper()
rc, err := s.Get(ctx, key)
if err != nil {
t.Fatalf("Get: %v", err)
}
defer func() { _ = rc.Close() }()
got, err := io.ReadAll(rc)
if err != nil {
t.Fatalf("reading object: %v", err)
}
return got
}
// TestRcloneStorerObjectAppearsOnlyWhenComplete checks that on a remote
// with a server-side move, such as local, nothing is at the key until the
// upload has finished, so a killed upload cannot leave a truncated object
// there.
//
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
func TestRcloneStorerObjectAppearsOnlyWhenComplete(t *testing.T) {
ctx := unbufferedUploadContext()
s, err := storage.NewRcloneStorer(ctx, ":local", t.TempDir())
if err != nil {
t.Fatalf("NewRcloneStorer: %v", err)
}
if objectAtKeyDuringUpload(ctx, t, s) {
t.Error("object was at its key before the upload finished")
}
}
// TestRcloneStorerListSkipsPartialFiles checks that a temporary file left
// by a killed upload is never listed as a key.
//
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
func TestRcloneStorerListSkipsPartialFiles(t *testing.T) {
dir := t.TempDir()
ctx := context.Background()
s, err := storage.NewRcloneStorer(ctx, ":local", dir)
if err != nil {
t.Fatalf("NewRcloneStorer: %v", err)
}
realKey := testBlobKey
err = s.Put(ctx, realKey, strings.NewReader("blob-bytes"))
if err != nil {
t.Fatalf("Put: %v", err)
}
leftover := filepath.Join(dir, realKey+"-123456.partial")
err = os.WriteFile(leftover, []byte("half"), 0o600)
if err != nil {
t.Fatalf("writing leftover temp file: %v", err)
}
keys, err := s.List(ctx, "blobs/")
if err != nil {
t.Fatalf("List: %v", err)
}
if len(keys) != 1 || keys[0] != realKey {
t.Fatalf("List should return only the real key, got %v", keys)
}
var streamed []string
for obj := range s.ListStream(ctx, "blobs/") {
if obj.Err != nil {
t.Fatalf("ListStream: %v", obj.Err)
}
streamed = append(streamed, obj.Key)
}
if len(streamed) != 1 || streamed[0] != realKey {
t.Fatalf("ListStream should return only the real key, got %v", streamed)
}
}
// newRcloneStorerOnWrappedLocal registers name as rclone's local backend
// wrapped by wrap, and builds an rclone backend on it rooted at a fresh
// temp directory. wrap changes the features the local backend reports, so
// that it behaves like a remote a unit test cannot reach.
func newRcloneStorerOnWrappedLocal(
ctx context.Context, t *testing.T, name string, wrap func(fs.Fs) fs.Fs,
) *storage.RcloneStorer {
t.Helper()
fs.Register(&fs.RegInfo{
Name: name,
NewFs: func(
ctx context.Context, _, root string, _ configmap.Mapper,
) (fs.Fs, error) {
local, err := fs.NewFs(ctx, ":local:"+root)
if err != nil {
return nil, err
}
return wrap(local), nil
},
})
s, err := storage.NewRcloneStorer(ctx, ":"+name, t.TempDir())
if err != nil {
t.Fatalf("NewRcloneStorer: %v", err)
}
return s
}
// withoutPartialUploads is rclone's local backend with the PartialUploads
// flag cleared. Like hdfs, it then has a server-side move and shows a file
// while it is written, without setting that flag.
type withoutPartialUploads struct {
fs.Fs
}
func (f *withoutPartialUploads) Features() *fs.Features {
features := *f.Fs.Features()
features.PartialUploads = false
return &features
}
// TestRcloneStorerMovesIntoPlaceWithoutPartialUploads checks that on a
// remote with a server-side move nothing is at the key until the upload has
// finished, even when rclone does not mark the remote as showing partial
// uploads.
//
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
func TestRcloneStorerMovesIntoPlaceWithoutPartialUploads(t *testing.T) {
ctx := unbufferedUploadContext()
s := newRcloneStorerOnWrappedLocal(ctx, t, "withoutpartialuploads",
func(local fs.Fs) fs.Fs { return &withoutPartialUploads{Fs: local} })
if objectAtKeyDuringUpload(ctx, t, s) {
t.Error("object was at its key before the upload finished")
}
}
// withoutMove is rclone's local backend reporting no server-side move.
type withoutMove struct {
fs.Fs
}
func (f *withoutMove) Features() *fs.Features {
features := *f.Fs.Features()
features.Move = nil
return &features
}
// TestRcloneStorerWritesInPlaceWithoutMove checks that on a remote with no
// server-side move an object is written straight to its key and reads back
// from there.
//
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
func TestRcloneStorerWritesInPlaceWithoutMove(t *testing.T) {
ctx := unbufferedUploadContext()
s := newRcloneStorerOnWrappedLocal(ctx, t, "withoutmove",
func(local fs.Fs) fs.Fs { return &withoutMove{Fs: local} })
if !objectAtKeyDuringUpload(ctx, t, s) {
t.Error("object was not at its key while it was uploaded")
}
}
var errNameConflict = errors.New("an object with this name already exists")
// moveRefusesExisting is rclone's local backend with a server-side move
// that, like dropbox's or onedrive's, refuses to move onto an existing
// object.
type moveRefusesExisting struct {
fs.Fs
}
func (f *moveRefusesExisting) Features() *fs.Features {
features := *f.Fs.Features()
features.Move = f.move
return &features
}
//nolint:ireturn // the signature is rclone's
func (f *moveRefusesExisting) move(
ctx context.Context, src fs.Object, remote string,
) (fs.Object, error) {
_, err := f.NewObject(ctx, remote)
if err == nil {
return nil, errNameConflict
}
return f.Fs.Features().Move(ctx, src, remote)
}
// TestRcloneStorerOverwritesWhereMoveRefusesExisting checks that writing a
// key twice replaces the object on a remote whose server-side move will not
// replace an existing object.
//
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
func TestRcloneStorerOverwritesWhereMoveRefusesExisting(t *testing.T) {
ctx := context.Background()
s := newRcloneStorerOnWrappedLocal(ctx, t, "moverefusesexisting",
func(local fs.Fs) fs.Fs { return &moveRefusesExisting{Fs: local} })
for _, content := range []string{"first", "second"} {
err := s.Put(ctx, testBlobKey, strings.NewReader(content))
if err != nil {
t.Fatalf("Put %q: %v", content, err)
}
}
got := readObject(ctx, t, s, testBlobKey)
if string(got) != "second" {
t.Errorf("object = %q, want %q", got, "second")
}
}
// TestNewRcloneStorerUnknownRemote checks that a remote that is not in the // TestNewRcloneStorerUnknownRemote checks that a remote that is not in the
// rclone config fails construction with the ErrRemoteNotFound sentinel, // rclone config fails construction with the ErrRemoteNotFound sentinel,
// rather than silently returning a backend pointed nowhere. // rather than silently returning a backend pointed nowhere.
-203
View File
@@ -1,20 +1,14 @@
package storage_test package storage_test
import ( import (
"bytes"
"context" "context"
"errors" "errors"
"net/http"
"net/http/httptest" "net/http/httptest"
"slices"
"strings"
"sync/atomic"
"testing" "testing"
"github.com/johannesboyne/gofakes3" "github.com/johannesboyne/gofakes3"
"github.com/johannesboyne/gofakes3/backend/s3mem" "github.com/johannesboyne/gofakes3/backend/s3mem"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/s3" "sneak.berlin/go/vaultik/internal/s3"
"sneak.berlin/go/vaultik/internal/storage" "sneak.berlin/go/vaultik/internal/storage"
) )
@@ -22,13 +16,6 @@ import (
// s3TestBucket is the bucket created for each in-process S3 server. // s3TestBucket is the bucket created for each in-process S3 server.
const s3TestBucket = "test-bucket" const s3TestBucket = "test-bucket"
// Credentials for the tests that build a storer from a config.Config. The
// in-process S3 server accepts any.
const (
s3TestAccessKeyID = "key"
s3TestSecretAccessKey = "secret"
)
// newS3Storer builds an s3:// backend backed by a fresh in-process // newS3Storer builds an s3:// backend backed by a fresh in-process
// S3 server. It reuses the same in-memory S3 harness (gofakes3 + s3mem // S3 server. It reuses the same in-memory S3 harness (gofakes3 + s3mem
// over httptest) that internal/s3 and the not-found regression test use, // over httptest) that internal/s3 and the not-found regression test use,
@@ -92,193 +79,3 @@ func TestS3StorerMissingKeyMapsToErrNotFound(t *testing.T) {
t.Errorf("Stat on missing key: got %v, want ErrNotFound", err) t.Errorf("Stat on missing key: got %v, want ErrNotFound", err)
} }
} }
// TestS3URLPrefixKeyLayout pins the bucket keys an s3:// URL reads and
// writes: the README's remote storage layout, with the prefix joined to
// each key by one "/". s3://b/p and s3://b/p/ must be the same
// destination, or a host that writes the URL the other way finds no
// snapshots. The listed object is put straight into the bucket, as
// another host would have written it. Both List and ListStream are
// checked: ListStream is what every snapshot listing goes through.
func TestS3URLPrefixKeyLayout(t *testing.T) {
t.Parallel()
const (
blobKey = "blobs/aa/bb/aabbccdd"
listPrefix = "metadata/"
manifestKey = listPrefix + "snap/manifest.json.zst"
manifestBody = "manifest"
)
cases := []struct {
urlPath string // URL path after the bucket name
keyPrefix string // what every key in the bucket must start with
}{
{urlPath: "/p", keyPrefix: "p/"},
{urlPath: "/p/", keyPrefix: "p/"},
{urlPath: "", keyPrefix: ""},
}
for _, tc := range cases {
storageURL := "s3://" + s3TestBucket + tc.urlPath
t.Run(storageURL, func(t *testing.T) {
t.Parallel()
backend := s3mem.New()
err := backend.CreateBucket(s3TestBucket)
if err != nil {
t.Fatalf("create bucket: %v", err)
}
srv := httptest.NewServer(gofakes3.New(backend).Server())
t.Cleanup(srv.Close)
storer, err := storage.NewStorer(&config.Config{
StorageURL: storageURL + "?endpoint=" + srv.URL,
S3: config.S3Config{
AccessKeyID: s3TestAccessKeyID,
SecretAccessKey: s3TestSecretAccessKey,
},
})
if err != nil {
t.Fatalf("NewStorer: %v", err)
}
ctx := context.Background()
err = storer.Put(ctx, blobKey, strings.NewReader("blob"))
if err != nil {
t.Fatalf("Put: %v", err)
}
_, err = backend.HeadObject(s3TestBucket, tc.keyPrefix+blobKey)
if err != nil {
t.Errorf("blob not stored at %q: %v", tc.keyPrefix+blobKey, err)
}
_, err = backend.PutObject(s3TestBucket, tc.keyPrefix+manifestKey,
nil, strings.NewReader(manifestBody), int64(len(manifestBody)))
if err != nil {
t.Fatalf("seed manifest: %v", err)
}
keys, err := storer.List(ctx, listPrefix)
if err != nil {
t.Fatalf("List: %v", err)
}
if !slices.Equal(keys, []string{manifestKey}) {
t.Errorf("List(%q) = %q, want [%q]", listPrefix, keys, manifestKey)
}
streamed := listStreamKeys(t, storer, listPrefix)
if !slices.Equal(streamed, []string{manifestKey}) {
t.Errorf("ListStream(%q) = %q, want [%q]", listPrefix, streamed, manifestKey)
}
})
}
}
// TestS3UploadUsesConfiguredPartSize checks that s3.part_size reaches the
// multipart uploader, through storage_url and through the s3.* fields. An
// object three parts long must arrive as three parts; at the SDK's default
// of 5 MiB it would arrive as four.
func TestS3UploadUsesConfiguredPartSize(t *testing.T) {
t.Parallel()
const (
partSize = 6 * 1024 * 1024
wantParts = 3
)
backend := s3mem.New()
err := backend.CreateBucket(s3TestBucket)
if err != nil {
t.Fatalf("create bucket: %v", err)
}
// Every part of a multipart upload is one request with a partNumber.
var parts atomic.Int32
fake := gofakes3.New(backend).Server()
srv := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
if r.URL.Query().Has("partNumber") {
parts.Add(1)
}
fake.ServeHTTP(w, r)
}))
t.Cleanup(srv.Close)
cases := []struct {
name string
cfg *config.Config
}{
{
name: "storage_url",
cfg: &config.Config{
StorageURL: "s3://" + s3TestBucket + "?endpoint=" + srv.URL,
S3: config.S3Config{
AccessKeyID: s3TestAccessKeyID,
SecretAccessKey: s3TestSecretAccessKey,
PartSize: partSize,
},
},
},
{
name: "s3.endpoint",
cfg: &config.Config{
S3: config.S3Config{
Endpoint: srv.URL,
Bucket: s3TestBucket,
AccessKeyID: s3TestAccessKeyID,
SecretAccessKey: s3TestSecretAccessKey,
PartSize: partSize,
},
},
},
}
for _, tc := range cases {
parts.Store(0)
storer, err := storage.NewStorer(tc.cfg)
if err != nil {
t.Fatalf("%s: NewStorer: %v", tc.name, err)
}
data := bytes.NewReader(make([]byte, wantParts*partSize))
err = storer.PutWithProgress(
context.Background(), "blob", data, data.Size(), nil)
if err != nil {
t.Fatalf("%s: PutWithProgress: %v", tc.name, err)
}
if got := parts.Load(); got != wantParts {
t.Errorf("%s: uploaded in %d parts, want %d", tc.name, got, wantParts)
}
}
}
// listStreamKeys returns the keys ListStream yields under a prefix, and
// fails the test on a listing error.
func listStreamKeys(t *testing.T, s storage.Storer, prefix string) []string {
t.Helper()
var keys []string
for obj := range s.ListStream(context.Background(), prefix) {
if obj.Err != nil {
t.Fatalf("ListStream %q: %v", prefix, obj.Err)
}
keys = append(keys, obj.Key)
}
return keys
}
+2 -1
View File
@@ -161,7 +161,8 @@ func rejectUnknownParams(query url.Values, allowed ...string) error {
// *url.Error that url.Parse returns embeds the raw URL in its message, so // *url.Error that url.Parse returns embeds the raw URL in its message, so
// wrapping it directly would echo a credential-bearing URL into logs. // wrapping it directly would echo a credential-bearing URL into logs.
func wrapParseError(err error) error { func wrapParseError(err error) error {
if uerr, ok := errors.AsType[*url.Error](err); ok { var uerr *url.Error
if errors.As(err, &uerr) {
return fmt.Errorf("invalid URL: %w", uerr.Err) return fmt.Errorf("invalid URL: %w", uerr.Err)
} }
+2 -2
View File
@@ -155,8 +155,8 @@ type BlobHash string
// FilePath represents an absolute path to a file or directory. // FilePath represents an absolute path to a file or directory.
type FilePath string type FilePath string
// SourcePath is the source directory a scan found a file under, made // SourcePath represents the root directory from which files are backed up.
// absolute and with symlinks resolved. // Used during restore to strip the source prefix from paths.
type SourcePath string type SourcePath string
// Hostname identifies a host machine. // Hostname identifies a host machine.
@@ -130,9 +130,10 @@ func assertThirdSnapshotRestores(
// up, and that snapshot is removed. The first snapshot keeps the file row, // up, and that snapshot is removed. The first snapshot keeps the file row,
// which now lists the appended content's chunks, while removal drops the // which now lists the appended content's chunks, while removal drops the
// blob that held them. // blob that held them.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestBackupAfterRemovingNewestSnapshotRestoresChangedFile(t *testing.T) { func TestBackupAfterRemovingNewestSnapshotRestoresChangedFile(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs() fs := afero.NewOsFs()
tempDir := t.TempDir() tempDir := t.TempDir()
@@ -177,9 +178,10 @@ func TestBackupAfterRemovingNewestSnapshotRestoresChangedFile(t *testing.T) {
// The next run's prune drops that incomplete snapshot and its blob, while // The next run's prune drops that incomplete snapshot and its blob, while
// the first snapshot keeps the file row, which now lists the appended // the first snapshot keeps the file row, which now lists the appended
// content's chunks. // content's chunks.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestBackupAfterInterruptedRunRestoresChangedFile(t *testing.T) { func TestBackupAfterInterruptedRunRestoresChangedFile(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs() fs := afero.NewOsFs()
tempDir := t.TempDir() tempDir := t.TempDir()
+22 -19
View File
@@ -38,9 +38,11 @@ import (
// (https://git.eeqj.de/sneak/vaultik/issues/130) and is not re-tested // (https://git.eeqj.de/sneak/vaultik/issues/130) and is not re-tested
// here; these tests target the layers above the backend. // here; these tests target the layers above the backend.
// //
// log.Initialize replaces the package-global logger that a running // The tests run serially, not with t.Parallel: each calls
// backup or restore reads, so each test calls it before t.Parallel, // log.Initialize, which replaces the package-global logger, and a
// while no parallel test is running yet. // backup or restore running concurrently reads that same logger. Under
// -race the two collide. Running one at a time is the same choice
// prune_count_test.go already makes for the same reason.
const ( const (
faultChunkSize = int64(64 * 1024) faultChunkSize = int64(64 * 1024)
@@ -163,19 +165,17 @@ func newReaderVaultik(
// Scenario 3: a stored blob's bytes are flipped before restore reads // Scenario 3: a stored blob's bytes are flipped before restore reads
// them. Restore must fail loudly, and no file must be left on the // them. Restore must fail loudly, and no file must be left on the
// restore target holding corrupt content. // restore target holding corrupt content.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestRestoreRejectsCorruptBlob(t *testing.T) { func TestRestoreRejectsCorruptBlob(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
assertRestoreRejectsDamagedBlob(t, faultstore.GetCorrupt, "corrupt") assertRestoreRejectsDamagedBlob(t, faultstore.GetCorrupt, "corrupt")
} }
// Scenario 4: a stored blob is truncated before restore reads it. Same // Scenario 4: a stored blob is truncated before restore reads it. Same
// contract as the corrupt case. // contract as the corrupt case.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestRestoreRejectsTruncatedBlob(t *testing.T) { func TestRestoreRejectsTruncatedBlob(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
assertRestoreRejectsDamagedBlob(t, faultstore.GetTruncate, "truncated") assertRestoreRejectsDamagedBlob(t, faultstore.GetTruncate, "truncated")
} }
@@ -188,6 +188,7 @@ func assertRestoreRejectsDamagedBlob(
t *testing.T, fault faultstore.GetFault, name string, t *testing.T, fault faultstore.GetFault, name string,
) { ) {
t.Helper() t.Helper()
log.Initialize(log.Config{})
fs := afero.NewOsFs() fs := afero.NewOsFs()
tempDir := t.TempDir() tempDir := t.TempDir()
@@ -231,9 +232,10 @@ func assertRestoreRejectsDamagedBlob(
// Scenario 6: the backend accepts blob uploads and reports success but // Scenario 6: the backend accepts blob uploads and reports success but
// stores nothing. verify --deep must catch it. // stores nothing. verify --deep must catch it.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestDeepVerifyCatchesLyingBackend(t *testing.T) { func TestDeepVerifyCatchesLyingBackend(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs() fs := afero.NewOsFs()
tempDir := t.TempDir() tempDir := t.TempDir()
@@ -283,9 +285,10 @@ func TestDeepVerifyCatchesLyingBackend(t *testing.T) {
// Scenario 1a: a blob upload fails partway through. The interrupted run // Scenario 1a: a blob upload fails partway through. The interrupted run
// must not record the blob as uploaded, must not reference it from the // must not record the blob as uploaded, must not reference it from the
// snapshot, and must leave no blob object at the destination. // snapshot, and must leave no blob object at the destination.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestInterruptedBlobUploadRecordsNoUploadedBlob(t *testing.T) { func TestInterruptedBlobUploadRecordsNoUploadedBlob(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs() fs := afero.NewOsFs()
tempDir := t.TempDir() tempDir := t.TempDir()
@@ -298,10 +301,6 @@ func TestInterruptedBlobUploadRecordsNoUploadedBlob(t *testing.T) {
writeFaultSourceTree(t, fs, dataDir) writeFaultSourceTree(t, fs, dataDir)
// No upload succeeds, so nothing creates the destination directory.
// It must exist for the listing below to show that no blob survived.
require.NoError(t, os.Mkdir(storeDir, 0o750))
inner, err := storage.NewFileStorer(storeDir) inner, err := storage.NewFileStorer(storeDir)
require.NoError(t, err) require.NoError(t, err)
@@ -358,9 +357,10 @@ func TestInterruptedBlobUploadRecordsNoUploadedBlob(t *testing.T) {
// chunks in a blob that was actually uploaded, so the retry re-chunks and // chunks in a blob that was actually uploaded, so the retry re-chunks and
// re-uploads the affected data instead of silently referencing data that // re-uploads the affected data instead of silently referencing data that
// never reached storage. // never reached storage.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestBackupRetryAfterInterruptedUploadIsRestorable(t *testing.T) { func TestBackupRetryAfterInterruptedUploadIsRestorable(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs() fs := afero.NewOsFs()
tempDir := t.TempDir() tempDir := t.TempDir()
@@ -422,9 +422,10 @@ func TestBackupRetryAfterInterruptedUploadIsRestorable(t *testing.T) {
// covered by TestBackupCompletesOnlyAfterMetadataExport // covered by TestBackupCompletesOnlyAfterMetadataExport
// (https://git.eeqj.de/sneak/vaultik/issues/177); this test exercises the // (https://git.eeqj.de/sneak/vaultik/issues/177); this test exercises the
// lower-level export path in isolation. // lower-level export path in isolation.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestBackupSurvivesMetadataExportInterruption(t *testing.T) { func TestBackupSurvivesMetadataExportInterruption(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs() fs := afero.NewOsFs()
tempDir := t.TempDir() tempDir := t.TempDir()
@@ -504,9 +505,10 @@ func TestBackupSurvivesMetadataExportInterruption(t *testing.T) {
// destination. Rerunning the backup must then prune the incomplete // destination. Rerunning the backup must then prune the incomplete
// snapshot, produce a snapshot whose destination metadata and local index // snapshot, produce a snapshot whose destination metadata and local index
// agree, and restore. See https://git.eeqj.de/sneak/vaultik/issues/177. // agree, and restore. See https://git.eeqj.de/sneak/vaultik/issues/177.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestBackupCompletesOnlyAfterMetadataExport(t *testing.T) { func TestBackupCompletesOnlyAfterMetadataExport(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs() fs := afero.NewOsFs()
tempDir := t.TempDir() tempDir := t.TempDir()
@@ -679,9 +681,10 @@ func faultScannerFactory(
// Scenario 5: the restore target runs out of space mid-file. Restore // Scenario 5: the restore target runs out of space mid-file. Restore
// must fail with an out-of-space error, and must not leave a truncated // must fail with an out-of-space error, and must not leave a truncated
// file at the target path presenting as a complete restore. // file at the target path presenting as a complete restore.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestRestoreReportsDiskFull(t *testing.T) { func TestRestoreReportsDiskFull(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel()
osFS := afero.NewOsFs() osFS := afero.NewOsFs()
tempDir := t.TempDir() tempDir := t.TempDir()
+18 -30
View File
@@ -9,7 +9,6 @@ import (
"time" "time"
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
"sneak.berlin/go/vaultik/internal/snapshot"
"sneak.berlin/go/vaultik/internal/types" "sneak.berlin/go/vaultik/internal/types"
) )
@@ -47,9 +46,12 @@ const (
year = 365 * day year = 365 * day
) )
// A snapshot ID split on "_" has at least a hostname and a trailing // Snapshot IDs split on "_" into hostname, optional name parts, and a
// timestamp. // trailing timestamp.
const minSnapshotIDParts = 2 const (
minSnapshotIDParts = 2
minSnapshotIDNameParts = 3
)
// SnapshotInfo contains information about a snapshot. // SnapshotInfo contains information about a snapshot.
// //
@@ -119,27 +121,20 @@ func parseSnapshotTimestamp(snapshotID string) (time.Time, error) {
return timestamp.UTC(), nil return timestamp.UTC(), nil
} }
// parseSnapshotName extracts the snapshot name from a snapshot ID of the // parseSnapshotName extracts the snapshot name from a snapshot ID.
// form hostname_name_timestamp, given the hostname stored with that // Format: hostname_snapshotname_timestamp — the middle part(s) between hostname
// snapshot. The hostname and the name may both contain underscores, so the // and the RFC3339 timestamp are the snapshot name (may contain underscores).
// name is what is left after removing the short hostname and its "_" from // Returns the snapshot name, or empty string if the ID is malformed.
// the front and the last "_" and the timestamp from the end. Returns "" for func parseSnapshotName(snapshotID string) string {
// an ID with no name (hostname_timestamp), and for an ID that does not start parts := strings.Split(snapshotID, "_")
// with that hostname, which CreateSnapshotWithName never writes. if len(parts) < minSnapshotIDNameParts {
func parseSnapshotName(snapshotID, hostname string) string { // Format: hostname_timestamp — no snapshot name
prefix := snapshot.ShortHostname(hostname) + "_"
rest, ok := strings.CutPrefix(snapshotID, prefix)
if !ok {
return "" return ""
} }
// Format: hostname_name_timestamp — middle parts are the name.
end := strings.LastIndex(rest, "_") // The last part is the RFC3339 timestamp, the first part is the hostname,
if end < 0 { // everything in between is the snapshot name (which may itself contain underscores).
return "" return strings.Join(parts[1:len(parts)-1], "_")
}
return rest[:end]
} }
// parseDuration parses a duration string with support for human-friendly units: // parseDuration parses a duration string with support for human-friendly units:
@@ -154,13 +149,6 @@ func parseDuration(s string) (time.Duration, error) {
return 0, errNegativeDuration return 0, errNegativeDuration
} }
// A bare number has no unit, but time.ParseDuration accepts 0, which
// would put the cutoff at now and select every snapshot.
_, err := strconv.ParseFloat(s, 64)
if err == nil {
return 0, fmt.Errorf("%w: %q", errInvalidDuration, s)
}
d, err := time.ParseDuration(s) d, err := time.ParseDuration(s)
if err == nil { if err == nil {
return d, nil return d, nil
+3 -27
View File
@@ -11,55 +11,33 @@ func TestParseSnapshotName(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
snapshotID string snapshotID string
hostname string
want string want string
}{ }{
{ {
name: "standard format with name", name: "standard format with name",
snapshotID: "myhost_home_2026-01-12T14:41:15Z", snapshotID: "myhost_home_2026-01-12T14:41:15Z",
hostname: "myhost",
want: "home", want: "home",
}, },
{ {
name: "standard format with different name", name: "standard format with different name",
snapshotID: "server1_system_2026-02-15T09:30:00Z", snapshotID: "server1_system_2026-02-15T09:30:00Z",
hostname: "server1",
want: "system", want: "system",
}, },
{ {
name: "name with underscores", name: "name with underscores",
snapshotID: "myhost_my_special_backup_2026-03-01T00:00:00Z", snapshotID: "myhost_my_special_backup_2026-03-01T00:00:00Z",
hostname: "myhost",
want: "my_special_backup", want: "my_special_backup",
}, },
{
name: "hostname with underscores",
snapshotID: "my_host_docs_2026-03-01T00:00:00Z",
hostname: "my_host",
want: "docs",
},
{
name: "stored hostname with domain",
snapshotID: "my_host_mail_2026-03-01T00:00:00Z",
hostname: "my_host.example.com",
want: "mail",
},
{
name: "no name",
snapshotID: "my_host_2026-03-01T00:00:00Z",
hostname: "my_host",
want: "",
},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
t.Parallel() t.Parallel()
got := parseSnapshotName(tt.snapshotID, tt.hostname) got := parseSnapshotName(tt.snapshotID)
if got != tt.want { if got != tt.want {
t.Errorf("parseSnapshotName(%q, %q) = %q, want %q", t.Errorf("parseSnapshotName(%q) = %q, want %q",
tt.snapshotID, tt.hostname, got, tt.want) tt.snapshotID, got, tt.want)
} }
}) })
} }
@@ -96,8 +74,6 @@ func TestParseDuration(t *testing.T) {
{"1y6mo", 365*24*time.Hour + 180*24*time.Hour, false}, {"1y6mo", 365*24*time.Hour + 180*24*time.Hour, false},
// Rejected inputs. // Rejected inputs.
{"6", 0, true}, // bare number, no unit {"6", 0, true}, // bare number, no unit
{"0", 0, true}, // bare number; time.ParseDuration accepts it
{"+0", 0, true}, // bare number; time.ParseDuration accepts it
{"5x", 0, true}, // unknown unit {"5x", 0, true}, // unknown unit
{"-5d", 0, true}, // negative, extended unit {"-5d", 0, true}, // negative, extended unit
{"-5h", 0, true}, // negative, Go unit {"-5h", 0, true}, // negative, Go unit
+24 -108
View File
@@ -183,11 +183,6 @@ type SnapshotMetadataInfo struct {
TotalSize int64 `json:"total_size"` TotalSize int64 `json:"total_size"`
BlobCount int `json:"blob_count"` BlobCount int `json:"blob_count"`
BlobsSize int64 `json:"blobs_size"` BlobsSize int64 `json:"blobs_size"`
// Set when the listing holds this snapshot's manifest.json.zst. A
// backup interrupted before its manifest upload leaves a directory
// without one, which prune does not treat as a snapshot.
hasManifest bool
} }
// RemoteInfoResult contains all remote storage information // RemoteInfoResult contains all remote storage information
@@ -211,20 +206,9 @@ type RemoteInfoResult struct {
ReferencedBlobCount int `json:"referenced_blob_count"` ReferencedBlobCount int `json:"referenced_blob_count"`
ReferencedBlobSize int64 `json:"referenced_blob_size"` ReferencedBlobSize int64 `json:"referenced_blob_size"`
// Orphaned blobs. Both stay nil (null in the JSON) when a manifest // Orphaned blobs
// was listed but not read, since that snapshot's blobs would be OrphanedBlobCount int `json:"orphaned_blob_count"`
// counted as orphaned. OrphanedBlobSize int64 `json:"orphaned_blob_size"`
OrphanedBlobCount *int `json:"orphaned_blob_count"`
OrphanedBlobSize *int64 `json:"orphaned_blob_size"`
// Remote key of each snapshot whose manifest could not be read
UnreadableManifests []string `json:"unreadable_manifests,omitempty"`
// Number of manifests not read because the name above them under
// metadata/ is not a remote key. The names themselves are not
// reported: they come from the destination store and may hold
// control characters.
SkippedManifestCount int `json:"skipped_manifest_count,omitempty"`
} }
// RemoteInfo displays information about remote storage // RemoteInfo displays information about remote storage
@@ -250,28 +234,16 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
v.stdoutf("Scanning snapshot metadata...\n") v.stdoutf("Scanning snapshot metadata...\n")
} }
snapshotMetadata, snapshotIDs, skippedManifestCount, err := v.collectSnapshotMetadata() snapshotMetadata, snapshotIDs, err := v.collectSnapshotMetadata()
if err != nil { if err != nil {
return err return err
} }
result.SkippedManifestCount = skippedManifestCount
if showText { if showText {
manifestCount := 0 v.stdoutf("Downloading %d manifest(s)...\n", len(snapshotIDs))
for _, info := range snapshotMetadata {
if info.hasManifest {
manifestCount++
}
}
v.stdoutf("Downloading %d manifest(s)...\n", manifestCount)
} }
referencedBlobs, unreadableManifests := v.collectReferencedBlobsFromManifests( referencedBlobs := v.collectReferencedBlobsFromManifests(snapshotIDs, snapshotMetadata)
snapshotIDs, snapshotMetadata)
result.UnreadableManifests = unreadableManifests
v.populateRemoteInfoResult(result, snapshotMetadata, snapshotIDs, referencedBlobs) v.populateRemoteInfoResult(result, snapshotMetadata, snapshotIDs, referencedBlobs)
@@ -284,7 +256,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
"snapshots", result.TotalMetadataCount, "snapshots", result.TotalMetadataCount,
"total_blobs", result.TotalBlobCount, "total_blobs", result.TotalBlobCount,
"referenced_blobs", result.ReferencedBlobCount, "referenced_blobs", result.ReferencedBlobCount,
"unreadable_manifests", len(result.UnreadableManifests)) "orphaned_blobs", result.OrphanedBlobCount)
if jsonOutput { if jsonOutput {
enc := json.NewEncoder(v.Stdout) enc := json.NewEncoder(v.Stdout)
@@ -301,18 +273,16 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
} }
// collectSnapshotMetadata scans remote metadata and returns // collectSnapshotMetadata scans remote metadata and returns
// per-snapshot info, sorted IDs and the number of manifests it skipped // per-snapshot info and sorted IDs.
// because the name above them is not a remote key.
func (v *Vaultik) collectSnapshotMetadata() ( func (v *Vaultik) collectSnapshotMetadata() (
map[string]*SnapshotMetadataInfo, []string, int, error, map[string]*SnapshotMetadataInfo, []string, error,
) { ) {
snapshotMetadata := make(map[string]*SnapshotMetadataInfo) snapshotMetadata := make(map[string]*SnapshotMetadataInfo)
skippedManifestCount := 0
metadataCh := v.Storage.ListStream(v.ctx, "metadata/") metadataCh := v.Storage.ListStream(v.ctx, "metadata/")
for obj := range metadataCh { for obj := range metadataCh {
if obj.Err != nil { if obj.Err != nil {
return nil, nil, 0, fmt.Errorf("listing metadata: %w", obj.Err) return nil, nil, fmt.Errorf("listing metadata: %w", obj.Err)
} }
parts := strings.Split(obj.Key, "/") parts := strings.Split(obj.Key, "/")
@@ -321,22 +291,6 @@ func (v *Vaultik) collectSnapshotMetadata() (
} }
snapshotID := parts[1] snapshotID := parts[1]
filename := parts[2]
isManifest := filename == "manifest.json.zst"
// The name comes from the destination store, which is not
// trusted, and is printed in the report. Accept it only in the
// form of a remote key.
if !isBlobHash(snapshotID) {
log.Warn("Skipping non-conforming key under metadata/",
"key", obj.Key)
if isManifest {
skippedManifestCount++
}
continue
}
if _, exists := snapshotMetadata[snapshotID]; !exists { if _, exists := snapshotMetadata[snapshotID]; !exists {
snapshotMetadata[snapshotID] = &SnapshotMetadataInfo{SnapshotID: snapshotID} snapshotMetadata[snapshotID] = &SnapshotMetadataInfo{SnapshotID: snapshotID}
@@ -344,10 +298,7 @@ func (v *Vaultik) collectSnapshotMetadata() (
info := snapshotMetadata[snapshotID] info := snapshotMetadata[snapshotID]
if isManifest { filename := parts[2]
info.hasManifest = true
}
if strings.HasPrefix(filename, "manifest") { if strings.HasPrefix(filename, "manifest") {
info.ManifestSize = obj.Size info.ManifestSize = obj.Size
} else if strings.HasPrefix(filename, "db") { } else if strings.HasPrefix(filename, "db") {
@@ -364,25 +315,17 @@ func (v *Vaultik) collectSnapshotMetadata() (
sort.Strings(snapshotIDs) sort.Strings(snapshotIDs)
return snapshotMetadata, snapshotIDs, skippedManifestCount, nil return snapshotMetadata, snapshotIDs, nil
} }
// collectReferencedBlobsFromManifests downloads the listed manifests // collectReferencedBlobsFromManifests downloads manifests and returns
// and returns referenced blob hashes with sizes, and the remote keys // referenced blob hashes with sizes.
// of the manifests it could not read.
func (v *Vaultik) collectReferencedBlobsFromManifests( func (v *Vaultik) collectReferencedBlobsFromManifests(
snapshotIDs []string, snapshotMetadata map[string]*SnapshotMetadataInfo, snapshotIDs []string, snapshotMetadata map[string]*SnapshotMetadataInfo,
) (map[string]int64, []string) { ) map[string]int64 {
referencedBlobs := make(map[string]int64) referencedBlobs := make(map[string]int64)
var unreadable []string
for _, snapshotID := range snapshotIDs { for _, snapshotID := range snapshotIDs {
info := snapshotMetadata[snapshotID]
if !info.hasManifest {
continue
}
// snapshotIDs here are remote keys, taken straight from the // snapshotIDs here are remote keys, taken straight from the
// metadata/ listing. downloadManifestByKey is the single reader // metadata/ listing. downloadManifestByKey is the single reader
// for remote manifests; see its doc comment. // for remote manifests; see its doc comment.
@@ -390,11 +333,10 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(
if err != nil { if err != nil {
log.Warn("Failed to read manifest", "snapshot", snapshotID, "error", err) log.Warn("Failed to read manifest", "snapshot", snapshotID, "error", err)
unreadable = append(unreadable, snapshotID)
continue continue
} }
info := snapshotMetadata[snapshotID]
info.BlobCount = manifest.BlobCount info.BlobCount = manifest.BlobCount
var blobsSize int64 var blobsSize int64
@@ -407,7 +349,7 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(
info.BlobsSize = blobsSize info.BlobsSize = blobsSize
} }
return referencedBlobs, unreadable return referencedBlobs
} }
// populateRemoteInfoResult fills in the result's snapshot and // populateRemoteInfoResult fills in the result's snapshot and
@@ -436,9 +378,8 @@ func (v *Vaultik) populateRemoteInfoResult(
} }
// scanRemoteBlobStorage lists all blobs on remote and computes orphan // scanRemoteBlobStorage lists all blobs on remote and computes orphan
// stats when every listed manifest was read. showText is true only // stats. showText is true only when the human report is being printed
// when the human report is being printed (not --json, not --quiet), // (not --json, not --quiet), gating the progress line.
// gating the progress line.
func (v *Vaultik) scanRemoteBlobStorage( func (v *Vaultik) scanRemoteBlobStorage(
result *RemoteInfoResult, referencedBlobs map[string]int64, showText bool, result *RemoteInfoResult, referencedBlobs map[string]int64, showText bool,
) error { ) error {
@@ -465,28 +406,13 @@ func (v *Vaultik) scanRemoteBlobStorage(
result.TotalBlobSize += obj.Size result.TotalBlobSize += obj.Size
} }
// A blob named only by a manifest that could not be read, or by one
// under a skipped name, would be counted as orphaned, so the orphan
// figures stay unknown.
if len(result.UnreadableManifests) > 0 || result.SkippedManifestCount > 0 {
return nil
}
var (
orphanedCount int
orphanedSize int64
)
for hash, size := range allBlobs { for hash, size := range allBlobs {
if _, referenced := referencedBlobs[hash]; !referenced { if _, referenced := referencedBlobs[hash]; !referenced {
orphanedCount++ result.OrphanedBlobCount++
orphanedSize += size result.OrphanedBlobSize += size
} }
} }
result.OrphanedBlobCount = &orphanedCount
result.OrphanedBlobSize = &orphanedSize
return nil return nil
} }
@@ -539,21 +465,11 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
v.stdoutf("Referenced by snapshots: %s (%s)\n", v.stdoutf("Referenced by snapshots: %s (%s)\n",
humanize.Comma(int64(result.ReferencedBlobCount)), humanize.Comma(int64(result.ReferencedBlobCount)),
ubytes(result.ReferencedBlobSize)) ubytes(result.ReferencedBlobSize))
if result.OrphanedBlobCount == nil {
v.stdoutf("Orphaned (unreferenced): unknown "+
"(%d manifest(s) could not be read, "+
"%d manifest(s) under a non-conforming name skipped)\n",
len(result.UnreadableManifests), result.SkippedManifestCount)
return
}
v.stdoutf("Orphaned (unreferenced): %s (%s)\n", v.stdoutf("Orphaned (unreferenced): %s (%s)\n",
humanize.Comma(int64(*result.OrphanedBlobCount)), humanize.Comma(int64(result.OrphanedBlobCount)),
ubytes(*result.OrphanedBlobSize)) ubytes(result.OrphanedBlobSize))
if *result.OrphanedBlobCount > 0 { if result.OrphanedBlobCount > 0 {
v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n") v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n")
} }
} }
+12 -11
View File
@@ -249,7 +249,7 @@ func verifyEndToEndBackupState(
assert.Positive(t, blobUploads, "Should upload at least one blob") assert.Positive(t, blobUploads, "Should upload at least one blob")
// Verify files in database // Verify files in database
files, err := repos.Files.ListUnderPath(ctx, "/home/user") files, err := repos.Files.ListByPrefix(ctx, "/home/user")
require.NoError(t, err) require.NoError(t, err)
// Count only regular files (not directories) // Count only regular files (not directories)
regularFiles := 0 regularFiles := 0
@@ -928,17 +928,17 @@ func setupDedupBackupEnv(
} }
} }
// runDedupSnapshot creates a snapshot with the given name, scans dataDir // runDedupSnapshot creates a "dedup" snapshot, scans dataDir into it,
// into it, completes it, and exports its metadata, returning the snapshot // completes it, and exports its metadata, returning the snapshot ID and
// ID and scan result. // scan result.
func runDedupSnapshot( func runDedupSnapshot(
ctx context.Context, t *testing.T, ctx context.Context, t *testing.T,
sm *snapshot.SnapshotManager, scanner *snapshot.Scanner, sm *snapshot.SnapshotManager, scanner *snapshot.Scanner,
hostname, name, dataDir, dbPath string, hostname, dataDir, dbPath string,
) (string, *snapshot.ScanResult) { ) (string, *snapshot.ScanResult) {
t.Helper() t.Helper()
id, err := sm.CreateSnapshotWithName(ctx, hostname, name, "v", "g") id, err := sm.CreateSnapshotWithName(ctx, hostname, "dedup", "v", "g")
require.NoError(t, err) require.NoError(t, err)
result, err := scanner.Scan(ctx, dataDir, id) result, err := scanner.Scan(ctx, dataDir, id)
@@ -980,15 +980,16 @@ func TestDedupOnlySnapshotRestores(t *testing.T) {
// First snapshot — uploads all blobs. // First snapshot — uploads all blobs.
_, r1 := runDedupSnapshot(ctx, t, sm, makeScanner(), _, r1 := runDedupSnapshot(ctx, t, sm, makeScanner(),
cfg.Hostname, "first", dataDir, dbPath) cfg.Hostname, dataDir, dbPath)
require.Positive(t, r1.BlobsCreated, require.Positive(t, r1.BlobsCreated,
"first snapshot should upload at least one blob") "first snapshot should upload at least one blob")
// Second snapshot — same data, every chunk dedups. Its own name gives // Second snapshot — same data, every chunk dedups. Sleep past the
// it a different snapshot ID without waiting for the one-second // second-precision timestamp so the snapshot IDs differ.
// timestamp in the ID to tick over. time.Sleep(1100 * time.Millisecond)
id2, r2 := runDedupSnapshot(ctx, t, sm, makeScanner(), id2, r2 := runDedupSnapshot(ctx, t, sm, makeScanner(),
cfg.Hostname, "second", dataDir, dbPath) cfg.Hostname, dataDir, dbPath)
require.Equal(t, 0, r2.BlobsCreated, require.Equal(t, 0, r2.BlobsCreated,
"second snapshot should upload zero new blobs (fully dedup'd)") "second snapshot should upload zero new blobs (fully dedup'd)")
@@ -1,171 +0,0 @@
package vaultik_test
import (
"bytes"
"context"
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/ui"
"sneak.berlin/go/vaultik/internal/vaultik"
)
// These tests cover https://git.eeqj.de/sneak/vaultik/issues/220: a
// file:// destination whose directory is missing, such as a USB stick
// that is not plugged in, cannot be listed. It is not an empty store, so
// no command may conclude from it that the local snapshots are gone.
// backUpToFileDestination backs up the snapshot named "first" to a
// file:// destination at storeDir, which need not exist yet. Everything
// the returned Vaultik prints after the backup goes to the returned
// buffer.
func backUpToFileDestination(
ctx context.Context, t *testing.T, storeDir string,
) (*vaultik.Vaultik, *database.Repositories, *bytes.Buffer) {
t.Helper()
osFs := afero.NewOsFs()
tempDir := t.TempDir()
dataDir := filepath.Join(tempDir, "src")
dbPath := filepath.Join(tempDir, "index.sqlite")
writeFaultSourceTree(t, osFs, dataDir)
store, err := storage.NewFileStorer(storeDir)
require.NoError(t, err)
db, err := database.New(ctx, dbPath)
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
repos := database.NewRepositories(db)
cfg := changedFileConfig(dataDir, dbPath)
v := newBackupVaultik(ctx, cfg, store, repos, db, osFs)
require.NoError(t, backUp(v, "first"))
out := &bytes.Buffer{}
v.Stdout = out
v.UI = ui.NewWithColor(out, false)
return v, repos, out
}
// backUpThenUnplug backs up to a file:// destination, then moves the
// destination directory away, as unplugging the volume it lives on would.
func backUpThenUnplug(
ctx context.Context, t *testing.T,
) (*vaultik.Vaultik, *database.Repositories, *bytes.Buffer) {
t.Helper()
storeDir := filepath.Join(t.TempDir(), "usbstick")
v, repos, out := backUpToFileDestination(ctx, t, storeDir)
require.NoError(t, os.Rename(storeDir, storeDir+"-unplugged"))
return v, repos, out
}
// TestFirstBackupCreatesDestinationDirectory checks that a first backup
// to a destination directory that does not exist yet creates it, and
// that the destination can be listed afterwards.
func TestFirstBackupCreatesDestinationDirectory(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
storeDir := filepath.Join(t.TempDir(), "volume", "backup")
v, _, out := backUpToFileDestination(ctx, t, storeDir)
require.NoError(t, v.ListSnapshots(false))
assert.NotContains(t, out.String(), "Could not list backup destination store")
assert.NotContains(t, out.String(), "not found in backup destination store")
}
// TestListSnapshotsWarnsWhenDestinationMissing checks that snapshot list
// warns and shows the local index alone, without reporting the local
// snapshot as missing from the destination.
func TestListSnapshotsWarnsWhenDestinationMissing(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
v, repos, out := backUpThenUnplug(ctx, t)
id := localSnapshotID(ctx, t, repos, "first")
require.NoError(t, v.ListSnapshots(false))
assert.Contains(t, out.String(), "Could not list backup destination store")
assert.Contains(t, out.String(), "Showing snapshots from the local index only.")
assert.Contains(t, out.String(), id)
assert.NotContains(t, out.String(), "not found in backup destination store")
}
// TestRemoveSnapshotWarnsWhenDestinationMissing checks that snapshot
// remove warns that the metadata could not be removed from the
// destination, instead of reporting that it was.
func TestRemoveSnapshotWarnsWhenDestinationMissing(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
v, repos, out := backUpThenUnplug(ctx, t)
result, err := v.RemoveSnapshot(localSnapshotID(ctx, t, repos, "first"),
&vaultik.RemoveOptions{Force: true})
require.NoError(t, err)
assert.False(t, result.RemoteRemoved)
assert.Contains(t, out.String(),
"Could not remove snapshot metadata from remote")
assert.NotContains(t, out.String(),
"Removed snapshot metadata from remote storage")
}
// TestPruneKeepsLocalRecordsWhenDestinationMissing checks that prune
// fails on a destination it cannot list and deletes no local snapshot
// record.
func TestPruneKeepsLocalRecordsWhenDestinationMissing(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
v, repos, _ := backUpThenUnplug(ctx, t)
err := v.Prune(&vaultik.PruneOptions{Force: true})
require.ErrorIs(t, err, fs.ErrNotExist)
require.ErrorContains(t, err, "listing remote snapshots")
snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
require.NoError(t, err)
assert.Len(t, snapshots, 1, "prune must delete no local snapshot record")
}
// TestPurgeSaysListingFailedOnceWhenDestinationMissing checks that
// snapshot purge fails on a destination it cannot list, with an error
// that says "listing remote snapshots" once.
func TestPurgeSaysListingFailedOnceWhenDestinationMissing(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
v, _, _ := backUpThenUnplug(ctx, t)
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true,
Force: true,
})
require.ErrorIs(t, err, fs.ErrNotExist)
assert.Equal(t, 1, strings.Count(err.Error(), "listing remote snapshots"),
err.Error())
}
+2 -1
View File
@@ -14,9 +14,10 @@ import (
// the discarded-error bug: getTableCount for a table its query cannot // the discarded-error bug: getTableCount for a table its query cannot
// resolve must not silently become 0. A count that could not be read is // resolve must not silently become 0. A count that could not be read is
// reported as unknown, which a reader can tell apart from an empty table. // reported as unknown, which a reader can tell apart from an empty table.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestTableCountForReportSurfacesReadFailure(t *testing.T) { func TestTableCountForReportSurfacesReadFailure(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background() ctx := context.Background()
@@ -42,7 +42,7 @@ func setupConsistencyTest(
completedAt := startedAt.Add(5 * time.Minute) completedAt := startedAt.Add(5 * time.Minute)
snap := &database.Snapshot{ snap := &database.Snapshot{
ID: types.SnapshotID(id), ID: types.SnapshotID(id),
Hostname: snapHostname, Hostname: testHostname,
VaultikVersion: testLabel, VaultikVersion: testLabel,
StartedAt: startedAt, StartedAt: startedAt,
CompletedAt: &completedAt, CompletedAt: &completedAt,
+12 -42
View File
@@ -17,10 +17,8 @@ import (
"sneak.berlin/go/vaultik/internal/vaultik" "sneak.berlin/go/vaultik/internal/vaultik"
) )
// Snapshot IDs reused across the purge tests, and the hostname they were // Snapshot IDs reused across the purge tests.
// taken on.
const ( const (
snapHostname = "testhost"
snapSystemT0 = "testhost_system_2026-01-01T00:00:00Z" snapSystemT0 = "testhost_system_2026-01-01T00:00:00Z"
snapHomeT0 = "testhost_home_2026-01-01T00:00:00Z" snapHomeT0 = "testhost_home_2026-01-01T00:00:00Z"
snapHomeT1 = "testhost_home_2026-01-01T01:00:00Z" snapHomeT1 = "testhost_home_2026-01-01T01:00:00Z"
@@ -28,12 +26,9 @@ const (
) )
// setupPurgeTest creates a Vaultik instance with an in-memory database and mock // setupPurgeTest creates a Vaultik instance with an in-memory database and mock
// storage pre-populated with the given snapshot IDs, all taken on hostname. // storage pre-populated with the given snapshot IDs. Each snapshot is marked as
// Each snapshot is marked as completed. Remote metadata stubs are created so // completed. Remote metadata stubs are created so syncWithRemote keeps them.
// syncWithRemote keeps them. func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik {
func setupPurgeTest(
t *testing.T, hostname string, snapshotIDs []string,
) *vaultik.Vaultik {
t.Helper() t.Helper()
ctx := context.Background() ctx := context.Background()
@@ -56,7 +51,7 @@ func setupPurgeTest(
completedAt := startedAt.Add(5 * time.Minute) completedAt := startedAt.Add(5 * time.Minute)
snap := &database.Snapshot{ snap := &database.Snapshot{
ID: types.SnapshotID(id), ID: types.SnapshotID(id),
Hostname: types.Hostname(hostname), Hostname: "testhost",
VaultikVersion: testLabel, VaultikVersion: testLabel,
StartedAt: startedAt, StartedAt: startedAt,
CompletedAt: &completedAt, CompletedAt: &completedAt,
@@ -125,7 +120,7 @@ func TestPurgeKeepLatest_PerName(t *testing.T) {
"testhost_system_2026-01-01T04:00:00Z", "testhost_system_2026-01-01T04:00:00Z",
} }
v := setupPurgeTest(t, snapHostname, snapshotIDs) v := setupPurgeTest(t, snapshotIDs)
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true, KeepLatest: true,
@@ -153,7 +148,7 @@ func TestPurgeKeepLatest_SingleName(t *testing.T) {
"testhost_home_2026-01-01T02:00:00Z", "testhost_home_2026-01-01T02:00:00Z",
} }
v := setupPurgeTest(t, snapHostname, snapshotIDs) v := setupPurgeTest(t, snapshotIDs)
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true, KeepLatest: true,
@@ -181,7 +176,7 @@ func TestPurgeKeepLatest_WithNameFilter(t *testing.T) {
"testhost_home_2026-01-01T04:00:00Z", "testhost_home_2026-01-01T04:00:00Z",
} }
v := setupPurgeTest(t, snapHostname, snapshotIDs) v := setupPurgeTest(t, snapshotIDs)
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true, KeepLatest: true,
@@ -203,7 +198,7 @@ func TestPurgeKeepLatest_NoSnapshots(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel() t.Parallel()
v := setupPurgeTest(t, snapHostname, nil) v := setupPurgeTest(t, nil)
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true, KeepLatest: true,
@@ -221,7 +216,7 @@ func TestPurgeKeepLatest_NameFilterNoMatch(t *testing.T) {
"testhost_system_2026-01-01T01:00:00Z", "testhost_system_2026-01-01T01:00:00Z",
} }
v := setupPurgeTest(t, snapHostname, snapshotIDs) v := setupPurgeTest(t, snapshotIDs)
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true, KeepLatest: true,
@@ -248,7 +243,7 @@ func TestPurgeOlderThan_WithNameFilter(t *testing.T) {
snapHomeT0, snapHomeT0,
} }
v := setupPurgeTest(t, snapHostname, snapshotIDs) v := setupPurgeTest(t, snapshotIDs)
// Purge only "home" snapshots older than 365 days // Purge only "home" snapshots older than 365 days
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
@@ -282,7 +277,7 @@ func TestPurgeKeepLatest_ThreeNames(t *testing.T) {
"testhost_home_2026-01-01T06:00:00Z", "testhost_home_2026-01-01T06:00:00Z",
} }
v := setupPurgeTest(t, snapHostname, snapshotIDs) v := setupPurgeTest(t, snapshotIDs)
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true, KeepLatest: true,
@@ -296,28 +291,3 @@ func TestPurgeKeepLatest_ThreeNames(t *testing.T) {
assert.Contains(t, remaining, "testhost_system_2026-01-01T04:00:00Z") assert.Contains(t, remaining, "testhost_system_2026-01-01T04:00:00Z")
assert.Contains(t, remaining, "testhost_media_2026-01-01T05:00:00Z") assert.Contains(t, remaining, "testhost_media_2026-01-01T05:00:00Z")
} }
// A hostname may contain underscores, so the snapshot name cannot be found
// by splitting the ID at them. A purge by name must still select "docs".
func TestPurgeKeepLatest_HostnameWithUnderscore(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
const (
system = "my_host_system_2026-01-01T00:00:00Z"
docsT1 = "my_host_docs_2026-01-01T01:00:00Z"
docsT2 = "my_host_docs_2026-01-01T02:00:00Z"
)
v := setupPurgeTest(t, "my_host", []string{system, docsT1, docsT2})
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
KeepLatest: true,
Force: true,
Names: []string{"docs"},
})
require.NoError(t, err)
assert.ElementsMatch(t, []string{system, docsT2},
listRemainingSnapshots(t, v))
}
-159
View File
@@ -1,159 +0,0 @@
package vaultik_test
import (
"bytes"
"context"
"encoding/json"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/snapshot"
)
// testBlobHashB is a blob that the manifest written by addRemote does
// not reference.
const testBlobHashB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
// TestRemoteInfo_UnreadableManifestLeavesOrphansUnknown checks that a
// manifest remote info cannot read makes the orphan figures unknown. A
// blob referenced only by that snapshot would otherwise be counted as
// orphaned, and the report would advise running prune.
func TestRemoteInfo_UnreadableManifestLeavesOrphansUnknown(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newListEnv(t)
// The readable manifest references blob A only.
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
addBlob(t, env.store.testStorer, testBlobHashA)
addBlob(t, env.store.testStorer, testBlobHashB)
// With every manifest readable, blob B is orphaned.
require.NoError(t, env.v.RemoteInfo(true))
var doc map[string]any
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
assert.InDelta(t, 1, doc["orphaned_blob_count"], 0)
// A second snapshot whose manifest cannot be decoded. Blob B may be
// one of its blobs.
unreadableKey := snapshot.RemoteSnapshotKey(listLocalID)
require.NoError(t, env.store.Put(context.Background(),
"metadata/"+unreadableKey+"/manifest.json.zst",
bytes.NewReader([]byte("not a valid manifest"))))
env.stdout.Reset()
require.NoError(t, env.v.RemoteInfo(false))
text := env.stdout.String()
assert.Contains(t, text, "Orphaned (unreferenced): unknown "+
"(1 manifest(s) could not be read, "+
"0 manifest(s) under a non-conforming name skipped)")
assert.NotContains(t, text, "vaultik prune")
env.stdout.Reset()
require.NoError(t, env.v.RemoteInfo(true))
doc = nil
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
assert.Contains(t, doc, "orphaned_blob_count")
assert.Nil(t, doc["orphaned_blob_count"])
assert.Contains(t, doc, "orphaned_blob_size")
assert.Nil(t, doc["orphaned_blob_size"])
assert.Equal(t, []any{unreadableKey}, doc["unreadable_manifests"])
}
// TestRemoteInfo_SkipsNonConformingMetadataName checks that a directory
// under metadata/ whose name is not a remote key is left out of the
// report, and that the orphan figures are unknown when it holds a
// manifest. The name comes from the destination store; printed raw, its
// control characters would reach the terminal. Its manifest is not
// read, so a blob only it references would otherwise be counted as
// orphaned.
func TestRemoteInfo_SkipsNonConformingMetadataName(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newListEnv(t)
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
addBlob(t, env.store.testStorer, testBlobHashA)
addBlob(t, env.store.testStorer, testBlobHashB)
require.NoError(t, env.store.Put(context.Background(),
"metadata/\x1b[31mred/manifest.json.zst",
bytes.NewReader([]byte("not a valid manifest"))))
require.NoError(t, env.v.RemoteInfo(false))
text := env.stdout.String()
assert.NotContains(t, text, "\x1b")
assert.NotContains(t, text, "31mred")
assert.Contains(t, text, "Total (1 snapshots)")
assert.Contains(t, text, "Orphaned (unreferenced): unknown "+
"(0 manifest(s) could not be read, "+
"1 manifest(s) under a non-conforming name skipped)")
assert.NotContains(t, text, "vaultik prune")
env.stdout.Reset()
require.NoError(t, env.v.RemoteInfo(true))
out := env.stdout.String()
assert.NotContains(t, out, "31mred")
var doc map[string]any
require.NoError(t, json.Unmarshal([]byte(out), &doc))
assert.Contains(t, doc, "orphaned_blob_count")
assert.Nil(t, doc["orphaned_blob_count"])
assert.Contains(t, doc, "orphaned_blob_size")
assert.Nil(t, doc["orphaned_blob_size"])
assert.InDelta(t, 1, doc["skipped_manifest_count"], 0)
assert.NotContains(t, doc, "unreadable_manifests")
}
// TestRemoteInfo_DirectoryWithoutManifestLeavesOrphansKnown checks that
// a directory under metadata/ holding no manifest.json.zst, such as one
// left by a backup interrupted before its manifest upload, leaves the
// orphan figures known. prune does not treat such a directory as a
// snapshot and deletes the blobs the report lists as orphaned.
func TestRemoteInfo_DirectoryWithoutManifestLeavesOrphansKnown(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newListEnv(t)
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
addBlob(t, env.store.testStorer, testBlobHashA)
addBlob(t, env.store.testStorer, testBlobHashB)
// One directory under a remote key and one under a non-conforming
// name, each holding only a database.
names := []string{snapshot.RemoteSnapshotKey(listLocalID), "\x1b[31mred"}
for _, name := range names {
require.NoError(t, env.store.Put(context.Background(),
"metadata/"+name+"/db.zst.age",
bytes.NewReader([]byte("not a valid database"))))
}
require.NoError(t, env.v.RemoteInfo(false))
text := env.stdout.String()
assert.NotContains(t, text, "\x1b")
assert.Contains(t, text, "Downloading 1 manifest(s)...")
assert.Contains(t, text, "Orphaned (unreferenced): 1 (")
assert.Contains(t, text, "Run 'vaultik prune' to remove orphaned blobs.")
env.stdout.Reset()
require.NoError(t, env.v.RemoteInfo(true))
var doc map[string]any
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
assert.InDelta(t, 1, doc["orphaned_blob_count"], 0)
assert.NotContains(t, doc, "unreadable_manifests")
assert.NotContains(t, doc, "skipped_manifest_count")
}
+38 -143
View File
@@ -10,13 +10,11 @@ import (
"math" "math"
"os" "os"
"path/filepath" "path/filepath"
"slices"
"strings" "strings"
"time" "time"
"filippo.io/age" "filippo.io/age"
"github.com/spf13/afero" "github.com/spf13/afero"
"golang.org/x/sys/unix"
"sneak.berlin/go/vaultik/internal/blobgen" "sneak.berlin/go/vaultik/internal/blobgen"
"sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
@@ -42,7 +40,6 @@ var (
errChunkNotInAnyBlob = errors.New("chunk not found in any blob") errChunkNotInAnyBlob = errors.New("chunk not found in any blob")
errBlobIDNotInHashIndex = errors.New("blob id missing from hash index") errBlobIDNotInHashIndex = errors.New("blob id missing from hash index")
errShortChunkRead = errors.New("short read") errShortChunkRead = errors.New("short read")
errChunkRowMissing = errors.New("chunk has no row in the chunks table")
errRestorePathEscapesTarget = errors.New( errRestorePathEscapesTarget = errors.New(
"refusing to restore path outside the target directory") "refusing to restore path outside the target directory")
errTrailingRestoreData = errors.New( errTrailingRestoreData = errors.New(
@@ -66,12 +63,6 @@ const snapshotDBFilename = "snapshot.db"
// directories themselves get their stored mode). // directories themselves get their stored mode).
const restoreDirMode = 0o755 const restoreDirMode = 0o755
// restoreDirCreateMode is the owner-only mode a directory from the
// snapshot is created with during restore, so its contents can be written
// whatever its stored mode. The stored mode is applied after the restore
// loop, by applyDirectoryMetadata.
const restoreDirCreateMode = 0o700
// restoreFileMode is the restrictive mode a regular file is created with // restoreFileMode is the restrictive mode a regular file is created with
// during restore. Content is written while the file holds this mode; the // during restore. Content is written while the file holds this mode; the
// stored mode is applied only after the file is fully written and closed, // stored mode is applied only after the file is fully written and closed,
@@ -341,8 +332,6 @@ func (v *Vaultik) restoreAllFiles(
return nil, err return nil, err
} }
session.applyDirectoryMetadata()
return result, nil return result, nil
} }
@@ -366,7 +355,7 @@ func (v *Vaultik) runRestoreLoop(
fileID, ready := plan.popReady() fileID, ready := plan.popReady()
if !ready { if !ready {
downloaded, err := session.downloadNextBlobSet(plan, filesByID) downloaded, err := session.downloadNextBlobSet(plan)
if err != nil { if err != nil {
return err return err
} }
@@ -422,13 +411,7 @@ func (v *Vaultik) runRestoreLoop(
// blob set and downloads its blobs; after each blob lands, the plan // blob set and downloads its blobs; after each blob lands, the plan
// moves any pending file whose set just emptied onto the ready queue. // moves any pending file whose set just emptied onto the ready queue.
// Returns false when nothing is pending download (the caller stops). // Returns false when nothing is pending download (the caller stops).
// func (s *restoreSession) downloadNextBlobSet(plan *restorePlan) (bool, error) {
// A blob that cannot be downloaded is reported through
// handleRestoreFileError for every pending file that references it, so
// it aborts the restore unless --skip-errors is set.
func (s *restoreSession) downloadNextBlobSet(
plan *restorePlan, filesByID map[types.FileID]*database.File,
) (bool, error) {
s.sweeper.sweep() s.sweeper.sweep()
next, ok := plan.pickNextDownload() next, ok := plan.pickNextDownload()
@@ -450,25 +433,7 @@ func (s *restoreSession) downloadNextBlobSet(
err := s.downloadBlobToCache(hash, blob.CompressedSize, blob.UncompressedSize) err := s.downloadBlobToCache(hash, blob.CompressedSize, blob.UncompressedSize)
if err != nil { if err != nil {
err = fmt.Errorf("downloading blob %s: %w", shortHash(hash), err) return false, fmt.Errorf("downloading blob %s: %w", shortHash(hash), err)
// On cancel the error says nothing about the blob, so it ends
// the restore instead of failing the files that need it.
if s.ctx.Err() != nil {
return false, err
}
for _, fileID := range plan.filesReferencingBlob(hash) {
fileErr := s.v.handleRestoreFileError(
plan, s.opts, s.result, filesByID[fileID], fileID, err)
if fileErr != nil {
return false, fileErr
}
}
// next is among the failed files, so the rest of its blob set
// is left for any other file that still needs it.
return true, nil
} }
s.result.BlobsDownloaded++ s.result.BlobsDownloaded++
@@ -830,9 +795,10 @@ func (v *Vaultik) getFilesToRestore(
// Normalize the filter path // Normalize the filter path
filter = filepath.Clean(filter) filter = filepath.Clean(filter)
files, err := repos.Files.ListUnderPath(ctx, filter) // Get files with this prefix
files, err := repos.Files.ListByPrefix(ctx, filter)
if err != nil { if err != nil {
return nil, fmt.Errorf("listing files under %s: %w", filter, err) return nil, fmt.Errorf("listing files with prefix %s: %w", filter, err)
} }
for _, file := range files { for _, file := range files {
@@ -911,9 +877,6 @@ type restoreSession struct {
// the call entirely as non-root and emit one warning at the end // the call entirely as non-root and emit one warning at the end
// of the restore explaining that ownership was not preserved. // of the restore explaining that ownership was not preserved.
runningAsRoot bool runningAsRoot bool
// directories holds every restored directory, for
// applyDirectoryMetadata to finish after the restore loop.
directories []*database.File
} }
// containedRestorePath resolves rel — a path read from the snapshot // containedRestorePath resolves rel — a path read from the snapshot
@@ -1020,8 +983,6 @@ func (s *restoreSession) restoreSymlink(file *database.File, targetPath string)
if err != nil { if err != nil {
return fmt.Errorf("creating symlink: %w", err) return fmt.Errorf("creating symlink: %w", err)
} }
s.applySymlinkMetadata(file, targetPath)
} else { } else {
log.Debug("Symlink creation not supported on this filesystem", log.Debug("Symlink creation not supported on this filesystem",
"path", file.Path, "target", file.LinkTarget) "path", file.Path, "target", file.LinkTarget)
@@ -1034,90 +995,35 @@ func (s *restoreSession) restoreSymlink(file *database.File, targetPath string)
return nil return nil
} }
// restoreDirectory creates a directory with restoreDirCreateMode. Its // restoreDirectory restores a directory with its permissions, mtime,
// stored mode, owner and mtime are applied after the restore loop by // and (on real filesystems, with sufficient privileges) ownership.
// applyDirectoryMetadata: a read-only stored mode would block writing
// its contents, and writing them changes its mtime.
func (s *restoreSession) restoreDirectory( func (s *restoreSession) restoreDirectory(
file *database.File, targetPath string, file *database.File, targetPath string,
) error { ) error {
err := s.v.Fs.MkdirAll(targetPath, restoreDirCreateMode) err := s.v.Fs.MkdirAll(targetPath, os.FileMode(file.Mode))
if err != nil { if err != nil {
return fmt.Errorf("creating directory: %w", err) return fmt.Errorf("creating directory: %w", err)
} }
s.directories = append(s.directories, file) // MkdirAll applies the process umask, so chmod to the exact stored
// mode. A failure here is non-fatal.
err = s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode))
if err != nil {
log.Debug("Failed to set permissions", "path", targetPath, "error", err)
}
s.applyFileMetadata(file, targetPath)
s.result.FilesRestored++ s.result.FilesRestored++
return nil return nil
} }
// applyDirectoryMetadata applies the stored owner, mtime and mode to every
// restored directory, each before its parent, so a parent whose stored
// mode denies search does not block its children. Failures are logged at
// debug level and do not abort the restore.
func (s *restoreSession) applyDirectoryMetadata() {
// A path sorts after its parent's, so reverse order puts every
// directory before its parent.
slices.SortFunc(s.directories, func(a, b *database.File) int {
return strings.Compare(b.Path.String(), a.Path.String())
})
for _, dir := range s.directories {
targetPath, err := containedRestorePath(
s.v.Fs, s.opts.TargetDir, dir.Path.String())
if err != nil {
log.Debug("Failed to set directory metadata",
"path", dir.Path, "error", err)
continue
}
// A later entry can have put a symlink in the directory's place,
// for example one stored as "/d/" next to the directory "/d". The
// calls below follow symlinks, so they would change its target.
info, err := lstatIfPossible(s.v.Fs, targetPath)
if err != nil || !info.IsDir() {
log.Debug("Not setting directory metadata: no longer a directory",
"path", targetPath, "error", err)
continue
}
s.applyFileMetadata(dir, targetPath)
err = s.v.Fs.Chmod(targetPath, os.FileMode(dir.Mode))
if err != nil {
log.Debug("Failed to set permissions", "path", targetPath, "error", err)
}
}
}
// applySymlinkMetadata applies ownership (when running as root) and mtime
// to a restored symlink itself; os.Chown and Chtimes would follow it.
// Failures are logged at debug level and do not abort the restore.
func (s *restoreSession) applySymlinkMetadata(file *database.File, targetPath string) {
if s.runningAsRoot {
err := os.Lchown(targetPath, int(file.UID), int(file.GID))
if err != nil {
log.Debug("Failed to set ownership", "path", targetPath, "error", err)
}
}
mtime := unix.NsecToTimeval(file.MTime.UnixNano())
err := unix.Lutimes(targetPath, []unix.Timeval{mtime, mtime})
if err != nil {
log.Debug("Failed to set mtime", "path", targetPath, "error", err)
}
}
// applyFileMetadata applies ownership (when running as root on a real // applyFileMetadata applies ownership (when running as root on a real
// filesystem) and mtime to a restored path. The caller applies the mode // filesystem) and mtime to a restored path. Permission mode is applied
// afterwards: on Linux a chown clears the setuid and setgid bits of a // separately by each caller, with different failure handling, so it is
// regular file. Failures are logged at debug level and do not abort the // not touched here. Failures are logged at debug level and do not abort
// restore. // the restore.
func (s *restoreSession) applyFileMetadata(file *database.File, targetPath string) { func (s *restoreSession) applyFileMetadata(file *database.File, targetPath string) {
if s.runningAsRoot { if s.runningAsRoot {
if _, ok := s.v.Fs.(*afero.OsFs); ok { if _, ok := s.v.Fs.(*afero.OsFs); ok {
@@ -1208,8 +1114,8 @@ func (s *restoreSession) restoreRegularFile(
return fmt.Errorf("closing output file: %w", err) return fmt.Errorf("closing output file: %w", err)
} }
s.applyFileMetadata(file, targetPath)
s.applyRestoredFileMode(file, targetPath) s.applyRestoredFileMode(file, targetPath)
s.applyFileMetadata(file, targetPath)
s.result.FilesRestored++ s.result.FilesRestored++
s.result.BytesRestored += bytesWritten s.result.BytesRestored += bytesWritten
@@ -1267,7 +1173,7 @@ func (s *restoreSession) writeFileChunks(
blobChunk, ok := s.chunkToBlobMap[chunkHashStr] blobChunk, ok := s.chunkToBlobMap[chunkHashStr]
if !ok { if !ok {
return bytesWritten, timings, fmt.Errorf( return bytesWritten, timings, fmt.Errorf(
"%w: %s", errChunkNotInAnyBlob, shortHash(chunkHashStr)) "%w: %s", errChunkNotInAnyBlob, chunkHashStr[:16])
} }
blobHash, ok := s.blobIDToHash[blobChunk.BlobID.String()] blobHash, ok := s.blobIDToHash[blobChunk.BlobID.String()]
@@ -1284,7 +1190,7 @@ func (s *restoreSession) writeFileChunks(
if err != nil { if err != nil {
return bytesWritten, timings, fmt.Errorf( return bytesWritten, timings, fmt.Errorf(
"reading chunk %s from cached blob %s: %w", "reading chunk %s from cached blob %s: %w",
shortHash(chunkHashStr), shortHash(blobHash), err) fc.ChunkHash[:16], blobHash[:16], err)
} }
t0 = time.Now() t0 = time.Now()
@@ -1482,44 +1388,33 @@ func (v *Vaultik) verifyFile(
chunk, err := repos.Chunks.GetByHash(ctx, fc.ChunkHash.String()) chunk, err := repos.Chunks.GetByHash(ctx, fc.ChunkHash.String())
if err != nil { if err != nil {
return bytesVerified, fmt.Errorf("getting chunk %s: %w", return bytesVerified, fmt.Errorf("getting chunk %s: %w",
shortHash(fc.ChunkHash.String()), err) fc.ChunkHash.String()[:16], err)
} }
if chunk == nil { // Read chunk data from file
return bytesVerified, fmt.Errorf("%w: %s", chunkData := make([]byte, chunk.Size)
errChunkRowMissing, shortHash(fc.ChunkHash.String()))
}
// chunk.Size comes from the snapshot database, which is not
// trusted: reject a negative size, and hash the chunk by
// streaming it rather than allocating that many bytes.
if chunk.Size < 0 {
return bytesVerified, fmt.Errorf("%w: chunk %d size %d",
errNegativeChunkLength, fc.Idx, chunk.Size)
}
hasher := sha256.New()
n, err := io.CopyN(hasher, f, chunk.Size)
if errors.Is(err, io.EOF) {
return bytesVerified, fmt.Errorf("%w: expected %d bytes, got %d",
errShortChunkRead, chunk.Size, n)
}
n, err := io.ReadFull(f, chunkData)
if err != nil { if err != nil {
return bytesVerified, fmt.Errorf("reading chunk data: %w", err) return bytesVerified, fmt.Errorf("reading chunk data: %w", err)
} }
actualHash := hex.EncodeToString(hasher.Sum(nil)) if int64(n) != chunk.Size {
return bytesVerified, fmt.Errorf("%w: expected %d bytes, got %d",
errShortChunkRead, chunk.Size, n)
}
// Calculate hash and compare
hash := sha256.Sum256(chunkData)
actualHash := hex.EncodeToString(hash[:])
expectedHash := fc.ChunkHash.String() expectedHash := fc.ChunkHash.String()
if actualHash != expectedHash { if actualHash != expectedHash {
return bytesVerified, fmt.Errorf("%w: chunk %d: expected %s, got %s", return bytesVerified, fmt.Errorf("%w: chunk %d: expected %s, got %s",
errChunkHashMismatch, fc.Idx, errChunkHashMismatch, fc.Idx, expectedHash[:16], actualHash[:16])
shortHash(expectedHash), shortHash(actualHash))
} }
bytesVerified += n bytesVerified += int64(n)
} }
// The stored chunks account for the whole file, so the reader must // The stored chunks account for the whole file, so the reader must
@@ -10,7 +10,6 @@ import (
"github.com/spf13/afero" "github.com/spf13/afero"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/cli"
"sneak.berlin/go/vaultik/internal/config" "sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
@@ -28,12 +27,10 @@ import (
// //
// The backup half writes a snapshot with one index and hostname. The // The backup half writes a snapshot with one index and hostname. The
// restore half throws that index away entirely: a fresh, empty index and // restore half throws that index away entirely: a fresh, empty index and
// a config written by `config init` and `config set storage_url`, as in // a config that shares nothing with the original but the storage location
// the README's steps for restoring on another machine, which shares // and the secret key. If restore or verify needed the original local
// nothing with the original but the storage location and the secret key. // index — or the human snapshot ID that only that index holds — this test
// If restore or verify needed the original local index — or // could not run, because the recovery host can know neither.
// the human snapshot ID that only that index holds — this test could not
// run, because the recovery host can know neither.
func TestRestoreOnAnotherMachine(t *testing.T) { func TestRestoreOnAnotherMachine(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel() t.Parallel()
@@ -61,12 +58,7 @@ func TestRestoreOnAnotherMachine(t *testing.T) {
// Recovery host: a fresh empty index, a different hostname, and no // Recovery host: a fresh empty index, a different hostname, and no
// age_recipients — only the secret key and the same storage location. // age_recipients — only the secret key and the same storage location.
configPath := filepath.Join(tempDir, "config.yml") recovery, stdout := newRecoveryHost(ctx, t, fs, storer)
runVaultikCommand(t, "--config", configPath, "config", "init")
runVaultikCommand(t, "--config", configPath,
"config", "set", "storage_url", "file://"+storeDir)
recovery, stdout := newRecoveryHost(ctx, t, fs, storer, configPath)
// The recovery index really is empty. This is the assertion that makes // The recovery index really is empty. This is the assertion that makes
// the test a guard against restore quietly depending on the original // the test a guard against restore quietly depending on the original
@@ -101,10 +93,6 @@ func TestRestoreOnAnotherMachine(t *testing.T) {
remote.RemoteKey, &vaultik.VerifyOptions{Deep: true})) remote.RemoteKey, &vaultik.VerifyOptions{Deep: true}))
assertRestoredTreeMatches(t, fs, restoreDir, sourceFiles) assertRestoredTreeMatches(t, fs, restoreDir, sourceFiles)
// With no public key configured, a backup must refuse to start.
err = recovery.CreateSnapshot(&vaultik.SnapshotCreateOptions{Cron: true})
require.ErrorContains(t, err, "age_recipients")
} }
// writeRecoverySourceTree writes a small source tree spanning several // writeRecoverySourceTree writes a small source tree spanning several
@@ -130,24 +118,14 @@ func writeRecoverySourceTree(
} }
// newRecoveryHost builds the Vaultik a replacement machine would run: an // newRecoveryHost builds the Vaultik a replacement machine would run: an
// empty in-memory index, a hostname different from the backup host, and // empty in-memory index, a hostname different from the backup host, no
// only the secret key plus the shared storer. Its config is read from // age_recipients, and only the secret key plus the shared storer. It
// configPath by config.Load, as every command reads it. It returns the // returns the instance and the buffer its stdout is wired to.
// instance and the buffer its stdout is wired to.
func newRecoveryHost( func newRecoveryHost(
ctx context.Context, t *testing.T, fs afero.Fs, storer storage.Storer, ctx context.Context, t *testing.T, fs afero.Fs, storer storage.Storer,
configPath string,
) (*vaultik.Vaultik, *bytes.Buffer) { ) (*vaultik.Vaultik, *bytes.Buffer) {
t.Helper() t.Helper()
cfg, err := config.Load(configPath)
require.NoError(t, err)
// Set directly rather than through VAULTIK_AGE_SECRET_KEY, which a
// parallel test cannot change.
cfg.AgeSecretKey = testAgeSecretKey
cfg.Hostname = "recovery-host"
recoveryDB, err := database.New(ctx, ":memory:") recoveryDB, err := database.New(ctx, ":memory:")
require.NoError(t, err) require.NoError(t, err)
t.Cleanup(func() { _ = recoveryDB.Close() }) t.Cleanup(func() { _ = recoveryDB.Close() })
@@ -155,7 +133,10 @@ func newRecoveryHost(
stdout := &bytes.Buffer{} stdout := &bytes.Buffer{}
recovery := &vaultik.Vaultik{ recovery := &vaultik.Vaultik{
Config: cfg, Config: &config.Config{
AgeSecretKey: testAgeSecretKey,
Hostname: "recovery-host",
},
Storage: storer, Storage: storer,
Fs: fs, Fs: fs,
Repositories: database.NewRepositories(recoveryDB), Repositories: database.NewRepositories(recoveryDB),
@@ -169,20 +150,6 @@ func newRecoveryHost(
return recovery, stdout return recovery, stdout
} }
// runVaultikCommand runs one vaultik command line in-process and fails the
// test if it returns an error. The command writes the cli package's global
// flag variables, so it must not be called from two tests that run at the
// same time.
func runVaultikCommand(t *testing.T, args ...string) {
t.Helper()
cmd := cli.NewRootCommand()
cmd.SetArgs(args)
cmd.SetOut(io.Discard)
cmd.SetErr(io.Discard)
require.NoError(t, cmd.Execute())
}
// assertRestoredTreeMatches byte-compares every restored file against its // assertRestoredTreeMatches byte-compares every restored file against its
// source content. // source content.
func assertRestoredTreeMatches( func assertRestoredTreeMatches(
@@ -1,7 +1,6 @@
package vaultik //nolint:testpackage // sets ctx/cancel and inspects scratch files package vaultik //nolint:testpackage // sets ctx/cancel and inspects scratch files
import ( import (
"bytes"
"context" "context"
"io" "io"
"path/filepath" "path/filepath"
@@ -12,7 +11,6 @@ import (
"time" "time"
"github.com/spf13/afero" "github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage" "sneak.berlin/go/vaultik/internal/storage"
@@ -159,50 +157,3 @@ func scratchEntries(t *testing.T, dir string) []string {
return matches return matches
} }
// TestRestoreSkipErrorsCancelDuringBlobDownload cancels a SkipErrors
// restore while a blob download is in progress. The download fails only
// because of the cancel, so Restore must return context.Canceled without
// reporting the file that needs the blob as failed.
func TestRestoreSkipErrorsCancelDuringBlobDownload(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs()
tempDir := t.TempDir()
cfg, storer, snapshotID, srcPath := backupOneFile(context.Background(),
t, fs, tempDir, "a.txt", []byte("hello vaultik"), 0o644)
gate := newBlockingBlobStorer(storer)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var out bytes.Buffer
v := newRestoreVaultik(ctx, cfg, gate, fs)
v.UI = ui.NewWithColor(&out, false)
restoreErr := make(chan error, 1)
go func() {
restoreErr <- v.Restore(&RestoreOptions{
SnapshotID: snapshotID,
TargetDir: filepath.Join(tempDir, "restored"),
SkipErrors: true,
})
}()
select {
case <-gate.entered:
case <-time.After(30 * time.Second):
t.Fatal("restore never reached the blob-download phase")
}
cancel()
require.ErrorIs(t, <-restoreErr, context.Canceled)
assert.NotContains(t, out.String(), srcPath,
"the cancel was reported as a failed file")
}
@@ -1,221 +0,0 @@
package vaultik //nolint:testpackage // drives unexported restore and verify steps
import (
"context"
"math"
"path/filepath"
"strings"
"testing"
"time"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/types"
)
// These tests feed restore and --verify a snapshot database written by
// hand, as a damaged or hostile store could serve one. Each malformed row
// must end in an error, not a panic.
// shortChunkHash is shorter than the hash prefix that error messages print.
const shortChunkHash = "abc"
// restoredFileContent is the content of the restored file under verify.
const restoredFileContent = "xyz"
// craftedSnapshotDB opens an empty snapshot database in a temp directory.
func craftedSnapshotDB(t *testing.T) (*database.DB, *database.Repositories) {
t.Helper()
db, err := database.New(context.Background(),
filepath.Join(t.TempDir(), "snapshot.db"))
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
return db, database.NewRepositories(db)
}
// craftedFile adds a regular file whose only chunk has the given hash.
// Adding the chunks row, if any, is left to the caller.
func craftedFile(
t *testing.T, repos *database.Repositories, chunkHash string,
) *database.File {
t.Helper()
ctx := context.Background()
file := &database.File{
Path: "/src/f",
MTime: time.Now().UTC(),
Size: int64(len(restoredFileContent)),
Mode: 0o644,
}
require.NoError(t, repos.Files.Create(ctx, nil, file))
require.NoError(t, repos.FileChunks.Create(ctx, nil, &database.FileChunk{
FileID: file.ID,
ChunkHash: types.ChunkHash(chunkHash),
}))
return file
}
// TestRestoreShortChunkHashInNoBlob proves a file whose short chunk hash
// has no blob_chunks row fails restore planning and the chunk write with
// an error.
func TestRestoreShortChunkHashInNoBlob(t *testing.T) {
t.Parallel()
ctx := context.Background()
_, repos := craftedSnapshotDB(t)
require.NoError(t, repos.Chunks.Create(ctx, nil,
&database.Chunk{ChunkHash: shortChunkHash, Size: 3}))
file := craftedFile(t, repos, shortChunkHash)
v := NewForTesting(nil)
chunkToBlobMap, err := v.buildChunkToBlobMap(ctx, repos)
require.NoError(t, err)
_, err = newRestorePlan(ctx, repos, []*database.File{file},
chunkToBlobMap, map[string]string{})
require.ErrorIs(t, err, errPlanChunkMissing)
fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID)
require.NoError(t, err)
out, err := afero.NewMemMapFs().Create("out")
require.NoError(t, err)
session := &restoreSession{
v: v.Vaultik, ctx: ctx, chunkToBlobMap: chunkToBlobMap,
}
_, _, err = session.writeFileChunks(out, fileChunks)
require.ErrorIs(t, err, errChunkNotInAnyBlob)
}
// TestRestoreShortChunkHashReadPastBlobEnd proves a short chunk hash
// whose blob_chunks row reads past the end of its blob fails the chunk
// write with an error.
func TestRestoreShortChunkHashReadPastBlobEnd(t *testing.T) {
t.Parallel()
ctx := context.Background()
_, repos := craftedSnapshotDB(t)
blobHash := strings.Repeat("b", blobHashHexLen)
blob := &database.Blob{
ID: types.NewBlobID(),
Hash: types.BlobHash(blobHash),
CreatedTS: time.Now().UTC(),
}
require.NoError(t, repos.Blobs.Create(ctx, nil, blob))
require.NoError(t, repos.Chunks.Create(ctx, nil,
&database.Chunk{ChunkHash: shortChunkHash, Size: 3}))
require.NoError(t, repos.BlobChunks.Create(ctx, nil, &database.BlobChunk{
BlobID: blob.ID,
ChunkHash: shortChunkHash,
Length: 100,
}))
file := craftedFile(t, repos, shortChunkHash)
cache, err := newBlobDiskCache(1 << 20)
require.NoError(t, err)
t.Cleanup(func() { _ = cache.Close() })
require.NoError(t, cache.Put(blobHash, []byte("abc")))
v := NewForTesting(nil)
chunkToBlobMap, err := v.buildChunkToBlobMap(ctx, repos)
require.NoError(t, err)
_, blobIDToHash, err := v.buildBlobIndexes(repos)
require.NoError(t, err)
fileChunks, err := repos.FileChunks.GetByFileID(ctx, file.ID)
require.NoError(t, err)
out, err := afero.NewMemMapFs().Create("out")
require.NoError(t, err)
session := &restoreSession{
v: v.Vaultik,
ctx: ctx,
chunkToBlobMap: chunkToBlobMap,
blobIDToHash: blobIDToHash,
blobCache: cache,
}
_, _, err = session.writeFileChunks(out, fileChunks)
require.ErrorIs(t, err, errCacheReadBeyondBlob)
}
// TestVerifyFileMalformedChunkRow proves --verify returns an error for a
// chunk with no chunks row, a short chunk hash, and a chunk size from the
// database that is negative or larger than the restored file.
func TestVerifyFileMalformedChunkRow(t *testing.T) {
t.Parallel()
fullHash := types.ChunkHash(strings.Repeat("c", blobHashHexLen))
tests := []struct {
name string
hash types.ChunkHash
chunk *database.Chunk // nil adds no chunks row
want error
}{
{
name: "missing chunk row",
hash: fullHash,
want: errChunkRowMissing,
},
{
name: "short hash",
hash: shortChunkHash,
chunk: &database.Chunk{ChunkHash: shortChunkHash, Size: 3},
want: errChunkHashMismatch,
},
{
name: "size larger than the file",
hash: fullHash,
chunk: &database.Chunk{ChunkHash: fullHash, Size: math.MaxInt64},
want: errShortChunkRead,
},
{
name: "negative size",
hash: fullHash,
chunk: &database.Chunk{ChunkHash: fullHash, Size: -1},
want: errNegativeChunkLength,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
ctx := context.Background()
db, repos := craftedSnapshotDB(t)
if tt.chunk == nil {
// A crafted database need not satisfy its foreign keys.
_, err := db.Conn().ExecContext(ctx, "PRAGMA foreign_keys = OFF")
require.NoError(t, err)
} else {
require.NoError(t, repos.Chunks.Create(ctx, nil, tt.chunk))
}
file := craftedFile(t, repos, tt.hash.String())
v := NewForTesting(nil)
v.Fs = afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(v.Fs, "/restore/f",
[]byte(restoredFileContent), 0o600))
_, err := v.verifyFile(ctx, repos, file, "/restore/f")
require.ErrorIs(t, err, tt.want)
})
}
}
-349
View File
@@ -1,349 +0,0 @@
package vaultik //nolint:testpackage // drives unexported restore internals
import (
"context"
"os"
"path/filepath"
"syscall"
"testing"
"time"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/types"
)
// These tests check that restore applies each entry's owner, mode and
// mtime in an order that keeps them.
const (
readOnlyDirMode = uint32(os.ModeDir | 0o555)
unsearchableMode = uint32(os.ModeDir | 0o600)
plainDirMode = uint32(os.ModeDir | 0o755)
plainFileMode = uint32(0o644)
setuidFileMode = uint32(os.ModeSetuid | 0o755)
otherOwnerID = uint32(4321)
writableTestMode = 0o755
symlinkTargetPath = "/nonexistent/target"
memTargetDir = "/restore"
// Owner bits a normal user needs on a directory to create an entry
// in it, and to change an entry in it.
ownerWriteAndSearch = os.FileMode(0o300)
ownerSearch = os.FileMode(0o100)
)
// normalUserFs refuses what the kernel refuses a normal user. make test
// runs as root, which a read-only or unsearchable directory does not
// stop, so without it the tests below could not fail there. Creating an
// entry needs owner write and search on the directory holding it;
// changing an entry's mode or times needs owner search. Only that one
// directory is checked, not every ancestor.
type normalUserFs struct {
afero.Fs
}
//nolint:ireturn // afero.Fs.OpenFile is defined to return the interface
func (fs normalUserFs) OpenFile(
name string, flag int, perm os.FileMode,
) (afero.File, error) {
if flag&os.O_CREATE != 0 {
err := fs.checkParent(name, ownerWriteAndSearch)
if err != nil {
return nil, err
}
}
return fs.Fs.OpenFile(name, flag, perm)
}
func (fs normalUserFs) MkdirAll(path string, perm os.FileMode) error {
_, err := fs.Stat(path)
if err != nil {
err = fs.checkParent(path, ownerWriteAndSearch)
if err != nil {
return err
}
}
return fs.Fs.MkdirAll(path, perm)
}
func (fs normalUserFs) Chmod(name string, mode os.FileMode) error {
err := fs.checkParent(name, ownerSearch)
if err != nil {
return err
}
return fs.Fs.Chmod(name, mode)
}
func (fs normalUserFs) Chtimes(name string, atime, mtime time.Time) error {
err := fs.checkParent(name, ownerSearch)
if err != nil {
return err
}
return fs.Fs.Chtimes(name, atime, mtime)
}
// checkParent returns a permission error when the directory holding name
// exists and its owner bits lack any of need.
func (fs normalUserFs) checkParent(name string, need os.FileMode) error {
info, err := fs.Stat(filepath.Dir(name))
if err == nil && info.Mode().Perm()&need != need {
return &os.PathError{Op: "access", Path: name, Err: os.ErrPermission}
}
return nil
}
// TestRestoreFillsReadOnlyDirectory checks that a read-only directory
// still receives the entries inside it, and ends with its stored mode.
func TestRestoreFillsReadOnlyDirectory(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
rows, repos := makeFiles(ctx, t, []*database.File{
{Path: "/ro", Mode: readOnlyDirMode},
{Path: "/ro/file", Mode: plainFileMode},
{Path: "/ro/sub", Mode: readOnlyDirMode},
{Path: "/ro/sub/file", Mode: plainFileMode},
})
fs := afero.NewMemMapFs()
v := newContainmentVaultik(ctx, normalUserFs{Fs: fs})
_, err := v.restoreAllFiles(rows, repos,
&RestoreOptions{TargetDir: memTargetDir}, nil, nil)
require.NoError(t, err)
for _, path := range []string{"/ro/file", "/ro/sub/file"} {
_, err := fs.Stat(filepath.Join(memTargetDir, path))
require.NoErrorf(t, err, "file inside a read-only directory: %s", path)
}
for _, dir := range []string{"/ro", "/ro/sub"} {
info, err := fs.Stat(filepath.Join(memTargetDir, dir))
require.NoError(t, err)
assert.Equalf(t, os.FileMode(0o555), info.Mode().Perm(), "mode of %s", dir)
}
}
// TestRestoreKeepsNonEmptyDirectoryMTime checks that a directory keeps
// its stored mtime although entries were written into it. It runs on the
// real filesystem, where writing an entry changes its directory's mtime.
func TestRestoreKeepsNonEmptyDirectoryMTime(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
targetDir := t.TempDir()
mtime := time.Date(2001, time.February, 3, 4, 5, 6, 0, time.UTC)
rows, repos := makeFiles(ctx, t, []*database.File{
{Path: "/dir", Mode: plainDirMode, MTime: mtime},
{Path: "/dir/file", Mode: plainFileMode, MTime: mtime},
})
v := newContainmentVaultik(ctx, afero.NewOsFs())
_, err := v.restoreAllFiles(rows, repos,
&RestoreOptions{TargetDir: targetDir}, nil, nil)
require.NoError(t, err)
info, err := os.Stat(filepath.Join(targetDir, "dir"))
require.NoError(t, err)
assert.Truef(t, info.ModTime().Equal(mtime),
"directory mtime is %s, stored %s", info.ModTime(), mtime)
}
// TestRestoreFinishesChildBeforeUnsearchableParent checks that a
// directory inside one whose stored mode denies search still gets its
// own stored mode and mtime.
func TestRestoreFinishesChildBeforeUnsearchableParent(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
mtime := time.Date(2001, time.February, 3, 4, 5, 6, 0, time.UTC)
rows, repos := makeFiles(ctx, t, []*database.File{
{Path: "/locked", Mode: unsearchableMode, MTime: mtime},
{Path: "/locked/sub", Mode: plainDirMode, MTime: mtime},
})
fs := afero.NewMemMapFs()
v := newContainmentVaultik(ctx, normalUserFs{Fs: fs})
_, err := v.restoreAllFiles(rows, repos,
&RestoreOptions{TargetDir: memTargetDir}, nil, nil)
require.NoError(t, err)
info, err := fs.Stat(filepath.Join(memTargetDir, "locked"))
require.NoError(t, err)
assert.Equal(t, os.FileMode(0o600), info.Mode().Perm())
info, err = fs.Stat(filepath.Join(memTargetDir, "locked", "sub"))
require.NoError(t, err)
assert.Equal(t, os.FileMode(0o755), info.Mode().Perm())
assert.Truef(t, info.ModTime().Equal(mtime),
"mtime of sub is %s, stored %s", info.ModTime(), mtime)
}
// TestRestoreLeavesSymlinkedDirectoryTargetAlone checks that a directory
// whose place a later entry takes with a symlink does not hand its stored
// owner, mode and mtime to whatever the symlink points at. "/d" and "/d/"
// are different stored paths for the same place on disk.
func TestRestoreLeavesSymlinkedDirectoryTargetAlone(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
tempDir := t.TempDir()
targetDir := filepath.Join(tempDir, "target")
outsideDir := filepath.Join(tempDir, "outside")
require.NoError(t, os.Mkdir(outsideDir, writableTestMode))
before, err := os.Stat(outsideDir)
require.NoError(t, err)
mtime := time.Date(2001, time.February, 3, 4, 5, 6, 0, time.UTC)
rows, repos := makeFiles(ctx, t, []*database.File{
{
Path: "/d",
Mode: readOnlyDirMode,
UID: otherOwnerID,
GID: otherOwnerID,
MTime: mtime,
},
{Path: "/d/", LinkTarget: types.FilePath(outsideDir), MTime: mtime},
})
v := newContainmentVaultik(ctx, afero.NewOsFs())
_, err = v.restoreAllFiles(rows, repos,
&RestoreOptions{TargetDir: targetDir}, nil, nil)
require.NoError(t, err)
after, err := os.Stat(outsideDir)
require.NoError(t, err)
assert.Equal(t, before.Mode(), after.Mode())
assert.Truef(t, after.ModTime().Equal(before.ModTime()),
"mtime changed from %s to %s", before.ModTime(), after.ModTime())
beforeOwner, ok := before.Sys().(*syscall.Stat_t)
require.True(t, ok)
afterOwner, ok := after.Sys().(*syscall.Stat_t)
require.True(t, ok)
assert.Equal(t, beforeOwner.Uid, afterOwner.Uid)
assert.Equal(t, beforeOwner.Gid, afterOwner.Gid)
}
// TestRestoreKeepsSetuidThroughChown checks that a setuid file keeps the
// bit when restore changes its owner. Linux clears setuid on any chown of
// a regular file, even one to its current owner, so the file is recorded
// with the current user as owner and the session is told it runs as
// root: the chown then needs no privilege.
func TestRestoreKeepsSetuidThroughChown(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
targetDir := t.TempDir()
info, err := os.Stat(targetDir)
require.NoError(t, err)
owner, ok := info.Sys().(*syscall.Stat_t)
require.True(t, ok)
rows, repos := makeFiles(ctx, t, []*database.File{{
Path: "/suid",
Mode: setuidFileMode,
UID: owner.Uid,
GID: owner.Gid,
MTime: time.Date(2001, time.February, 3, 4, 5, 6, 0, time.UTC),
}})
session := &restoreSession{
v: newContainmentVaultik(ctx, afero.NewOsFs()),
ctx: ctx,
repos: repos,
opts: &RestoreOptions{TargetDir: targetDir},
result: &RestoreResult{},
runningAsRoot: true,
}
require.NoError(t, session.restoreFile(rows[0]))
info, err = os.Stat(filepath.Join(targetDir, "suid"))
require.NoError(t, err)
assert.Equal(t, os.FileMode(setuidFileMode),
info.Mode()&(os.ModeSetuid|os.ModePerm))
}
// TestRestoreSetsSymlinkMTime checks that a restored symlink gets its
// stored mtime on the link itself. The link dangles, so a call that
// follows it would fail.
func TestRestoreSetsSymlinkMTime(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
targetDir := t.TempDir()
mtime := time.Date(2001, time.February, 3, 4, 5, 6, 0, time.UTC)
rows, repos := makeFiles(ctx, t, []*database.File{{
Path: "/link",
LinkTarget: types.FilePath(symlinkTargetPath),
MTime: mtime,
}})
v := newContainmentVaultik(ctx, afero.NewOsFs())
_, err := v.restoreAllFiles(rows, repos,
&RestoreOptions{TargetDir: targetDir}, nil, nil)
require.NoError(t, err)
info, err := os.Lstat(filepath.Join(targetDir, "link"))
require.NoError(t, err)
assert.Truef(t, info.ModTime().Equal(mtime),
"symlink mtime is %s, stored %s", info.ModTime(), mtime)
}
// TestRestoreSetsSymlinkOwnerAsRoot checks that a symlink restored as
// root gets its stored owner on the link itself.
func TestRestoreSetsSymlinkOwnerAsRoot(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
if os.Geteuid() != 0 {
t.Skip("giving a file to another user needs root")
}
ctx := context.Background()
targetDir := t.TempDir()
rows, repos := makeFiles(ctx, t, []*database.File{{
Path: "/link",
LinkTarget: types.FilePath(symlinkTargetPath),
UID: otherOwnerID,
GID: otherOwnerID,
MTime: time.Date(2001, time.February, 3, 4, 5, 6, 0, time.UTC),
}})
v := newContainmentVaultik(ctx, afero.NewOsFs())
_, err := v.restoreAllFiles(rows, repos,
&RestoreOptions{TargetDir: targetDir}, nil, nil)
require.NoError(t, err)
info, err := os.Lstat(filepath.Join(targetDir, "link"))
require.NoError(t, err)
owner, ok := info.Sys().(*syscall.Stat_t)
require.True(t, ok)
assert.Equal(t, otherOwnerID, owner.Uid)
assert.Equal(t, otherOwnerID, owner.Gid)
}
+1 -15
View File
@@ -75,7 +75,7 @@ func newRestorePlan(
bc, ok := chunkToBlobMap[fc.ChunkHash.String()] bc, ok := chunkToBlobMap[fc.ChunkHash.String()]
if !ok { if !ok {
return nil, fmt.Errorf("planning %s: %w: %s", return nil, fmt.Errorf("planning %s: %w: %s",
f.Path, errPlanChunkMissing, shortHash(fc.ChunkHash.String())) f.Path, errPlanChunkMissing, fc.ChunkHash.String()[:16])
} }
hash, ok := blobIDToHash[bc.BlobID.String()] hash, ok := blobIDToHash[bc.BlobID.String()]
@@ -214,20 +214,6 @@ func (p *restorePlan) blobsNeeded(fileID types.FileID) []string {
return out return out
} }
// filesReferencingBlob returns the pending files that reference the
// named blob, in any order. The result is a copy, so the caller may
// finishFile each of them while ranging over it.
func (p *restorePlan) filesReferencingBlob(blobHash string) []types.FileID {
files := p.blobFiles[blobHash]
out := make([]types.FileID, 0, len(files))
for id := range files {
out = append(out, id)
}
return out
}
// hasPending reports whether any unfinished files remain. // hasPending reports whether any unfinished files remain.
func (p *restorePlan) hasPending() bool { func (p *restorePlan) hasPending() bool {
return len(p.fileBlobs) > 0 return len(p.fileBlobs) > 0
@@ -1,195 +0,0 @@
package vaultik_test
import (
"bytes"
"context"
"fmt"
"path/filepath"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/ui"
"sneak.berlin/go/vaultik/internal/vaultik"
)
// A file no larger than the chunker's minimum chunk size (a quarter of
// faultChunkSize) is stored as one chunk, so it lives in exactly one blob.
// More of them than fit in faultMaxBlobSize make the snapshot span two
// blobs.
const (
missingBlobFileBytes = int(faultChunkSize / 4)
missingBlobFileCount = 20
)
// missingBlobBackup is a snapshot of single-chunk files spread over two
// blobs, with one of those blobs deleted from the store.
type missingBlobBackup struct {
fs afero.Fs
cfg *config.Config
storer storage.Storer
snapshotID string
restoreDir string
// files holds the original content by source path.
files map[string][]byte
// lost holds the source paths whose chunk is in the deleted blob.
lost map[string]bool
}
// TestRestoreSkipErrorsSkipsFilesOfMissingBlob restores with SkipErrors
// after one blob of a two-blob snapshot was deleted. Every file stored in
// that blob must be reported as failed and left absent, every other file
// must be restored intact, and Restore must still return an error.
func TestRestoreSkipErrorsSkipsFilesOfMissingBlob(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
backup := backupThenDeleteOneBlob(ctx, t)
var out bytes.Buffer
v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs)
v.UI = ui.NewWithColor(&out, false)
err := v.Restore(&vaultik.RestoreOptions{
SnapshotID: backup.snapshotID,
TargetDir: backup.restoreDir,
SkipErrors: true,
})
require.Error(t, err, "restore must fail when files were skipped")
assert.Contains(t, err.Error(),
fmt.Sprintf("%d file(s) failed to restore", len(backup.lost)))
for path, content := range backup.files {
restored := filepath.Join(backup.restoreDir, path)
if backup.lost[path] {
assert.Containsf(t, out.String(), path,
"%s needs the deleted blob and must be reported", path)
assert.NoFileExists(t, restored)
continue
}
got, err := afero.ReadFile(backup.fs, restored)
require.NoErrorf(t, err, "%s does not need the deleted blob", path)
assert.Equalf(t, content, got, "%s restored with wrong content", path)
}
}
// TestRestoreMissingBlobAbortsWithoutSkipErrors checks that a deleted blob
// still ends the restore with an error when SkipErrors is not set.
func TestRestoreMissingBlobAbortsWithoutSkipErrors(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
backup := backupThenDeleteOneBlob(ctx, t)
v := newReaderVaultik(ctx, backup.cfg, backup.storer, nil, backup.fs)
err := v.Restore(&vaultik.RestoreOptions{
SnapshotID: backup.snapshotID,
TargetDir: backup.restoreDir,
})
require.ErrorIs(t, err, storage.ErrNotFound)
}
// backupThenDeleteOneBlob backs up missingBlobFileCount single-chunk
// files, then deletes from the store the blob holding the first of them.
func backupThenDeleteOneBlob(
ctx context.Context, t *testing.T,
) *missingBlobBackup {
t.Helper()
fs := afero.NewOsFs()
tempDir := t.TempDir()
dataDir := filepath.Join(tempDir, "src")
dbPath := filepath.Join(tempDir, "index.sqlite")
cfg := faultTestConfig()
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
files := make(map[string][]byte, missingBlobFileCount)
for i := range missingBlobFileCount {
path := filepath.Join(dataDir, fmt.Sprintf("file-%02d.bin", i))
files[path] = bytesPattern(
fmt.Sprintf("file-%02d-", i), missingBlobFileBytes)
require.NoError(t, afero.WriteFile(fs, path, files[path], 0o644))
}
storer, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
require.NoError(t, err)
db, err := database.New(ctx, dbPath)
require.NoError(t, err)
repos := database.NewRepositories(db)
id := fullFaultBackup(
ctx, t, fs, storer, cfg, repos, dataDir, dbPath, "missingblob")
blobOfFile := make(map[string]string, len(files))
for path := range files {
blobOfFile[path] = blobHashOfSingleChunkFile(ctx, t, repos, path)
}
require.NoError(t, db.Close())
deleted := blobOfFile[filepath.Join(dataDir, "file-00.bin")]
require.NoError(t, storer.Delete(ctx, fmt.Sprintf(
"blobs/%s/%s/%s", deleted[:2], deleted[2:4], deleted)))
lost := make(map[string]bool)
for path, hash := range blobOfFile {
if hash == deleted {
lost[path] = true
}
}
require.Less(t, len(lost), len(files),
"the snapshot must span more than one blob")
return &missingBlobBackup{
fs: fs,
cfg: cfg,
storer: storer,
snapshotID: id,
restoreDir: filepath.Join(tempDir, "restored"),
files: files,
lost: lost,
}
}
// blobHashOfSingleChunkFile returns the hash of the blob holding the one
// chunk of the file at path, as recorded in the local index.
func blobHashOfSingleChunkFile(
ctx context.Context, t *testing.T,
repos *database.Repositories, path string,
) string {
t.Helper()
chunks, err := repos.FileChunks.GetByPath(ctx, path)
require.NoError(t, err)
require.Lenf(t, chunks, 1, "%s must be a single chunk", path)
blobChunk, err := repos.BlobChunks.GetByChunkHash(
ctx, chunks[0].ChunkHash.String())
require.NoError(t, err)
require.NotNilf(t, blobChunk, "chunk of %s is in no blob", path)
blob, err := repos.Blobs.GetByID(ctx, blobChunk.BlobID.String())
require.NoError(t, err)
return blob.Hash.String()
}
@@ -1,72 +0,0 @@
package vaultik_test
import (
"bytes"
"context"
"path/filepath"
"testing"
"time"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/vaultik"
)
// A file rewritten with its size unchanged and a new mtime in the same
// second as the mtime the index holds must still be backed up. See
// https://git.eeqj.de/sneak/vaultik/issues/226.
func TestBackupOfSameSecondRewriteRestoresNewContent(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
fs := afero.NewOsFs()
tempDir := t.TempDir()
dataDir := filepath.Join(tempDir, "src")
storeDir := filepath.Join(tempDir, "remote")
restoreDir := filepath.Join(tempDir, "restored")
dbPath := filepath.Join(tempDir, "index.sqlite")
rewrittenPath := filepath.Join(dataDir, "small.txt")
ctx := context.Background()
files := writeFaultSourceTree(t, fs, dataDir)
cfg := changedFileConfig(dataDir, dbPath)
firstMTime := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC).
Add(100 * time.Millisecond)
secondMTime := firstMTime.Add(800 * time.Millisecond)
require.NoError(t, fs.Chtimes(rewrittenPath, firstMTime, firstMTime))
store, err := storage.NewFileStorer(storeDir)
require.NoError(t, err)
db, err := database.New(ctx, dbPath)
require.NoError(t, err)
repos := database.NewRepositories(db)
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
require.NoError(t, backUp(v, "first"))
// Upper-casing ASCII text keeps its size.
files[rewrittenPath] = bytes.ToUpper(files[rewrittenPath])
require.NoError(t, afero.WriteFile(fs, rewrittenPath, files[rewrittenPath], 0o644))
require.NoError(t, fs.Chtimes(rewrittenPath, secondMTime, secondMTime))
require.NoError(t, backUp(v, "second"))
id := localSnapshotID(ctx, t, repos, "second")
require.NoError(t, db.Close())
reader := newReaderVaultik(ctx, cfg, store, nil, fs)
require.NoError(t, reader.Restore(&vaultik.RestoreOptions{
SnapshotID: id,
TargetDir: restoreDir,
Verify: true,
}))
assertRestoredTree(t, fs, restoreDir, files)
}
+75 -101
View File
@@ -14,7 +14,6 @@ import (
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/snapshot" "sneak.berlin/go/vaultik/internal/snapshot"
"sneak.berlin/go/vaultik/internal/types"
) )
// Sentinel errors for snapshot management. // Sentinel errors for snapshot management.
@@ -24,9 +23,6 @@ var (
errSnapshotVerifyFailed = errors.New("verification failed") errSnapshotVerifyFailed = errors.New("verification failed")
errRemoveAllNeedsForce = errors.New("--all requires --force") errRemoveAllNeedsForce = errors.New("--all requires --force")
errInvalidTableName = errors.New("invalid table name") errInvalidTableName = errors.New("invalid table name")
errNoAgeRecipients = errors.New(
"creating a snapshot needs at least one public key in " +
"age_recipients (generate a keypair with: age-keygen)")
) )
// listRecentLimit caps how many snapshot rows are fetched from the // listRecentLimit caps how many snapshot rows are fetched from the
@@ -46,12 +42,6 @@ type SnapshotCreateOptions struct {
// CreateSnapshot executes the snapshot creation operation // CreateSnapshot executes the snapshot creation operation
func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
// config.Load accepts an empty list, since listing, verifying and
// restoring need no public key.
if len(v.Config.AgeRecipients) == 0 {
return errNoAgeRecipients
}
overallStartTime := time.Now() overallStartTime := time.Now()
log.Info("Starting snapshot creation", log.Info("Starting snapshot creation",
@@ -190,11 +180,6 @@ type snapshotStats struct {
totalBytesUploaded int64 totalBytesUploaded int64
totalBlobsUploaded int totalBlobsUploaded int
uploadDuration time.Duration uploadDuration time.Duration
// The sizes of all blobs the snapshot references, set by
// finalizeSnapshotMetadata once snapshot_blobs is populated.
blobSize int64
blobUncompressedSize int64
} }
// createNamedSnapshot creates a single named snapshot // createNamedSnapshot creates a single named snapshot
@@ -234,6 +219,8 @@ func (v *Vaultik) createNamedSnapshot(
return err return err
} }
v.collectUploadStats(scanner, stats)
err = v.finalizeSnapshotMetadata(snapshotID, stats) err = v.finalizeSnapshotMetadata(snapshotID, stats)
if err != nil { if err != nil {
return err return err
@@ -285,11 +272,6 @@ func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) {
func (v *Vaultik) scanAllDirectories( func (v *Vaultik) scanAllDirectories(
scanner *snapshot.Scanner, resolvedDirs []string, snapshotID string, scanner *snapshot.Scanner, resolvedDirs []string, snapshotID string,
) (*snapshotStats, error) { ) (*snapshotStats, error) {
if progress := scanner.GetProgress(); progress != nil {
progress.Start()
defer progress.Stop()
}
stats := &snapshotStats{} stats := &snapshotStats{}
for i, dir := range resolvedDirs { for i, dir := range resolvedDirs {
@@ -318,9 +300,6 @@ func (v *Vaultik) scanAllDirectories(
stats.totalBytesSkipped += result.BytesSkipped stats.totalBytesSkipped += result.BytesSkipped
stats.totalFilesDeleted += result.FilesDeleted stats.totalFilesDeleted += result.FilesDeleted
stats.totalBytesDeleted += result.BytesDeleted stats.totalBytesDeleted += result.BytesDeleted
stats.totalBlobsUploaded += result.BlobsUploaded
stats.totalBytesUploaded += result.BytesUploaded
stats.uploadDuration += result.UploadDuration
log.Info("Directory scan complete", log.Info("Directory scan complete",
"path", dir, "path", dir,
@@ -336,6 +315,18 @@ func (v *Vaultik) scanAllDirectories(
return stats, nil return stats, nil
} }
// collectUploadStats gathers upload statistics from the scanner's
// progress reporter.
func (v *Vaultik) collectUploadStats(scanner *snapshot.Scanner, stats *snapshotStats) {
if s := scanner.GetProgress(); s != nil {
progressStats := s.GetStats()
stats.totalBytesUploaded = progressStats.BytesUploaded.Load()
stats.totalBlobsUploaded = int(progressStats.BlobsUploaded.Load())
stats.uploadDuration = time.Duration(
progressStats.UploadDurationMs.Load()) * time.Millisecond
}
}
// finalizeSnapshotMetadata updates stats, exports metadata, and only then // finalizeSnapshotMetadata updates stats, exports metadata, and only then
// marks the snapshot complete. Recording completion last is deliberate: an // marks the snapshot complete. Recording completion last is deliberate: an
// export interrupted by a crash leaves the snapshot incomplete rather than // export interrupted by a crash leaves the snapshot incomplete rather than
@@ -345,39 +336,31 @@ func (v *Vaultik) scanAllDirectories(
func (v *Vaultik) finalizeSnapshotMetadata( func (v *Vaultik) finalizeSnapshotMetadata(
snapshotID string, stats *snapshotStats, snapshotID string, stats *snapshotStats,
) error { ) error {
// snapshot_blobs must be populated before the blob sizes below, which
// total the snapshot's blobs, and before the export, which builds the
// manifest and the trimmed metadata database from it.
err := v.SnapshotManager.PopulateSnapshotBlobs(v.ctx, snapshotID)
if err != nil {
return fmt.Errorf("populating snapshot blobs: %w", err)
}
stats.blobSize, stats.blobUncompressedSize, err =
v.Repositories.Snapshots.GetSnapshotBlobSizes(v.ctx, snapshotID)
if err != nil {
return fmt.Errorf("getting snapshot blob sizes: %w", err)
}
extStats := snapshot.ExtendedBackupStats{ extStats := snapshot.ExtendedBackupStats{
BackupStats: snapshot.BackupStats{ BackupStats: snapshot.BackupStats{
FilesScanned: stats.totalFiles, FilesScanned: stats.totalFiles,
TotalSize: stats.totalBytes + stats.totalBytesSkipped, BytesScanned: stats.totalBytes,
ChunksCreated: stats.totalChunks, ChunksCreated: stats.totalChunks,
BlobsCreated: stats.totalBlobs, BlobsCreated: stats.totalBlobs,
BytesUploaded: stats.totalBytesUploaded, BytesUploaded: stats.totalBytesUploaded,
}, },
BlobSize: stats.blobSize, BlobUncompressedSize: 0,
BlobUncompressedSize: stats.blobUncompressedSize,
CompressionLevel: v.Config.CompressionLevel, CompressionLevel: v.Config.CompressionLevel,
UploadDurationMs: stats.uploadDuration.Milliseconds(), UploadDurationMs: stats.uploadDuration.Milliseconds(),
} }
err = v.SnapshotManager.UpdateSnapshotStatsExtended(v.ctx, snapshotID, extStats) err := v.SnapshotManager.UpdateSnapshotStatsExtended(v.ctx, snapshotID, extStats)
if err != nil { if err != nil {
return fmt.Errorf("updating snapshot stats: %w", err) return fmt.Errorf("updating snapshot stats: %w", err)
} }
// snapshot_blobs must be populated before the export, which builds the
// manifest and the trimmed metadata database from it.
err = v.SnapshotManager.PopulateSnapshotBlobs(v.ctx, snapshotID)
if err != nil {
return fmt.Errorf("populating snapshot blobs: %w", err)
}
err = v.SnapshotManager.ExportSnapshotMetadata( err = v.SnapshotManager.ExportSnapshotMetadata(
v.ctx, v.Config.IndexPath, snapshotID) v.ctx, v.Config.IndexPath, snapshotID)
if err != nil { if err != nil {
@@ -412,10 +395,12 @@ func (v *Vaultik) printSnapshotSummary(
totalFilesChanged := stats.totalFiles - stats.totalFilesSkipped totalFilesChanged := stats.totalFiles - stats.totalFilesSkipped
totalBytesAll := stats.totalBytes + stats.totalBytesSkipped totalBytesAll := stats.totalBytes + stats.totalBytesSkipped
// Get total blob sizes from database
compressedSize, uncompressedSize := v.getSnapshotBlobSizes(snapshotID)
var compressionRatio float64 var compressionRatio float64
if stats.blobUncompressedSize > 0 { if uncompressedSize > 0 {
compressionRatio = float64(stats.blobSize) / compressionRatio = float64(compressedSize) / float64(uncompressedSize)
float64(stats.blobUncompressedSize)
} else { } else {
compressionRatio = 1.0 compressionRatio = 1.0
} }
@@ -443,8 +428,8 @@ func (v *Vaultik) printSnapshotSummary(
if stats.totalBlobsUploaded > 0 { if stats.totalBlobsUploaded > 0 {
v.UI.Detailf("Storage: %s compressed from %s (%.2fx ratio).", v.UI.Detailf("Storage: %s compressed from %s (%.2fx ratio).",
v.UI.Size(stats.blobSize), v.UI.Size(compressedSize),
v.UI.Size(stats.blobUncompressedSize), v.UI.Size(uncompressedSize),
compressionRatio) compressionRatio)
v.UI.Detailf("Upload: %d blobs, %s in %s (%s).", v.UI.Detailf("Upload: %d blobs, %s in %s (%s).",
stats.totalBlobsUploaded, stats.totalBlobsUploaded,
@@ -456,6 +441,27 @@ func (v *Vaultik) printSnapshotSummary(
v.UI.Detailf("Snapshot create duration: %s.", v.UI.Duration(snapshotDuration)) v.UI.Detailf("Snapshot create duration: %s.", v.UI.Duration(snapshotDuration))
} }
// getSnapshotBlobSizes returns total compressed and uncompressed blob
// sizes for a snapshot.
func (v *Vaultik) getSnapshotBlobSizes(snapshotID string) (int64, int64) {
var compressed, uncompressed int64
blobHashes, err := v.Repositories.Snapshots.GetBlobHashes(v.ctx, snapshotID)
if err != nil {
return 0, 0
}
for _, hash := range blobHashes {
blob, err := v.Repositories.Blobs.GetByHash(v.ctx, hash)
if err == nil && blob != nil {
compressed += blob.CompressedSize
uncompressed += blob.UncompressedSize
}
}
return compressed, uncompressed
}
// SnapshotPurgeOptions contains options for the snapshot purge command. // SnapshotPurgeOptions contains options for the snapshot purge command.
type SnapshotPurgeOptions struct { type SnapshotPurgeOptions struct {
KeepLatest bool // Keep only the most recent snapshot per name KeepLatest bool // Keep only the most recent snapshot per name
@@ -496,23 +502,19 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
nameFilter[n] = struct{}{} nameFilter[n] = struct{}{}
} }
// Collect completed snapshots and their names, applying the name filter. // Collect completed snapshots, applying the name filter.
snapshots := make([]SnapshotInfo, 0, len(dbSnapshots)) snapshots := make([]SnapshotInfo, 0, len(dbSnapshots))
names := make(map[types.SnapshotID]string, len(dbSnapshots))
for _, s := range dbSnapshots { for _, s := range dbSnapshots {
if s.CompletedAt == nil { if s.CompletedAt == nil {
continue continue
} }
name := parseSnapshotName(s.ID.String(), s.Hostname.String())
if len(nameFilter) > 0 { if len(nameFilter) > 0 {
if _, ok := nameFilter[name]; !ok { if _, ok := nameFilter[parseSnapshotName(s.ID.String())]; !ok {
continue continue
} }
} }
names[s.ID] = name
snapshots = append(snapshots, SnapshotInfo{ snapshots = append(snapshots, SnapshotInfo{
ID: s.ID, ID: s.ID,
Timestamp: s.StartedAt, Timestamp: s.StartedAt,
@@ -525,7 +527,7 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
return snapshots[i].Timestamp.After(snapshots[j].Timestamp) return snapshots[i].Timestamp.After(snapshots[j].Timestamp)
}) })
toDelete, err := selectSnapshotsToPurge(snapshots, names, opts) toDelete, err := selectSnapshotsToPurge(snapshots, opts)
if err != nil { if err != nil {
return err return err
} }
@@ -543,11 +545,9 @@ func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error {
// selectSnapshotsToPurge applies the purge retention criteria to the // selectSnapshotsToPurge applies the purge retention criteria to the
// newest-first sorted snapshot list and returns the deletion // newest-first sorted snapshot list and returns the deletion
// candidates. names maps each snapshot's ID to its snapshot name. // candidates.
func selectSnapshotsToPurge( func selectSnapshotsToPurge(
snapshots []SnapshotInfo, snapshots []SnapshotInfo, opts *SnapshotPurgeOptions,
names map[types.SnapshotID]string,
opts *SnapshotPurgeOptions,
) ([]SnapshotInfo, error) { ) ([]SnapshotInfo, error) {
var toDelete []SnapshotInfo var toDelete []SnapshotInfo
@@ -558,7 +558,7 @@ func selectSnapshotsToPurge(
seen := make(map[string]bool) seen := make(map[string]bool)
for _, snap := range snapshots { for _, snap := range snapshots {
name := names[snap.ID] name := parseSnapshotName(snap.ID.String())
if seen[name] { if seen[name] {
toDelete = append(toDelete, snap) toDelete = append(toDelete, snap)
@@ -718,7 +718,7 @@ func (v *Vaultik) VerifySnapshotWithOptions(
result.BlobCount = manifest.BlobCount result.BlobCount = manifest.BlobCount
result.TotalSize = manifest.TotalCompressedSize result.TotalSize = manifest.TotalCompressedSize
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON {
v.stdoutf("Snapshot information:\n") v.stdoutf("Snapshot information:\n")
v.stdoutf(" Blob count: %d\n", manifest.BlobCount) v.stdoutf(" Blob count: %d\n", manifest.BlobCount)
v.stdoutf(" Total size: %s\n", ubytes(manifest.TotalCompressedSize)) v.stdoutf(" Total size: %s\n", ubytes(manifest.TotalCompressedSize))
@@ -773,7 +773,7 @@ func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) {
snapshotTime = t snapshotTime = t
} }
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON {
v.stdoutf("Verifying snapshot %s\n", snapshotID) v.stdoutf("Verifying snapshot %s\n", snapshotID)
if !snapshotTime.IsZero() { if !snapshotTime.IsZero() {
@@ -813,7 +813,7 @@ func (v *Vaultik) verifyManifestBlobs(
stat, err := v.Storage.Stat(v.ctx, blobPath) stat, err := v.Storage.Stat(v.ctx, blobPath)
switch { switch {
case err != nil: case err != nil:
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON {
v.stdoutf(" Missing: %s (%s)\n", v.stdoutf(" Missing: %s (%s)\n",
blob.Hash, ubytes(blob.CompressedSize)) blob.Hash, ubytes(blob.CompressedSize))
} }
@@ -821,7 +821,7 @@ func (v *Vaultik) verifyManifestBlobs(
missing++ missing++
missingSize += blob.CompressedSize missingSize += blob.CompressedSize
case stat.Size != blob.CompressedSize: case stat.Size != blob.CompressedSize:
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON {
v.stdoutf(" Wrong size: %s (store has %s, manifest lists %s)\n", v.stdoutf(" Wrong size: %s (store has %s, manifest lists %s)\n",
blob.Hash, ubytes(stat.Size), ubytes(blob.CompressedSize)) blob.Hash, ubytes(stat.Size), ubytes(blob.CompressedSize))
} }
@@ -853,22 +853,6 @@ func (v *Vaultik) formatVerifyResult(
return v.outputVerifyJSON(result) return v.outputVerifyJSON(result)
} }
// Under --quiet a failure is still returned, and the cli layer
// prints it on stderr.
if !v.UI.Quiet() {
v.printVerifySummary(result, failure)
}
if failure != "" {
return fmt.Errorf("%w: %s", errSnapshotVerifyFailed, failure)
}
return nil
}
// printVerifySummary prints the counts and the status line that end the
// human-readable shallow verify report. failure is empty when it passed.
func (v *Vaultik) printVerifySummary(result *VerifyResult, failure string) {
v.stdoutf("\nVerification complete:\n") v.stdoutf("\nVerification complete:\n")
v.stdoutf(" Present with listed size: %d blobs\n", result.Verified) v.stdoutf(" Present with listed size: %d blobs\n", result.Verified)
@@ -890,12 +874,14 @@ func (v *Vaultik) printVerifySummary(result *VerifyResult, failure string) {
if failure != "" { if failure != "" {
v.stdoutf("FAILED - %s\n", failure) v.stdoutf("FAILED - %s\n", failure)
return return fmt.Errorf("%w: %s", errSnapshotVerifyFailed, failure)
} }
// Report only what was actually checked: presence and size, not contents. // Report only what was actually checked: presence and size, not contents.
v.stdoutf("OK - all %d blobs listed in the manifest are present with the "+ v.stdoutf("OK - all %d blobs listed in the manifest are present with the "+
"listed size; contents not checked (use --deep)\n", result.Verified) "listed size; contents not checked (use --deep)\n", result.Verified)
return nil
} }
// shallowVerifyFailure returns a human-readable description of everything // shallowVerifyFailure returns a human-readable description of everything
@@ -1052,7 +1038,7 @@ func (v *Vaultik) syncWithRemote() error {
// every local snapshot record (issue #160). // every local snapshot record (issue #160).
remoteKeys, err := v.listAllRemoteSnapshotKeys() remoteKeys, err := v.listAllRemoteSnapshotKeys()
if err != nil { if err != nil {
return err return fmt.Errorf("listing remote snapshots: %w", err)
} }
remoteKeySet := make(map[string]bool, len(remoteKeys)) remoteKeySet := make(map[string]bool, len(remoteKeys))
@@ -1117,12 +1103,6 @@ type RemoveResult struct {
// just-removed snapshot left behind on the destination store. // just-removed snapshot left behind on the destination store.
const pruneCommandHint = "vaultik prune" const pruneCommandHint = "vaultik prune"
// snapshotRemoveCommandHint is the command suggested, with the
// snapshot's ID, when a remove could not reach the destination store:
// running it again removes the snapshot's metadata there, which
// `vaultik prune` never does.
const snapshotRemoveCommandHint = "vaultik snapshot remove"
// RemoveSnapshot removes a snapshot from the local index database and, // RemoveSnapshot removes a snapshot from the local index database and,
// unless LocalOnly is set, also strips the snapshot's metadata from the // unless LocalOnly is set, also strips the snapshot's metadata from the
// destination store. Blobs are NOT touched: removing a snapshot's // destination store. Blobs are NOT touched: removing a snapshot's
@@ -1159,7 +1139,7 @@ func (v *Vaultik) RemoveSnapshot(
} }
if !opts.LocalOnly { if !opts.LocalOnly {
result.RemoteRemoved = v.removeSnapshotRemote(snapshotID, opts) result.RemoteRemoved = v.removeSnapshotRemote(snapshotID)
} }
if v.SnapshotManager != nil { if v.SnapshotManager != nil {
@@ -1242,11 +1222,9 @@ func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions)
// removeSnapshotRemote strips the snapshot's metadata from the // removeSnapshotRemote strips the snapshot's metadata from the
// destination store, warning and proceeding on failure: the local-DB // destination store, warning and proceeding on failure: the local-DB
// removal has already happened, so the user is told the remote half // removal has already happened, so the user is told the remote half
// didn't finish and to run `vaultik snapshot remove` for the snapshot // didn't finish and can retry with `vaultik prune` once the destination
// again once the destination store is reachable (`vaultik prune` never // store is reachable. Returns true when the remote removal succeeded.
// removes snapshot metadata). Returns true when the remote removal func (v *Vaultik) removeSnapshotRemote(snapshotID string) bool {
// succeeded.
func (v *Vaultik) removeSnapshotRemote(snapshotID string, opts *RemoveOptions) bool {
log.Info("Removing snapshot metadata from remote storage", log.Info("Removing snapshot metadata from remote storage",
"snapshot_id", snapshotID) "snapshot_id", snapshotID)
@@ -1254,17 +1232,13 @@ func (v *Vaultik) removeSnapshotRemote(snapshotID string, opts *RemoveOptions) b
err := v.deleteRemoteSnapshotByKey(remoteKey) err := v.deleteRemoteSnapshotByKey(remoteKey)
if err != nil { if err != nil {
log.Warn("Could not remove snapshot metadata from remote storage; "+ log.Warn("Could not remove snapshot metadata from remote storage",
"run '"+snapshotRemoveCommandHint+"' with the snapshot's ID "+ "error", err)
"again once the remote is reachable",
"snapshot_id", snapshotID, "error", err)
// The UI writes to stdout, which under --json holds only the if v.UI != nil {
// document; the log record above is the warning on stderr.
if v.UI != nil && !opts.JSON {
v.UI.Warningf("Could not remove snapshot metadata from remote: "+ v.UI.Warningf("Could not remove snapshot metadata from remote: "+
"%v. Run '%s %s' again once the remote is reachable.", "%v. Run '%s' once the remote is reachable to finish cleanup.",
err, snapshotRemoveCommandHint, snapshotID) err, pruneCommandHint)
} }
return false return false
-286
View File
@@ -1,286 +0,0 @@
package vaultik_test
import (
"bytes"
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/storage/faultstore"
"sneak.berlin/go/vaultik/internal/ui"
"sneak.berlin/go/vaultik/internal/vaultik"
)
// These tests cover https://git.eeqj.de/sneak/vaultik/issues/225: the
// summary printed after a backup, and the statistics stored in the
// snapshots table, count each file, byte and upload once, and a --cron
// run records its uploads.
// summaryUploadDelay slows every blob upload, so a run's upload time is
// at least this long per blob even on a local store.
const summaryUploadDelay = 20 * time.Millisecond
// summaryEnv is a backup setup whose user-facing output is kept in out.
type summaryEnv struct {
v *vaultik.Vaultik
db *database.DB
repos *database.Repositories
out *bytes.Buffer
// aPath is a.bin, whose content copy.bin repeats; aSize is its size
// and totalSize the size of all three source files.
aPath string
aSize int64
totalSize int64
}
// newSummaryEnv writes src/one/a.bin, src/one/small.txt and
// src/two/copy.bin, a copy of a.bin. Every chunk of copy.bin is therefore
// already stored by the time the backup reaches it.
//
// The snapshot names "first" and "second" back up src; "split" backs up
// src/one and src/two as two paths.
func newSummaryEnv(t *testing.T) *summaryEnv {
t.Helper()
fs := afero.NewOsFs()
tempDir := t.TempDir()
srcDir := filepath.Join(tempDir, "src")
dirOne := filepath.Join(srcDir, "one")
dirTwo := filepath.Join(srcDir, "two")
dbPath := filepath.Join(tempDir, "index.sqlite")
ctx := context.Background()
aContent := bytesPattern("a-", int(3*faultChunkSize))
smallContent := []byte("hello vaultik")
files := map[string][]byte{
filepath.Join(dirOne, "a.bin"): aContent,
filepath.Join(dirOne, "small.txt"): smallContent,
filepath.Join(dirTwo, "copy.bin"): aContent,
}
for path, content := range files {
require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
}
cfg := faultTestConfig()
cfg.IndexPath = dbPath
cfg.ChunkSize = config.Size(faultChunkSize)
cfg.Snapshots = map[string]config.SnapshotConfig{
"first": {Paths: []string{srcDir}},
"second": {Paths: []string{srcDir}},
"split": {Paths: []string{dirOne, dirTwo}},
}
inner, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
require.NoError(t, err)
store := faultstore.New(inner)
store.OnPut = func(key string) faultstore.PutAction {
if strings.HasPrefix(key, "blobs/") {
time.Sleep(summaryUploadDelay)
}
return faultstore.PutNormal
}
db, err := database.New(ctx, dbPath)
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
repos := database.NewRepositories(db)
out := &bytes.Buffer{}
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
v.UI = ui.NewWithColor(out, false)
return &summaryEnv{
v: v,
db: db,
repos: repos,
out: out,
aPath: filepath.Join(dirOne, "a.bin"),
aSize: int64(len(aContent)),
totalSize: int64(2*len(aContent) + len(smallContent)),
}
}
// backUp runs a backup of the named snapshot and returns its output.
func (e *summaryEnv) backUp(t *testing.T, name string, cron bool) string {
t.Helper()
e.out.Reset()
require.NoError(t, e.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
Cron: cron,
Snapshots: []string{name},
}))
return e.out.String()
}
// snapshot returns the local snapshots row of the snapshot named name.
func (e *summaryEnv) snapshot(t *testing.T, name string) *database.Snapshot {
t.Helper()
ctx := context.Background()
snap, err := e.repos.Snapshots.GetByID(ctx,
localSnapshotID(ctx, t, e.repos, name))
require.NoError(t, err)
require.NotNil(t, snap)
return snap
}
// uploads returns how many blobs the snapshot uploaded and their
// total size, as recorded in the uploads table.
func (e *summaryEnv) uploads(t *testing.T, snapshotID string) (int64, int64) {
t.Helper()
var count, size int64
err := e.db.Conn().QueryRowContext(context.Background(), `
SELECT COUNT(*), COALESCE(SUM(size), 0)
FROM uploads WHERE snapshot_id = ?`, snapshotID).Scan(&count, &size)
require.NoError(t, err)
return count, size
}
// referencedBlobSizes returns the compressed and uncompressed sizes of
// all blobs the snapshot references.
func (e *summaryEnv) referencedBlobSizes(
t *testing.T, snapshotID string,
) (int64, int64) {
t.Helper()
var compressed, uncompressed int64
err := e.db.Conn().QueryRowContext(context.Background(), `
SELECT COALESCE(SUM(b.compressed_size), 0),
COALESCE(SUM(b.uncompressed_size), 0)
FROM snapshot_blobs sb JOIN blobs b ON b.blob_hash = sb.blob_hash
WHERE sb.snapshot_id = ?`, snapshotID).Scan(&compressed, &uncompressed)
require.NoError(t, err)
return compressed, uncompressed
}
// filesLine returns the summary's line of file counts.
func filesLine(examined, backedUp, unchanged int) string {
return fmt.Sprintf("Files: %d examined, %d backed up, %d unchanged.",
examined, backedUp, unchanged)
}
// dataLine returns the summary's line of byte counts.
func (e *summaryEnv) dataLine(total, backedUp int64) string {
return fmt.Sprintf("Data: %s total (%s backed up).",
e.v.UI.Size(total), e.v.UI.Size(backedUp))
}
// A first backup stores copy.bin's chunks while backing up a.bin, so
// copy.bin's chunks are deduplicated within the run. Each file and byte
// is still counted once.
func TestSnapshotSummaryFirstRun(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newSummaryEnv(t)
summary := env.backUp(t, "first", false)
assert.Contains(t, summary, filesLine(3, 3, 0))
assert.Contains(t, summary, env.dataLine(env.totalSize, env.totalSize))
snap := env.snapshot(t, "first")
uploadCount, uploadBytes := env.uploads(t, snap.ID.String())
require.Positive(t, uploadCount)
assert.Contains(t, summary, fmt.Sprintf("Upload: %d blobs, %s in ",
uploadCount, env.v.UI.Size(uploadBytes)))
assert.Equal(t, int64(3), snap.FileCount)
assert.Equal(t, env.totalSize, snap.TotalSize)
assert.Equal(t, uploadCount, snap.BlobCount)
assert.Equal(t, uploadBytes, snap.UploadBytes)
}
// An incremental backup where a.bin's mtime changed but its content did
// not: a.bin is backed up again and every one of its chunks is already
// stored.
func TestSnapshotSummaryIncrementalRunWithDeduplicatedChunks(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newSummaryEnv(t)
env.backUp(t, "first", false)
later := time.Now().Add(time.Hour)
require.NoError(t, os.Chtimes(env.aPath, later, later))
summary := env.backUp(t, "second", false)
assert.Contains(t, summary, filesLine(3, 1, 2))
assert.Contains(t, summary, env.dataLine(env.totalSize, env.aSize))
assert.NotContains(t, summary, "Upload:")
snap := env.snapshot(t, "second")
compressed, uncompressed := env.referencedBlobSizes(t, snap.ID.String())
require.Positive(t, compressed)
assert.Equal(t, env.totalSize, snap.TotalSize)
assert.Zero(t, snap.ChunkCount)
assert.Zero(t, snap.BlobCount)
assert.Zero(t, snap.UploadBytes)
assert.Equal(t, compressed, snap.BlobSize,
"blob_size must total the blobs the snapshot references")
assert.Equal(t, uncompressed, snap.BlobUncompressedSize)
}
// Under --cron the progress reporter is off; the upload figures must
// still reach the summary and the snapshots row. The snapshot has two
// paths, each backed up by its own scan.
func TestSnapshotSummaryCronRunRecordsUploads(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newSummaryEnv(t)
summary := env.backUp(t, "split", true)
snap := env.snapshot(t, "split")
uploadCount, uploadBytes := env.uploads(t, snap.ID.String())
require.Positive(t, uploadCount)
assert.Contains(t, summary, filesLine(3, 3, 0))
assert.Contains(t, summary, env.dataLine(env.totalSize, env.totalSize))
assert.Contains(t, summary, fmt.Sprintf("Upload: %d blobs, %s in ",
uploadCount, env.v.UI.Size(uploadBytes)))
assert.Equal(t, env.totalSize, snap.TotalSize)
assert.Equal(t, uploadCount, snap.BlobCount,
"blob_count must count each blob once, however many paths the "+
"snapshot has")
assert.Equal(t, uploadBytes, snap.UploadBytes)
assert.GreaterOrEqual(t, snap.UploadDurationMs,
uploadCount*summaryUploadDelay.Milliseconds())
compressed, uncompressed := env.referencedBlobSizes(t, snap.ID.String())
require.Positive(t, uncompressed)
assert.Equal(t, compressed, snap.BlobSize)
assert.Equal(t, uncompressed, snap.BlobUncompressedSize)
assert.InDelta(t, float64(compressed)/float64(uncompressed),
snap.CompressionRatio, 1e-9)
}
@@ -1,70 +0,0 @@
package vaultik_test
import (
"context"
"maps"
"path/filepath"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/vaultik"
)
// A backup without --cron runs the progress reporter while one scanner
// scans each path of the snapshot in turn. See
// https://git.eeqj.de/sneak/vaultik/issues/253.
func TestBackupWithoutCronOfTwoPathSnapshotRestoresBothPaths(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
const snapshotName = "data"
fs := afero.NewOsFs()
tempDir := t.TempDir()
firstDir := filepath.Join(tempDir, "first")
secondDir := filepath.Join(tempDir, "second")
storeDir := filepath.Join(tempDir, "remote")
restoreDir := filepath.Join(tempDir, "restored")
dbPath := filepath.Join(tempDir, "index.sqlite")
ctx := context.Background()
files := writeFaultSourceTree(t, fs, firstDir)
maps.Copy(files, writeFaultSourceTree(t, fs, secondDir))
cfg := faultTestConfig()
cfg.IndexPath = dbPath
cfg.ChunkSize = config.Size(faultChunkSize)
cfg.Snapshots = map[string]config.SnapshotConfig{
snapshotName: {Paths: []string{firstDir, secondDir}},
}
store, err := storage.NewFileStorer(storeDir)
require.NoError(t, err)
db, err := database.New(ctx, dbPath)
require.NoError(t, err)
repos := database.NewRepositories(db)
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
require.NoError(t, v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
Snapshots: []string{snapshotName},
}))
id := localSnapshotID(ctx, t, repos, snapshotName)
require.NoError(t, db.Close())
reader := newReaderVaultik(ctx, cfg, store, nil, fs)
require.NoError(t, reader.Restore(&vaultik.RestoreOptions{
SnapshotID: id,
TargetDir: restoreDir,
Verify: true,
}))
assertRestoredTree(t, fs, restoreDir, files)
}
+12 -15
View File
@@ -103,7 +103,7 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep") log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep")
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON {
v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID) v.stdoutf("Deep verification of snapshot: %s\n\n", snapshotID)
} }
@@ -143,13 +143,10 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
log.Info("✓ Verification completed successfully", log.Info("✓ Verification completed successfully",
"snapshot_id", snapshotID, "mode", "deep", "blobs_verified", len(dbBlobs)) "snapshot_id", snapshotID, "mode", "deep", "blobs_verified", len(dbBlobs))
v.stdoutf("\n✓ Verification completed successfully\n")
if !v.UI.Quiet() { v.stdoutf(" Snapshot: %s\n", snapshotID)
v.stdoutf("\n✓ Verification completed successfully\n") v.stdoutf(" Blobs verified: %d\n", len(dbBlobs))
v.stdoutf(" Snapshot: %s\n", snapshotID) v.stdoutf(" Total size: %s\n", ubytes(totalSize))
v.stdoutf(" Blobs verified: %d\n", len(dbBlobs))
v.stdoutf(" Total size: %s\n", ubytes(totalSize))
}
return nil return nil
} }
@@ -173,7 +170,7 @@ func (v *Vaultik) loadVerificationData(
// remote manifests; see its doc comment. // remote manifests; see its doc comment.
log.Info("Downloading manifest", "remote_key", remoteKey) log.Info("Downloading manifest", "remote_key", remoteKey)
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON {
v.stdoutf("Downloading manifest...\n") v.stdoutf("Downloading manifest...\n")
} }
@@ -188,7 +185,7 @@ func (v *Vaultik) loadVerificationData(
"manifest_blob_count", manifest.BlobCount, "manifest_blob_count", manifest.BlobCount,
"manifest_total_size", ubytes(manifest.TotalCompressedSize)) "manifest_total_size", ubytes(manifest.TotalCompressedSize))
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON {
v.stdoutf("Manifest loaded: %d blobs (%s)\n", v.stdoutf("Manifest loaded: %d blobs (%s)\n",
manifest.BlobCount, ubytes(manifest.TotalCompressedSize)) manifest.BlobCount, ubytes(manifest.TotalCompressedSize))
v.stdoutf("Downloading and decrypting database...\n") v.stdoutf("Downloading and decrypting database...\n")
@@ -218,7 +215,7 @@ func (v *Vaultik) loadVerificationData(
"db_blob_count", len(dbBlobs), "db_blob_count", len(dbBlobs),
"db_total_size", ubytes(dbTotalSize)) "db_total_size", ubytes(dbTotalSize))
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON {
v.stdoutf("Database loaded: %d blobs (%s)\n", v.stdoutf("Database loaded: %d blobs (%s)\n",
len(dbBlobs), ubytes(dbTotalSize)) len(dbBlobs), ubytes(dbTotalSize))
} }
@@ -276,7 +273,7 @@ func (v *Vaultik) runVerificationSteps(
totalSize int64, totalSize int64,
identities []age.Identity, identities []age.Identity,
) error { ) error {
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON {
v.stdoutf("Verifying manifest against database...\n") v.stdoutf("Verifying manifest against database...\n")
} }
@@ -285,7 +282,7 @@ func (v *Vaultik) runVerificationSteps(
return v.deepVerifyFailure(result, opts, err.Error(), err) return v.deepVerifyFailure(result, opts, err.Error(), err)
} }
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON {
v.stdoutf("Manifest verified.\n") v.stdoutf("Manifest verified.\n")
v.stdoutf("Checking blob existence in remote storage...\n") v.stdoutf("Checking blob existence in remote storage...\n")
} }
@@ -295,7 +292,7 @@ func (v *Vaultik) runVerificationSteps(
return v.deepVerifyFailure(result, opts, err.Error(), err) return v.deepVerifyFailure(result, opts, err.Error(), err)
} }
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON {
v.stdoutf("All blobs exist.\n") v.stdoutf("All blobs exist.\n")
v.stdoutf("Downloading and verifying blob contents (%d blobs, %s)...\n", v.stdoutf("Downloading and verifying blob contents (%d blobs, %s)...\n",
len(dbBlobs), ubytes(totalSize)) len(dbBlobs), ubytes(totalSize))
@@ -751,7 +748,7 @@ func (v *Vaultik) performDeepVerificationFromDB(
"eta", eta.Round(time.Second), "eta", eta.Round(time.Second),
) )
if !opts.JSON && !v.UI.Quiet() { if !opts.JSON {
v.stdoutf(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n", v.stdoutf(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n",
i+1, len(blobs), remaining, i+1, len(blobs), remaining,
ubytes(bytesProcessed), ubytes(bytesProcessed),

Some files were not shown because too many files have changed in this diff Show More