Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c22363cc3c |
+10
-15
@@ -65,27 +65,22 @@ RUN echo "check epoch: ${CHECK_EPOCH}" && make fmt-check
|
|||||||
RUN echo "check epoch: ${CHECK_EPOCH}" && make test
|
RUN echo "check epoch: ${CHECK_EPOCH}" && make test
|
||||||
|
|
||||||
# Version, commit and build date are computed on the host by
|
# Version, commit and build date are computed on the host by
|
||||||
# script/docker and script/cibuild (where .git exists) and passed in as
|
# script/docker (where .git exists) and passed in as build args. The
|
||||||
# build args. The build context excludes .git (see .dockerignore), so
|
# build context excludes .git (see .dockerignore), so the build cannot
|
||||||
# the build cannot derive them itself: it used to try, with `git
|
# derive them itself: it used to try, with `git rev-parse` inside this
|
||||||
# rev-parse` inside this stage, and always got "unknown". VERSION comes
|
# stage, and always got "unknown". VERSION comes from script/version,
|
||||||
# from script/version, the source of truth shared with the Makefile, so
|
# the source of truth shared with the Makefile, so it carries the same
|
||||||
# it carries the same tag / dev-<sha> / -dirty rules and a Docker image
|
# tag / dev-<sha> / -dirty rules and a Docker image reports the same
|
||||||
# reports the same string a local build of the same tree would.
|
# string a local build of the same tree would.
|
||||||
#
|
|
||||||
# The defaults are the fallback for a bare `docker build .` that passes
|
|
||||||
# none of them: an unset arg would otherwise stamp an empty string and
|
|
||||||
# produce an image that cannot report its own version, commit or date.
|
|
||||||
# They match what an out-of-git build reports elsewhere.
|
|
||||||
#
|
#
|
||||||
# These ARGs sit here, after the checks, rather than at the top of the
|
# These ARGs sit here, after the checks, rather than at the top of the
|
||||||
# stage: every commit changes their values, and a value change
|
# stage: every commit changes their values, and a value change
|
||||||
# invalidates all layers below the ARG. Declared up top they would bust
|
# invalidates all layers below the ARG. Declared up top they would bust
|
||||||
# `go mod download`; here they only rekey this build layer, which the
|
# `go mod download`; here they only rekey this build layer, which the
|
||||||
# COPY of the sources above already rebuilds on any change anyway.
|
# COPY of the sources above already rebuilds on any change anyway.
|
||||||
ARG VERSION=dev
|
ARG VERSION
|
||||||
ARG COMMIT=unknown
|
ARG COMMIT
|
||||||
ARG COMMIT_DATE=unknown
|
ARG COMMIT_DATE
|
||||||
|
|
||||||
# Build (pure Go, no CGO required since we use modernc.org/sqlite)
|
# Build (pure Go, no CGO required since we use modernc.org/sqlite)
|
||||||
RUN CGO_ENABLED=0 go build -ldflags "-X 'sneak.berlin/go/vaultik/internal/globals.Version=${VERSION}' -X 'sneak.berlin/go/vaultik/internal/globals.Commit=${COMMIT}' -X 'sneak.berlin/go/vaultik/internal/globals.CommitDate=${COMMIT_DATE}'" -o /vaultik ./cmd/vaultik
|
RUN CGO_ENABLED=0 go build -ldflags "-X 'sneak.berlin/go/vaultik/internal/globals.Version=${VERSION}' -X 'sneak.berlin/go/vaultik/internal/globals.Commit=${COMMIT}' -X 'sneak.berlin/go/vaultik/internal/globals.CommitDate=${COMMIT_DATE}'" -o /vaultik ./cmd/vaultik
|
||||||
|
|||||||
+5
-5
@@ -72,11 +72,11 @@ RUN [ -n "$CHECK_EPOCH" ] || exit 1
|
|||||||
# running, and exits 0 reporting `0 issues.` on a tree the real config
|
# running, and exits 0 reporting `0 issues.` on a tree the real config
|
||||||
# fails. Demonstrated on this repo at this pin, recorded on
|
# fails. Demonstrated on this repo at this pin, recorded on
|
||||||
# https://git.eeqj.de/sneak/vaultik/pulls/114: with a planted
|
# https://git.eeqj.de/sneak/vaultik/pulls/114: with a planted
|
||||||
# over-length line, `script/lint` exits 1 naming the `revive` finding
|
# over-length line, `script/lint` exits 1 naming the `lll` finding with
|
||||||
# with `linters:` and exits 0 with `linterz:`. A set-but-ineffective
|
# `linters:` and exits 0 with `linterz:`. A set-but-ineffective config
|
||||||
# config quietly falling back to defaults is precisely the false-green
|
# quietly falling back to defaults is precisely the false-green class
|
||||||
# class this gate exists to eliminate, so it must not sit in the gate's
|
# this gate exists to eliminate, so it must not sit in the gate's own
|
||||||
# own configuration.
|
# configuration.
|
||||||
#
|
#
|
||||||
# `config verify` catches it, and it does so OFFLINE at this pinned
|
# `config verify` catches it, and it does so OFFLINE at this pinned
|
||||||
# version -- verified, not assumed. Under `docker run --network none`
|
# version -- verified, not assumed. Under `docker run --network none`
|
||||||
|
|||||||
@@ -251,8 +251,7 @@ local index alone, and still exits zero.
|
|||||||
per-snapshot-name (`--keep-latest` keeps the latest of each name, not the
|
per-snapshot-name (`--keep-latest` keeps the latest of each name, not the
|
||||||
latest globally).
|
latest globally).
|
||||||
* `--keep-latest`: Keep only the most recent snapshot of each name
|
* `--keep-latest`: Keep only the most recent snapshot of each name
|
||||||
* `--older-than <duration>`: Remove snapshots older than duration (e.g. `30d`,
|
* `--older-than <duration>`: Remove snapshots older than duration (e.g. `30d`, `6m`, `1y`)
|
||||||
`4w`, `6mo`, `1y`; `m` is minutes, `mo` is months)
|
|
||||||
* `--snapshot <name>`: Restrict to specific snapshot names (repeat for multiple)
|
* `--snapshot <name>`: Restrict to specific snapshot names (repeat for multiple)
|
||||||
* `--force`: Skip confirmation prompt
|
* `--force`: Skip confirmation prompt
|
||||||
|
|
||||||
|
|||||||
@@ -25,25 +25,6 @@ release" is exactly the contradiction
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- 2026-09-21: Made the s3 storage backend report a missing object as
|
|
||||||
`storage.ErrNotFound`, like the `file` and `rclone` backends and as the
|
|
||||||
`Storer` interface documents. `S3Storer.Get` and `Stat` returned the raw
|
|
||||||
AWS SDK error, so `errors.Is(err, storage.ErrNotFound)` was false on s3
|
|
||||||
and callers branched differently per backend. Added a small `s3.IsNotFound`
|
|
||||||
helper (reused by `HeadObject`) and a test that a missing key maps to
|
|
||||||
`ErrNotFound`
|
|
||||||
([issue #129](https://git.eeqj.de/sneak/vaultik/issues/129)).
|
|
||||||
|
|
||||||
- 2026-09-21: Fixed `verify --deep` reporting healthy snapshots as
|
|
||||||
corrupt. Its final blob-integrity check hashed the encrypted
|
|
||||||
downloaded bytes with a single SHA256 and compared that to the blob
|
|
||||||
ID, which is the double SHA256 of the plaintext, so the two could
|
|
||||||
never match. It now hashes the decompressed plaintext and compares the
|
|
||||||
double SHA256. Added a test that backs up a real snapshot, deep-verifies
|
|
||||||
it, then flips a byte in one stored blob and confirms deep verification
|
|
||||||
then fails
|
|
||||||
([issue #131](https://git.eeqj.de/sneak/vaultik/issues/131)).
|
|
||||||
|
|
||||||
- 2026-09-21: Made `snapshot create` VACUUM the per-snapshot metadata
|
- 2026-09-21: Made `snapshot create` VACUUM the per-snapshot metadata
|
||||||
database through the `modernc.org/sqlite` driver instead of shelling
|
database through the `modernc.org/sqlite` driver instead of shelling
|
||||||
out to the external `sqlite` command-line binary (issue #120). A
|
out to the external `sqlite` command-line binary (issue #120). A
|
||||||
@@ -69,24 +50,6 @@ release" is exactly the contradiction
|
|||||||
keeps that exact compiler from auto-switching. Bumping Go now touches
|
keeps that exact compiler from auto-switching. Bumping Go now touches
|
||||||
`go.mod`, the checksum, and the `Dockerfile` `golang` digest together.
|
`go.mod`, the checksum, and the `Dockerfile` `golang` digest together.
|
||||||
|
|
||||||
- 2026-09-21: Collapsed the two duration parsers into one and fixed the
|
|
||||||
`--older-than` months example
|
|
||||||
([issue #123](https://git.eeqj.de/sneak/vaultik/issues/123)). Two
|
|
||||||
functions named `parseDuration` existed with different grammars;
|
|
||||||
`snapshot purge --older-than` and `--keep-newer-than` both already went
|
|
||||||
through the one in `internal/vaultik`, while the richer copy in
|
|
||||||
`internal/cli/duration.go` was reachable only from its own test. Kept
|
|
||||||
the live-path parser and deleted the unused one, so no flag's accepted
|
|
||||||
grammar changes. The trap the issue was filed over: `README.md`
|
|
||||||
documented `6m` as the months example for `--older-than`, but `m` is
|
|
||||||
minutes, so the documented command deleted every snapshot older than
|
|
||||||
six minutes on a destructive flag. Corrected the doc to `6mo` and put
|
|
||||||
both flags' help text on one example list that states `m` is minutes
|
|
||||||
and `mo` is months. The surviving parser now rejects negatives, which
|
|
||||||
it previously accepted (`-5h`) or silently made positive (`-5d`).
|
|
||||||
Table-driven tests cover every unit, `6m` as six minutes, `6mo` as 180
|
|
||||||
days, and rejection of a bare number, an unknown unit, and a negative.
|
|
||||||
|
|
||||||
- 2026-08-10: Moved every lint run into its own container, as a build
|
- 2026-08-10: Moved every lint run into its own container, as a build
|
||||||
step ([issue #113](https://git.eeqj.de/sneak/vaultik/issues/113)).
|
step ([issue #113](https://git.eeqj.de/sneak/vaultik/issues/113)).
|
||||||
New root `Dockerfile.lint`, built by `script/lint`, runs
|
New root `Dockerfile.lint`, built by `script/lint`, runs
|
||||||
@@ -115,11 +78,10 @@ release" is exactly the contradiction
|
|||||||
into each check command, and a fresh `$(date +%s%N)$$` per invocation
|
into each check command, and a fresh `$(date +%s%N)$$` per invocation
|
||||||
computed as a bare assignment. `cmd/vaultik/lintdocker_test.go`
|
computed as a bare assignment. `cmd/vaultik/lintdocker_test.go`
|
||||||
parses both Dockerfiles and both scripts and fails if any part of
|
parses both Dockerfiles and both scripts and fails if any part of
|
||||||
that is dropped, because every way of losing it is silent. No test
|
that is dropped, because every way of losing it is silent. Its
|
||||||
asserts that no script runs the host linter: `script/lint` is the one
|
host-lint assertion is structural — no script runs `golangci-lint`
|
||||||
lint entry point and runs `golangci-lint` only inside the container,
|
except through `docker` — rather than a search for the one retired
|
||||||
and keeping it that way is a review matter, not something a test
|
variable name, which nothing could ever reintroduce.
|
||||||
proves.
|
|
||||||
|
|
||||||
The product `Dockerfile` lost its lint stage rather than gaining a
|
The product `Dockerfile` lost its lint stage rather than gaining a
|
||||||
second linter pin: `make lint` is now `docker build`, so the stage
|
second linter pin: `make lint` is now `docker build`, so the stage
|
||||||
|
|||||||
+153
-13
@@ -28,11 +28,6 @@ import (
|
|||||||
// -- that a real finding actually fails the build -- is verified by
|
// -- that a real finding actually fails the build -- is verified by
|
||||||
// hand against a deliberately broken tree, recorded on the pull
|
// hand against a deliberately broken tree, recorded on the pull
|
||||||
// request.
|
// request.
|
||||||
//
|
|
||||||
// One property is deliberately NOT tested here: that no script runs the
|
|
||||||
// linter on the host. script/lint is the only lint entry point, and it
|
|
||||||
// runs golangci-lint only inside the container; keeping it that way is a
|
|
||||||
// review matter, not something a test in this file establishes.
|
|
||||||
|
|
||||||
// The files under guard, relative to the repository root.
|
// The files under guard, relative to the repository root.
|
||||||
const (
|
const (
|
||||||
@@ -42,8 +37,9 @@ const (
|
|||||||
cibuildScript = "script/cibuild"
|
cibuildScript = "script/cibuild"
|
||||||
)
|
)
|
||||||
|
|
||||||
// linterBinary is the linter's command name, used to locate the
|
// linterBinary is the linter's command name. Every occurrence of it in
|
||||||
// config-verify and lint steps in Dockerfile.lint.
|
// executable shell in this repo must be inside a docker invocation; see
|
||||||
|
// TestNoHostLintPathRemains.
|
||||||
const linterBinary = "golangci-lint"
|
const linterBinary = "golangci-lint"
|
||||||
|
|
||||||
// checkEpochARG is the declaration, with no default value. A default
|
// checkEpochARG is the declaration, with no default value. A default
|
||||||
@@ -223,6 +219,90 @@ func TestCibuildBuildsBothDockerfilesWithFreshEpochs(t *testing.T) {
|
|||||||
"%s must build %s", cibuildScript, lintDockerfile)
|
"%s must build %s", cibuildScript, lintDockerfile)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestNoHostLintPathRemains fails if any escape hatch to a host linter
|
||||||
|
// comes back. The owner's ruling is that every lint run happens inside
|
||||||
|
// a container; a PATH binary that happens to match the pinned version
|
||||||
|
// is a different build reached by a different code path, and admitting
|
||||||
|
// it is what lets a local pass disagree with CI.
|
||||||
|
//
|
||||||
|
// This asserts the PROPERTY -- no script invokes the linter except
|
||||||
|
// through docker -- rather than the absence of any particular variable
|
||||||
|
// name. An earlier version of this test looked only for the literal
|
||||||
|
// VAULTIK_LINT_IN_CONTAINER, the name of the hatch that was removed
|
||||||
|
// alongside it, so nothing could ever trip it again: a hatch under any
|
||||||
|
// other name left it passing. A structural test that passes on a broken
|
||||||
|
// tree is worse than no test, because it is what a later reader trusts
|
||||||
|
// instead of re-deriving the invariant.
|
||||||
|
//
|
||||||
|
// script/lint-fix is not exempted. It is the one script that runs the
|
||||||
|
// linter as a container rather than as a build step, but it still runs
|
||||||
|
// it in one, so the same property holds of it.
|
||||||
|
func TestNoHostLintPathRemains(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
root := repoRoot(t)
|
||||||
|
|
||||||
|
entries, err := os.ReadDir(filepath.Join(root, "script"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotEmpty(t, entries, "no scripts found to scan")
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
name := filepath.Join("script", entry.Name())
|
||||||
|
for _, line := range shellCode(readRepoFile(t, name)) {
|
||||||
|
assertLinterIsContainerised(t, name, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertLinterIsContainerised fails if the line runs the linter without
|
||||||
|
// handing it to docker first. Position matters: docker has to come
|
||||||
|
// before the binary, or the line is running the host linter and merely
|
||||||
|
// mentioning docker afterwards.
|
||||||
|
func assertLinterIsContainerised(t *testing.T, name, line string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
at := strings.Index(line, linterBinary)
|
||||||
|
if at < 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
docker := strings.Index(line, "docker")
|
||||||
|
|
||||||
|
assert.True(t, docker >= 0 && docker < at,
|
||||||
|
"%s runs %s on the host; every lint run happens in a container"+
|
||||||
|
" (line: %s)", name, linterBinary, line)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestShellCodeSeesCodeAndNotProse keeps the scanner above honest. It
|
||||||
|
// has to ignore comments and here-document bodies, because script/lint
|
||||||
|
// and script/bootstrap both NAME golangci-lint in prose -- in comments,
|
||||||
|
// and in the error text they print -- precisely to say that the host
|
||||||
|
// binary is never used. A scanner that went blind, by over-eager
|
||||||
|
// stripping or by failing to join continuation lines, would make
|
||||||
|
// TestNoHostLintPathRemains pass on everything.
|
||||||
|
func TestShellCodeSeesCodeAndNotProse(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
script := strings.Join([]string{
|
||||||
|
"#!/bin/sh",
|
||||||
|
"# a comment naming golangci-lint",
|
||||||
|
"cat >&2 <<EOF",
|
||||||
|
"prose naming golangci-lint, printed not executed",
|
||||||
|
"EOF",
|
||||||
|
"docker run --rm \\",
|
||||||
|
" \"$image\" \\",
|
||||||
|
" golangci-lint run ./...",
|
||||||
|
}, "\n")
|
||||||
|
|
||||||
|
assert.Equal(t,
|
||||||
|
[]string{"cat >&2 <<EOF", `docker run --rm "$image" golangci-lint run ./...`},
|
||||||
|
shellCode(script))
|
||||||
|
}
|
||||||
|
|
||||||
// assertEpochExpandedInto fails unless some instruction runs the named
|
// assertEpochExpandedInto fails unless some instruction runs the named
|
||||||
// command with the epoch expanded into it. Expansion, not mere
|
// command with the epoch expanded into it. Expansion, not mere
|
||||||
// declaration: an ARG that no instruction references is not guaranteed
|
// declaration: an ARG that no instruction references is not guaranteed
|
||||||
@@ -304,14 +384,10 @@ func instructionText(contents string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// indexOf returns the position of the first instruction equal to, or
|
// indexOf returns the position of the first instruction equal to, or
|
||||||
// beginning with, want; -1 if there is none. An `ARG NAME=default`
|
// beginning with, want; -1 if there is none.
|
||||||
// counts as beginning with `ARG NAME`, so a declared arg is found
|
|
||||||
// whether or not it carries a default.
|
|
||||||
func indexOf(found []string, want string) int {
|
func indexOf(found []string, want string) int {
|
||||||
for i, instruction := range found {
|
for i, instruction := range found {
|
||||||
if instruction == want ||
|
if instruction == want || strings.HasPrefix(instruction, want+" ") {
|
||||||
strings.HasPrefix(instruction, want+" ") ||
|
|
||||||
strings.HasPrefix(instruction, want+"=") {
|
|
||||||
return i
|
return i
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -331,6 +407,70 @@ func indexContaining(found []string, want string) int {
|
|||||||
return -1
|
return -1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// shellCode returns a POSIX shell script's executable lines: comments
|
||||||
|
// dropped, here-document bodies dropped, and backslash continuations
|
||||||
|
// joined so a multi-line command is a single string. Whitespace is
|
||||||
|
// collapsed, as it is for Dockerfile instructions.
|
||||||
|
//
|
||||||
|
// Both exclusions are load-bearing rather than tidiness. The scripts
|
||||||
|
// name golangci-lint in prose to state that the host binary is never
|
||||||
|
// used, and joining continuations is what lets the one legitimate
|
||||||
|
// container invocation -- script/lint-fix's `docker run`, whose linter
|
||||||
|
// command sits several lines below the word `docker` -- be recognised
|
||||||
|
// as containerised.
|
||||||
|
func shellCode(contents string) []string {
|
||||||
|
var (
|
||||||
|
out []string
|
||||||
|
joined string
|
||||||
|
terminate string
|
||||||
|
)
|
||||||
|
|
||||||
|
for line := range strings.SplitSeq(contents, "\n") {
|
||||||
|
trimmed := strings.TrimSpace(line)
|
||||||
|
|
||||||
|
if terminate != "" {
|
||||||
|
if trimmed == terminate {
|
||||||
|
terminate = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if joined == "" && (trimmed == "" || strings.HasPrefix(trimmed, "#")) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
joined += strings.TrimSuffix(trimmed, `\`) + " "
|
||||||
|
if strings.HasSuffix(trimmed, `\`) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
joined = strings.Join(strings.Fields(joined), " ")
|
||||||
|
terminate = heredocTerminator(joined)
|
||||||
|
|
||||||
|
out = append(out, joined)
|
||||||
|
joined = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// heredocTerminator returns the terminator of the here-document a
|
||||||
|
// command opens, or "" if it opens none. Only the first on a line is
|
||||||
|
// recognised; nothing in script/ opens two.
|
||||||
|
func heredocTerminator(line string) string {
|
||||||
|
_, after, opens := strings.Cut(line, "<<")
|
||||||
|
if !opens {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// `<<-` strips leading tabs from the body; the terminator word is
|
||||||
|
// the same either way, and callers compare against trimmed lines.
|
||||||
|
word, _, _ := strings.Cut(strings.TrimPrefix(after, "-"), " ")
|
||||||
|
|
||||||
|
return strings.Trim(word, `'"`)
|
||||||
|
}
|
||||||
|
|
||||||
// readRepoFile reads a file by its path relative to the repository
|
// readRepoFile reads a file by its path relative to the repository
|
||||||
// root.
|
// root.
|
||||||
func readRepoFile(t *testing.T, name string) string {
|
func readRepoFile(t *testing.T, name string) string {
|
||||||
|
|||||||
@@ -0,0 +1,126 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Approximate lengths of the extended calendar units accepted by
|
||||||
|
// parseDuration.
|
||||||
|
const (
|
||||||
|
durationDay = 24 * time.Hour
|
||||||
|
durationWeek = 7 * durationDay
|
||||||
|
durationMonth = 30 * durationDay
|
||||||
|
durationYear = 365 * durationDay
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
errNegativeDuration = errors.New("negative durations are not supported")
|
||||||
|
errInvalidDuration = errors.New("invalid duration format")
|
||||||
|
errUnknownTimeUnit = errors.New("unknown time unit")
|
||||||
|
)
|
||||||
|
|
||||||
|
// parseDuration parses duration strings. Supports standard Go duration format
|
||||||
|
// (e.g., "3h30m", "1h45m30s") as well as extended units:
|
||||||
|
// - d: days (e.g., "30d", "7d")
|
||||||
|
// - w: weeks (e.g., "2w", "4w")
|
||||||
|
// - mo: months (30 days) (e.g., "6mo", "1mo")
|
||||||
|
// - y: years (365 days) (e.g., "1y", "2y")
|
||||||
|
//
|
||||||
|
// Can combine units: "1y6mo", "2w3d", "1d12h30m"
|
||||||
|
func parseDuration(s string) (time.Duration, error) {
|
||||||
|
// First try standard Go duration parsing
|
||||||
|
d, err := time.ParseDuration(s)
|
||||||
|
if err == nil {
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extended duration parsing
|
||||||
|
// Check for negative values
|
||||||
|
if strings.HasPrefix(strings.TrimSpace(s), "-") {
|
||||||
|
return 0, errNegativeDuration
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pattern matches: number + unit, repeated
|
||||||
|
re := regexp.MustCompile(`(\d+(?:\.\d+)?)\s*([a-zA-Z]+)`)
|
||||||
|
matches := re.FindAllStringSubmatch(s, -1)
|
||||||
|
|
||||||
|
if len(matches) == 0 {
|
||||||
|
return 0, fmt.Errorf("%w: %q", errInvalidDuration, s)
|
||||||
|
}
|
||||||
|
|
||||||
|
var total time.Duration
|
||||||
|
|
||||||
|
for _, match := range matches {
|
||||||
|
valueStr := match[1]
|
||||||
|
unit := strings.ToLower(match[2])
|
||||||
|
|
||||||
|
value, err := strconv.ParseFloat(valueStr, 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("invalid number %q: %w", valueStr, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
d, err := durationForUnit(value, unit)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
total += d
|
||||||
|
}
|
||||||
|
|
||||||
|
return total, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// durationForUnit converts a value with a (case-normalized) unit suffix
|
||||||
|
// into a time.Duration, accepting Go's standard units plus the extended
|
||||||
|
// calendar units.
|
||||||
|
func durationForUnit(value float64, unit string) (time.Duration, error) {
|
||||||
|
switch unit {
|
||||||
|
// Standard time units
|
||||||
|
case "ns", "nanosecond", "nanoseconds":
|
||||||
|
return time.Duration(value), nil
|
||||||
|
case "us", "µs", "microsecond", "microseconds":
|
||||||
|
return time.Duration(value * float64(time.Microsecond)), nil
|
||||||
|
case "ms", "millisecond", "milliseconds":
|
||||||
|
return time.Duration(value * float64(time.Millisecond)), nil
|
||||||
|
case "s", "sec", "second", "seconds":
|
||||||
|
return time.Duration(value * float64(time.Second)), nil
|
||||||
|
case "m", "min", "minute", "minutes":
|
||||||
|
return time.Duration(value * float64(time.Minute)), nil
|
||||||
|
case "h", "hr", "hour", "hours":
|
||||||
|
return time.Duration(value * float64(time.Hour)), nil
|
||||||
|
// Extended units
|
||||||
|
case "d", "day", "days":
|
||||||
|
return time.Duration(value * float64(durationDay)), nil
|
||||||
|
case "w", "week", "weeks":
|
||||||
|
return time.Duration(value * float64(durationWeek)), nil
|
||||||
|
case "mo", "month", "months":
|
||||||
|
// Using 30 days as approximation
|
||||||
|
return time.Duration(value * float64(durationMonth)), nil
|
||||||
|
case "y", "year", "years":
|
||||||
|
// Using 365 days as approximation
|
||||||
|
return time.Duration(value * float64(durationYear)), nil
|
||||||
|
default:
|
||||||
|
// Try parsing as standard Go duration unit
|
||||||
|
testStr := "1" + unit
|
||||||
|
|
||||||
|
_, err := time.ParseDuration(testStr)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("%w: %q", errUnknownTimeUnit, unit)
|
||||||
|
}
|
||||||
|
|
||||||
|
// It's a valid Go duration unit, parse the full value
|
||||||
|
fullStr := fmt.Sprintf("%g%s", value, unit)
|
||||||
|
|
||||||
|
d, err := time.ParseDuration(fullStr)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("invalid duration %q: %w", fullStr, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,299 @@
|
|||||||
|
package cli //nolint:testpackage // needs access to unexported parseDuration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
type parseDurationCase struct {
|
||||||
|
name string
|
||||||
|
input string
|
||||||
|
expected time.Duration
|
||||||
|
wantErr bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// runParseDurationCases executes a table of parseDuration cases as
|
||||||
|
// parallel subtests.
|
||||||
|
func runParseDurationCases(t *testing.T, tests []parseDurationCase) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
got, err := parseDuration(tt.input)
|
||||||
|
|
||||||
|
if tt.wantErr {
|
||||||
|
require.Error(t, err, "expected error for input %q", tt.input)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, err, "unexpected error for input %q", tt.input)
|
||||||
|
assert.Equal(t, tt.expected, got, "duration mismatch for input %q", tt.input)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseDurationStandard(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
runParseDurationCases(t, []parseDurationCase{
|
||||||
|
{
|
||||||
|
name: "standard seconds",
|
||||||
|
input: "30s",
|
||||||
|
expected: 30 * time.Second,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "standard minutes",
|
||||||
|
input: "45m",
|
||||||
|
expected: 45 * time.Minute,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "standard hours",
|
||||||
|
input: "2h",
|
||||||
|
expected: 2 * time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "standard combined",
|
||||||
|
input: "3h30m",
|
||||||
|
expected: 3*time.Hour + 30*time.Minute,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "standard complex",
|
||||||
|
input: "1h45m30s",
|
||||||
|
expected: 1*time.Hour + 45*time.Minute + 30*time.Second,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "standard with milliseconds",
|
||||||
|
input: "1s500ms",
|
||||||
|
expected: 1*time.Second + 500*time.Millisecond,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseDurationExtendedUnits(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
runParseDurationCases(t, []parseDurationCase{
|
||||||
|
// Extended units - days
|
||||||
|
{
|
||||||
|
name: "single day",
|
||||||
|
input: "1d",
|
||||||
|
expected: 24 * time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "multiple days",
|
||||||
|
input: "7d",
|
||||||
|
expected: 7 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "fractional days",
|
||||||
|
input: "1.5d",
|
||||||
|
expected: 36 * time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "days spelled out",
|
||||||
|
input: "3days",
|
||||||
|
expected: 3 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
// Extended units - weeks
|
||||||
|
{
|
||||||
|
name: "single week",
|
||||||
|
input: "1w",
|
||||||
|
expected: 7 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "multiple weeks",
|
||||||
|
input: "4w",
|
||||||
|
expected: 4 * 7 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "weeks spelled out",
|
||||||
|
input: "2weeks",
|
||||||
|
expected: 2 * 7 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
// Extended units - months
|
||||||
|
{
|
||||||
|
name: "single month",
|
||||||
|
input: "1mo",
|
||||||
|
expected: 30 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "multiple months",
|
||||||
|
input: "6mo",
|
||||||
|
expected: 6 * 30 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "months spelled out",
|
||||||
|
input: "3months",
|
||||||
|
expected: 3 * 30 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
// Extended units - years
|
||||||
|
{
|
||||||
|
name: "single year",
|
||||||
|
input: "1y",
|
||||||
|
expected: 365 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "multiple years",
|
||||||
|
input: "2y",
|
||||||
|
expected: 2 * 365 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "years spelled out",
|
||||||
|
input: "1year",
|
||||||
|
expected: 365 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseDurationCombinedAndErrors(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
runParseDurationCases(t, []parseDurationCase{
|
||||||
|
// Combined extended units
|
||||||
|
{
|
||||||
|
name: "weeks and days",
|
||||||
|
input: "2w3d",
|
||||||
|
expected: 2*7*24*time.Hour + 3*24*time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "years and months",
|
||||||
|
input: "1y6mo",
|
||||||
|
expected: 365*24*time.Hour + 6*30*24*time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "days and hours",
|
||||||
|
input: "1d12h",
|
||||||
|
expected: 24*time.Hour + 12*time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "complex combination",
|
||||||
|
input: "1y2mo3w4d5h6m7s",
|
||||||
|
expected: 365*24*time.Hour + 2*30*24*time.Hour +
|
||||||
|
3*7*24*time.Hour + 4*24*time.Hour +
|
||||||
|
5*time.Hour + 6*time.Minute + 7*time.Second,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "with spaces",
|
||||||
|
input: "1d 12h 30m",
|
||||||
|
expected: 24*time.Hour + 12*time.Hour + 30*time.Minute,
|
||||||
|
},
|
||||||
|
// Edge cases
|
||||||
|
{
|
||||||
|
name: "zero duration",
|
||||||
|
input: "0s",
|
||||||
|
expected: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "large duration",
|
||||||
|
input: "10y",
|
||||||
|
expected: 10 * 365 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
// Error cases
|
||||||
|
{
|
||||||
|
name: "empty string",
|
||||||
|
input: "",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid format",
|
||||||
|
input: "abc",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown unit",
|
||||||
|
input: "5x",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid number",
|
||||||
|
input: "xyzd",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "negative not supported",
|
||||||
|
input: "-5d",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseDurationSpecialCases(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// Test that standard Go durations work exactly as expected
|
||||||
|
standardDurations := []string{
|
||||||
|
"300ms",
|
||||||
|
"1.5h",
|
||||||
|
"2h45m",
|
||||||
|
"72h",
|
||||||
|
"1us",
|
||||||
|
"1µs",
|
||||||
|
"1ns",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, d := range standardDurations {
|
||||||
|
expected, err := time.ParseDuration(d)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
got, err := parseDuration(d)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, expected, got, "standard duration %q should parse identically", d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseDurationRealWorldExamples(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// Test real-world snapshot purge scenarios
|
||||||
|
tests := []struct {
|
||||||
|
description string
|
||||||
|
input string
|
||||||
|
olderThan time.Duration
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
description: "keep snapshots from last 30 days",
|
||||||
|
input: "30d",
|
||||||
|
olderThan: 30 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "keep snapshots from last 6 months",
|
||||||
|
input: "6mo",
|
||||||
|
olderThan: 6 * 30 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "keep snapshots from last year",
|
||||||
|
input: "1y",
|
||||||
|
olderThan: 365 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "keep snapshots from last week and a half",
|
||||||
|
input: "1w3d",
|
||||||
|
olderThan: 10 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
description: "keep snapshots from last 90 days",
|
||||||
|
input: "90d",
|
||||||
|
olderThan: 90 * 24 * time.Hour,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.description, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
got, err := parseDuration(tt.input)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, tt.olderThan, got)
|
||||||
|
|
||||||
|
// Verify the duration makes sense for snapshot purging
|
||||||
|
assert.Greater(t, got, time.Hour,
|
||||||
|
"snapshot purge duration should be at least an hour")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -108,8 +108,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
|||||||
"orphaned blobs")
|
"orphaned blobs")
|
||||||
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "",
|
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "",
|
||||||
"With --prune: keep snapshots newer than this duration "+
|
"With --prune: keep snapshots newer than this duration "+
|
||||||
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+
|
"(e.g. 4w, 30d, 6mo) instead of only the latest")
|
||||||
"instead of only the latest")
|
|
||||||
|
|
||||||
return cmd
|
return cmd
|
||||||
}
|
}
|
||||||
@@ -172,8 +171,7 @@ restrict the operation to specific snapshot names.`,
|
|||||||
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false,
|
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false,
|
||||||
"Keep only the latest snapshot of each name")
|
"Keep only the latest snapshot of each name")
|
||||||
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "",
|
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "",
|
||||||
"Remove snapshots older than duration "+
|
"Remove snapshots older than duration (e.g., 30d, 6m, 1y)")
|
||||||
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months)")
|
|
||||||
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt")
|
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt")
|
||||||
cmd.Flags().StringArrayVar(&opts.Names, "snapshot", nil,
|
cmd.Flags().StringArrayVar(&opts.Names, "snapshot", nil,
|
||||||
"Restrict to snapshots with these names (repeat for multiple)")
|
"Restrict to snapshots with these names (repeat for multiple)")
|
||||||
|
|||||||
+5
-13
@@ -219,7 +219,11 @@ func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) {
|
|||||||
Key: aws.String(fullKey),
|
Key: aws.String(fullKey),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if IsNotFound(err) {
|
var (
|
||||||
|
notFound *s3types.NotFound
|
||||||
|
noSuchKey *s3types.NoSuchKey
|
||||||
|
)
|
||||||
|
if errors.As(err, ¬Found) || errors.As(err, &noSuchKey) {
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -229,18 +233,6 @@ func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) {
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsNotFound reports whether err indicates that an object does not exist.
|
|
||||||
// Head and Get requests surface a missing object as different SDK types,
|
|
||||||
// so both are checked here.
|
|
||||||
func IsNotFound(err error) bool {
|
|
||||||
var (
|
|
||||||
notFound *s3types.NotFound
|
|
||||||
noSuchKey *s3types.NoSuchKey
|
|
||||||
)
|
|
||||||
|
|
||||||
return errors.As(err, ¬Found) || errors.As(err, &noSuchKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ObjectInfo contains information about an S3 object.
|
// ObjectInfo contains information about an S3 object.
|
||||||
// It is used by ListObjectsStream to return object metadata
|
// It is used by ListObjectsStream to return object metadata
|
||||||
// along with any errors encountered during listing.
|
// along with any errors encountered during listing.
|
||||||
|
|||||||
+1
-16
@@ -38,29 +38,14 @@ func (s *S3Storer) PutWithProgress(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Get retrieves data from the specified key.
|
// Get retrieves data from the specified key.
|
||||||
// Returns ErrNotFound if the object does not exist.
|
|
||||||
func (s *S3Storer) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
func (s *S3Storer) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||||
rc, err := s.client.GetObject(ctx, key)
|
return s.client.GetObject(ctx, key)
|
||||||
if err != nil {
|
|
||||||
if s3.IsNotFound(err) {
|
|
||||||
return nil, fmt.Errorf("get %q: %w", key, ErrNotFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return rc, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stat returns metadata about an object without retrieving its contents.
|
// Stat returns metadata about an object without retrieving its contents.
|
||||||
// Returns ErrNotFound if the object does not exist.
|
|
||||||
func (s *S3Storer) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
|
func (s *S3Storer) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
|
||||||
info, err := s.client.StatObject(ctx, key)
|
info, err := s.client.StatObject(ctx, key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if s3.IsNotFound(err) {
|
|
||||||
return nil, fmt.Errorf("stat %q: %w", key, ErrNotFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,59 +0,0 @@
|
|||||||
package storage_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/johannesboyne/gofakes3"
|
|
||||||
"github.com/johannesboyne/gofakes3/backend/s3mem"
|
|
||||||
|
|
||||||
"sneak.berlin/go/vaultik/internal/s3"
|
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestS3StorerMissingKeyMapsToErrNotFound verifies that the s3 backend reports
|
|
||||||
// a missing object as storage.ErrNotFound, matching the file and rclone
|
|
||||||
// backends and the Storer contract. Without the mapping, Get and Stat leak the
|
|
||||||
// raw SDK error and errors.Is(err, storage.ErrNotFound) is false.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // shares an in-process S3 server via t.Cleanup
|
|
||||||
func TestS3StorerMissingKeyMapsToErrNotFound(t *testing.T) {
|
|
||||||
const bucket = "test-bucket"
|
|
||||||
|
|
||||||
backend := s3mem.New()
|
|
||||||
|
|
||||||
err := backend.CreateBucket(bucket)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("create bucket: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
srv := httptest.NewServer(gofakes3.New(backend).Server())
|
|
||||||
t.Cleanup(srv.Close)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
client, err := s3.NewClient(ctx, s3.Config{
|
|
||||||
Endpoint: srv.URL,
|
|
||||||
Bucket: bucket,
|
|
||||||
AccessKeyID: "test",
|
|
||||||
SecretAccessKey: "test",
|
|
||||||
Region: "us-east-1",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("new client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
storer := storage.NewS3Storer(client)
|
|
||||||
|
|
||||||
_, err = storer.Get(ctx, "does-not-exist")
|
|
||||||
if !errors.Is(err, storage.ErrNotFound) {
|
|
||||||
t.Errorf("Get on missing key: got %v, want ErrNotFound", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = storer.Stat(ctx, "does-not-exist")
|
|
||||||
if !errors.Is(err, storage.ErrNotFound) {
|
|
||||||
t.Errorf("Stat on missing key: got %v, want ErrNotFound", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,108 +0,0 @@
|
|||||||
package vaultik_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
|
||||||
"sneak.berlin/go/vaultik/internal/ui"
|
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestDeepVerifyAcceptsHealthyAndRejectsCorruptBlob backs up a real
|
|
||||||
// snapshot with the on-disk storage backend, runs deep verification on
|
|
||||||
// it, then flips a byte inside one stored blob and runs deep
|
|
||||||
// verification again. A healthy snapshot must pass; a corrupted blob
|
|
||||||
// must fail. The healthy case is the regression guard: deep
|
|
||||||
// verification used to hash the encrypted blob bytes and compare them
|
|
||||||
// to the blob's ID (the double SHA256 of the plaintext), so it reported
|
|
||||||
// every healthy blob as corrupt.
|
|
||||||
func TestDeepVerifyAcceptsHealthyAndRejectsCorruptBlob(t *testing.T) {
|
|
||||||
log.Initialize(log.Config{})
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewOsFs()
|
|
||||||
tempDir := t.TempDir()
|
|
||||||
|
|
||||||
dataDir := filepath.Join(tempDir, "source")
|
|
||||||
storeDir := filepath.Join(tempDir, "remote")
|
|
||||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
|
||||||
|
|
||||||
chunkSize := int64(64 * 1024)
|
|
||||||
maxBlobSize := int64(512 * 1024)
|
|
||||||
|
|
||||||
// One file large enough to span several chunks within a single blob.
|
|
||||||
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
|
||||||
require.NoError(t, afero.WriteFile(fs,
|
|
||||||
filepath.Join(dataDir, "data.bin"),
|
|
||||||
bytesPattern("deep-", int(chunkSize*3)), 0o644))
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
// runFileStorageBackup writes a real snapshot to storeDir and closes
|
|
||||||
// the source index, so verification runs from remote bytes only.
|
|
||||||
cfg, storer, snapshotID := runFileStorageBackup(
|
|
||||||
ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
|
|
||||||
|
|
||||||
newVerifier := func() *vaultik.Vaultik {
|
|
||||||
v := &vaultik.Vaultik{
|
|
||||||
Config: cfg,
|
|
||||||
Storage: storer,
|
|
||||||
Fs: fs,
|
|
||||||
Stdout: io.Discard,
|
|
||||||
Stderr: io.Discard,
|
|
||||||
UI: ui.NewWithColor(io.Discard, false),
|
|
||||||
}
|
|
||||||
v.SetContext(ctx)
|
|
||||||
|
|
||||||
return v
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t,
|
|
||||||
newVerifier().RunDeepVerify(snapshotID, &vaultik.VerifyOptions{Deep: true}),
|
|
||||||
"deep verify should pass on a healthy snapshot")
|
|
||||||
|
|
||||||
// Flip a byte inside one blob without changing its length, so the
|
|
||||||
// blob-existence and size checks still pass and verification reaches
|
|
||||||
// the blob-content stage.
|
|
||||||
corruptOneBlob(t, fs, filepath.Join(storeDir, "blobs"))
|
|
||||||
|
|
||||||
require.Error(t,
|
|
||||||
newVerifier().RunDeepVerify(snapshotID, &vaultik.VerifyOptions{Deep: true}),
|
|
||||||
"deep verify should fail on a corrupted blob")
|
|
||||||
}
|
|
||||||
|
|
||||||
// corruptOneBlob flips a middle byte of the first blob file found under
|
|
||||||
// blobsDir, leaving the file length unchanged.
|
|
||||||
func corruptOneBlob(t *testing.T, fs afero.Fs, blobsDir string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var blobPath string
|
|
||||||
|
|
||||||
err := afero.Walk(fs, blobsDir,
|
|
||||||
func(path string, info os.FileInfo, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if blobPath == "" && !info.IsDir() {
|
|
||||||
blobPath = path
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotEmpty(t, blobPath, "expected at least one blob on disk")
|
|
||||||
|
|
||||||
data, err := afero.ReadFile(fs, blobPath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotEmpty(t, data)
|
|
||||||
|
|
||||||
data[len(data)/2] ^= 0xff
|
|
||||||
require.NoError(t, afero.WriteFile(fs, blobPath, data, 0o644))
|
|
||||||
}
|
|
||||||
@@ -33,9 +33,8 @@ func ubytes(n int64) string {
|
|||||||
var (
|
var (
|
||||||
errMalformedSnapshotID = errors.New(
|
errMalformedSnapshotID = errors.New(
|
||||||
"invalid snapshot ID format: expected hostname_snapshotname_timestamp")
|
"invalid snapshot ID format: expected hostname_snapshotname_timestamp")
|
||||||
errInvalidDuration = errors.New("invalid duration")
|
errInvalidDuration = errors.New("invalid duration")
|
||||||
errUnknownTimeUnit = errors.New("unknown time unit")
|
errUnknownTimeUnit = errors.New("unknown time unit")
|
||||||
errNegativeDuration = errors.New("negative durations are not supported")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Time-unit lengths used by parseDuration.
|
// Time-unit lengths used by parseDuration.
|
||||||
@@ -139,13 +138,8 @@ func parseSnapshotName(snapshotID string) string {
|
|||||||
|
|
||||||
// parseDuration parses a duration string with support for human-friendly units:
|
// parseDuration parses a duration string with support for human-friendly units:
|
||||||
// d/day/days, w/week/weeks, mo/month/months, y/year/years, plus standard Go
|
// d/day/days, w/week/weeks, mo/month/months, y/year/years, plus standard Go
|
||||||
// duration units. Following Go, m is minutes and mo is months. A bare number,
|
// duration units (h, m, s).
|
||||||
// an unknown unit, and a negative value are all rejected.
|
|
||||||
func parseDuration(s string) (time.Duration, error) {
|
func parseDuration(s string) (time.Duration, error) {
|
||||||
if strings.HasPrefix(strings.TrimSpace(s), "-") {
|
|
||||||
return 0, errNegativeDuration
|
|
||||||
}
|
|
||||||
|
|
||||||
d, err := time.ParseDuration(s)
|
d, err := time.ParseDuration(s)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return d, nil
|
return d, nil
|
||||||
|
|||||||
@@ -51,32 +51,13 @@ func TestParseDuration(t *testing.T) {
|
|||||||
want time.Duration
|
want time.Duration
|
||||||
err bool
|
err bool
|
||||||
}{
|
}{
|
||||||
// Go units, including the m-is-minutes / mo-is-months distinction
|
|
||||||
// that this parser exists to keep straight.
|
|
||||||
{"10ns", 10 * time.Nanosecond, false},
|
|
||||||
{"10us", 10 * time.Microsecond, false},
|
|
||||||
{"500ms", 500 * time.Millisecond, false},
|
|
||||||
{"30s", 30 * time.Second, false},
|
|
||||||
{"6m", 6 * time.Minute, false},
|
|
||||||
{"1h", time.Hour, false},
|
|
||||||
// Extended calendar units.
|
|
||||||
{"30d", 30 * 24 * time.Hour, false},
|
{"30d", 30 * 24 * time.Hour, false},
|
||||||
{"3days", 3 * 24 * time.Hour, false},
|
|
||||||
{"4w", 4 * 7 * 24 * time.Hour, false},
|
{"4w", 4 * 7 * 24 * time.Hour, false},
|
||||||
{"2weeks", 2 * 7 * 24 * time.Hour, false},
|
{"6mo", 6 * 30 * 24 * time.Hour, false},
|
||||||
{"6mo", 180 * 24 * time.Hour, false},
|
|
||||||
{"1month", 30 * 24 * time.Hour, false},
|
|
||||||
{"1y", 365 * 24 * time.Hour, false},
|
{"1y", 365 * 24 * time.Hour, false},
|
||||||
{"2years", 2 * 365 * 24 * time.Hour, false},
|
|
||||||
// Combined units.
|
|
||||||
{"2w3d", 2*7*24*time.Hour + 3*24*time.Hour, false},
|
{"2w3d", 2*7*24*time.Hour + 3*24*time.Hour, false},
|
||||||
{"1y6mo", 365*24*time.Hour + 180*24*time.Hour, false},
|
{"1h", time.Hour, false},
|
||||||
// Rejected inputs.
|
{"30s", 30 * time.Second, false},
|
||||||
{"6", 0, true}, // bare number, no unit
|
|
||||||
{"5x", 0, true}, // unknown unit
|
|
||||||
{"-5d", 0, true}, // negative, extended unit
|
|
||||||
{"-5h", 0, true}, // negative, Go unit
|
|
||||||
{"", 0, true}, // empty
|
|
||||||
{"garbage", 0, true},
|
{"garbage", 0, true},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+14
-19
@@ -344,8 +344,12 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error {
|
|||||||
return fmt.Errorf("failed to get decryptor: %w", err)
|
return fmt.Errorf("failed to get decryptor: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Decrypt blob
|
// Hash the encrypted blob data as it streams through to decryption
|
||||||
decryptedReader, err := decryptor.DecryptStream(reader)
|
blobHasher := sha256.New()
|
||||||
|
teeReader := io.TeeReader(reader, blobHasher)
|
||||||
|
|
||||||
|
// Decrypt blob (reading through teeReader to hash encrypted data)
|
||||||
|
decryptedReader, err := decryptor.DecryptStream(teeReader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to decrypt: %w", err)
|
return fmt.Errorf("failed to decrypt: %w", err)
|
||||||
}
|
}
|
||||||
@@ -357,19 +361,12 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error {
|
|||||||
}
|
}
|
||||||
defer decompressor.Close()
|
defer decompressor.Close()
|
||||||
|
|
||||||
// A blob's hash — its remote name — is the double SHA256 of its
|
chunkCount, err := v.verifyBlobChunks(db, blobInfo.Hash, decompressor)
|
||||||
// decompressed plaintext (see blobgen.Writer.Sum256), not of the
|
|
||||||
// encrypted bytes. Hash the plaintext as chunk verification streams
|
|
||||||
// it, then compare on completion.
|
|
||||||
plaintextHasher := sha256.New()
|
|
||||||
hashedStream := io.TeeReader(decompressor, plaintextHasher)
|
|
||||||
|
|
||||||
chunkCount, err := v.verifyBlobChunks(db, blobInfo.Hash, hashedStream)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = v.verifyBlobFinalIntegrity(hashedStream, plaintextHasher, blobInfo.Hash)
|
err = v.verifyBlobFinalIntegrity(decompressor, blobHasher, blobInfo.Hash)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -473,13 +470,14 @@ func (v *Vaultik) verifyBlobChunks(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// verifyBlobFinalIntegrity checks that no trailing data exists in the
|
// verifyBlobFinalIntegrity checks that no trailing data exists in the
|
||||||
// decompressed stream and that the blob hash matches the expected value.
|
// decompressed stream and that the encrypted blob hash matches the
|
||||||
|
// expected value.
|
||||||
func (v *Vaultik) verifyBlobFinalIntegrity(
|
func (v *Vaultik) verifyBlobFinalIntegrity(
|
||||||
plaintext io.Reader, plaintextHasher hash.Hash, expectedHash string,
|
decompressor io.Reader, blobHasher hash.Hash, expectedHash string,
|
||||||
) error {
|
) error {
|
||||||
// Verify no remaining data in blob - if the chunk list is accurate,
|
// Verify no remaining data in blob - if the chunk list is accurate,
|
||||||
// the blob should be fully consumed.
|
// the blob should be fully consumed.
|
||||||
remaining, err := io.Copy(io.Discard, plaintext)
|
remaining, err := io.Copy(io.Discard, decompressor)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to check for remaining blob data: %w", err)
|
return fmt.Errorf("failed to check for remaining blob data: %w", err)
|
||||||
}
|
}
|
||||||
@@ -488,11 +486,8 @@ func (v *Vaultik) verifyBlobFinalIntegrity(
|
|||||||
return fmt.Errorf("%w: %d bytes", errTrailingBlobData, remaining)
|
return fmt.Errorf("%w: %d bytes", errTrailingBlobData, remaining)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The blob hash is the double SHA256 of its plaintext content.
|
// Verify blob hash matches the encrypted data we downloaded
|
||||||
firstHash := plaintextHasher.Sum(nil)
|
calculatedBlobHash := hex.EncodeToString(blobHasher.Sum(nil))
|
||||||
secondHash := sha256.Sum256(firstHash)
|
|
||||||
calculatedBlobHash := hex.EncodeToString(secondHash[:])
|
|
||||||
|
|
||||||
if calculatedBlobHash != expectedHash {
|
if calculatedBlobHash != expectedHash {
|
||||||
return fmt.Errorf("%w: calculated %s, expected %s",
|
return fmt.Errorf("%w: calculated %s, expected %s",
|
||||||
errBlobHashMismatch, calculatedBlobHash, expectedHash)
|
errBlobHashMismatch, calculatedBlobHash, expectedHash)
|
||||||
|
|||||||
+1
-16
@@ -56,23 +56,8 @@ main() {
|
|||||||
docker build --output=type=cacheonly \
|
docker build --output=type=cacheonly \
|
||||||
--build-arg CHECK_EPOCH="$epoch" -f Dockerfile.lint .
|
--build-arg CHECK_EPOCH="$epoch" -f Dockerfile.lint .
|
||||||
|
|
||||||
# Version, commit and build date are computed here on the host, the
|
|
||||||
# same way script/docker does, and passed into the product build so
|
|
||||||
# the CI-built image reports its real source. The build context
|
|
||||||
# excludes .git (see .dockerignore), so the build cannot derive them
|
|
||||||
# itself; without these it would stamp the Dockerfile's dev/unknown
|
|
||||||
# fallbacks. VERSION comes from script/version, the source of truth
|
|
||||||
# shared with the Makefile.
|
|
||||||
version="$("$ROOT/script/version")"
|
|
||||||
commit="$(git rev-parse HEAD 2>/dev/null || echo unknown)"
|
|
||||||
commit_date="$(git show -s --format=%cs HEAD 2>/dev/null || echo unknown)"
|
|
||||||
|
|
||||||
epoch="$(date +%s%N)$$"
|
epoch="$(date +%s%N)$$"
|
||||||
docker build --build-arg CHECK_EPOCH="$epoch" \
|
docker build --build-arg CHECK_EPOCH="$epoch" .
|
||||||
--build-arg VERSION="$version" \
|
|
||||||
--build-arg COMMIT="$commit" \
|
|
||||||
--build-arg COMMIT_DATE="$commit_date" \
|
|
||||||
.
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
Reference in New Issue
Block a user