1 Commits
Author SHA1 Message Date
sneak d199ff53ce Harden the lint-guard shell scanner against silent evasions (closes #121)
check / check (pull_request) Failing after 1s
The guard test's shell scanner was weaker than its commit message
claimed. Two holes are closed.

shellCode now treats `<<` as a here-document only when it is a real
redirection: outside single and double quotes, not past an unquoted
word-initial `#` that begins an inline comment, and followed by a
delimiter word. A `<<` inside a quoted string or an inline comment no
longer opens a phantom here-document that swallows the rest of the
file -- including the silent case where the fake terminator recurs
later as a line of its own -- and a here-document still open at end of
file is a loud error rather than a silent truncation.

assertLinterIsContainerised now cuts the joined line into the simple
commands the shell would run -- on `;`, `&&`, `||` and `|` -- and
requires the command that names the linter to begin with docker. So
`docker info; golangci-lint run` and `docker info || golangci-lint run`
are rejected, while script/lint-fix's `docker run ... golangci-lint`
still passes.

The scanner comment now names the inline-comment exception alongside
the quoted-string and arithmetic ones, and the inherent limits of a
text scan. Dockerfile.lint's citation is corrected from `lll` to
`revive`, the finding the recorded evidence actually named.

model: claude-opus-4-8
2026-09-21 17:34:41 +00:00
11 changed files with 258 additions and 363 deletions
+25 -18
View File
@@ -20,21 +20,33 @@ jobs:
# check.yml runs script/cibuild, which does all of its work inside # check.yml runs script/cibuild, which does all of its work inside
# the digest-pinned Dockerfile images -- so without this step the # the digest-pinned Dockerfile images -- so without this step the
# release either fails at the before-hook or, worse, ships binaries # release either fails at the before-hook or, worse, ships binaries
# built by whatever Go the runner happens to carry. # built by whatever unpinned Go the runner happens to carry.
# REPO_POLICIES.md requires every external reference to be pinned,
# and script/release already refuses a goreleaser that is not the
# pinned build; the compiler that actually produces the artifacts
# is the last thing that should be exempt from that.
# #
# actions/setup-go would pin the action by commit sha, but the Go # go-version-file rather than a literal: go.mod's `go 1.26.1` is
# tarball it downloads at runtime is verified against no value in # the single source of truth for the toolchain, the same way the
# this repo, and the action exposes no checksum input. # Dockerfile FROM line is the single source of truth for the
# REPO_POLICIES.md requires every external reference to be pinned # linter version that script/lint enforces. It is a three-component
# by hash with no exceptions, and this is the compiler that # version, so setup-go resolves it exactly -- no silent drift onto
# produces the published binaries -- the input where a substituted # a newer patch release.
# artifact matters most. So Go is installed the way goreleaser is: #
# script/install-go downloads the exact archive for go.mod's `go` # actions/setup-go v5.6.0, 2025-12-15. Pinned by commit sha, like
# directive and refuses it unless its sha256 matches the value # the checkout above. v5.x is a node20 action, matching the node20
# committed in the script, then puts .tool/go/bin on PATH for the # actions/checkout v4 already in use here; the v6/v7 line requires
# steps below. # a node24 runner, which this Gitea runner has never been asked
# for and cannot be assumed to provide.
- name: Install Go - name: Install Go
run: script/install-go uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff
with:
go-version-file: go.mod
# setup-go's module cache needs a runner-side cache backend.
# A release is cut rarely and a cold module download costs
# seconds; a release failing because a cache service is absent
# costs a re-tag. Off, deliberately.
cache: false
- name: Install goreleaser - name: Install goreleaser
run: script/install-goreleaser run: script/install-goreleaser
- name: Release - name: Release
@@ -46,8 +58,3 @@ jobs:
# It is deliberately not the runner's automatic token, which is # It is deliberately not the runner's automatic token, which is
# not guaranteed to carry that scope. # not guaranteed to carry that scope.
GITEA_TOKEN: ${{ secrets.RELEASE_TOKEN }} GITEA_TOKEN: ${{ secrets.RELEASE_TOKEN }}
# Build with the toolchain install-go just verified, never a
# different one auto-downloaded from a `toolchain` directive:
# the point of the hash pin is that this exact compiler makes
# the release.
GOTOOLCHAIN: local
+3 -5
View File
@@ -22,10 +22,8 @@ FROM golang:1.26.1-alpine@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f
ARG VERSION=dev ARG VERSION=dev
# Build tooling: make, plus a C toolchain because `go test -race` needs cgo. # Install build dependencies for CGO (mattn/go-sqlite3) and sqlite3 CLI (tests)
# The sqlite driver is pure Go (modernc.org/sqlite), so no sqlite library or RUN apk add --no-cache make build-base sqlite
# CLI is required.
RUN apk add --no-cache make build-base
WORKDIR /src WORKDIR /src
@@ -73,7 +71,7 @@ RUN CGO_ENABLED=0 go build -ldflags "-X 'sneak.berlin/go/vaultik/internal/global
# alpine:3.21, 2026-02-25 # alpine:3.21, 2026-02-25
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709 FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
RUN apk add --no-cache ca-certificates RUN apk add --no-cache ca-certificates sqlite
# Copy binary from builder # Copy binary from builder
COPY --from=builder /vaultik /usr/local/bin/vaultik COPY --from=builder /vaultik /usr/local/bin/vaultik
+5 -5
View File
@@ -72,11 +72,11 @@ RUN [ -n "$CHECK_EPOCH" ] || exit 1
# running, and exits 0 reporting `0 issues.` on a tree the real config # running, and exits 0 reporting `0 issues.` on a tree the real config
# fails. Demonstrated on this repo at this pin, recorded on # fails. Demonstrated on this repo at this pin, recorded on
# https://git.eeqj.de/sneak/vaultik/pulls/114: with a planted # https://git.eeqj.de/sneak/vaultik/pulls/114: with a planted
# over-length line, `script/lint` exits 1 naming the `lll` finding with # over-length line, `script/lint` exits 1 naming the `revive` finding
# `linters:` and exits 0 with `linterz:`. A set-but-ineffective config # with `linters:` and exits 0 with `linterz:`. A set-but-ineffective
# quietly falling back to defaults is precisely the false-green class # config quietly falling back to defaults is precisely the false-green
# this gate exists to eliminate, so it must not sit in the gate's own # class this gate exists to eliminate, so it must not sit in the gate's
# configuration. # own configuration.
# #
# `config verify` catches it, and it does so OFFLINE at this pinned # `config verify` catches it, and it does so OFFLINE at this pinned
# version -- verified, not assumed. Under `docker run --network none` # version -- verified, not assumed. Under `docker run --network none`
+3 -10
View File
@@ -613,6 +613,7 @@ regardless of color setting (emoji are not color).
and the pre-commit hook both run it. A `golangci-lint` installed on and the pre-commit hook both run it. A `golangci-lint` installed on
`PATH` is not a substitute and is never used on a host, whatever its `PATH` is not a substitute and is never used on a host, whatever its
version. version.
* `sqlite3` CLI, which the test suite shells out to
* S3-compatible object storage (or local filesystem, or rclone remote) * S3-compatible object storage (or local filesystem, or rclone remote)
## development workflow ## development workflow
@@ -643,8 +644,8 @@ standard: normalized scripts in `script/` are the entrypoints for the
development workflow, and the Makefile targets are thin shims that call development workflow, and the Makefile targets are thin shims that call
them. We provide: them. We provide:
* `script/bootstrap` — install all development dependencies (go, Go * `script/bootstrap` — install all development dependencies (go, sqlite3,
module download). It deliberately does not install `golangci-lint`; Go module download). It deliberately does not install `golangci-lint`;
see `script/lint` below. see `script/lint` below.
* `script/setup` — make a fresh clone ready for development: runs * `script/setup` — make a fresh clone ready for development: runs
`script/bootstrap`, then `script/install-precommit` `script/bootstrap`, then `script/install-precommit`
@@ -658,14 +659,6 @@ them. We provide:
called by `script/bootstrap`; the release workflow calls it directly called by `script/bootstrap`; the release workflow calls it directly
because it needs `goreleaser` but not the Docker daemon because it needs `goreleaser` but not the Docker daemon
`script/bootstrap` insists on. `script/bootstrap` insists on.
* `script/install-go` — install the Go toolchain named by `go.mod`'s
`go` directive into `.tool/go` from a sha256-verified `go.dev`
archive, and put it on `PATH`. Idempotent. Called only by the release
workflow, which needs a host Go for `goreleaser` to shell out to;
nothing else on the release runner does. `actions/setup-go` is not
used because it verifies the downloaded toolchain against no value in
this repo. Bumping Go edits `go.mod`, the checksum in this script, and
the `Dockerfile` `golang` digest together.
* `script/release` — cross-compile and publish the release artifacts * `script/release` — cross-compile and publish the release artifacts
with the pinned `goreleaser`. Refuses a `goreleaser` on `PATH` whose with the pinned `goreleaser`. Refuses a `goreleaser` on `PATH` whose
version is not the pinned one, on the same reasoning as `script/lint`. version is not the pinned one, on the same reasoning as `script/lint`.
+1 -20
View File
@@ -25,31 +25,12 @@ release" is exactly the contradiction
# Completed Steps # Completed Steps
- 2026-09-21: Made `snapshot create` VACUUM the per-snapshot metadata
database through the `modernc.org/sqlite` driver instead of shelling
out to the external `sqlite` command-line binary (issue #120). A
backup no longer needs that binary on `PATH`, so `make check` passes
on a stock `go install` host; `script/bootstrap` and the `Dockerfile`
(both the test-build and the shipped runtime stage) no longer install
it, and a new test asserts the uploaded database keeps no pages from
deleted rows. Dropped the now-false note on the 2026-08-07 entry below
that said bootstrap installs it.
- 2026-09-21: Made `.gitea/workflows/check.yml` run on pushes to `main` - 2026-09-21: Made `.gitea/workflows/check.yml` run on pushes to `main`
and `next` and on pull requests against either, so unit PRs (whose and `next` and on pull requests against either, so unit PRs (whose
base is `next`) and `next` itself get a CI run instead of relying on a base is `next`) and `next` itself get a CI run instead of relying on a
local `make check` local `make check`
([issue #122](https://git.eeqj.de/sneak/vaultik/issues/122)). ([issue #122](https://git.eeqj.de/sneak/vaultik/issues/122)).
- 2026-09-21: Hash-verified the Go toolchain in the release workflow
([issue #105](https://git.eeqj.de/sneak/vaultik/issues/105)). New
`script/install-go` downloads the exact `go.dev` archive for `go.mod`'s
`go` directive and refuses it unless its sha256 matches a value
committed in the script; `.gitea/workflows/release.yml` calls it
instead of `actions/setup-go`, which verified the downloaded toolchain
against nothing in the repo. `GOTOOLCHAIN: local` on the release step
keeps that exact compiler from auto-switching. Bumping Go now touches
`go.mod`, the checksum, and the `Dockerfile` `golang` digest together.
- 2026-08-10: Moved every lint run into its own container, as a build - 2026-08-10: Moved every lint run into its own container, as a build
step ([issue #113](https://git.eeqj.de/sneak/vaultik/issues/113)). step ([issue #113](https://git.eeqj.de/sneak/vaultik/issues/113)).
New root `Dockerfile.lint`, built by `script/lint`, runs New root `Dockerfile.lint`, built by `script/lint`, runs
@@ -548,7 +529,7 @@ release" is exactly the contradiction
was green was wrong. was green was wrong.
- 2026-08-07: Added the standard `.golangci.yml` and `.editorconfig` - 2026-08-07: Added the standard `.golangci.yml` and `.editorconfig`
(issue #59); lint findings under the new config are tracked in issue (issue #59); lint findings under the new config are tracked in issue
#61. #61. `script/bootstrap` now installs sqlite3 (needed by tests).
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, - 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
Makefile shims, README Entrypoints section Makefile shims, README Entrypoints section
- 2026-07-02: Consolidated CLI verbs, retired overlapping commands; bound - 2026-07-02: Consolidated CLI verbs, retired overlapping commands; bound
+212 -30
View File
@@ -1,6 +1,8 @@
package main_test package main_test
import ( import (
"errors"
"fmt"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -252,29 +254,64 @@ func TestNoHostLintPathRemains(t *testing.T) {
} }
name := filepath.Join("script", entry.Name()) name := filepath.Join("script", entry.Name())
for _, line := range shellCode(readRepoFile(t, name)) {
lines, err := shellCode(readRepoFile(t, name))
require.NoError(t, err, "scanning %s", name)
for _, line := range lines {
assertLinterIsContainerised(t, name, line) assertLinterIsContainerised(t, name, line)
} }
} }
} }
// assertLinterIsContainerised fails if the line runs the linter without // assertLinterIsContainerised fails unless every command that names the
// handing it to docker first. Position matters: docker has to come // linter on this joined line is a docker command. Merely mentioning
// before the binary, or the line is running the host linter and merely // docker somewhere on the line is not enough; see linterRunsInDocker.
// mentioning docker afterwards.
func assertLinterIsContainerised(t *testing.T, name, line string) { func assertLinterIsContainerised(t *testing.T, name, line string) {
t.Helper() t.Helper()
at := strings.Index(line, linterBinary) assert.True(t, linterRunsInDocker(line),
if at < 0 { "%s runs %s outside a container; every command that names the"+
return " linter must begin with docker (line: %s)", name, linterBinary,
line)
}
// linterRunsInDocker reports whether the linter, wherever it appears on
// this joined shell line, is only ever the argument of a docker command.
// The line is cut into the simple commands the shell would run -- on
// `;`, `&&`, `||` and `|` -- and every command that names the linter
// must begin with `docker`. This is what distinguishes the one
// legitimate invocation, script/lint-fix's `docker run ... golangci-lint
// run ...`, from evasions like `docker info; golangci-lint run` or
// `docker info || golangci-lint run`, where the linter sits in a command
// of its own that docker does not introduce.
func linterRunsInDocker(line string) bool {
for _, command := range splitShellCommands(line) {
if !strings.Contains(command, linterBinary) {
continue
} }
docker := strings.Index(line, "docker") if !strings.HasPrefix(strings.TrimSpace(command), "docker") {
return false
}
}
assert.True(t, docker >= 0 && docker < at, return true
"%s runs %s on the host; every lint run happens in a container"+ }
" (line: %s)", name, linterBinary, line)
// splitShellCommands breaks a joined shell line into the separate simple
// commands the shell would run, cutting at the `;`, `&&`, `||` and `|`
// operators (`||` before `|`, so the two-character operator is not split
// twice). It is deliberately blind to quoting and to `$(...)`: no line
// under guard puts one of these operators inside a string, and a scan
// that tried to account for that would be the kind of half-parser this
// file avoids.
func splitShellCommands(line string) []string {
for _, op := range []string{"&&", "||", "|", ";"} {
line = strings.ReplaceAll(line, op, "\n")
}
return strings.Split(line, "\n")
} }
// TestShellCodeSeesCodeAndNotProse keeps the scanner above honest. It // TestShellCodeSeesCodeAndNotProse keeps the scanner above honest. It
@@ -287,20 +324,94 @@ func assertLinterIsContainerised(t *testing.T, name, line string) {
func TestShellCodeSeesCodeAndNotProse(t *testing.T) { func TestShellCodeSeesCodeAndNotProse(t *testing.T) {
t.Parallel() t.Parallel()
// A `<<` inside quotes is not a here-document, so the code after it
// is still scanned; a real `<<EOF` opens one and its body is dropped.
script := strings.Join([]string{ script := strings.Join([]string{
"#!/bin/sh", "#!/bin/sh",
"# a comment naming golangci-lint", "# a comment naming golangci-lint",
"cat >&2 <<EOF", "cat >&2 <<EOF",
"prose naming golangci-lint, printed not executed", "prose naming golangci-lint, printed not executed",
"EOF", "EOF",
`echo "a left shift << is not a here-document"`,
"docker run --rm \\", "docker run --rm \\",
" \"$image\" \\", " \"$image\" \\",
" golangci-lint run ./...", " golangci-lint run ./...",
}, "\n") }, "\n")
lines, err := shellCode(script)
require.NoError(t, err)
assert.Equal(t, assert.Equal(t,
[]string{"cat >&2 <<EOF", `docker run --rm "$image" golangci-lint run ./...`}, []string{
shellCode(script)) "cat >&2 <<EOF",
`echo "a left shift << is not a here-document"`,
`docker run --rm "$image" golangci-lint run ./...`,
},
lines)
// A `<<` inside an inline comment is not a here-document either: an
// unquoted, word-initial `#` begins a comment that runs to end of
// line, so the `<< STOP` is prose. The code that follows is still
// scanned -- here, a host golangci-lint that TestNoHostLintPathRemains
// must then see rather than have swallowed. The fake terminator even
// recurs later as a line of its own; a phantom here-document would
// swallow everything up to it silently, past the end-of-file error
// that only catches a terminator which never recurs.
inlineComment := strings.Join([]string{
": # housekeeping marker << STOP",
"golangci-lint run --config .golangci.yml ./...",
"STOP",
}, "\n")
lines, err = shellCode(inlineComment)
require.NoError(t, err)
assert.Equal(t,
[]string{
": # housekeeping marker << STOP",
"golangci-lint run --config .golangci.yml ./...",
"STOP",
},
lines)
// A here-document still open at end of file must be a loud error,
// not a silent truncation of everything the scanner has yet to see.
unterminated := strings.Join([]string{
"cat <<EOF",
"body line naming golangci-lint, no terminator follows",
}, "\n")
_, err = shellCode(unterminated)
require.Error(t, err)
}
// TestLinterCommandMustBeginWithDocker pins the property that a mention
// of docker somewhere on the line is not enough: the command that
// actually runs the linter has to be a docker command. The two evasions
// from the issue place the linter in a command of its own, joined to a
// harmless docker command by `;` or `||`; both must be rejected. The
// containerised invocation script/lint-fix writes -- docker run with the
// linter as its argument -- must still be accepted.
func TestLinterCommandMustBeginWithDocker(t *testing.T) {
t.Parallel()
rejected := []string{
"docker info >/dev/null; golangci-lint run ./...",
"docker info || golangci-lint run ./...",
"docker build . && golangci-lint run ./... | tee log",
}
for _, line := range rejected {
assert.False(t, linterRunsInDocker(line),
"a linter command docker does not introduce must be rejected: %s",
line)
}
accepted := []string{
`docker run --rm "$image" golangci-lint run ./...`,
`docker run --rm --user x --volume "$ROOT:/src" img golangci-lint run --fix ./...`,
}
for _, line := range accepted {
assert.True(t, linterRunsInDocker(line),
"a docker-introduced linter command must be accepted: %s", line)
}
} }
// assertEpochExpandedInto fails unless some instruction runs the named // assertEpochExpandedInto fails unless some instruction runs the named
@@ -410,7 +521,9 @@ func indexContaining(found []string, want string) int {
// shellCode returns a POSIX shell script's executable lines: comments // shellCode returns a POSIX shell script's executable lines: comments
// dropped, here-document bodies dropped, and backslash continuations // dropped, here-document bodies dropped, and backslash continuations
// joined so a multi-line command is a single string. Whitespace is // joined so a multi-line command is a single string. Whitespace is
// collapsed, as it is for Dockerfile instructions. // collapsed, as it is for Dockerfile instructions. A here-document left
// open at end of file is an error rather than a silent truncation of
// everything after its opener.
// //
// Both exclusions are load-bearing rather than tidiness. The scripts // Both exclusions are load-bearing rather than tidiness. The scripts
// name golangci-lint in prose to state that the host binary is never // name golangci-lint in prose to state that the host binary is never
@@ -418,7 +531,11 @@ func indexContaining(found []string, want string) int {
// container invocation -- script/lint-fix's `docker run`, whose linter // container invocation -- script/lint-fix's `docker run`, whose linter
// command sits several lines below the word `docker` -- be recognised // command sits several lines below the word `docker` -- be recognised
// as containerised. // as containerised.
func shellCode(contents string) []string { //
// This is a text scan, not a shell: it cannot see a linter name
// assembled at runtime, one split across a continuation, a script in a
// subdirectory of script/, or anything in the Makefile.
func shellCode(contents string) ([]string, error) {
var ( var (
out []string out []string
joined string joined string
@@ -452,23 +569,88 @@ func shellCode(contents string) []string {
joined = "" joined = ""
} }
return out if terminate != "" {
} return nil, fmt.Errorf("%w: terminator %q", errUnterminatedHeredoc,
terminate)
// heredocTerminator returns the terminator of the here-document a
// command opens, or "" if it opens none. Only the first on a line is
// recognised; nothing in script/ opens two.
func heredocTerminator(line string) string {
_, after, opens := strings.Cut(line, "<<")
if !opens {
return ""
} }
// `<<-` strips leading tabs from the body; the terminator word is return out, nil
// the same either way, and callers compare against trimmed lines. }
word, _, _ := strings.Cut(strings.TrimPrefix(after, "-"), " ")
return strings.Trim(word, `'"`) // errUnterminatedHeredoc is what shellCode returns when a here-document
// is still open at end of file. Its callers require its absence, so an
// unterminated body -- which would otherwise be swallowed silently --
// fails the guard loudly.
var errUnterminatedHeredoc = errors.New(
"here-document opened but never closed before end of file")
// heredocTerminator returns the delimiter word of the here-document the
// command opens, or "" if it opens none. A `<<` only opens one when it
// is a real redirection: outside single and double quotes, not in an
// inline comment, and followed by a delimiter word. A `<<` inside a
// quoted string, past an unquoted word-initial `#` (which begins a
// comment that runs to end of line), or in an arithmetic left shift
// like `$((x << 2))`, is not a here-document; the first two are cases
// this guards, the last appears in no script here. Only the first
// opener on a line is recognised; nothing in script/ opens two.
func heredocTerminator(line string) string {
var quote byte // 0 when outside quotes, else '\'' or '"'
for i := 0; i+1 < len(line); i++ {
c := line[i]
switch {
case quote != 0:
if c == quote {
quote = 0
}
case c == '\'' || c == '"':
quote = c
case c == '#' && (i == 0 || line[i-1] == ' '):
// A word-initial `#` starts a comment; the rest of the
// line, `<<` included, is prose, not a redirection.
return ""
case c == '<' && line[i+1] == '<':
return heredocWord(line[i+2:])
}
}
return ""
}
// heredocWord extracts the delimiter that follows `<<` or `<<-`: it drops
// an optional `-`, skips blanks, then reads the delimiter -- quoted or
// bare -- and returns it with quotes removed. `<<-'EOF'` and `<< EOF`
// both yield "EOF". It returns "" when no word follows, so a bare `<<`
// opens nothing.
func heredocWord(after string) string {
after = strings.TrimLeft(strings.TrimPrefix(after, "-"), " \t")
var (
word strings.Builder
quote byte
)
for i := range len(after) {
c := after[i]
switch {
case quote != 0:
if c == quote {
quote = 0
} else {
word.WriteByte(c)
}
case c == '\'' || c == '"':
quote = c
case c == ' ' || c == '\t':
return word.String()
default:
word.WriteByte(c)
}
}
return word.String()
} }
// readRepoFile reads a file by its path relative to the repository // readRepoFile reads a file by its path relative to the repository
+1 -1
View File
@@ -192,7 +192,7 @@ Tracks blob upload metrics.
After a snapshot is completed: After a snapshot is completed:
1. Copy database to temporary file 1. Copy database to temporary file
2. Clean temporary database to contain only current snapshot data 2. Clean temporary database to contain only current snapshot data
3. VACUUM the trimmed database so deleted rows leave no pages behind 3. Export to SQL dump using sqlite3
4. Compress with zstd and encrypt with age 4. Compress with zstd and encrypt with age
5. Upload to S3 as `metadata/{remote-key}/db.zst.age` 5. Upload to S3 as `metadata/{remote-key}/db.zst.age`
6. Generate blob manifest and upload as `metadata/{remote-key}/manifest.json.zst` 6. Generate blob manifest and upload as `metadata/{remote-key}/manifest.json.zst`
+5 -21
View File
@@ -44,6 +44,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"os/exec"
"path/filepath" "path/filepath"
"strings" "strings"
"time" "time"
@@ -668,31 +669,14 @@ func (sm *SnapshotManager) collectCleanupStats(
// vacuumDatabase runs VACUUM on the database to remove deleted data and compact // vacuumDatabase runs VACUUM on the database to remove deleted data and compact
// This is critical for security - ensures no stale/deleted data pages are uploaded // This is critical for security - ensures no stale/deleted data pages are uploaded
//
// VACUUM runs through the modernc.org/sqlite driver, on a freshly opened
// connection with no transaction in flight (VACUUM cannot run inside one).
// The database opens in WAL mode, so VACUUM's rewrite lands in the WAL; the
// checkpoint on Close flushes it into the main file, which is the file we
// then compress and upload.
func (sm *SnapshotManager) vacuumDatabase(ctx context.Context, dbPath string) error { func (sm *SnapshotManager) vacuumDatabase(ctx context.Context, dbPath string) error {
log.Debug("Running VACUUM on database", "path", dbPath) log.Debug("Running VACUUM on database", "path", dbPath)
//nolint:gosec // G204: fixed argv; dbPath is our own temp file path
cmd := exec.CommandContext(ctx, "sqlite3", dbPath, "VACUUM;")
db, err := database.New(ctx, dbPath) output, err := cmd.CombinedOutput()
if err != nil { if err != nil {
return fmt.Errorf("opening database for VACUUM: %w", err) return fmt.Errorf("running VACUUM: %w (output: %s)", err, string(output))
}
defer func() {
cerr := db.Close()
if cerr != nil {
log.Debug("Failed to close database after VACUUM",
"path", dbPath, "error", cerr)
}
}()
_, err = db.ExecWithLog(ctx, "VACUUM")
if err != nil {
return fmt.Errorf("running VACUUM: %w", err)
} }
return nil return nil
-92
View File
@@ -2,7 +2,6 @@
package snapshot package snapshot
import ( import (
"bytes"
"context" "context"
"database/sql" "database/sql"
"io" "io"
@@ -97,97 +96,6 @@ func verifyCleanedDB(
} }
} }
// TestVacuumDatabaseRemovesDeletedData proves the export path uploads a
// compacted database: after rows carrying a recognizable marker are deleted
// and vacuumDatabase runs, no page holding that marker survives in the file
// on disk (the file compressFile later reads for upload).
func TestVacuumDatabaseRemovesDeletedData(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
ctx := context.Background()
fs := afero.NewOsFs()
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "snapshot.db")
db, err := database.New(ctx, dbPath)
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
// A marker distinctive enough that its presence in the raw file can only
// come from the rows inserted below.
marker := []byte("VACUUM_PROBE_DEADBEEF_DELETED_ROW")
payload := bytes.Repeat(marker, 128) // ~4 KiB per row
_, err = db.Conn().ExecContext(ctx,
"CREATE TABLE vacuum_probe (id INTEGER PRIMARY KEY, payload BLOB)")
if err != nil {
t.Fatalf("failed to create probe table: %v", err)
}
for range 512 {
_, err = db.Conn().ExecContext(ctx,
"INSERT INTO vacuum_probe (payload) VALUES (?)", payload)
if err != nil {
t.Fatalf("failed to insert probe row: %v", err)
}
}
_, err = db.Conn().ExecContext(ctx, "DELETE FROM vacuum_probe")
if err != nil {
t.Fatalf("failed to delete probe rows: %v", err)
}
// Close so the deletes reach the main file, mirroring the state
// prepareExportDB hands to vacuumDatabase.
err = db.Close()
if err != nil {
t.Fatalf("failed to close database: %v", err)
}
beforeInfo, err := fs.Stat(dbPath)
if err != nil {
t.Fatalf("failed to stat database before vacuum: %v", err)
}
beforeBytes, err := afero.ReadFile(fs, dbPath)
if err != nil {
t.Fatalf("failed to read database before vacuum: %v", err)
}
if !bytes.Contains(beforeBytes, marker) {
t.Fatalf("expected deleted-row data to linger before vacuum")
}
sm := &SnapshotManager{fs: fs}
err = sm.vacuumDatabase(ctx, dbPath)
if err != nil {
t.Fatalf("vacuumDatabase failed: %v", err)
}
afterBytes, err := afero.ReadFile(fs, dbPath)
if err != nil {
t.Fatalf("failed to read database after vacuum: %v", err)
}
if bytes.Contains(afterBytes, marker) {
t.Fatalf("deleted-row data survived vacuum in the uploaded file")
}
afterInfo, err := fs.Stat(dbPath)
if err != nil {
t.Fatalf("failed to stat database after vacuum: %v", err)
}
if afterInfo.Size() >= beforeInfo.Size() {
t.Fatalf("expected vacuum to shrink the file: before=%d after=%d",
beforeInfo.Size(), afterInfo.Size())
}
}
func TestCleanSnapshotDBEmptySnapshot(t *testing.T) { func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
// Initialize logger // Initialize logger
log.Initialize(log.Config{}) log.Initialize(log.Config{})
+3
View File
@@ -114,6 +114,9 @@ main() {
# from CI. Nothing on the host is ever used as a linter, at any # from CI. Nothing on the host is ever used as a linter, at any
# version, so installing one here would buy nothing. # version, so installing one here would buy nothing.
# sqlite3 CLI: the test suite shells out to it (VACUUM).
if missing sqlite3; then pkg_install sqlite sqlite3 sqlite sqlite; fi
# goreleaser, at the version pinned by script/install-goreleaser and # goreleaser, at the version pinned by script/install-goreleaser and
# verified against a hardcoded sha256. Package managers are not used # verified against a hardcoded sha256. Package managers are not used
# for it: they ship whatever version they happen to carry, and the # for it: they ship whatever version they happen to carry, and the
-161
View File
@@ -1,161 +0,0 @@
#!/bin/sh
# script/install-go: install the Go toolchain pinned by go.mod into the
# repo-local tool directory, verified against a committed sha256. Our
# own extension to scripts-to-rule-them-all. Idempotent: exits at once
# when the pinned toolchain is already installed.
#
# Only .gitea/workflows/release.yml calls this. goreleaser is not a
# compiler: it shells out to `go` for the `before:` hook and for every
# one of the four cross-compiles, so the release runner needs a Go
# toolchain on PATH. check.yml never does -- it builds inside the
# digest-pinned Dockerfile images -- so this is the release path's only
# host Go, and per REPO_POLICIES.md it must be pinned by hash.
# actions/setup-go exposes no checksum input, so Go is installed the way
# script/install-goreleaser installs goreleaser: download the exact
# archive from go.dev and refuse it unless its sha256 matches the value
# committed below.
#
# The version is go.mod's `go` directive, the single source of truth for
# the toolchain. GO_VERSION below MUST equal it, and this script fails
# when they disagree -- so bumping Go is one reviewed change touching
# go.mod, the checksum here, and the Dockerfile golang digest together.
#
# Linux only, because that is what the release runner is. A darwin dev
# building a snapshot uses their own Go; supporting an OS means adding
# its checksums.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Go 1.26.1, 2026-09-21. Checksums are the sha256 values go.dev publishes
# for each archive at https://go.dev/dl/ (also in its ?mode=json
# manifest).
GO_VERSION="1.26.1"
SHA256_LINUX_AMD64="031f088e5d955bab8657ede27ad4e3bc5b7c1ba281f05f245bcc304f327c987a"
SHA256_LINUX_ARM64="a290581cfe4fe28ddd737dde3095f3dbeb7f2e4065cab4eae44dfc53b760c2f7"
GOROOT_DIR="$ROOT/.tool/go"
GOCMD="$GOROOT_DIR/bin/go"
# The `go` directive in go.mod, e.g. "1.26.1" from `go 1.26.1`.
gomod_go_version() {
sed -n 's/^go \([0-9][0-9.]*\).*/\1/p' "$ROOT/go.mod" | head -n 1
}
# Print the version of the go at $1 as "1.26.1", or nothing if it is not
# usable. `go version` prints "go version go1.26.1 linux/amd64".
go_version() {
[ -x "$1" ] || return 0
"$1" version 2>/dev/null |
sed -n 's/^go version go\([0-9][0-9.]*\) .*/\1/p' |
head -n 1
}
verify_sha256() {
file="$1"
want="$2"
if command -v sha256sum >/dev/null 2>&1; then
got="$(sha256sum "$file" | cut -d' ' -f1)"
elif command -v shasum >/dev/null 2>&1; then
got="$(shasum -a 256 "$file" | cut -d' ' -f1)"
else
echo "install-go: no sha256sum or shasum available" >&2
return 1
fi
if [ "$got" != "$want" ]; then
echo "install-go: checksum mismatch for $file" >&2
echo " expected: $want" >&2
echo " actual: $got" >&2
return 1
fi
}
# On a Gitea/GitHub Actions runner, put the toolchain on PATH for the
# steps that follow by appending to the file named by $GITHUB_PATH. A
# no-op off CI, where the caller manages its own PATH.
export_ci_path() {
[ -n "${GITHUB_PATH:-}" ] || return 0
echo "$GOROOT_DIR/bin" >>"$GITHUB_PATH"
}
main() {
cd "$ROOT"
want="$(gomod_go_version)"
if [ "$want" != "$GO_VERSION" ]; then
echo "install-go: go.mod says go $want but this script pins" \
"$GO_VERSION." >&2
echo " Update GO_VERSION and the checksums in this script to" \
"match go.mod." >&2
exit 1
fi
# Already installed from a previous run? Then just fix PATH and stop.
if [ "$(go_version "$GOCMD")" = "$GO_VERSION" ]; then
echo "go $GO_VERSION already installed in .tool/go"
export_ci_path
return 0
fi
os="$(uname -s)"
arch="$(uname -m)"
case "$os" in
Linux) os="linux" ;;
*)
echo "install-go: unsupported OS $os (release runner is Linux)" >&2
exit 1
;;
esac
case "$arch" in
x86_64 | amd64)
arch="amd64"
sum="$SHA256_LINUX_AMD64"
;;
arm64 | aarch64)
arch="arm64"
sum="$SHA256_LINUX_ARM64"
;;
*)
echo "install-go: no pinned checksum for architecture $arch" >&2
exit 1
;;
esac
archive="go${GO_VERSION}.${os}-${arch}.tar.gz"
url="https://go.dev/dl/${archive}"
if ! command -v curl >/dev/null 2>&1; then
echo "install-go: curl is required" >&2
exit 1
fi
dl="$(mktemp -d)"
mkdir -p "$ROOT/.tool"
stage="$(mktemp -d "$ROOT/.tool/.go-install.XXXXXX")"
# shellcheck disable=SC2064 # expand the paths now, not at trap time
trap "rm -rf '$dl' '$stage'" EXIT INT TERM
echo "installing go $GO_VERSION for ${os}-${arch}"
curl -fsSL --retry 3 -o "$dl/$archive" "$url"
verify_sha256 "$dl/$archive" "$sum"
# The archive unpacks to a top-level `go/` directory. Extract it into
# a staging directory on the same filesystem as the destination, then
# rename it into place so a concurrent run never observes a
# half-written toolchain.
tar -xzf "$dl/$archive" -C "$stage"
rm -rf "$GOROOT_DIR"
mv "$stage/go" "$GOROOT_DIR"
installed="$(go_version "$GOCMD")"
if [ "$installed" != "$GO_VERSION" ]; then
echo "install-go: installed toolchain reports '$installed'," \
"expected '$GO_VERSION'" >&2
exit 1
fi
echo "go $GO_VERSION installed to .tool/go"
export_ci_path
}
main "$@"