Compare commits
2 Commits
8690cf9311
...
99968231ad
| Author | SHA1 | Date | |
|---|---|---|---|
| 99968231ad | |||
| c378690977 |
@@ -3,6 +3,11 @@
|
|||||||
# stage of the Dockerfile.
|
# stage of the Dockerfile.
|
||||||
.git/
|
.git/
|
||||||
bin/
|
bin/
|
||||||
|
# Third-party browser assets are fetched and hash-verified inside the build by
|
||||||
|
# script/fetch-assets. Excluding any host copy keeps a developer's working tree
|
||||||
|
# from supplying the bytes that get shipped. The script and its
|
||||||
|
# static/vendor.sha256 manifest stay in the context.
|
||||||
|
static/js/alpine.min.js
|
||||||
*.md
|
*.md
|
||||||
LICENSE
|
LICENSE
|
||||||
.editorconfig
|
.editorconfig
|
||||||
|
|||||||
5
.gitignore
vendored
5
.gitignore
vendored
@@ -45,3 +45,8 @@ temp/
|
|||||||
|
|
||||||
# CI cache barrier, written into the build context by the check workflow
|
# CI cache barrier, written into the build context by the check workflow
|
||||||
.ci-fingerprint
|
.ci-fingerprint
|
||||||
|
|
||||||
|
# Third-party browser assets, fetched and hash-verified by
|
||||||
|
# script/fetch-assets against static/vendor.sha256. Not committed:
|
||||||
|
# REPO_POLICIES.md forbids minified bundles in version control.
|
||||||
|
/static/js/alpine.min.js
|
||||||
10
Dockerfile
10
Dockerfile
@@ -32,7 +32,7 @@ FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a349228
|
|||||||
# Depend on lint stage passing
|
# Depend on lint stage passing
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends make && rm -rf /var/lib/apt/lists/*
|
RUN apt-get update && apt-get install -y --no-install-recommends make curl ca-certificates && rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
WORKDIR /build
|
WORKDIR /build
|
||||||
|
|
||||||
@@ -44,6 +44,14 @@ RUN go mod download
|
|||||||
# the lint stage above.
|
# the lint stage above.
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
# Fetch the third-party browser assets the UI serves. They are not committed
|
||||||
|
# (REPO_POLICIES.md forbids minified bundles in version control) and
|
||||||
|
# .dockerignore keeps any host copy out of the build context, so this step is
|
||||||
|
# the only way they enter the image. Each download is checked against a
|
||||||
|
# hardcoded sha256 and the build fails on mismatch; make test re-checks the
|
||||||
|
# hashes against the bytes go:embed actually put in the binary.
|
||||||
|
RUN script/fetch-assets
|
||||||
|
|
||||||
# Run tests and build
|
# Run tests and build
|
||||||
RUN make test
|
RUN make test
|
||||||
RUN make build
|
RUN make build
|
||||||
|
|||||||
5
Makefile
5
Makefile
@@ -1,4 +1,4 @@
|
|||||||
.PHONY: bootstrap setup test lint fmt fmt-check check build run dev deps docker clean hooks css
|
.PHONY: bootstrap setup assets test lint fmt fmt-check check build run dev deps docker clean hooks css
|
||||||
|
|
||||||
# Default target
|
# Default target
|
||||||
.DEFAULT_GOAL := check
|
.DEFAULT_GOAL := check
|
||||||
@@ -9,6 +9,9 @@ bootstrap:
|
|||||||
setup:
|
setup:
|
||||||
@script/setup
|
@script/setup
|
||||||
|
|
||||||
|
assets:
|
||||||
|
@script/fetch-assets
|
||||||
|
|
||||||
test:
|
test:
|
||||||
@script/test
|
@script/test
|
||||||
|
|
||||||
|
|||||||
85
README.md
85
README.md
@@ -40,6 +40,7 @@ make docker
|
|||||||
```bash
|
```bash
|
||||||
make bootstrap # Install all dependencies (idempotent)
|
make bootstrap # Install all dependencies (idempotent)
|
||||||
make setup # Bootstrap + install git pre-commit hook
|
make setup # Bootstrap + install git pre-commit hook
|
||||||
|
make assets # Fetch + verify third-party browser assets
|
||||||
make fmt # Format code (gofmt + goimports)
|
make fmt # Format code (gofmt + goimports)
|
||||||
make lint # Run golangci-lint
|
make lint # Run golangci-lint
|
||||||
make test # Run tests with race detection
|
make test # Run tests with race detection
|
||||||
@@ -247,6 +248,8 @@ them. We provide:
|
|||||||
- `script/setup` — make a fresh clone ready for development
|
- `script/setup` — make a fresh clone ready for development
|
||||||
(bootstrap, then install-precommit)
|
(bootstrap, then install-precommit)
|
||||||
- `script/projectname` — output the project name ("webhooker")
|
- `script/projectname` — output the project name ("webhooker")
|
||||||
|
- `script/fetch-assets` — download the third-party browser assets into
|
||||||
|
`static/`, verifying each against its pinned sha256
|
||||||
- `script/test` — run the test suite
|
- `script/test` — run the test suite
|
||||||
- `script/lint` — run golangci-lint
|
- `script/lint` — run golangci-lint
|
||||||
- `script/fmt` — format all code (writes)
|
- `script/fmt` — format all code (writes)
|
||||||
@@ -260,6 +263,27 @@ them. We provide:
|
|||||||
- `script/install-precommit` — install the git pre-commit hook that
|
- `script/install-precommit` — install the git pre-commit hook that
|
||||||
runs `script/precommit`
|
runs `script/precommit`
|
||||||
|
|
||||||
|
## Third-party browser assets
|
||||||
|
|
||||||
|
The web UI serves one third-party script, Alpine.js. It is **not** committed:
|
||||||
|
a minified bundle in the tree is unreviewable, and `REPO_POLICIES.md` bars
|
||||||
|
both committed build artifacts and unpinned external references.
|
||||||
|
|
||||||
|
Instead `script/fetch-assets` downloads it from a pinned URL, checks the
|
||||||
|
download against a hardcoded sha256, and installs it under `static/`. The
|
||||||
|
sha256 of every installed asset is recorded in `static/vendor.sha256`, and
|
||||||
|
`static/vendor_test.go` re-hashes the bytes `go:embed` put in the binary
|
||||||
|
against that manifest — so the pin is enforced on what actually ships, not
|
||||||
|
merely written down. Any mismatch fails the build.
|
||||||
|
|
||||||
|
`make bootstrap` runs the fetch for local development, and the Dockerfile
|
||||||
|
runs it in the build stage; `.gitignore` and `.dockerignore` keep the
|
||||||
|
artifact out of both the repo and the build context.
|
||||||
|
|
||||||
|
To move to a new version: update the version, URL, and tarball sha256 in
|
||||||
|
`script/fetch-assets` and the asset sha256 in `static/vendor.sha256`, then
|
||||||
|
run `make assets && make check`.
|
||||||
|
|
||||||
## Rationale
|
## Rationale
|
||||||
|
|
||||||
Webhook integrations between services are inherently fragile. The
|
Webhook integrations between services are inherently fragile. The
|
||||||
@@ -904,8 +928,65 @@ requests and has the rest of its aggregate budget rejected there, so
|
|||||||
the aggregate limit is what bounds those `WARN` lines — to under ten
|
the aggregate limit is what bounds those `WARN` lines — to under ten
|
||||||
times `RECEIVER_RATE_LIMIT` per minute per client IP, 1080 at the
|
times `RECEIVER_RATE_LIMIT` per minute per client IP, 1080 at the
|
||||||
defaults, where before it there was no bound at all. The access log is
|
defaults, where before it there was no bound at all. The access log is
|
||||||
bounded by neither limit: every request is recorded once at `INFO` with
|
bounded by neither limit: every request is recorded once at `INFO`,
|
||||||
its full URL, served or rejected alike.
|
served or rejected alike.
|
||||||
|
|
||||||
|
What the access log does bound is the _content_ of those lines. A 3xx
|
||||||
|
or 4xx response logs the chi route pattern — `/webhook/{uuid}`,
|
||||||
|
`/user/{username}//`, or the literal `(unmatched)` when the request hit
|
||||||
|
no route at all — in place of the concrete URL. Those are the outcomes
|
||||||
|
an unauthenticated client can drive for free: 404 and 429 on any
|
||||||
|
invented receiver path, a login redirect on any invented profile path.
|
||||||
|
Logging the URL there would let a flood write text of its own choosing,
|
||||||
|
at a length of its own choosing, into the log. 2xx and 5xx responses
|
||||||
|
keep the concrete path — a success resolved against a static route or
|
||||||
|
against the operator's own data (on the receiver, against a stored
|
||||||
|
entrypoint UUID), and a 5xx is a bug in this service, where the exact
|
||||||
|
path is the evidence and no client can provoke one at will.
|
||||||
|
|
||||||
|
The query string is never logged; it is replaced by the fixed marker
|
||||||
|
`?(redacted)`. It is client-chosen on every route, and
|
||||||
|
`/.well-known/healthcheck` and `/s/*` answer 200 to anyone with no rate
|
||||||
|
limiter in front of them, so a query on a fixed 200 URL would otherwise
|
||||||
|
buy the same amplification as an invented path. Nothing debuggable is
|
||||||
|
lost: `page`, on the authenticated pagination links, is the only query
|
||||||
|
parameter this service reads.
|
||||||
|
|
||||||
|
The remaining client-supplied fields are truncated rather than dropped,
|
||||||
|
each to a fixed budget: 512 bytes for `url`, `useragent` and `referer`,
|
||||||
|
128 for `request_id` (chi passes an inbound `X-Request-Id` header
|
||||||
|
through), and 32 for `method`. A truncated `User-Agent` is still worth
|
||||||
|
reading; an absent one is not. A cut value ends in `[truncated]`, which
|
||||||
|
is charged on top of the budget rather than inside it.
|
||||||
|
|
||||||
|
Each budget is spent in _encoded_ bytes, not in the bytes the client
|
||||||
|
sent. The log handler escapes a quotation mark, a backslash and a tab
|
||||||
|
to two bytes each and a non-printable rune to six, and Go's header
|
||||||
|
parser accepts all of them in a header value, so a budget counted raw
|
||||||
|
would buy a field twice its nominal size — and the line, not the
|
||||||
|
header, is what an operator has to store. Plain ASCII encodes one byte
|
||||||
|
for one, so a real browser's `User-Agent` still fits whole; a value
|
||||||
|
built out of escapes keeps a proportionally shorter prefix, which is
|
||||||
|
the right trade.
|
||||||
|
|
||||||
|
Net: **one `INFO` line per request, of at most 2,560 bytes.** That
|
||||||
|
ceiling is arithmetic, not an observation: 3 × (512 + 11) for `url`,
|
||||||
|
`useragent` and `referer`, plus 128 + 11 for `request_id`, plus 32 + 11
|
||||||
|
for `method`, plus a 336-byte fixed portion (the field names, the
|
||||||
|
punctuation, both timestamps at their longest, an IPv6 `remoteIP` with
|
||||||
|
a zone, the status and the latency) — 2,087 bytes, stated at 2,560 so
|
||||||
|
the figure has headroom. `internal/middleware/accesslog_test.go`
|
||||||
|
asserts it against 8 KB of client-chosen text in the path, in the
|
||||||
|
query, and in each of `User-Agent`, `Referer` and `X-Request-Id`,
|
||||||
|
including cases built from the characters the handler escapes, and
|
||||||
|
against the widest line the service can be made to write: a 5xx that
|
||||||
|
keeps its concrete path while all three header fields are also at their
|
||||||
|
budget. Measured over a real connection, that line is 1,972 bytes.
|
||||||
|
|
||||||
|
Multiply that ceiling by the request rate to size log storage. Note
|
||||||
|
that the rate is not bounded by the limits above on every route:
|
||||||
|
`/.well-known/healthcheck` and `/s/*` sit behind no limiter, so there
|
||||||
|
the multiplier is whatever the deployment will serve.
|
||||||
|
|
||||||
Every limiter here — receiver, login, and password change — identifies
|
Every limiter here — receiver, login, and password change — identifies
|
||||||
the client the same way, through one shared key function: the
|
the client the same way, through one shared key function: the
|
||||||
|
|||||||
542
internal/middleware/accesslog_test.go
Normal file
542
internal/middleware/accesslog_test.go
Normal file
@@ -0,0 +1,542 @@
|
|||||||
|
package middleware_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi"
|
||||||
|
chimw "github.com/go-chi/chi/middleware"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
|
)
|
||||||
|
|
||||||
|
// floodRequests is the number of distinct invented paths each flood
|
||||||
|
// test drives through the access log.
|
||||||
|
const floodRequests = 64
|
||||||
|
|
||||||
|
// attackerMarker is embedded in every invented path. No access log
|
||||||
|
// line for a redirected or rejected request may contain it.
|
||||||
|
const attackerMarker = "QQATTACKERTEXTQQ"
|
||||||
|
|
||||||
|
// maxLineBytes bounds a single access log line whose client-supplied
|
||||||
|
// fields are of ordinary size. Well above what the fixed fields need,
|
||||||
|
// well below the length of the oversized input the amplification tests
|
||||||
|
// send.
|
||||||
|
const maxLineBytes = 1024
|
||||||
|
|
||||||
|
// maxCappedLineBytes bounds a single access log line when every
|
||||||
|
// client-supplied field arrives oversized and is truncated to its
|
||||||
|
// budget. This is the number the README quotes as the per-line cost an
|
||||||
|
// operator sizes log storage against, and it is a bound on the
|
||||||
|
// ENCODED line, which is what the operator's disk holds.
|
||||||
|
const maxCappedLineBytes = 2560
|
||||||
|
|
||||||
|
// oversizedSegmentBytes is the length of the single attacker-chosen
|
||||||
|
// path segment, query string or header used to show line size does not
|
||||||
|
// track input size.
|
||||||
|
const oversizedSegmentBytes = 8192
|
||||||
|
|
||||||
|
// tailMarker is placed at the END of an oversized header value, so its
|
||||||
|
// absence from the log proves the value was truncated rather than
|
||||||
|
// merely being short.
|
||||||
|
const tailMarker = "QQTRUNCATEDTAILQQ"
|
||||||
|
|
||||||
|
// These mirror the middleware's own budgets, which are unexported.
|
||||||
|
// They are duplicated rather than exported so that widening a budget
|
||||||
|
// in the middleware has to be restated here deliberately.
|
||||||
|
const (
|
||||||
|
maxFieldBytes = 512
|
||||||
|
maxRequestIDBytes = 128
|
||||||
|
truncationSuffix = "[truncated]"
|
||||||
|
unmatchedRouteLiteral = "(unmatched)"
|
||||||
|
)
|
||||||
|
|
||||||
|
// capturingMiddleware returns a Middleware whose logger writes JSON
|
||||||
|
// lines into the returned buffer, so the access log can be asserted
|
||||||
|
// on directly.
|
||||||
|
func capturingMiddleware(t *testing.T) (*middleware.Middleware, *bytes.Buffer) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
buf := new(bytes.Buffer)
|
||||||
|
log := slog.New(slog.NewJSONHandler(
|
||||||
|
buf,
|
||||||
|
&slog.HandlerOptions{Level: slog.LevelInfo},
|
||||||
|
))
|
||||||
|
|
||||||
|
cfg := &config.Config{Environment: config.EnvironmentDev}
|
||||||
|
|
||||||
|
return middleware.NewForTest(log, cfg, nil), buf
|
||||||
|
}
|
||||||
|
|
||||||
|
// accessLogRouter mirrors the production route shapes that an
|
||||||
|
// unauthenticated client can reach: the public receiver, the
|
||||||
|
// authenticated profile route (which redirects to login rather than
|
||||||
|
// rejecting outright), the health check (which answers 200 to anyone,
|
||||||
|
// behind no rate limiter at all), and a plain static route.
|
||||||
|
func accessLogRouter(m *middleware.Middleware) *chi.Mux {
|
||||||
|
router := chi.NewRouter()
|
||||||
|
// Production registers RequestID ahead of Logging, and chi's
|
||||||
|
// RequestID passes an inbound X-Request-Id header straight
|
||||||
|
// through, so the request_id field is client-supplied too.
|
||||||
|
router.Use(chimw.RequestID)
|
||||||
|
router.Use(m.Logging())
|
||||||
|
|
||||||
|
router.Get(
|
||||||
|
"/.well-known/healthcheck",
|
||||||
|
func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
router.HandleFunc(
|
||||||
|
"/webhook/{uuid}",
|
||||||
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
// Stands in for the real handler: an unknown entrypoint
|
||||||
|
// UUID 404s, a known one succeeds.
|
||||||
|
if chi.URLParam(r, "uuid") != "known" {
|
||||||
|
http.Error(w, "not found", http.StatusNotFound)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
router.Route("/user/{username}", func(r chi.Router) {
|
||||||
|
r.Get("/", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
http.Redirect(
|
||||||
|
w, r, "/pages/login", http.StatusSeeOther,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
boom := func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
http.Error(w, "boom", http.StatusInternalServerError)
|
||||||
|
}
|
||||||
|
|
||||||
|
router.Get("/boom", boom)
|
||||||
|
// The 5xx branch keeps the concrete path, so it needs a route that
|
||||||
|
// answers 500 to a path of the client's choosing: that is where the
|
||||||
|
// url field and the header fields are both at their budget on the
|
||||||
|
// same line.
|
||||||
|
router.Get("/boom/*", boom)
|
||||||
|
|
||||||
|
return router
|
||||||
|
}
|
||||||
|
|
||||||
|
// accessLogEntries decodes the captured buffer into one map per
|
||||||
|
// logged line, holding every line to maxLineBytes.
|
||||||
|
func accessLogEntries(
|
||||||
|
t *testing.T,
|
||||||
|
buf *bytes.Buffer,
|
||||||
|
) []map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
return accessLogEntriesWithin(t, buf, maxLineBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// accessLogEntriesWithin decodes the captured buffer into one map per
|
||||||
|
// logged line, holding every line to bound bytes.
|
||||||
|
func accessLogEntriesWithin(
|
||||||
|
t *testing.T,
|
||||||
|
buf *bytes.Buffer,
|
||||||
|
bound int,
|
||||||
|
) []map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var entries []map[string]any
|
||||||
|
|
||||||
|
for line := range strings.SplitSeq(
|
||||||
|
strings.TrimSpace(buf.String()), "\n",
|
||||||
|
) {
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
require.LessOrEqual(
|
||||||
|
t, len(line), bound,
|
||||||
|
"access log line exceeded its bound",
|
||||||
|
)
|
||||||
|
|
||||||
|
var entry map[string]any
|
||||||
|
|
||||||
|
require.NoError(t, json.Unmarshal([]byte(line), &entry))
|
||||||
|
|
||||||
|
entries = append(entries, entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
return entries
|
||||||
|
}
|
||||||
|
|
||||||
|
// get drives one GET through the router.
|
||||||
|
func get(t *testing.T, router *chi.Mux, target string) int {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
return getWithHeaders(t, router, target, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// getWithHeaders drives one GET through the router with the supplied
|
||||||
|
// request headers set.
|
||||||
|
func getWithHeaders(
|
||||||
|
t *testing.T,
|
||||||
|
router *chi.Mux,
|
||||||
|
target string,
|
||||||
|
headers map[string]string,
|
||||||
|
) int {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
context.Background(), http.MethodGet, target, nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
for name, value := range headers {
|
||||||
|
req.Header.Set(name, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
return w.Code
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertFloodIsBounded drives floodRequests distinct invented paths
|
||||||
|
// built by pathFor and asserts every logged line names wantURL, that
|
||||||
|
// none carries the invented text, and that the line count is exactly
|
||||||
|
// one per request.
|
||||||
|
func assertFloodIsBounded(
|
||||||
|
t *testing.T,
|
||||||
|
pathFor func(i int) string,
|
||||||
|
wantStatus int,
|
||||||
|
wantURL string,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
m, buf := capturingMiddleware(t)
|
||||||
|
router := accessLogRouter(m)
|
||||||
|
|
||||||
|
for i := range floodRequests {
|
||||||
|
assert.Equal(t, wantStatus, get(t, router, pathFor(i)))
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.NotContains(
|
||||||
|
t, buf.String(), attackerMarker,
|
||||||
|
"access log carried attacker-chosen path text",
|
||||||
|
)
|
||||||
|
|
||||||
|
entries := accessLogEntries(t, buf)
|
||||||
|
require.Len(t, entries, floodRequests)
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
assert.Equal(t, wantURL, entry["url"])
|
||||||
|
assert.InDelta(
|
||||||
|
t, float64(wantStatus), entry["status"], 0,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessLog_InventedReceiverPathsLogRoutePattern(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
assertFloodIsBounded(
|
||||||
|
t,
|
||||||
|
func(i int) string {
|
||||||
|
return "/webhook/" + attackerMarker +
|
||||||
|
strings.Repeat("x", i) + "?q=" + attackerMarker
|
||||||
|
},
|
||||||
|
http.StatusNotFound,
|
||||||
|
"/webhook/{uuid}",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessLog_InventedProfilePathsLogRoutePattern(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// The login redirect is a 3xx, not a 4xx, but it is just as free
|
||||||
|
// for an unauthenticated client to drive with invented input.
|
||||||
|
// The doubled slash is what chi's RoutePattern yields for a
|
||||||
|
// mounted subrouter's index route.
|
||||||
|
assertFloodIsBounded(
|
||||||
|
t,
|
||||||
|
func(i int) string {
|
||||||
|
return "/user/" + attackerMarker +
|
||||||
|
strings.Repeat("x", i) + "/"
|
||||||
|
},
|
||||||
|
http.StatusSeeOther,
|
||||||
|
"/user/{username}//",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessLog_UnroutablePathsLogFixedLiteral(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
assertFloodIsBounded(
|
||||||
|
t,
|
||||||
|
func(i int) string {
|
||||||
|
return "/" + attackerMarker + strings.Repeat("x", i)
|
||||||
|
},
|
||||||
|
http.StatusNotFound,
|
||||||
|
"(unmatched)",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// oversizedValue builds an 8 KB header value out of repetitions of ch,
|
||||||
|
// with the tail marker at its end.
|
||||||
|
//
|
||||||
|
// The leading 'x' is load-bearing for tab: net/textproto strips leading
|
||||||
|
// and trailing whitespace from a header value, so a value that were
|
||||||
|
// nothing but tabs would arrive empty over a real connection and the
|
||||||
|
// case would prove nothing.
|
||||||
|
func oversizedValue(ch string) string {
|
||||||
|
return "x" + strings.Repeat(ch, oversizedSegmentBytes) + tailMarker
|
||||||
|
}
|
||||||
|
|
||||||
|
// oversizedHeaders fills every client-supplied header the access log
|
||||||
|
// reads with the same value.
|
||||||
|
func oversizedHeaders(value string) map[string]string {
|
||||||
|
return map[string]string{
|
||||||
|
"User-Agent": value,
|
||||||
|
"Referer": value,
|
||||||
|
"X-Request-Id": value,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sizeCase is one way of pointing 8 KB of client-chosen text at the
|
||||||
|
// access log.
|
||||||
|
type sizeCase struct {
|
||||||
|
target string
|
||||||
|
headers map[string]string
|
||||||
|
wantStatus int
|
||||||
|
wantURL string
|
||||||
|
bound int
|
||||||
|
}
|
||||||
|
|
||||||
|
// lineSizeCases enumerates every part of a request that reaches the
|
||||||
|
// access log, at 8 KB apiece.
|
||||||
|
func lineSizeCases() map[string]sizeCase {
|
||||||
|
cases := map[string]sizeCase{
|
||||||
|
"oversized path segment": {
|
||||||
|
target: "/webhook/" + attackerMarker +
|
||||||
|
strings.Repeat("x", oversizedSegmentBytes),
|
||||||
|
wantStatus: http.StatusNotFound,
|
||||||
|
wantURL: "/webhook/{uuid}",
|
||||||
|
bound: maxLineBytes,
|
||||||
|
},
|
||||||
|
// /.well-known/healthcheck answers 200 to anyone and has no
|
||||||
|
// rate limiter in front of it, so an oversized query appended
|
||||||
|
// to it would otherwise buy the same amplification as an
|
||||||
|
// invented 404 path, unauthenticated and unthrottled.
|
||||||
|
"oversized query on an unauthenticated 200": {
|
||||||
|
target: "/.well-known/healthcheck?q=" + attackerMarker +
|
||||||
|
strings.Repeat("x", oversizedSegmentBytes),
|
||||||
|
wantStatus: http.StatusOK,
|
||||||
|
wantURL: "/.well-known/healthcheck?(redacted)",
|
||||||
|
bound: maxLineBytes,
|
||||||
|
},
|
||||||
|
// These reach the line on every request, including one whose
|
||||||
|
// url field is correctly redacted.
|
||||||
|
"oversized headers": {
|
||||||
|
target: "/" + attackerMarker,
|
||||||
|
headers: oversizedHeaders(oversizedValue("h")),
|
||||||
|
wantStatus: http.StatusNotFound,
|
||||||
|
wantURL: unmatchedRouteLiteral,
|
||||||
|
bound: maxCappedLineBytes,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// The url field on a 5xx keeps the concrete path, so it reaches its
|
||||||
|
// own budget on the same line as the three header fields. That is
|
||||||
|
// the widest line the service can be made to write.
|
||||||
|
longPath := "/boom/" + strings.Repeat("x", oversizedSegmentBytes)
|
||||||
|
wantLongURL := longPath[:maxFieldBytes] + truncationSuffix
|
||||||
|
|
||||||
|
// escapeChars are the bytes Go's header parser accepts in a header
|
||||||
|
// value and the log handler then escapes, one byte in and two or
|
||||||
|
// more out. A budget counted in raw bytes lets any of them buy a
|
||||||
|
// field twice its nominal size, so every one of them gets a case.
|
||||||
|
escapeChars := map[string]string{
|
||||||
|
"quote": `"`,
|
||||||
|
"backslash": `\`,
|
||||||
|
"tab": "\t",
|
||||||
|
}
|
||||||
|
|
||||||
|
for kind, char := range escapeChars {
|
||||||
|
fill := oversizedValue(char)
|
||||||
|
|
||||||
|
cases["oversized "+kind+" headers"] = sizeCase{
|
||||||
|
target: "/" + attackerMarker,
|
||||||
|
headers: oversizedHeaders(fill),
|
||||||
|
wantStatus: http.StatusNotFound,
|
||||||
|
wantURL: unmatchedRouteLiteral,
|
||||||
|
bound: maxCappedLineBytes,
|
||||||
|
}
|
||||||
|
|
||||||
|
cases["oversized "+kind+" headers with a 5xx concrete url"] =
|
||||||
|
sizeCase{
|
||||||
|
target: longPath,
|
||||||
|
headers: oversizedHeaders(fill),
|
||||||
|
wantStatus: http.StatusInternalServerError,
|
||||||
|
wantURL: wantLongURL,
|
||||||
|
bound: maxCappedLineBytes,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return cases
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAccessLog_LineSizeDoesNotTrackInputSize drives 8 KB of
|
||||||
|
// client-chosen text at the access log through each part of the
|
||||||
|
// request that reaches it, and holds the resulting line to a fixed
|
||||||
|
// bound in every case.
|
||||||
|
//
|
||||||
|
// The bound is on the ENCODED line, so the cases built out of
|
||||||
|
// characters the handler escapes are the ones that matter: a budget
|
||||||
|
// spent in raw bytes passes every plain-ASCII case here and still
|
||||||
|
// writes a line half again as long as the stated ceiling.
|
||||||
|
func TestAccessLog_LineSizeDoesNotTrackInputSize(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
require.Equal(
|
||||||
|
t, middleware.MaxAccessLogLineBytes, maxCappedLineBytes,
|
||||||
|
"the README quotes this ceiling and the middleware derives "+
|
||||||
|
"it; they have to agree",
|
||||||
|
)
|
||||||
|
|
||||||
|
for name, tc := range lineSizeCases() {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m, buf := capturingMiddleware(t)
|
||||||
|
router := accessLogRouter(m)
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
tc.wantStatus,
|
||||||
|
getWithHeaders(t, router, tc.target, tc.headers),
|
||||||
|
)
|
||||||
|
|
||||||
|
// accessLogEntriesWithin enforces the bound, which is
|
||||||
|
// orders of magnitude smaller than the input just sent.
|
||||||
|
entries := accessLogEntriesWithin(t, buf, tc.bound)
|
||||||
|
require.Len(t, entries, 1)
|
||||||
|
assert.Equal(t, tc.wantURL, entries[0]["url"])
|
||||||
|
|
||||||
|
// The markers sit at the far end of the client-chosen
|
||||||
|
// text, so their absence is what proves the redaction and
|
||||||
|
// the truncation actually ran.
|
||||||
|
assert.NotContains(
|
||||||
|
t, buf.String(), attackerMarker,
|
||||||
|
"access log carried attacker-chosen text",
|
||||||
|
)
|
||||||
|
assert.NotContains(
|
||||||
|
t, buf.String(), tailMarker,
|
||||||
|
"access log carried an untruncated client field",
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAccessLog_OversizedHeadersKeepATruncatedPrefix checks the other
|
||||||
|
// half of the header cap: the fields are cut, not dropped, so a
|
||||||
|
// truncated User-Agent is still worth reading.
|
||||||
|
func TestAccessLog_OversizedHeadersKeepATruncatedPrefix(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m, buf := capturingMiddleware(t)
|
||||||
|
router := accessLogRouter(m)
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
http.StatusNotFound,
|
||||||
|
getWithHeaders(
|
||||||
|
t, router, "/nope",
|
||||||
|
oversizedHeaders(oversizedValue("h")),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
entries := accessLogEntriesWithin(t, buf, maxCappedLineBytes)
|
||||||
|
require.Len(t, entries, 1)
|
||||||
|
|
||||||
|
for key, budget := range map[string]int{
|
||||||
|
"useragent": maxFieldBytes,
|
||||||
|
"referer": maxFieldBytes,
|
||||||
|
"request_id": maxRequestIDBytes,
|
||||||
|
} {
|
||||||
|
value, ok := entries[0][key].(string)
|
||||||
|
require.True(t, ok, key)
|
||||||
|
assert.LessOrEqual(
|
||||||
|
t, len(value), budget+len(truncationSuffix), key,
|
||||||
|
)
|
||||||
|
assert.Contains(t, value, truncationSuffix, key)
|
||||||
|
assert.Contains(t, value, "hhhh", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessLog_SuccessKeepsConcretePathAndRedactsQuery(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m, buf := capturingMiddleware(t)
|
||||||
|
router := accessLogRouter(m)
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t, http.StatusOK, get(t, router, "/webhook/known?src=ci"),
|
||||||
|
)
|
||||||
|
|
||||||
|
// The path resolved against a stored entrypoint, so it stays. The
|
||||||
|
// query never does: see TestAccessLog_UnauthenticatedSuccess...
|
||||||
|
entries := accessLogEntries(t, buf)
|
||||||
|
require.Len(t, entries, 1)
|
||||||
|
assert.Equal(t, "/webhook/known?(redacted)", entries[0]["url"])
|
||||||
|
assert.NotContains(t, buf.String(), "src=ci")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessLog_ServerErrorKeepsConcreteURL(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m, buf := capturingMiddleware(t)
|
||||||
|
router := accessLogRouter(m)
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t, http.StatusInternalServerError, get(t, router, "/boom"),
|
||||||
|
)
|
||||||
|
|
||||||
|
entries := accessLogEntries(t, buf)
|
||||||
|
require.Len(t, entries, 1)
|
||||||
|
assert.Equal(t, "/boom", entries[0]["url"])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessLog_RetainsEveryOtherField(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m, buf := capturingMiddleware(t)
|
||||||
|
router := accessLogRouter(m)
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
http.StatusNotFound,
|
||||||
|
get(t, router, "/webhook/"+attackerMarker),
|
||||||
|
)
|
||||||
|
|
||||||
|
entries := accessLogEntries(t, buf)
|
||||||
|
require.Len(t, entries, 1)
|
||||||
|
|
||||||
|
for _, key := range []string{
|
||||||
|
"request_start", "method", "url", "useragent", "request_id",
|
||||||
|
"referer", "proto", "remoteIP", "status", "latency_ms",
|
||||||
|
} {
|
||||||
|
assert.Contains(t, entries[0], key)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, http.MethodGet, entries[0]["method"])
|
||||||
|
assert.Equal(t, "HTTP/1.1", entries[0]["proto"])
|
||||||
|
}
|
||||||
@@ -6,9 +6,13 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
basicauth "github.com/99designs/basicauth-go"
|
basicauth "github.com/99designs/basicauth-go"
|
||||||
|
"github.com/go-chi/chi"
|
||||||
"github.com/go-chi/chi/middleware"
|
"github.com/go-chi/chi/middleware"
|
||||||
"github.com/go-chi/cors"
|
"github.com/go-chi/cors"
|
||||||
metrics "github.com/slok/go-http-metrics/metrics/prometheus"
|
metrics "github.com/slok/go-http-metrics/metrics/prometheus"
|
||||||
@@ -25,6 +29,71 @@ const (
|
|||||||
// corsMaxAge is the maximum time (in seconds) that a
|
// corsMaxAge is the maximum time (in seconds) that a
|
||||||
// preflight response can be cached.
|
// preflight response can be cached.
|
||||||
corsMaxAge = 300
|
corsMaxAge = 300
|
||||||
|
|
||||||
|
// unmatchedRoute is logged in the access log's url field when a
|
||||||
|
// redirected or rejected request matched no route pattern at
|
||||||
|
// all. Every byte of such a path is client-chosen, so none of it
|
||||||
|
// is logged.
|
||||||
|
unmatchedRoute = "(unmatched)"
|
||||||
|
|
||||||
|
// redactedQuery stands in for the query string on the access log
|
||||||
|
// branches that keep the concrete URL. The query is client-chosen
|
||||||
|
// on every route, including the ones that answer an
|
||||||
|
// unauthenticated 200, so logging it verbatim would let a client
|
||||||
|
// pick the size of the line it writes.
|
||||||
|
redactedQuery = "?(redacted)"
|
||||||
|
|
||||||
|
// maxLogFieldBytes bounds each access log field whose value the
|
||||||
|
// client supplies outright: the URL, the User-Agent and the
|
||||||
|
// Referer. The budget is spent in ENCODED bytes (see
|
||||||
|
// truncateLogField), so 512 still holds a real browser's User-Agent
|
||||||
|
// whole — those are plain ASCII, which encodes one byte for one —
|
||||||
|
// while a value built from characters the encoder escapes keeps a
|
||||||
|
// shorter prefix. That is the intended trade: 500 quotation marks
|
||||||
|
// are not a debugging asset.
|
||||||
|
maxLogFieldBytes = 512
|
||||||
|
|
||||||
|
// maxLogRequestIDBytes bounds the request id, which is also
|
||||||
|
// client-supplied: chi's RequestID middleware passes an inbound
|
||||||
|
// X-Request-Id header through verbatim. Its generated form is an
|
||||||
|
// order of magnitude shorter than this.
|
||||||
|
maxLogRequestIDBytes = 128
|
||||||
|
|
||||||
|
// maxLogMethodBytes bounds the method. Go accepts any RFC 7230
|
||||||
|
// token there, bounded only by the header size limit, so it is
|
||||||
|
// client-chosen text like the rest. The longest registered method
|
||||||
|
// is half this.
|
||||||
|
maxLogMethodBytes = 32
|
||||||
|
|
||||||
|
// truncationMarker is appended to any field the access log cut, so
|
||||||
|
// a short value and a truncated one cannot be confused. It is
|
||||||
|
// charged on top of the budget, not inside it.
|
||||||
|
truncationMarker = "[truncated]"
|
||||||
|
|
||||||
|
// MaxAccessLogLineBytes is the ceiling on one JSON access log line,
|
||||||
|
// and the number an operator multiplies by the request rate to size
|
||||||
|
// log storage. It is not an observation of a sample: it is the sum
|
||||||
|
// of the budgets above, each of which truncateLogField enforces in
|
||||||
|
// ENCODED bytes, plus the part of the line no client can influence.
|
||||||
|
//
|
||||||
|
// url, useragent, referer 3*(512+11) = 1569
|
||||||
|
// request_id 128+11 = 139
|
||||||
|
// method 32+11 = 43
|
||||||
|
// fixed portion = 336
|
||||||
|
// ----
|
||||||
|
// 2087
|
||||||
|
//
|
||||||
|
// The fixed portion is the JSON punctuation, the field names, the
|
||||||
|
// level and the message, both timestamps at their longest, an IPv6
|
||||||
|
// remoteIP with a zone, a three-digit status and a full-width int64
|
||||||
|
// latency. Stated at 2560 so the figure carries headroom rather
|
||||||
|
// than sitting on the arithmetic.
|
||||||
|
//
|
||||||
|
// The tty text handler in internal/logger is covered by the same
|
||||||
|
// figure: encodedLogFieldBytes charges the worse of the two
|
||||||
|
// handlers' escapes, and the text handler's fixed portion is the
|
||||||
|
// smaller of the two.
|
||||||
|
MaxAccessLogLineBytes = 2560
|
||||||
)
|
)
|
||||||
|
|
||||||
//nolint:revive // MiddlewareParams is a standard fx naming convention.
|
//nolint:revive // MiddlewareParams is a standard fx naming convention.
|
||||||
@@ -94,6 +163,161 @@ func (lrw *loggingResponseWriter) WriteHeader(code int) {
|
|||||||
lrw.ResponseWriter.WriteHeader(code)
|
lrw.ResponseWriter.WriteHeader(code)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// encodedLogFieldBytes is what r costs on the line once the log
|
||||||
|
// handler has escaped it, taking the worse of the two handlers
|
||||||
|
// internal/logger configures.
|
||||||
|
//
|
||||||
|
// slog's JSON handler escapes quote, backslash, newline, carriage
|
||||||
|
// return and tab to two bytes each, and every other C0 control plus
|
||||||
|
// LINE SEPARATOR and PARAGRAPH SEPARATOR to a six-byte \u escape. Its
|
||||||
|
// text handler quotes with strconv.Quote, which spells any
|
||||||
|
// non-printable rune the same six-byte way. Both pass printable runes
|
||||||
|
// through as their own UTF-8, so unicode.IsPrint separates the two
|
||||||
|
// cases for either handler. Go's header parser accepts quote,
|
||||||
|
// backslash, tab and non-printable multi-byte runes in a header value,
|
||||||
|
// so every one of these is reachable from a request.
|
||||||
|
func encodedLogFieldBytes(r rune) int {
|
||||||
|
const (
|
||||||
|
// A backslash and the character itself.
|
||||||
|
shortEscapeBytes = 2
|
||||||
|
// \uXXXX, which is also the width of \u00XX.
|
||||||
|
escapedRuneBytes = 6
|
||||||
|
)
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case r == '"' || r == '\\' || r == '\n' || r == '\r' || r == '\t':
|
||||||
|
return shortEscapeBytes
|
||||||
|
case !unicode.IsPrint(r):
|
||||||
|
return escapedRuneBytes
|
||||||
|
default:
|
||||||
|
return utf8.RuneLen(r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// truncateLogField caps s at maxBytes of ENCODED output, marking the
|
||||||
|
// value when it cuts.
|
||||||
|
//
|
||||||
|
// Budgeting raw bytes would not bound the line. Escaping only ever
|
||||||
|
// grows a value, so a raw budget spent on characters the encoder
|
||||||
|
// escapes buys a field several times its nominal size — and the line
|
||||||
|
// is the thing an operator is told to multiply by their request rate.
|
||||||
|
// Charging each rune what it will actually cost is what makes
|
||||||
|
// MaxAccessLogLineBytes true rather than merely larger. The visible
|
||||||
|
// consequence is that an escape-heavy value keeps a shorter prefix
|
||||||
|
// than a plain one, which is the correct trade.
|
||||||
|
//
|
||||||
|
// The result is always valid UTF-8. A cut on a byte boundary can split
|
||||||
|
// a multi-byte rune, and a header can carry bytes that were never
|
||||||
|
// valid UTF-8 to begin with; both are dropped rather than kept, since
|
||||||
|
// an encoder would otherwise spend six bytes replacing each one.
|
||||||
|
func truncateLogField(s string, maxBytes int) string {
|
||||||
|
// No rune encodes to fewer bytes than it occupies, so nothing past
|
||||||
|
// maxBytes raw can fit the budget. Slicing first bounds the scan
|
||||||
|
// below to the budget rather than to the size of the header the
|
||||||
|
// client sent.
|
||||||
|
window, cut := s, false
|
||||||
|
if len(window) > maxBytes {
|
||||||
|
window, cut = window[:maxBytes], true
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
kept strings.Builder
|
||||||
|
spent int
|
||||||
|
)
|
||||||
|
|
||||||
|
for i := 0; i < len(window); {
|
||||||
|
r, size := utf8.DecodeRuneInString(window[i:])
|
||||||
|
if r == utf8.RuneError && size == 1 {
|
||||||
|
i += size
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
cost := encodedLogFieldBytes(r)
|
||||||
|
if spent+cost > maxBytes {
|
||||||
|
cut = true
|
||||||
|
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
spent += cost
|
||||||
|
|
||||||
|
kept.WriteString(window[i : i+size])
|
||||||
|
|
||||||
|
i += size
|
||||||
|
}
|
||||||
|
|
||||||
|
if !cut {
|
||||||
|
return kept.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
return kept.String() + truncationMarker
|
||||||
|
}
|
||||||
|
|
||||||
|
// concreteLogURL renders the request's own URL for the access log
|
||||||
|
// branches that keep it, with the query string replaced by a fixed
|
||||||
|
// marker.
|
||||||
|
//
|
||||||
|
// The path on those branches is bounded by the service's routes or by
|
||||||
|
// the operator's data — a 2xx on the receiver means the UUID named a
|
||||||
|
// stored entrypoint, a 2xx under /s means the file is in the embedded
|
||||||
|
// tree. The query is not bounded by anything: /.well-known/healthcheck
|
||||||
|
// and /s/* take no authentication and sit behind no rate limiter, and
|
||||||
|
// /pages/login behind only the login limiter, so any of them will
|
||||||
|
// answer 200 to a URL carrying an arbitrary number of arbitrary bytes
|
||||||
|
// after the '?'. Keeping the path and dropping the query is what makes
|
||||||
|
// this branch as bounded as the pattern branches below.
|
||||||
|
//
|
||||||
|
// Nothing debuggable is lost. One route in the service reads a query
|
||||||
|
// parameter at all — `page`, on the authenticated pagination links in
|
||||||
|
// internal/handlers/source_management.go — and the alternatives that
|
||||||
|
// would preserve more (a key count, a key allowlist) all require
|
||||||
|
// parsing an attacker-sized query on every request, which is work an
|
||||||
|
// unauthenticated client would then be choosing for us.
|
||||||
|
func concreteLogURL(r *http.Request) string {
|
||||||
|
path := r.URL.EscapedPath()
|
||||||
|
|
||||||
|
if r.URL.RawQuery == "" && !r.URL.ForceQuery {
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
return path + redactedQuery
|
||||||
|
}
|
||||||
|
|
||||||
|
// accessLogURL returns the value for the access log's url field.
|
||||||
|
//
|
||||||
|
// 2xx and 5xx responses get the concrete path (see concreteLogURL). A
|
||||||
|
// success resolved against a static route or against the operator's
|
||||||
|
// own data — on the receiver, a 2xx means the UUID named a stored
|
||||||
|
// entrypoint — and a server error is our own bug, where the exact URL
|
||||||
|
// is the primary evidence and which no client can provoke at will.
|
||||||
|
//
|
||||||
|
// 3xx and 4xx responses get the chi route pattern instead. Those are
|
||||||
|
// the outcomes an unauthenticated client drives for free: 404 or 429
|
||||||
|
// on any invented /webhook/ path, 303 to the login page on any
|
||||||
|
// invented /user/ path. Logging the concrete URL there lets a flood
|
||||||
|
// write attacker-chosen text, of attacker-chosen length, into the
|
||||||
|
// operator's log at one line per request. The pattern comes from the
|
||||||
|
// router's own table, so it is bounded by the service's routes while
|
||||||
|
// still naming which class of request was rejected.
|
||||||
|
//
|
||||||
|
// The pattern is only populated once routing has run, so this must be
|
||||||
|
// called after the handler returns, not before.
|
||||||
|
func accessLogURL(r *http.Request, status int) string {
|
||||||
|
if status < http.StatusMultipleChoices ||
|
||||||
|
status >= http.StatusInternalServerError {
|
||||||
|
return concreteLogURL(r)
|
||||||
|
}
|
||||||
|
|
||||||
|
if rc := chi.RouteContext(r.Context()); rc != nil {
|
||||||
|
if pattern := rc.RoutePattern(); pattern != "" {
|
||||||
|
return pattern
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return unmatchedRoute
|
||||||
|
}
|
||||||
|
|
||||||
// Logging returns middleware that logs each HTTP request with
|
// Logging returns middleware that logs each HTTP request with
|
||||||
// timing and metadata.
|
// timing and metadata.
|
||||||
func (s *Middleware) Logging() func(http.Handler) http.Handler {
|
func (s *Middleware) Logging() func(http.Handler) http.Handler {
|
||||||
@@ -118,13 +342,27 @@ func (s *Middleware) Logging() func(http.Handler) http.Handler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Every field below that a client can influence is
|
||||||
|
// truncated to a fixed budget, so the size of this
|
||||||
|
// line does not track the size of the request.
|
||||||
s.log.Info("http request",
|
s.log.Info("http request",
|
||||||
"request_start", start,
|
"request_start", start,
|
||||||
"method", r.Method,
|
"method", truncateLogField(
|
||||||
"url", r.URL.String(),
|
r.Method, maxLogMethodBytes,
|
||||||
"useragent", r.UserAgent(),
|
),
|
||||||
"request_id", requestID,
|
"url", truncateLogField(
|
||||||
"referer", r.Referer(),
|
accessLogURL(r, lrw.statusCode),
|
||||||
|
maxLogFieldBytes,
|
||||||
|
),
|
||||||
|
"useragent", truncateLogField(
|
||||||
|
r.UserAgent(), maxLogFieldBytes,
|
||||||
|
),
|
||||||
|
"request_id", truncateLogField(
|
||||||
|
requestID, maxLogRequestIDBytes,
|
||||||
|
),
|
||||||
|
"referer", truncateLogField(
|
||||||
|
r.Referer(), maxLogFieldBytes,
|
||||||
|
),
|
||||||
"proto", r.Proto,
|
"proto", r.Proto,
|
||||||
"remoteIP", ipFromHostPort(r.RemoteAddr),
|
"remoteIP", ipFromHostPort(r.RemoteAddr),
|
||||||
"status", lrw.statusCode,
|
"status", lrw.statusCode,
|
||||||
|
|||||||
50
internal/server/static_assets_test.go
Normal file
50
internal/server/static_assets_test.go
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
package server_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"regexp"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"sneak.berlin/go/webhooker/templates"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestBaseTemplateScriptsAreServed walks every /s/ script the base
|
||||||
|
// template loads on each page and fetches it through the real router.
|
||||||
|
// Alpine.js is fetched at build time rather than committed, so nothing
|
||||||
|
// in the repo guarantees it is present: this is the check that the page
|
||||||
|
// still gets the JavaScript it asks for.
|
||||||
|
func TestBaseTemplateScriptsAreServed(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// scriptSrc matches the src of every <script> tag pointing at the
|
||||||
|
// /s/ static mount.
|
||||||
|
scriptSrc := regexp.MustCompile(`<script[^>]+src="(/s/[^"]+)"`)
|
||||||
|
|
||||||
|
base, err := templates.Templates.ReadFile("base.html")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
matches := scriptSrc.FindAllStringSubmatch(string(base), -1)
|
||||||
|
require.NotEmpty(t, matches, "base.html should load scripts from /s/")
|
||||||
|
|
||||||
|
env := newTestEnv(t)
|
||||||
|
|
||||||
|
for _, m := range matches {
|
||||||
|
src := m[1]
|
||||||
|
t.Run(src, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
w := env.get(src, nil)
|
||||||
|
|
||||||
|
require.Equalf(
|
||||||
|
t, http.StatusOK, w.Code,
|
||||||
|
"base.html loads %s but the server does not serve it", src,
|
||||||
|
)
|
||||||
|
assert.NotEmptyf(
|
||||||
|
t, w.Body.Bytes(), "%s is served but empty", src,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,7 +5,8 @@
|
|||||||
# or apk (detected in that order); assumes NOTHING is present (not git,
|
# or apk (detected in that order); assumes NOTHING is present (not git,
|
||||||
# make, or go). golangci-lint is packaged in nix, brew, and apk; on apt
|
# make, or go). golangci-lint is packaged in nix, brew, and apk; on apt
|
||||||
# it is installed from a hash-verified GitHub release archive (never
|
# it is installed from a hash-verified GitHub release archive (never
|
||||||
# curl | sh).
|
# curl | sh). Finishes by running script/fetch-assets, which installs the
|
||||||
|
# hash-pinned third-party browser assets the repo does not commit.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
@@ -115,6 +116,11 @@ main() {
|
|||||||
|
|
||||||
go mod download
|
go mod download
|
||||||
|
|
||||||
|
# Third-party browser assets are not committed; fetch and verify them
|
||||||
|
# so a fresh clone can build and test.
|
||||||
|
if missing curl; then pkg_install curl curl curl curl; fi
|
||||||
|
"$ROOT/script/fetch-assets"
|
||||||
|
|
||||||
echo "bootstrap complete"
|
echo "bootstrap complete"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
104
script/fetch-assets
Executable file
104
script/fetch-assets
Executable file
@@ -0,0 +1,104 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/fetch-assets: download the third-party browser assets the web UI
|
||||||
|
# ships and install them under static/. Minified bundles are not committed
|
||||||
|
# (REPO_POLICIES.md: no build artifacts in version control), so the build
|
||||||
|
# fetches them here. Every download is verified against a hardcoded sha256
|
||||||
|
# before it is installed, and any mismatch aborts. Idempotent: an asset
|
||||||
|
# already present with its pinned hash is left alone.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
# The sha256 of each installed asset lives in static/vendor.sha256, in
|
||||||
|
# sha256sum(1) format, with paths relative to static/. That file is the
|
||||||
|
# single source of truth: this script verifies against it, and
|
||||||
|
# static/vendor_test.go asserts the bytes embedded into the binary match
|
||||||
|
# it, so the hash cannot rot into a value nothing checks.
|
||||||
|
MANIFEST="static/vendor.sha256"
|
||||||
|
|
||||||
|
# Alpine.js 3.14.9, 2026-08-17. Fetched from registry.npmjs.org, the
|
||||||
|
# publisher of record; the jsDelivr and unpkg copies are mirrors of this
|
||||||
|
# same tarball. dist/cdn.min.js is the browser build Alpine publishes for
|
||||||
|
# a <script> tag.
|
||||||
|
ALPINE_VERSION="3.14.9"
|
||||||
|
ALPINE_URL="https://registry.npmjs.org/alpinejs/-/alpinejs-${ALPINE_VERSION}.tgz"
|
||||||
|
# sha256 of alpinejs-3.14.9.tgz
|
||||||
|
ALPINE_TARBALL_SHA256="97dad7c0c81e659cfc8e7700055da9770f8186187cb9a8a76efb57e00d5ce52a"
|
||||||
|
ALPINE_MEMBER="package/dist/cdn.min.js"
|
||||||
|
ALPINE_DEST="js/alpine.min.js"
|
||||||
|
|
||||||
|
sha256_of() {
|
||||||
|
if command -v sha256sum >/dev/null 2>&1; then
|
||||||
|
sha256sum "$1" | cut -d' ' -f1
|
||||||
|
else
|
||||||
|
shasum -a 256 "$1" | cut -d' ' -f1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# expected_sha256 <path-relative-to-static>
|
||||||
|
expected_sha256() {
|
||||||
|
awk -v want="$1" '$2 == want { print $1; found = 1 }
|
||||||
|
END { if (!found) exit 1 }' "$ROOT/$MANIFEST"
|
||||||
|
}
|
||||||
|
|
||||||
|
# verify <file> <expected-sha256> <what>
|
||||||
|
verify() {
|
||||||
|
actual="$(sha256_of "$1")"
|
||||||
|
if [ "$actual" != "$2" ]; then
|
||||||
|
echo "fetch-assets: sha256 mismatch for $3" >&2
|
||||||
|
echo " expected: $2" >&2
|
||||||
|
echo " actual: $actual" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# up_to_date <path-relative-to-static> <expected-sha256>
|
||||||
|
up_to_date() {
|
||||||
|
[ -f "$ROOT/static/$1" ] || return 1
|
||||||
|
[ "$(sha256_of "$ROOT/static/$1")" = "$2" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
fetch_alpine() {
|
||||||
|
want="$(expected_sha256 "$ALPINE_DEST")"
|
||||||
|
|
||||||
|
if up_to_date "$ALPINE_DEST" "$want"; then
|
||||||
|
echo "fetch-assets: static/$ALPINE_DEST already at $want"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "fetch-assets: fetching Alpine.js $ALPINE_VERSION from $ALPINE_URL"
|
||||||
|
tmp="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$tmp"' EXIT INT TERM
|
||||||
|
curl -fsSL -o "$tmp/alpine.tgz" "$ALPINE_URL"
|
||||||
|
verify "$tmp/alpine.tgz" "$ALPINE_TARBALL_SHA256" "alpinejs-${ALPINE_VERSION}.tgz"
|
||||||
|
tar -xzOf "$tmp/alpine.tgz" "$ALPINE_MEMBER" >"$tmp/alpine.min.js"
|
||||||
|
verify "$tmp/alpine.min.js" "$want" "$ALPINE_MEMBER from alpinejs-${ALPINE_VERSION}.tgz"
|
||||||
|
|
||||||
|
mkdir -p "$(dirname "$ROOT/static/$ALPINE_DEST")"
|
||||||
|
cp "$tmp/alpine.min.js" "$ROOT/static/$ALPINE_DEST"
|
||||||
|
rm -rf "$tmp"
|
||||||
|
trap - EXIT INT TERM
|
||||||
|
echo "fetch-assets: installed static/$ALPINE_DEST ($want)"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Re-check every manifest entry against what is now on disk, so an entry
|
||||||
|
# no script installs fails loudly instead of passing silently.
|
||||||
|
verify_manifest() {
|
||||||
|
while read -r want path; do
|
||||||
|
case "$want" in '' | '#'*) continue ;; esac
|
||||||
|
if [ ! -f "$ROOT/static/$path" ]; then
|
||||||
|
echo "fetch-assets: $MANIFEST lists static/$path, which is missing" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
verify "$ROOT/static/$path" "$want" "static/$path"
|
||||||
|
done <"$ROOT/$MANIFEST"
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
fetch_alpine
|
||||||
|
verify_manifest
|
||||||
|
echo "fetch-assets: all assets in $MANIFEST verified"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
5
static/js/alpine.min.js
vendored
5
static/js/alpine.min.js
vendored
File diff suppressed because one or more lines are too long
1
static/vendor.sha256
Normal file
1
static/vendor.sha256
Normal file
@@ -0,0 +1 @@
|
|||||||
|
3ed1eed252488921df65e363d6715deb04d7f92aaedb9e52199fdf73cb1e0ad3 js/alpine.min.js
|
||||||
92
static/vendor_test.go
Normal file
92
static/vendor_test.go
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
package static_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
|
||||||
|
"sneak.berlin/go/webhooker/static"
|
||||||
|
)
|
||||||
|
|
||||||
|
const manifestPath = "vendor.sha256"
|
||||||
|
|
||||||
|
// fetchHint is appended to every failure here: the assets the manifest
|
||||||
|
// covers are fetched by the build, not committed, so a fresh clone that
|
||||||
|
// has not run script/fetch-assets fails this test and should be told why.
|
||||||
|
const fetchHint = "run `script/fetch-assets` (or `make assets`) to install " +
|
||||||
|
"the pinned third-party assets"
|
||||||
|
|
||||||
|
// TestVendoredAssetsMatchManifest asserts that every asset listed in
|
||||||
|
// static/vendor.sha256 is embedded in the binary with exactly the pinned
|
||||||
|
// bytes. script/fetch-assets verifies the same hashes at download time;
|
||||||
|
// this test verifies them again on what actually ships, so a build that
|
||||||
|
// skipped, cached, or subverted the fetch cannot produce a binary serving
|
||||||
|
// unpinned third-party JavaScript.
|
||||||
|
func TestVendoredAssetsMatchManifest(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
entries := readManifest(t)
|
||||||
|
require.NotEmpty(t, entries, "%s lists no assets", manifestPath)
|
||||||
|
|
||||||
|
for path, want := range entries {
|
||||||
|
t.Run(path, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
data, err := static.Static.ReadFile(path)
|
||||||
|
require.NoErrorf(
|
||||||
|
t, err,
|
||||||
|
"%s is listed in %s but is not embedded; %s",
|
||||||
|
path, manifestPath, fetchHint,
|
||||||
|
)
|
||||||
|
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
got := hex.EncodeToString(sum[:])
|
||||||
|
require.Equalf(
|
||||||
|
t, want, got,
|
||||||
|
"embedded %s does not match its pinned sha256 in %s; %s",
|
||||||
|
path, manifestPath, fetchHint,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// readManifest parses static/vendor.sha256, which is in sha256sum(1)
|
||||||
|
// format with paths relative to static/.
|
||||||
|
func readManifest(t *testing.T) map[string]string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
f, err := os.Open(manifestPath)
|
||||||
|
require.NoError(t, err, "opening %s", manifestPath)
|
||||||
|
|
||||||
|
defer func() { require.NoError(t, f.Close()) }()
|
||||||
|
|
||||||
|
entries := make(map[string]string)
|
||||||
|
scanner := bufio.NewScanner(f)
|
||||||
|
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := strings.TrimSpace(scanner.Text())
|
||||||
|
if line == "" || strings.HasPrefix(line, "#") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
fields := strings.Fields(line)
|
||||||
|
require.Lenf(
|
||||||
|
t, fields, 2,
|
||||||
|
"%s: malformed entry %q, want \"<sha256> <path>\"",
|
||||||
|
manifestPath, line,
|
||||||
|
)
|
||||||
|
|
||||||
|
sum, path := fields[0], fields[1]
|
||||||
|
require.Lenf(t, sum, 64, "%s: %q is not a sha256", manifestPath, sum)
|
||||||
|
entries[path] = sum
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, scanner.Err(), "reading %s", manifestPath)
|
||||||
|
|
||||||
|
return entries
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user