Compare commits
1
Commits
next
..
033b2f2c14
| Author | SHA1 | Date | |
|---|---|---|---|
 clawbot
|
033b2f2c14 |
@@ -1,2 +0,0 @@
|
||||
node_modules/
|
||||
yarn.lock
|
||||
@@ -1,4 +0,0 @@
|
||||
{
|
||||
"tabWidth": 4,
|
||||
"proseWrap": "always"
|
||||
}
|
||||
+3
-19
@@ -47,9 +47,7 @@ FROM golang:1.26.4-alpine@sha256:3ad57304ad93bbec8548a0437ad9e06a455660655d9af01
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
COPY --from=test /src/go.sum /dev/null
|
||||
|
||||
RUN apk add --no-cache git
|
||||
# A tar-stream context keeps the sender's file owners, which git refuses.
|
||||
RUN git config --system --add safe.directory /src
|
||||
ARG VERSION=dev
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
@@ -60,22 +58,8 @@ RUN go mod download
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Build (pure Go, no CGO required since we use modernc.org/sqlite).
|
||||
# The VERSION build arg when one is given, otherwise
|
||||
# `git describe --tags --always` on the .git in the build context. With
|
||||
# .git present, a version that is still empty, dev or unknown fails the
|
||||
# build: git is missing or could not read the checkout.
|
||||
ARG VERSION
|
||||
RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \
|
||||
if [ -e .git ]; then \
|
||||
case "$VERSION" in ""|dev|unknown) \
|
||||
echo "version is '$VERSION' although .git is present" >&2; \
|
||||
exit 1 ;; \
|
||||
esac; \
|
||||
fi; \
|
||||
CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
||||
-o /bsdaily ./cmd/bsdaily
|
||||
# Build (pure Go, no CGO required since we use modernc.org/sqlite)
|
||||
RUN CGO_ENABLED=0 go build -o /bsdaily ./cmd/bsdaily
|
||||
|
||||
# Runtime stage
|
||||
# alpine:3.21, 2026-06-28
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
.PHONY: all bootstrap setup check test lint fmt fmt-check build clean deps test-coverage test-integration install release release-snapshot docker hooks
|
||||
|
||||
# Stamped into the binary: the same `git describe` a plain `docker build .`
|
||||
# runs, or dev when it prints nothing (outside a git checkout, or where git is
|
||||
# missing). ?= so that a VERSION already in the environment takes precedence.
|
||||
VERSION ?= $(or $(shell git describe --tags --always 2>/dev/null),dev)
|
||||
# Version number
|
||||
VERSION := 0.1.0-dev
|
||||
|
||||
# Default target
|
||||
all: bsdaily
|
||||
@@ -38,7 +36,7 @@ lint:
|
||||
|
||||
# Build binary (pure Go; no CGO required since we use modernc.org/sqlite).
|
||||
bsdaily: internal/*/*.go cmd/bsdaily/*.go
|
||||
CGO_ENABLED=0 go build -ldflags "-X main.Version=$(VERSION)" -o $@ ./cmd/bsdaily
|
||||
CGO_ENABLED=0 go build -o $@ ./cmd/bsdaily
|
||||
|
||||
# Clean build artifacts.
|
||||
clean:
|
||||
|
||||
@@ -1,26 +1,28 @@
|
||||
# bsdaily
|
||||
|
||||
[bsdaily](https://git.eeqj.de/sneak/bsdaily) is a command-line utility written
|
||||
in [Go](https://golang.org) that carves a single day (or a range of days) of
|
||||
[Bluesky](https://bsky.app) firehose data out of a large, continuously-growing
|
||||
SQLite database and writes it out as a self-contained,
|
||||
[bsdaily](https://git.eeqj.de/sneak/bsdaily) is a command-line utility
|
||||
written in [Go](https://golang.org) that carves a single day (or a range of
|
||||
days) of [Bluesky](https://bsky.app) firehose data out of a large,
|
||||
continuously-growing SQLite database and writes it out as a self-contained,
|
||||
[zstd](https://facebook.github.io/zstd/)-compressed SQL dump. The dumps are
|
||||
named by date (e.g. `2026-06-27.sql.zst`), organized into per-month directories,
|
||||
and are designed to be published, archived, mirrored, and later re-merged back
|
||||
into a single database.
|
||||
named by date (e.g. `2026-06-27.sql.zst`), organized into per-month
|
||||
directories, and are designed to be published, archived, mirrored, and later
|
||||
re-merged back into a single database.
|
||||
|
||||
The source database is read from a read-only [ZFS](https://openzfs.org)
|
||||
snapshot, so extraction never contends with the live firehose ingester that is
|
||||
writing to the original database. The tool is operationally conservative: it
|
||||
checks free disk space before starting, copies the snapshot to fast scratch
|
||||
storage, processes one day at a time to avoid SQLite lock contention, verifies
|
||||
every compressed output before publishing it, and writes output atomically via a
|
||||
temp-file-and-rename so a partial run never leaves a corrupt `.sql.zst` behind.
|
||||
snapshot, so extraction never contends with the live firehose ingester that
|
||||
is writing to the original database. The tool is operationally
|
||||
conservative: it checks free disk space before starting, copies the snapshot
|
||||
to fast scratch storage, processes one day at a time to avoid SQLite lock
|
||||
contention, verifies every compressed output before publishing it, and
|
||||
writes output atomically via a temp-file-and-rename so a partial run never
|
||||
leaves a corrupt `.sql.zst` behind.
|
||||
|
||||
This project was written by [@sneak](https://sneak.berlin) to produce a daily,
|
||||
mergeable, publicly-mirrorable archive of the Bluesky firehose. It is currently
|
||||
a one-person effort. The current version is pre-1.0 and there has not yet been a
|
||||
versioned release; [SemVer](https://semver.org) will be used for releases.
|
||||
This project was written by [@sneak](https://sneak.berlin) to produce a
|
||||
daily, mergeable, publicly-mirrorable archive of the Bluesky firehose. It is
|
||||
currently a one-person effort. The current version is pre-1.0 and there has
|
||||
not yet been a versioned release; [SemVer](https://semver.org) will be used
|
||||
for releases.
|
||||
|
||||
# Build Status
|
||||
|
||||
@@ -31,14 +33,14 @@ branch must always be green.
|
||||
|
||||
Primary development happens on a privately-run Gitea instance at
|
||||
[https://git.eeqj.de/sneak/bsdaily](https://git.eeqj.de/sneak/bsdaily) and
|
||||
issues are [tracked there](https://git.eeqj.de/sneak/bsdaily/issues).
|
||||
issues are [tracked
|
||||
there](https://git.eeqj.de/sneak/bsdaily/issues).
|
||||
|
||||
Changes must always be formatted with `make fmt` (`go fmt` for Go, prettier for
|
||||
Markdown), syntactically valid, and must pass the linting defined in the
|
||||
repository (the shared `.golangci.yml` from `sneak/prompts`), which can be run
|
||||
with a `make lint`. The `main` branch is protected and all changes must be made
|
||||
via [pull requests](https://git.eeqj.de/sneak/bsdaily/pulls) and pass CI to be
|
||||
merged.
|
||||
Changes must always be formatted with a standard `go fmt`, syntactically
|
||||
valid, and must pass the linting defined in the repository (the shared
|
||||
`.golangci.yml` from `sneak/prompts`), which can be run with a `make lint`.
|
||||
The `main` branch is protected and all changes must be made via [pull
|
||||
requests](https://git.eeqj.de/sneak/bsdaily/pulls) and pass CI to be merged.
|
||||
|
||||
See [`REPO_POLICIES.md`](REPO_POLICIES.md) for detailed coding standards,
|
||||
tooling requirements, and workflow conventions.
|
||||
@@ -48,55 +50,55 @@ tooling requirements, and workflow conventions.
|
||||
This repository adheres to the
|
||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||
standard: normalized scripts in `script/` are the entrypoints for the
|
||||
development workflow, and the Makefile targets are thin shims that call them. We
|
||||
provide:
|
||||
development workflow, and the Makefile targets are thin shims that call
|
||||
them. We provide:
|
||||
|
||||
- `script/bootstrap` — install all development dependencies (go, Go module
|
||||
download, node and yarn at pinned versions when absent, and the prettier
|
||||
pinned in `package.json` and `yarn.lock`); a node or yarn already installed is
|
||||
used whatever its version; the linter is not installed on the host
|
||||
- `script/bootstrap` — install all development dependencies (go, Go
|
||||
module download); the linter is not installed on the host
|
||||
- `script/setup` — make a fresh clone ready for development: runs
|
||||
`script/bootstrap`, then `script/install-precommit`
|
||||
- `script/projectname` — print the project name (used for the Docker image tags)
|
||||
- `script/test` — build the Dockerfile's `test` phase, which runs the test suite
|
||||
with `-race` (verbose rerun on failure)
|
||||
- `script/projectname` — print the project name (used for the Docker
|
||||
image tags)
|
||||
- `script/test` — build the Dockerfile's `test` phase, which runs the
|
||||
test suite with `-race` (verbose rerun on failure)
|
||||
- `script/lint` — build the Dockerfile's `lint` phase, which runs
|
||||
`golangci-lint run ./...`
|
||||
- `script/fmt` — format the Go code with `go fmt` and every Markdown file with
|
||||
prettier (writes)
|
||||
- `script/fmt-check` — check the Go formatting with `gofmt` and the Markdown
|
||||
formatting with prettier (read-only)
|
||||
- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`
|
||||
- `script/docker` — build the Docker image tagged via `script/projectname`; the
|
||||
build runs the `lint` and `test` phases first
|
||||
- `script/cibuild` — CI entrypoint: runs `script/bootstrap` and `script/check`,
|
||||
then builds the Docker image tagged via `script/projectname`
|
||||
- `script/precommit` — pre-commit gate: `go mod tidy` (must not change `go.mod`
|
||||
or `go.sum`) and `go fmt`, then `script/check`
|
||||
- `script/install-precommit` — install the git pre-commit hook that runs
|
||||
`script/precommit`
|
||||
- `script/fmt` — format the Go code with `go fmt` (writes)
|
||||
- `script/fmt-check` — check the Go formatting with `gofmt` (read-only)
|
||||
- `script/check` — run `script/test`, `script/lint`, and
|
||||
`script/fmt-check`
|
||||
- `script/docker` — build the Docker image tagged via
|
||||
`script/projectname`; the build runs the `lint` and `test` phases
|
||||
first
|
||||
- `script/cibuild` — CI entrypoint: runs `script/bootstrap` and
|
||||
`script/check`, then builds the Docker image tagged via
|
||||
`script/projectname`
|
||||
- `script/precommit` — pre-commit gate: `go mod tidy` (must not change
|
||||
`go.mod` or `go.sum`) and `go fmt`, then `script/check`
|
||||
- `script/install-precommit` — install the git pre-commit hook that
|
||||
runs `script/precommit`
|
||||
|
||||
Every Docker build in `script/` is uncached, so the `lint` and `test` phases
|
||||
always run rather than being served from the build cache.
|
||||
Every Docker build in `script/` is uncached, so the `lint` and `test`
|
||||
phases always run rather than being served from the build cache.
|
||||
|
||||
# Problem Statement
|
||||
|
||||
A Bluesky firehose ingester writes every observed post (and associated users,
|
||||
hashtags, URLs, and media references) into a single ever-growing SQLite
|
||||
database, `firehose.db`. This database has several properties that make it
|
||||
awkward to publish or archive directly:
|
||||
A Bluesky firehose ingester writes every observed post (and associated
|
||||
users, hashtags, URLs, and media references) into a single ever-growing
|
||||
SQLite database, `firehose.db`. This database has several properties that
|
||||
make it awkward to publish or archive directly:
|
||||
|
||||
- It is **large and always growing**, so re-publishing the whole thing every day
|
||||
is wasteful.
|
||||
- It is **large and always growing**, so re-publishing the whole thing every
|
||||
day is wasteful.
|
||||
- It is **continuously written**, so reading from it directly risks lock
|
||||
contention with the live ingester and inconsistent reads.
|
||||
- It is **monolithic**, so there is no natural unit at which to mirror, share,
|
||||
or distribute "just yesterday's posts".
|
||||
- It is **monolithic**, so there is no natural unit at which to mirror,
|
||||
share, or distribute "just yesterday's posts".
|
||||
|
||||
What is wanted instead is a stable, immutable, per-day artifact: a small file
|
||||
containing exactly one calendar day of firehose data, cheap to publish, cheap to
|
||||
mirror, and trivially re-mergeable into a full database by anyone who collects a
|
||||
set of them.
|
||||
What is wanted instead is a stable, immutable, per-day artifact: a small
|
||||
file containing exactly one calendar day of firehose data, cheap to publish,
|
||||
cheap to mirror, and trivially re-mergeable into a full database by anyone
|
||||
who collects a set of them.
|
||||
|
||||
# Proposed Solution
|
||||
|
||||
@@ -106,86 +108,81 @@ A tool, `bsdaily`, that:
|
||||
filesystem, so it reads from a consistent point-in-time copy that the live
|
||||
ingester cannot be writing to;
|
||||
- copies the snapshot's database files to fast scratch storage;
|
||||
- **extracts** a single day's `posts` (and all rows reachable from them) into a
|
||||
fresh, minimal per-day SQLite database;
|
||||
- **dumps** that per-day database to SQL and pipes it through multithreaded zstd
|
||||
compression;
|
||||
- **verifies** the compressed output (zstd integrity check plus a sanity check
|
||||
that the decompressed stream actually looks like SQL);
|
||||
- **extracts** a single day's `posts` (and all rows reachable from them) into
|
||||
a fresh, minimal per-day SQLite database;
|
||||
- **dumps** that per-day database to SQL and pipes it through multithreaded
|
||||
zstd compression;
|
||||
- **verifies** the compressed output (zstd integrity check plus a sanity
|
||||
check that the decompressed stream actually looks like SQL);
|
||||
- **publishes** the result atomically as
|
||||
`DailiesBase/YYYY-MM/YYYY-MM-DD.sql.zst`.
|
||||
|
||||
Each daily dump is emitted with `INSERT` statements over the full schema
|
||||
(including the deduplicated `users`, `hashtags`, and `urls` lookup tables), so
|
||||
any collection of daily dumps can be merged into a single database by rewriting
|
||||
`INSERT INTO` to `INSERT OR IGNORE INTO` and replaying them in sequence. Two
|
||||
helper scripts ([`merge_daily_dumps.sh`](merge_daily_dumps.sh) and
|
||||
[`regenerate_auxiliary_tables.sql`](regenerate_auxiliary_tables.sql)) are
|
||||
(including the deduplicated `users`, `hashtags`, and `urls` lookup tables),
|
||||
so any collection of daily dumps can be merged into a single database by
|
||||
rewriting `INSERT INTO` to `INSERT OR IGNORE INTO` and replaying them in
|
||||
sequence. Two helper scripts ([`merge_daily_dumps.sh`](merge_daily_dumps.sh)
|
||||
and [`regenerate_auxiliary_tables.sql`](regenerate_auxiliary_tables.sql)) are
|
||||
included to do exactly this and to rebuild the aggregate statistics
|
||||
(`use_count`, `first_seen`, user `resolved_at`/`updated_at`) afterward.
|
||||
|
||||
# Design Goals
|
||||
|
||||
- **Never disturb the live ingester.** All reads come from a ZFS snapshot, never
|
||||
the live database.
|
||||
- **Never disturb the live ingester.** All reads come from a ZFS snapshot,
|
||||
never the live database.
|
||||
- **Crash-safe, idempotent runs.** Output is written to a temp file and
|
||||
atomically renamed; a day whose final output already exists is skipped, so
|
||||
re-running a range is safe and resumable.
|
||||
- **Mergeable output.** Daily dumps re-combine losslessly into a full database
|
||||
via `INSERT OR IGNORE`.
|
||||
- **Mergeable output.** Daily dumps re-combine losslessly into a full
|
||||
database via `INSERT OR IGNORE`.
|
||||
- **Operationally cautious.** Free-space preflight checks on both scratch and
|
||||
output filesystems; explicit verification of every artifact before it is
|
||||
published.
|
||||
- **Fast where it's free.** Large snapshot copies use a 256MiB buffer,
|
||||
pre-allocate the destination, and (on Linux) issue `posix_fadvise`
|
||||
sequential/willneed hints; extraction uses aggressive, crash-unsafe-by-design
|
||||
SQLite pragmas because the working data lives only in disposable scratch
|
||||
space.
|
||||
sequential/willneed hints; extraction uses aggressive,
|
||||
crash-unsafe-by-design SQLite pragmas because the working data lives only
|
||||
in disposable scratch space.
|
||||
|
||||
# Non-Goals
|
||||
|
||||
- **Real-time export.** `bsdaily` operates on daily snapshots; the freshest day
|
||||
it can produce is the snapshot date minus one.
|
||||
- **Schema ownership.** The schema is defined by the upstream firehose ingester;
|
||||
[`schema.sql`](schema.sql) is included for reference only. `bsdaily` copies
|
||||
whatever table and index DDL it finds in the source.
|
||||
- **Cross-platform deployment.** It is built and run on Linux (the free-space
|
||||
check and fadvise hints use `golang.org/x/sys/unix`; a non-Linux build
|
||||
compiles but is a no-op for the fadvise hints). The hard-coded paths assume
|
||||
the production host's ZFS layout.
|
||||
- **Real-time export.** `bsdaily` operates on daily snapshots; the freshest
|
||||
day it can produce is the snapshot date minus one.
|
||||
- **Schema ownership.** The schema is defined by the upstream firehose
|
||||
ingester; [`schema.sql`](schema.sql) is included for reference only.
|
||||
`bsdaily` copies whatever table and index DDL it finds in the source.
|
||||
- **Cross-platform deployment.** It is built and run on Linux (the
|
||||
free-space check and fadvise hints use `golang.org/x/sys/unix`; a non-Linux
|
||||
build compiles but is a no-op for the fadvise hints). The hard-coded paths
|
||||
assume the production host's ZFS layout.
|
||||
|
||||
# How It Works
|
||||
|
||||
`cmd/bsdaily/main.go` only passes the version to `internal/cli` and exits with
|
||||
the status it returns. `internal/cli` holds the command line: the flags, the
|
||||
rules for combining them and the parsing of the dates they name.
|
||||
`internal/bsdaily` does the extraction.
|
||||
|
||||
A single run proceeds as follows:
|
||||
|
||||
1. **Find the snapshot.** Scan `SnapshotBase` for directories matching
|
||||
`zfs-auto-snap_daily-YYYY-MM-DD-NNNN`, pick the most recent, and confirm it
|
||||
contains `firehose.db`.
|
||||
2. **Determine target days.** Default to the snapshot date minus one day; or use
|
||||
`--date`, or every day in the inclusive `--from`/`--to` range.
|
||||
`zfs-auto-snap_daily-YYYY-MM-DD-NNNN`, pick the most recent, and confirm
|
||||
it contains `firehose.db`.
|
||||
2. **Determine target days.** Default to the snapshot date minus one day;
|
||||
or use `--date`, or every day in the inclusive `--from`/`--to` range.
|
||||
3. **Preflight disk space.** Require at least 500GiB free on the scratch
|
||||
filesystem and 20GiB free on the output filesystem.
|
||||
4. **Copy the database to scratch.** Copy `firehose.db`, its `-wal`, and (if
|
||||
present) its `-shm` from the snapshot into a fresh temp directory under
|
||||
`TmpBase`.
|
||||
5. **Per day**, processed strictly one at a time to avoid SQLite contention:
|
||||
- skip the day if its final output file already exists;
|
||||
- `ATTACH` the copied source DB to a new empty per-day DB, recreate the
|
||||
table DDL, and `INSERT ... SELECT` the target day's `posts` plus all rows
|
||||
reachable from them (`posts_hashtags`, `posts_urls`, `hashtags`, `urls`,
|
||||
`users`, and `media` if that table exists);
|
||||
- abort the day cleanly if there are zero posts (`ErrNoPosts`), rather than
|
||||
emitting an empty dump;
|
||||
- recreate indexes, detach the source, and verify the inserted row count;
|
||||
- `sqlite3 .dump | zstdmt` into a hidden temp file;
|
||||
- run a zstd integrity check and confirm the decompressed head looks like
|
||||
SQL;
|
||||
- atomically rename into place and delete the per-day scratch DB.
|
||||
- skip the day if its final output file already exists;
|
||||
- `ATTACH` the copied source DB to a new empty per-day DB, recreate the
|
||||
table DDL, and `INSERT ... SELECT` the target day's `posts` plus all
|
||||
rows reachable from them (`posts_hashtags`, `posts_urls`, `hashtags`,
|
||||
`urls`, `users`, and `media` if that table exists);
|
||||
- abort the day cleanly if there are zero posts (`ErrNoPosts`), rather
|
||||
than emitting an empty dump;
|
||||
- recreate indexes, detach the source, and verify the inserted row count;
|
||||
- `sqlite3 .dump | zstdmt` into a hidden temp file;
|
||||
- run a zstd integrity check and confirm the decompressed head looks like
|
||||
SQL;
|
||||
- atomically rename into place and delete the per-day scratch DB.
|
||||
6. **Clean up** the temp directory and log a processed/skipped/total summary.
|
||||
|
||||
# Usage
|
||||
@@ -194,7 +191,6 @@ A single run proceeds as follows:
|
||||
bsdaily # extract the snapshot date minus one day
|
||||
bsdaily --date 2026-06-27 # extract a single specific day
|
||||
bsdaily --from 2026-06-01 --to 2026-06-27 # extract an inclusive range
|
||||
bsdaily --version # print the version and exit
|
||||
```
|
||||
|
||||
Flags:
|
||||
@@ -203,24 +199,9 @@ Flags:
|
||||
`--from`/`--to`.
|
||||
- `--from YYYY-MM-DD` — start of an inclusive range (requires `--to`).
|
||||
- `--to YYYY-MM-DD` — end of an inclusive range (requires `--from`).
|
||||
- `-v`, `--version` — print the version and exit.
|
||||
|
||||
With no flags, the tool extracts the day before the latest snapshot. All
|
||||
progress is logged as structured `slog` text to stderr; the first line of every
|
||||
run carries the version.
|
||||
|
||||
The version is set at link time and depends on how the binary was built:
|
||||
|
||||
- `docker build .` takes it from the `VERSION` build argument when one is given,
|
||||
otherwise from `git describe --tags --always` on the `.git` in the build
|
||||
context. The build fails if `.git` is there and the version still comes out
|
||||
empty, `dev` or `unknown`. With neither `.git` nor `VERSION`, the binary
|
||||
reports `dev`.
|
||||
- `script/docker`, `script/cibuild` and `make docker` pass the host's
|
||||
`git describe --tags --always --dirty` as `VERSION`, so on a modified tree the
|
||||
version ends in `-dirty`. When that prints nothing, they pass `unknown`.
|
||||
- `make` stamps the host's `git describe --tags --always`, without `-dirty`, or
|
||||
`dev` when that prints nothing.
|
||||
progress is logged as structured `slog` text to stderr.
|
||||
|
||||
## Merging dumps back into a database
|
||||
|
||||
@@ -238,9 +219,9 @@ sqlite3 merged.db < regenerate_auxiliary_tables.sql
|
||||
- **Go** (see [`go.mod`](go.mod) for the toolchain version) to build.
|
||||
- **Linux** for production use (ZFS snapshots, `statfs` free-space checks,
|
||||
`posix_fadvise` hints).
|
||||
- The **`sqlite3`** and **`zstdmt`** (multithreaded zstd) binaries on `PATH`;
|
||||
`zstdcat` is used for verification. SQLite reads/writes during extraction use
|
||||
the pure-Go [`modernc.org/sqlite`](https://pkg.go.dev/modernc.org/sqlite)
|
||||
- The **`sqlite3`** and **`zstdmt`** (multithreaded zstd) binaries on
|
||||
`PATH`; `zstdcat` is used for verification. SQLite reads/writes during
|
||||
extraction use the pure-Go [`modernc.org/sqlite`](https://pkg.go.dev/modernc.org/sqlite)
|
||||
driver, so no cgo is required for that part.
|
||||
|
||||
# Configuration
|
||||
@@ -260,21 +241,21 @@ elsewhere.
|
||||
|
||||
# Data Model
|
||||
|
||||
The firehose schema (reference copy in [`schema.sql`](schema.sql)) centers on a
|
||||
`posts` table, with `users` keyed by DID and many-to-many junction tables
|
||||
linking posts to deduplicated `hashtags` and `urls`. An optional `media` table
|
||||
tracks downloaded blobs by content hash. `bsdaily` does not own this schema; it
|
||||
reflects whatever DDL exists in the source snapshot and selects forward from
|
||||
`posts` along the foreign-key relationships to produce a referentially-complete
|
||||
per-day slice.
|
||||
The firehose schema (reference copy in [`schema.sql`](schema.sql)) centers
|
||||
on a `posts` table, with `users` keyed by DID and many-to-many junction
|
||||
tables linking posts to deduplicated `hashtags` and `urls`. An optional
|
||||
`media` table tracks downloaded blobs by content hash. `bsdaily` does not
|
||||
own this schema; it reflects whatever DDL exists in the source snapshot and
|
||||
selects forward from `posts` along the foreign-key relationships to produce a
|
||||
referentially-complete per-day slice.
|
||||
|
||||
# Use Cases
|
||||
|
||||
## Daily public archive
|
||||
|
||||
Publish one small, immutable file per day to static HTTP (or IPFS, or a mirror
|
||||
network) so that anyone can fetch exactly the day(s) they want and re-merge them
|
||||
locally.
|
||||
Publish one small, immutable file per day to static HTTP (or IPFS, or a
|
||||
mirror network) so that anyone can fetch exactly the day(s) they want and
|
||||
re-merge them locally.
|
||||
|
||||
## Backfilling a range
|
||||
|
||||
@@ -284,17 +265,16 @@ safe to re-run.
|
||||
|
||||
## Reconstituting a full database
|
||||
|
||||
Collect any set of daily dumps and merge them with `INSERT OR IGNORE` to rebuild
|
||||
a complete, queryable SQLite database, then regenerate the aggregate statistics
|
||||
tables.
|
||||
Collect any set of daily dumps and merge them with `INSERT OR IGNORE` to
|
||||
rebuild a complete, queryable SQLite database, then regenerate the aggregate
|
||||
statistics tables.
|
||||
|
||||
# See Also
|
||||
|
||||
## Links
|
||||
|
||||
- Repo: [https://git.eeqj.de/sneak/bsdaily](https://git.eeqj.de/sneak/bsdaily)
|
||||
- Issues:
|
||||
[https://git.eeqj.de/sneak/bsdaily/issues](https://git.eeqj.de/sneak/bsdaily/issues)
|
||||
- Issues: [https://git.eeqj.de/sneak/bsdaily/issues](https://git.eeqj.de/sneak/bsdaily/issues)
|
||||
- Bluesky: [https://bsky.app](https://bsky.app)
|
||||
- zstd: [https://facebook.github.io/zstd/](https://facebook.github.io/zstd/)
|
||||
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
# Workflow
|
||||
|
||||
- branch (from `main`)
|
||||
- do the work in Next Step
|
||||
- move Next Step to the top of Completed Steps
|
||||
- move the top item of Future Steps into Next Step
|
||||
- commit (`TODO.md` changes in the same commit as the work)
|
||||
- merge to `main` if the branch is not protected, otherwise open a PR
|
||||
- push
|
||||
* branch (from `main`)
|
||||
* do the work in Next Step
|
||||
* move Next Step to the top of Completed Steps
|
||||
* move the top item of Future Steps into Next Step
|
||||
* commit (`TODO.md` changes in the same commit as the work)
|
||||
* merge to `main` if the branch is not protected, otherwise open a PR
|
||||
* push
|
||||
|
||||
# Status
|
||||
|
||||
@@ -14,46 +14,35 @@ pre-1.0
|
||||
|
||||
# Next Step
|
||||
|
||||
Expand the `internal/bsdaily` tests beyond the compilation smoke test: unit
|
||||
tests for the extraction, verification, and atomic-publish paths.
|
||||
Format Markdown with prettier in `script/fmt` and `script/fmt-check`
|
||||
(https://git.eeqj.de/sneak/bsdaily/issues/7).
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-06: Moved the command line (flags, the rules for combining them, date
|
||||
parsing) from `cmd/bsdaily` into `internal/cli`, with unit tests for the flag
|
||||
rules and the dates; `cmd/bsdaily/main.go` is now a single call into it
|
||||
(https://git.eeqj.de/sneak/bsdaily/issues/8).
|
||||
- 2026-10-06: A plain `docker build .` and a host `make` build stamp the git tag
|
||||
or short commit into the binary, which `bsdaily` logs on the first line of
|
||||
every run and prints with `--version`
|
||||
(https://git.eeqj.de/sneak/bsdaily/issues/4).
|
||||
- 2026-10-06: Added the canonical `.golangci.yml`, moved the lint phase to
|
||||
golangci-lint v2.14.0, and fixed the code to pass it
|
||||
(https://git.eeqj.de/sneak/bsdaily/issues/6).
|
||||
- 2026-10-06: Formatted Markdown with prettier: `script/fmt` writes and
|
||||
`script/fmt-check` checks every Markdown file; prettier pinned in
|
||||
`package.json` and `yarn.lock`, installed by `script/bootstrap`, which
|
||||
installs node and yarn at pinned versions when absent and otherwise uses the
|
||||
ones already installed; existing Markdown reformatted.
|
||||
- 2026-10-05: Brought the repo up to the standard layout: canonical
|
||||
`.gitignore`, `.dockerignore` and `.editorconfig`; `lint` and `test` phases in
|
||||
the `Dockerfile`, built by `script/lint` and `script/test`; canonical
|
||||
`script/cibuild`, `script/docker` and CI workflow; no linter installed on the
|
||||
host; re-vendored `REPO_POLICIES.md`.
|
||||
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile
|
||||
shims, README Entrypoints section
|
||||
- 2026-06-28: Fixed errcheck lint failures; added compilation smoke test; tidied
|
||||
go.mod.
|
||||
- 2026-06-28: Added repo scaffolding: README, LICENSE, Makefile, Dockerfile,
|
||||
REPO_POLICIES.md, and Gitea CI.
|
||||
- 2026-02-12: Fixed SQLite database locking by removing parallel processing;
|
||||
fixed Linux build via golang.org/x/sys/unix Fadvise.
|
||||
- 2026-02-12: Optimized file copy for large databases; moved temp directory to
|
||||
NVMe scratch storage.
|
||||
`.gitignore`, `.dockerignore` and `.editorconfig`; `lint` and `test`
|
||||
phases in the `Dockerfile`, built by `script/lint` and `script/test`;
|
||||
canonical `script/cibuild`, `script/docker` and CI workflow; no linter
|
||||
installed on the host; re-vendored `REPO_POLICIES.md`.
|
||||
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
|
||||
Makefile shims, README Entrypoints section
|
||||
- 2026-06-28: Fixed errcheck lint failures; added compilation smoke test;
|
||||
tidied go.mod.
|
||||
- 2026-06-28: Added repo scaffolding: README, LICENSE, Makefile,
|
||||
Dockerfile, REPO_POLICIES.md, and Gitea CI.
|
||||
- 2026-02-12: Fixed SQLite database locking by removing parallel
|
||||
processing; fixed Linux build via golang.org/x/sys/unix Fadvise.
|
||||
- 2026-02-12: Optimized file copy for large databases; moved temp
|
||||
directory to NVMe scratch storage.
|
||||
- 2026-02-11: Added date range support.
|
||||
- 2026-02-09: Initial implementation: single-day extraction, specific-date
|
||||
targeting, faster pruning of throwaway database copies.
|
||||
|
||||
# Future Steps
|
||||
|
||||
- Expand tests beyond the compilation smoke test: unit tests for the
|
||||
extraction, verification, and atomic-publish paths.
|
||||
- Cut a first SemVer release once compliance and test coverage land.
|
||||
|
||||
+113
-8
@@ -3,18 +3,123 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/bsdaily/internal/cli"
|
||||
"git.eeqj.de/sneak/bsdaily/internal/bsdaily"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Version is the git tag or short commit, set at link time with
|
||||
// -X main.Version=... by the Dockerfile and the Makefile. A build that sets
|
||||
// nothing, or sets it empty, reports dev.
|
||||
//
|
||||
//nolint:gochecknoglobals // -X can only set a package-level variable
|
||||
var Version string
|
||||
var (
|
||||
errDateExclusive = errors.New("--date and --from/--to are mutually exclusive")
|
||||
errFromRequiresTo = errors.New("--from requires --to")
|
||||
errToRequiresFrom = errors.New("--to requires --from")
|
||||
errFromAfterTo = errors.New("--from is after --to")
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(cli.Main(Version))
|
||||
logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
|
||||
Level: slog.LevelInfo,
|
||||
}))
|
||||
slog.SetDefault(logger)
|
||||
|
||||
var dateFlag, fromFlag, toFlag string
|
||||
|
||||
rootCmd := &cobra.Command{
|
||||
Use: "bsdaily",
|
||||
Short: "Extract a single day's data from the latest daily snapshot",
|
||||
SilenceUsage: true,
|
||||
RunE: func(_ *cobra.Command, _ []string) error {
|
||||
targetDates, err := parseTargetDates(dateFlag, fromFlag, toFlag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = bsdaily.Run(targetDates)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
slog.Info("completed successfully")
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
rootCmd.Flags().StringVarP(&dateFlag, "date", "d", "",
|
||||
"target date to extract (YYYY-MM-DD); "+
|
||||
"defaults to snapshot date minus one day")
|
||||
rootCmd.Flags().StringVar(&fromFlag, "from", "",
|
||||
"start of date range to extract (YYYY-MM-DD, inclusive); use with --to")
|
||||
rootCmd.Flags().StringVar(&toFlag, "to", "",
|
||||
"end of date range to extract (YYYY-MM-DD, inclusive); use with --from")
|
||||
|
||||
err := rootCmd.Execute()
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// parseTargetDates turns the --date, --from and --to flags into the days
|
||||
// to extract. It returns nil when none of them is set, which Run takes
|
||||
// to mean the snapshot date minus one day.
|
||||
func parseTargetDates(dateFlag, fromFlag, toFlag string) ([]time.Time, error) {
|
||||
hasDate := dateFlag != ""
|
||||
hasFrom := fromFlag != ""
|
||||
hasTo := toFlag != ""
|
||||
|
||||
// Validate mutual exclusivity
|
||||
if hasDate && (hasFrom || hasTo) {
|
||||
return nil, errDateExclusive
|
||||
}
|
||||
|
||||
if hasFrom != hasTo {
|
||||
if hasFrom {
|
||||
return nil, errFromRequiresTo
|
||||
}
|
||||
|
||||
return nil, errToRequiresFrom
|
||||
}
|
||||
|
||||
if hasDate {
|
||||
t, err := time.Parse("2006-01-02", dateFlag)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"invalid --date %q (expected YYYY-MM-DD): %w", dateFlag, err)
|
||||
}
|
||||
|
||||
return []time.Time{t}, nil
|
||||
}
|
||||
|
||||
if !hasFrom {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
from, err := time.Parse("2006-01-02", fromFlag)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"invalid --from %q (expected YYYY-MM-DD): %w", fromFlag, err)
|
||||
}
|
||||
|
||||
to, err := time.Parse("2006-01-02", toFlag)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"invalid --to %q (expected YYYY-MM-DD): %w", toFlag, err)
|
||||
}
|
||||
|
||||
if from.After(to) {
|
||||
return nil, fmt.Errorf("%w (--from %s, --to %s)",
|
||||
errFromAfterTo, fromFlag, toFlag)
|
||||
}
|
||||
|
||||
var targetDates []time.Time
|
||||
|
||||
for d := from; !d.After(to); d = d.AddDate(0, 0, 1) {
|
||||
targetDates = append(targetDates, d)
|
||||
}
|
||||
|
||||
return targetDates, nil
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -25,8 +26,7 @@ func CopyFile(src, dst string) (err error) {
|
||||
|
||||
slog.Info("copying file", "src", src, "dst", dst)
|
||||
|
||||
//nolint:gosec // src is a path this package built
|
||||
srcFile, err := os.Open(src)
|
||||
srcFile, err := os.Open(filepath.Clean(src))
|
||||
if err != nil {
|
||||
return fmt.Errorf("opening source %s: %w", src, err)
|
||||
}
|
||||
@@ -48,8 +48,7 @@ func CopyFile(src, dst string) (err error) {
|
||||
applyFileAdvice(srcFile, srcInfo.Size())
|
||||
}
|
||||
|
||||
//nolint:gosec // dst is a path this package built
|
||||
dstFile, err := os.Create(dst)
|
||||
dstFile, err := os.Create(filepath.Clean(dst))
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating destination %s: %w", dst, err)
|
||||
}
|
||||
|
||||
@@ -21,8 +21,7 @@ func CheckFreeSpace(path string, minBytes uint64, label string) error {
|
||||
return fmt.Errorf("statfs %s (%s): %w", path, label, err)
|
||||
}
|
||||
|
||||
//nolint:gosec,unconvert // Bsize is never negative; Bavail is signed on FreeBSD
|
||||
free := uint64(stat.Bavail) * uint64(stat.Bsize)
|
||||
free := stat.Bavail * uint64(stat.Bsize) //nolint:gosec // Bsize is never negative
|
||||
freeGB := float64(free) / float64(bytesPerGB)
|
||||
minGB := float64(minBytes) / float64(bytesPerGB)
|
||||
|
||||
|
||||
@@ -29,8 +29,7 @@ func DumpAndCompress(dbPath, outputPath string) (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
//nolint:gosec // outputPath is a path this package built
|
||||
outFile, err := os.Create(outputPath)
|
||||
outFile, err := os.Create(filepath.Clean(outputPath))
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating output file: %w", err)
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ func ExtractDay(srcDBPath, dstDBPath string, targetDay time.Time) error {
|
||||
return fmt.Errorf("attaching source database: %w", err)
|
||||
}
|
||||
|
||||
err = createTables(ctx, db)
|
||||
err = copyTables(ctx, db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -94,9 +94,9 @@ func ExtractDay(srcDBPath, dstDBPath string, targetDay time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// createTables creates every table of the attached source database, empty,
|
||||
// copyTables creates every table of the attached source database, empty,
|
||||
// in the destination database.
|
||||
func createTables(ctx context.Context, db *sql.DB) error {
|
||||
func copyTables(ctx context.Context, db *sql.DB) error {
|
||||
// Copy table DDL from source
|
||||
slog.Info("copying table DDL from source")
|
||||
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
// Package cli is the bsdaily command line: the command, its flags, the
|
||||
// rules for combining them and the dates they name. The extraction itself
|
||||
// is in package bsdaily.
|
||||
package cli
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/bsdaily/internal/bsdaily"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Main runs the bsdaily command on the program's command-line arguments
|
||||
// and returns the status for the process to exit with. version is the
|
||||
// build's version; an empty one is reported as dev.
|
||||
func Main(version string) int {
|
||||
if version == "" {
|
||||
version = "dev"
|
||||
}
|
||||
|
||||
logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
|
||||
Level: slog.LevelInfo,
|
||||
}))
|
||||
slog.SetDefault(logger)
|
||||
|
||||
var dateFlag, fromFlag, toFlag string
|
||||
|
||||
rootCmd := &cobra.Command{
|
||||
Use: "bsdaily",
|
||||
Short: "Extract a single day's data from the latest daily snapshot",
|
||||
Version: version,
|
||||
SilenceUsage: true,
|
||||
RunE: func(_ *cobra.Command, _ []string) error {
|
||||
slog.Info("starting", "version", version)
|
||||
|
||||
targetDates, err := ParseTargetDates(dateFlag, fromFlag, toFlag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = bsdaily.Run(targetDates)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
slog.Info("completed successfully")
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
rootCmd.Flags().StringVarP(&dateFlag, "date", "d", "",
|
||||
"target date to extract (YYYY-MM-DD); "+
|
||||
"defaults to snapshot date minus one day")
|
||||
rootCmd.Flags().StringVar(&fromFlag, "from", "",
|
||||
"start of date range to extract (YYYY-MM-DD, inclusive); use with --to")
|
||||
rootCmd.Flags().StringVar(&toFlag, "to", "",
|
||||
"end of date range to extract (YYYY-MM-DD, inclusive); use with --from")
|
||||
|
||||
err := rootCmd.Execute()
|
||||
if err != nil {
|
||||
return 1
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
errDateExclusive = errors.New("--date and --from/--to are mutually exclusive")
|
||||
errFromRequiresTo = errors.New("--from requires --to")
|
||||
errToRequiresFrom = errors.New("--to requires --from")
|
||||
errFromAfterTo = errors.New("is after --to")
|
||||
)
|
||||
|
||||
// ParseTargetDates turns the --date, --from and --to flags into the days
|
||||
// to extract. It returns nil when none of them is set, which bsdaily.Run
|
||||
// takes to mean the snapshot date minus one day.
|
||||
func ParseTargetDates(dateFlag, fromFlag, toFlag string) ([]time.Time, error) {
|
||||
hasDate := dateFlag != ""
|
||||
hasFrom := fromFlag != ""
|
||||
hasTo := toFlag != ""
|
||||
|
||||
// Validate mutual exclusivity
|
||||
if hasDate && (hasFrom || hasTo) {
|
||||
return nil, errDateExclusive
|
||||
}
|
||||
|
||||
if hasFrom != hasTo {
|
||||
if hasFrom {
|
||||
return nil, errFromRequiresTo
|
||||
}
|
||||
|
||||
return nil, errToRequiresFrom
|
||||
}
|
||||
|
||||
if hasDate {
|
||||
t, err := time.Parse("2006-01-02", dateFlag)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"invalid --date %q (expected YYYY-MM-DD): %w", dateFlag, err)
|
||||
}
|
||||
|
||||
return []time.Time{t}, nil
|
||||
}
|
||||
|
||||
if !hasFrom {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
from, err := time.Parse("2006-01-02", fromFlag)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"invalid --from %q (expected YYYY-MM-DD): %w", fromFlag, err)
|
||||
}
|
||||
|
||||
to, err := time.Parse("2006-01-02", toFlag)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"invalid --to %q (expected YYYY-MM-DD): %w", toFlag, err)
|
||||
}
|
||||
|
||||
if from.After(to) {
|
||||
return nil, fmt.Errorf("--from %s %w %s", fromFlag, errFromAfterTo, toFlag)
|
||||
}
|
||||
|
||||
var targetDates []time.Time
|
||||
|
||||
for d := from; !d.After(to); d = d.AddDate(0, 0, 1) {
|
||||
targetDates = append(targetDates, d)
|
||||
}
|
||||
|
||||
return targetDates, nil
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
package cli_test
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.eeqj.de/sneak/bsdaily/internal/cli"
|
||||
)
|
||||
|
||||
func TestParseTargetDates(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const oneDay = "2026-05-14"
|
||||
|
||||
// want lists the expected days as YYYY-MM-DD.
|
||||
tests := []struct {
|
||||
name string
|
||||
date string
|
||||
from string
|
||||
to string
|
||||
want []string
|
||||
}{
|
||||
{name: "no flags"},
|
||||
{name: "single date", date: "2026-06-27", want: []string{"2026-06-27"}},
|
||||
{
|
||||
name: "range across a month end", from: "2026-06-29", to: "2026-07-02",
|
||||
want: []string{"2026-06-29", "2026-06-30", "2026-07-01", "2026-07-02"},
|
||||
},
|
||||
{name: "range of one day", from: oneDay, to: oneDay, want: []string{oneDay}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got, err := cli.ParseTargetDates(tt.date, tt.from, tt.to)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
days := make([]string, 0, len(got))
|
||||
for _, day := range got {
|
||||
days = append(days, day.Format("2006-01-02"))
|
||||
}
|
||||
|
||||
if !slices.Equal(days, tt.want) {
|
||||
t.Errorf("days = %v, want %v", days, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTargetDatesErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const dateExclusive = "--date and --from/--to are mutually exclusive"
|
||||
|
||||
// wantErr is the whole error message. With startsWith set it is only how
|
||||
// the message starts: for a malformed date, the date parser's own
|
||||
// explanation follows it.
|
||||
tests := []struct {
|
||||
name string
|
||||
date string
|
||||
from string
|
||||
to string
|
||||
wantErr string
|
||||
startsWith bool
|
||||
}{
|
||||
{
|
||||
name: "from after to", from: "2026-03-02", to: "2026-03-01",
|
||||
wantErr: "--from 2026-03-02 is after --to 2026-03-01",
|
||||
},
|
||||
{name: "from without to", from: "2026-04-01", wantErr: "--from requires --to"},
|
||||
{name: "to without from", to: "2026-04-02", wantErr: "--to requires --from"},
|
||||
{
|
||||
name: "date with from", date: "2026-01-05", from: "2026-01-06",
|
||||
wantErr: dateExclusive,
|
||||
},
|
||||
{
|
||||
name: "date with to", date: "2026-01-13", to: "2026-01-14",
|
||||
wantErr: dateExclusive,
|
||||
},
|
||||
{
|
||||
name: "date with from and to", date: "2026-01-07",
|
||||
from: "2026-01-08", to: "2026-01-09",
|
||||
wantErr: dateExclusive,
|
||||
},
|
||||
{
|
||||
name: "malformed date", date: "27.06.2026",
|
||||
wantErr: `invalid --date "27.06.2026" (expected YYYY-MM-DD): `,
|
||||
startsWith: true,
|
||||
},
|
||||
{
|
||||
name: "date that does not exist", date: "2026-02-30",
|
||||
wantErr: `invalid --date "2026-02-30" (expected YYYY-MM-DD): `,
|
||||
startsWith: true,
|
||||
},
|
||||
{
|
||||
name: "malformed from", from: "2026-1-10", to: "2026-01-11",
|
||||
wantErr: `invalid --from "2026-1-10" (expected YYYY-MM-DD): `,
|
||||
startsWith: true,
|
||||
},
|
||||
{
|
||||
name: "malformed to", from: "2026-01-12", to: "tomorrow",
|
||||
wantErr: `invalid --to "tomorrow" (expected YYYY-MM-DD): `,
|
||||
startsWith: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := cli.ParseTargetDates(tt.date, tt.from, tt.to)
|
||||
|
||||
switch {
|
||||
case err == nil:
|
||||
t.Errorf("no error, want %q", tt.wantErr)
|
||||
case tt.startsWith && !strings.HasPrefix(err.Error(), tt.wantErr):
|
||||
t.Errorf("error = %q, want one starting %q", err, tt.wantErr)
|
||||
case !tt.startsWith && err.Error() != tt.wantErr:
|
||||
t.Errorf("error = %q, want %q", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
{
|
||||
"devDependencies": {
|
||||
"prettier": "3.8.1"
|
||||
}
|
||||
}
|
||||
+1
-79
@@ -3,20 +3,11 @@
|
||||
# this repo. Idempotent: every install is guarded by a check so already
|
||||
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
||||
# or apk (detected in that order); assumes NOTHING is present (not git,
|
||||
# make, go, or node). Node is used directly if installed; otherwise it
|
||||
# is installed at a pinned version via nvm (installing nvm itself first,
|
||||
# from a hash-verified release archive, never curl | sh).
|
||||
# make, or go).
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# Pinned versions, 2026-07-06
|
||||
NODE_VERSION="22.17.0"
|
||||
NVM_VERSION="0.40.3"
|
||||
# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz
|
||||
NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0"
|
||||
YARN_VERSION="1.22.22"
|
||||
|
||||
PKGMGR=""
|
||||
SUDO=""
|
||||
APT_UPDATED=""
|
||||
@@ -65,69 +56,6 @@ missing() {
|
||||
! command -v "$1" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# verify_sha256 <file> <expected-hash>
|
||||
verify_sha256() {
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
actual="$(sha256sum "$1" | cut -d' ' -f1)"
|
||||
else
|
||||
actual="$(shasum -a 256 "$1" | cut -d' ' -f1)"
|
||||
fi
|
||||
if [ "$actual" != "$2" ]; then
|
||||
echo "bootstrap: sha256 mismatch for $1" >&2
|
||||
echo " expected: $2" >&2
|
||||
echo " actual: $actual" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# nvm is a bash script; run a command in a bash with nvm loaded
|
||||
nvm_sh() {
|
||||
bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*"
|
||||
}
|
||||
|
||||
ensure_nvm() {
|
||||
[ -s "$HOME/.nvm/nvm.sh" ] && return 0
|
||||
# nvm prerequisites; nvm itself requires bash
|
||||
if missing bash; then pkg_install bash bash bash bash; fi
|
||||
if missing curl; then pkg_install curl curl curl curl; fi
|
||||
if missing git; then pkg_install git git git git; fi
|
||||
tmp="$(mktemp -d)"
|
||||
curl -fsSL -o "$tmp/nvm.tar.gz" \
|
||||
"https://github.com/nvm-sh/nvm/archive/refs/tags/v${NVM_VERSION}.tar.gz"
|
||||
verify_sha256 "$tmp/nvm.tar.gz" "$NVM_SHA256"
|
||||
mkdir -p "$HOME/.nvm"
|
||||
tar -xzf "$tmp/nvm.tar.gz" -C "$HOME/.nvm" --strip-components=1
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
ensure_node() {
|
||||
if ! missing node; then return 0; fi
|
||||
ensure_nvm
|
||||
nvm_sh "nvm install $NODE_VERSION"
|
||||
}
|
||||
|
||||
ensure_yarn() {
|
||||
if ! missing yarn; then return 0; fi
|
||||
if ! missing corepack; then
|
||||
corepack enable
|
||||
corepack prepare "yarn@$YARN_VERSION" --activate
|
||||
elif [ -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
|
||||
corepack prepare yarn@$YARN_VERSION --activate"
|
||||
else
|
||||
npm install -g "yarn@$YARN_VERSION"
|
||||
fi
|
||||
}
|
||||
|
||||
install_js_deps() {
|
||||
if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \
|
||||
yarn install --frozen-lockfile"
|
||||
else
|
||||
yarn install --frozen-lockfile
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
|
||||
@@ -140,12 +68,6 @@ main() {
|
||||
|
||||
go mod download
|
||||
|
||||
# Node and yarn, then the prettier pinned in package.json and
|
||||
# yarn.lock, which script/fmt and script/fmt-check run
|
||||
ensure_node
|
||||
ensure_yarn
|
||||
install_js_deps
|
||||
|
||||
echo "bootstrap complete"
|
||||
}
|
||||
|
||||
|
||||
+1
-23
@@ -1,34 +1,12 @@
|
||||
#!/bin/sh
|
||||
# script/fmt: format all files (writes): the Go code with go fmt and
|
||||
# every Markdown file with prettier.
|
||||
# script/fmt: format all files (writes).
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# Must match the pin in script/bootstrap.
|
||||
NODE_VERSION="22.17.0"
|
||||
|
||||
# script/bootstrap installs node and yarn under nvm and leaves neither
|
||||
# on the PATH of the shell that called it, so resolve the pinned
|
||||
# toolchain here the way bootstrap's own install step does. nvm is a
|
||||
# bash script, hence the subshell.
|
||||
run_yarn() {
|
||||
if command -v yarn >/dev/null 2>&1; then
|
||||
exec yarn "$@"
|
||||
fi
|
||||
if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
echo "fmt: no yarn; run script/bootstrap first" >&2
|
||||
exit 1
|
||||
fi
|
||||
exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null &&
|
||||
shift && exec yarn "$@"' bash "$NODE_VERSION" "$@"
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
go fmt ./...
|
||||
# run_yarn replaces this shell, so it stays the last step.
|
||||
run_yarn run prettier --write '**/*.md' --tab-width 4 --prose-wrap always
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
+1
-23
@@ -1,30 +1,10 @@
|
||||
#!/bin/sh
|
||||
# script/fmt-check: check formatting (read-only). Same scope as
|
||||
# script/fmt, the Go code and every Markdown file, but fails instead of
|
||||
# writing.
|
||||
# script/fmt, but fails instead of writing.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# Must match the pin in script/bootstrap.
|
||||
NODE_VERSION="22.17.0"
|
||||
|
||||
# script/bootstrap installs node and yarn under nvm and leaves neither
|
||||
# on the PATH of the shell that called it, so resolve the pinned
|
||||
# toolchain here the way bootstrap's own install step does. nvm is a
|
||||
# bash script, hence the subshell.
|
||||
run_yarn() {
|
||||
if command -v yarn >/dev/null 2>&1; then
|
||||
exec yarn "$@"
|
||||
fi
|
||||
if [ ! -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
echo "fmt-check: no yarn; run script/bootstrap first" >&2
|
||||
exit 1
|
||||
fi
|
||||
exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null &&
|
||||
shift && exec yarn "$@"' bash "$NODE_VERSION" "$@"
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
unformatted="$(gofmt -l .)"
|
||||
@@ -33,8 +13,6 @@ main() {
|
||||
echo "$unformatted" >&2
|
||||
exit 1
|
||||
fi
|
||||
# run_yarn replaces this shell, so it stays the last step.
|
||||
run_yarn run prettier --check '**/*.md' --tab-width 4 --prose-wrap always
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
|
||||
# yarn lockfile v1
|
||||
|
||||
|
||||
prettier@3.8.1:
|
||||
version "3.8.1"
|
||||
resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.8.1.tgz#edf48977cf991558f4fcbd8a3ba6015ba2a3a173"
|
||||
integrity sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==
|
||||
Reference in New Issue
Block a user