From 7316054b077ee579e64986c5ef67c3ea20e88904 Mon Sep 17 00:00:00 2001 From: sneak Date: Sun, 28 Jun 2026 10:08:44 +0200 Subject: [PATCH] Add README, LICENSE, Makefile, Dockerfile, and CI Add a detailed README, WTFPL LICENSE, and build/CI tooling modeled on the vaultik repo (Makefile, multi-stage digest-pinned Dockerfile, .gitea/workflows/check.yml). Bump Go to 1.26.4 and pin golangci-lint to v2.12.2. gofmt existing sources so the new fmt-check gate passes. --- .gitea/workflows/check.yml | 14 ++ Dockerfile | 57 ++++++++ LICENSE | 13 ++ Makefile | 71 ++++++++++ README.md | 251 +++++++++++++++++++++++++++++++++ cmd/bsdaily/main.go | 4 +- go.mod | 2 +- internal/bsdaily/copy.go | 2 +- internal/bsdaily/copy_linux.go | 4 +- internal/bsdaily/copy_other.go | 2 +- 10 files changed, 413 insertions(+), 7 deletions(-) create mode 100644 .gitea/workflows/check.yml create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml new file mode 100644 index 0000000..fb6ef70 --- /dev/null +++ b/.gitea/workflows/check.yml @@ -0,0 +1,14 @@ +name: check +on: + push: + branches: [main] + pull_request: + branches: [main] +jobs: + check: + runs-on: ubuntu-latest + steps: + # actions/checkout v4, 2024-09-16 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + - name: Build and check + run: docker build . diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6104a62 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,57 @@ +# Lint stage +# golangci/golangci-lint:v2.12.2-alpine +FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint + +RUN apk add --no-cache make build-base + +WORKDIR /src + +# Copy go mod files first for better layer caching +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source code +COPY . . + +# Run formatting check and linter +RUN make fmt-check +RUN make lint + +# Build stage +# golang:1.26.4-alpine +FROM golang:1.26.4-alpine@sha256:3ad57304ad93bbec8548a0437ad9e06a455660655d9af011d58b993f6f615648 AS builder + +# Depend on lint stage passing +COPY --from=lint /src/go.sum /dev/null + +ARG VERSION=dev + +# Install build deps plus the sqlite3 and zstd CLIs the tests/tool shell out to +RUN apk add --no-cache make build-base sqlite zstd + +WORKDIR /src + +# Copy go mod files first for better layer caching +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source code +COPY . . + +# Run tests +RUN make test + +# Build (pure Go, no CGO required since we use modernc.org/sqlite) +RUN CGO_ENABLED=0 go build -o /bsdaily ./cmd/bsdaily + +# Runtime stage +# alpine:3.21 +FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d + +# bsdaily shells out to sqlite3, zstdmt, zstdcat at runtime +RUN apk add --no-cache ca-certificates sqlite zstd + +# Copy binary from builder +COPY --from=builder /bsdaily /usr/local/bin/bsdaily + +ENTRYPOINT ["/usr/local/bin/bsdaily"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5c93f45 --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ee3f835 --- /dev/null +++ b/Makefile @@ -0,0 +1,71 @@ +.PHONY: all check test lint fmt fmt-check build clean deps test-coverage test-integration install release release-snapshot docker hooks + +# Version number +VERSION := 0.1.0-dev + +# Default target +all: bsdaily + +# Combined pre-commit/CI gate: lint, format check, then tests. +check: lint fmt-check test + +# Run tests only. +test: + go test -race -timeout 30s ./... + +# Check if code is formatted (read-only). +fmt-check: + @test -z "$$(gofmt -l .)" || (echo "Files not formatted:" && gofmt -l . && exit 1) + +# Format code. +fmt: + go fmt ./... + +# Run linter only. +lint: + golangci-lint run ./... + +# Build binary (pure Go; no CGO required since we use modernc.org/sqlite). +bsdaily: internal/*/*.go cmd/bsdaily/*.go + CGO_ENABLED=0 go build -o $@ ./cmd/bsdaily + +# Clean build artifacts. +clean: + rm -f bsdaily + go clean + +# Install dependencies. +deps: + go mod download + go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest + +# Run tests with coverage. +test-coverage: + go test -v -coverprofile=coverage.out ./... + go tool cover -html=coverage.out -o coverage.html + +# Run integration tests. +test-integration: + go test -v -tags=integration ./... + +install: bsdaily + cp ./bsdaily $(HOME)/bin/ + +# Build and publish release artifacts via goreleaser. +release: + goreleaser release --clean + +# Dry-run a release build without publishing or tagging. +release-snapshot: + goreleaser release --clean --snapshot + +# Build Docker image. +docker: + docker build -t bsdaily . + +# Install pre-commit hook. +hooks: + @printf '#!/bin/sh\nset -e\n' > .git/hooks/pre-commit + @printf 'go mod tidy\ngo fmt ./...\ngit diff --exit-code -- go.mod go.sum || { echo "go mod tidy changed files; please stage and retry"; exit 1; }\n' >> .git/hooks/pre-commit + @printf 'make check\n' >> .git/hooks/pre-commit + @chmod +x .git/hooks/pre-commit diff --git a/README.md b/README.md new file mode 100644 index 0000000..01a2967 --- /dev/null +++ b/README.md @@ -0,0 +1,251 @@ +# bsdaily + +[bsdaily](https://git.eeqj.de/sneak/bsdaily) is a command-line utility +written in [Go](https://golang.org) that carves a single day (or a range of +days) of [Bluesky](https://bsky.app) firehose data out of a large, +continuously-growing SQLite database and writes it out as a self-contained, +[zstd](https://facebook.github.io/zstd/)-compressed SQL dump. The dumps are +named by date (e.g. `2026-06-27.sql.zst`), organized into per-month +directories, and are designed to be published, archived, mirrored, and later +re-merged back into a single database. + +The source database is read from a read-only [ZFS](https://openzfs.org) +snapshot, so extraction never contends with the live firehose ingester that +is writing to the original database. The tool is operationally +conservative: it checks free disk space before starting, copies the snapshot +to fast scratch storage, processes one day at a time to avoid SQLite lock +contention, verifies every compressed output before publishing it, and +writes output atomically via a temp-file-and-rename so a partial run never +leaves a corrupt `.sql.zst` behind. + +This project was written by [@sneak](https://sneak.berlin) to produce a +daily, mergeable, publicly-mirrorable archive of the Bluesky firehose. It is +currently a one-person effort. The current version is pre-1.0 and there has +not yet been a versioned release; [SemVer](https://semver.org) will be used +for releases. + +# Build Status + +CI runs the standard `make check` (formatting, linting, tests). The `main` +branch must always be green. + +# Participation + +Primary development happens on a privately-run Gitea instance at +[https://git.eeqj.de/sneak/bsdaily](https://git.eeqj.de/sneak/bsdaily) and +issues are [tracked +there](https://git.eeqj.de/sneak/bsdaily/issues). + +Changes must always be formatted with a standard `go fmt`, syntactically +valid, and must pass the linting defined in the repository (presently the +`golangci-lint` defaults), which can be run with a `make lint`. The `main` +branch is protected and all changes must be made via [pull +requests](https://git.eeqj.de/sneak/bsdaily/pulls) and pass CI to be merged. + +See [`REPO_POLICIES.md`](REPO_POLICIES.md) for detailed coding standards, +tooling requirements, and workflow conventions. + +# Problem Statement + +A Bluesky firehose ingester writes every observed post (and associated +users, hashtags, URLs, and media references) into a single ever-growing +SQLite database, `firehose.db`. This database has several properties that +make it awkward to publish or archive directly: + +- It is **large and always growing**, so re-publishing the whole thing every + day is wasteful. +- It is **continuously written**, so reading from it directly risks lock + contention with the live ingester and inconsistent reads. +- It is **monolithic**, so there is no natural unit at which to mirror, + share, or distribute "just yesterday's posts". + +What is wanted instead is a stable, immutable, per-day artifact: a small +file containing exactly one calendar day of firehose data, cheap to publish, +cheap to mirror, and trivially re-mergeable into a full database by anyone +who collects a set of them. + +# Proposed Solution + +A tool, `bsdaily`, that: + +- locates the most recent read-only **ZFS daily snapshot** of the firehose + filesystem, so it reads from a consistent point-in-time copy that the live + ingester cannot be writing to; +- copies the snapshot's database files to fast scratch storage; +- **extracts** a single day's `posts` (and all rows reachable from them) into + a fresh, minimal per-day SQLite database; +- **dumps** that per-day database to SQL and pipes it through multithreaded + zstd compression; +- **verifies** the compressed output (zstd integrity check plus a sanity + check that the decompressed stream actually looks like SQL); +- **publishes** the result atomically as + `DailiesBase/YYYY-MM/YYYY-MM-DD.sql.zst`. + +Each daily dump is emitted with `INSERT` statements over the full schema +(including the deduplicated `users`, `hashtags`, and `urls` lookup tables), +so any collection of daily dumps can be merged into a single database by +rewriting `INSERT INTO` to `INSERT OR IGNORE INTO` and replaying them in +sequence. Two helper scripts ([`merge_daily_dumps.sh`](merge_daily_dumps.sh) +and [`regenerate_auxiliary_tables.sql`](regenerate_auxiliary_tables.sql)) are +included to do exactly this and to rebuild the aggregate statistics +(`use_count`, `first_seen`, user `resolved_at`/`updated_at`) afterward. + +# Design Goals + +- **Never disturb the live ingester.** All reads come from a ZFS snapshot, + never the live database. +- **Crash-safe, idempotent runs.** Output is written to a temp file and + atomically renamed; a day whose final output already exists is skipped, so + re-running a range is safe and resumable. +- **Mergeable output.** Daily dumps re-combine losslessly into a full + database via `INSERT OR IGNORE`. +- **Operationally cautious.** Free-space preflight checks on both scratch and + output filesystems; explicit verification of every artifact before it is + published. +- **Fast where it's free.** Large snapshot copies use a 256MiB buffer, + pre-allocate the destination, and (on Linux) issue `posix_fadvise` + sequential/willneed hints; extraction uses aggressive, + crash-unsafe-by-design SQLite pragmas because the working data lives only + in disposable scratch space. + +# Non-Goals + +- **Real-time export.** `bsdaily` operates on daily snapshots; the freshest + day it can produce is the snapshot date minus one. +- **Schema ownership.** The schema is defined by the upstream firehose + ingester; [`schema.sql`](schema.sql) is included for reference only. + `bsdaily` copies whatever table and index DDL it finds in the source. +- **Cross-platform deployment.** It is built and run on Linux (the + free-space check and fadvise hints use `golang.org/x/sys/unix`; a non-Linux + build compiles but is a no-op for the fadvise hints). The hard-coded paths + assume the production host's ZFS layout. + +# How It Works + +A single run proceeds as follows: + +1. **Find the snapshot.** Scan `SnapshotBase` for directories matching + `zfs-auto-snap_daily-YYYY-MM-DD-NNNN`, pick the most recent, and confirm + it contains `firehose.db`. +2. **Determine target days.** Default to the snapshot date minus one day; + or use `--date`, or every day in the inclusive `--from`/`--to` range. +3. **Preflight disk space.** Require at least 500GiB free on the scratch + filesystem and 20GiB free on the output filesystem. +4. **Copy the database to scratch.** Copy `firehose.db`, its `-wal`, and (if + present) its `-shm` from the snapshot into a fresh temp directory under + `TmpBase`. +5. **Per day**, processed strictly one at a time to avoid SQLite contention: + - skip the day if its final output file already exists; + - `ATTACH` the copied source DB to a new empty per-day DB, recreate the + table DDL, and `INSERT ... SELECT` the target day's `posts` plus all + rows reachable from them (`posts_hashtags`, `posts_urls`, `hashtags`, + `urls`, `users`, and `media` if that table exists); + - abort the day cleanly if there are zero posts (`ErrNoPosts`), rather + than emitting an empty dump; + - recreate indexes, detach the source, and verify the inserted row count; + - `sqlite3 .dump | zstdmt` into a hidden temp file; + - run a zstd integrity check and confirm the decompressed head looks like + SQL; + - atomically rename into place and delete the per-day scratch DB. +6. **Clean up** the temp directory and log a processed/skipped/total summary. + +# Usage + +``` +bsdaily # extract the snapshot date minus one day +bsdaily --date 2026-06-27 # extract a single specific day +bsdaily --from 2026-06-01 --to 2026-06-27 # extract an inclusive range +``` + +Flags: + +- `-d`, `--date YYYY-MM-DD` — extract a single day. Mutually exclusive with + `--from`/`--to`. +- `--from YYYY-MM-DD` — start of an inclusive range (requires `--to`). +- `--to YYYY-MM-DD` — end of an inclusive range (requires `--from`). + +With no flags, the tool extracts the day before the latest snapshot. All +progress is logged as structured `slog` text to stderr. + +## Merging dumps back into a database + +``` +# Using the helper script: +./merge_daily_dumps.sh merged.db daily_dumps/*.sql.zst + +# Or by hand: +zstdcat *.sql.zst | sed 's/INSERT INTO/INSERT OR IGNORE INTO/g' | sqlite3 merged.db +sqlite3 merged.db < regenerate_auxiliary_tables.sql +``` + +# Requirements + +- **Go** (see [`go.mod`](go.mod) for the toolchain version) to build. +- **Linux** for production use (ZFS snapshots, `statfs` free-space checks, + `posix_fadvise` hints). +- The **`sqlite3`** and **`zstdmt`** (multithreaded zstd) binaries on + `PATH`; `zstdcat` is used for verification. SQLite reads/writes during + extraction use the pure-Go [`modernc.org/sqlite`](https://pkg.go.dev/modernc.org/sqlite) + driver, so no cgo is required for that part. + +# Configuration + +Operational parameters are compile-time constants in +[`internal/bsdaily/config.go`](internal/bsdaily/config.go): + +- `SnapshotBase` — `/srv/berlin.sneak.fs.blueskyarchive/.zfs/snapshot` +- `TmpBase` — `/srv/storage/tmp` (fast scratch space for copies + per-day DBs) +- `DailiesBase` — `/srv/berlin.sneak.fs.bluesky-dailies` (output root) +- `MinTmpFreeGB` / `MinDailiesFreeGB` — `500` / `20` +- `zstdCompressionLevel` — `15` +- `sqliteCacheSizeKB` — `200000` (≈200MB) + +These are tuned for one specific production host; adjust and rebuild to run +elsewhere. + +# Data Model + +The firehose schema (reference copy in [`schema.sql`](schema.sql)) centers +on a `posts` table, with `users` keyed by DID and many-to-many junction +tables linking posts to deduplicated `hashtags` and `urls`. An optional +`media` table tracks downloaded blobs by content hash. `bsdaily` does not +own this schema; it reflects whatever DDL exists in the source snapshot and +selects forward from `posts` along the foreign-key relationships to produce a +referentially-complete per-day slice. + +# Use Cases + +## Daily public archive + +Publish one small, immutable file per day to static HTTP (or IPFS, or a +mirror network) so that anyone can fetch exactly the day(s) they want and +re-merge them locally. + +## Backfilling a range + +Run `--from`/`--to` over a span of dates to (re)generate any missing daily +dumps; already-published days are skipped, so the operation is resumable and +safe to re-run. + +## Reconstituting a full database + +Collect any set of daily dumps and merge them with `INSERT OR IGNORE` to +rebuild a complete, queryable SQLite database, then regenerate the aggregate +statistics tables. + +# See Also + +## Links + +- Repo: [https://git.eeqj.de/sneak/bsdaily](https://git.eeqj.de/sneak/bsdaily) +- Issues: [https://git.eeqj.de/sneak/bsdaily/issues](https://git.eeqj.de/sneak/bsdaily/issues) +- Bluesky: [https://bsky.app](https://bsky.app) +- zstd: [https://facebook.github.io/zstd/](https://facebook.github.io/zstd/) + +# Authors + +- [@sneak <sneak@sneak.berlin>](mailto:sneak@sneak.berlin) + +# License + +- [WTFPL](https://wtfpl.net) diff --git a/cmd/bsdaily/main.go b/cmd/bsdaily/main.go index fa565bf..7d5409e 100644 --- a/cmd/bsdaily/main.go +++ b/cmd/bsdaily/main.go @@ -21,8 +21,8 @@ func main() { var toFlag string rootCmd := &cobra.Command{ - Use: "bsdaily", - Short: "Extract a single day's data from the latest daily snapshot", + Use: "bsdaily", + Short: "Extract a single day's data from the latest daily snapshot", SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { hasDate := dateFlag != "" diff --git a/go.mod b/go.mod index 07e7bb1..ded4198 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.eeqj.de/sneak/bsdaily -go 1.25.5 +go 1.26.4 require ( golang.org/x/sys v0.41.0 diff --git a/internal/bsdaily/copy.go b/internal/bsdaily/copy.go index 9e3d4cc..6532b78 100644 --- a/internal/bsdaily/copy.go +++ b/internal/bsdaily/copy.go @@ -69,4 +69,4 @@ func CopyFile(src, dst string) (err error) { "elapsed", elapsed.Round(time.Millisecond), "throughput_mbps", fmt.Sprintf("%.1f", throughputMBps)) return nil -} \ No newline at end of file +} diff --git a/internal/bsdaily/copy_linux.go b/internal/bsdaily/copy_linux.go index 12733ad..d513062 100644 --- a/internal/bsdaily/copy_linux.go +++ b/internal/bsdaily/copy_linux.go @@ -3,8 +3,8 @@ package bsdaily import ( - "os" "golang.org/x/sys/unix" + "os" ) func applyFileAdvice(file *os.File, size int64) { @@ -13,4 +13,4 @@ func applyFileAdvice(file *os.File, size int64) { _ = unix.Fadvise(fd, 0, size, 2) // POSIX_FADV_WILLNEED = 3 - prefetch file into cache _ = unix.Fadvise(fd, 0, size, 3) -} \ No newline at end of file +} diff --git a/internal/bsdaily/copy_other.go b/internal/bsdaily/copy_other.go index b56da7b..6fc0a04 100644 --- a/internal/bsdaily/copy_other.go +++ b/internal/bsdaily/copy_other.go @@ -6,4 +6,4 @@ import "os" func applyFileAdvice(file *os.File, size int64) { // Fadvise not available on this platform -} \ No newline at end of file +}