Add README, LICENSE, Makefile, Dockerfile, and CI

Add a detailed README, WTFPL LICENSE, and build/CI tooling modeled on
the vaultik repo (Makefile, multi-stage digest-pinned Dockerfile,
.gitea/workflows/check.yml). Bump Go to 1.26.4 and pin golangci-lint
to v2.12.2. gofmt existing sources so the new fmt-check gate passes.

.gitea/workflows/check.yml

@@ -0,0 +1,14 @@
+name: check
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      # actions/checkout v4, 2024-09-16
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
+      - name: Build and check
+        run: docker build .

Dockerfile

@@ -0,0 +1,57 @@
+# Lint stage
+# golangci/golangci-lint:v2.12.2-alpine
+FROM golangci/golangci-lint:v2.12.2-alpine@sha256:91b27804074a0bacea298707f016911e60cf0cdbc6c7bf5ccacb5f0606d18d60 AS lint
+
+RUN apk add --no-cache make build-base
+
+WORKDIR /src
+
+# Copy go mod files first for better layer caching
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy source code
+COPY . .
+
+# Run formatting check and linter
+RUN make fmt-check
+RUN make lint
+
+# Build stage
+# golang:1.26.4-alpine
+FROM golang:1.26.4-alpine@sha256:3ad57304ad93bbec8548a0437ad9e06a455660655d9af011d58b993f6f615648 AS builder
+
+# Depend on lint stage passing
+COPY --from=lint /src/go.sum /dev/null
+
+ARG VERSION=dev
+
+# Install build deps plus the sqlite3 and zstd CLIs the tests/tool shell out to
+RUN apk add --no-cache make build-base sqlite zstd
+
+WORKDIR /src
+
+# Copy go mod files first for better layer caching
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy source code
+COPY . .
+
+# Run tests
+RUN make test
+
+# Build (pure Go, no CGO required since we use modernc.org/sqlite)
+RUN CGO_ENABLED=0 go build -o /bsdaily ./cmd/bsdaily
+
+# Runtime stage
+# alpine:3.21
+FROM alpine:3.21@sha256:48b0309ca019d89d40f670aa1bc06e426dc0931948452e8491e3d65087abc07d
+
+# bsdaily shells out to sqlite3, zstdmt, zstdcat at runtime
+RUN apk add --no-cache ca-certificates sqlite zstd
+
+# Copy binary from builder
+COPY --from=builder /bsdaily /usr/local/bin/bsdaily
+
+ENTRYPOINT ["/usr/local/bin/bsdaily"]

LICENSE

@@ -0,0 +1,13 @@
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+                    Version 2, December 2004
+
+ Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
+
+ Everyone is permitted to copy and distribute verbatim or modified
+ copies of this license document, and changing it is allowed as long
+ as the name is changed.
+
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. You just DO WHAT THE FUCK YOU WANT TO.

Makefile

@@ -0,0 +1,71 @@
+.PHONY: all check test lint fmt fmt-check build clean deps test-coverage test-integration install release release-snapshot docker hooks
+
+# Version number
+VERSION := 0.1.0-dev
+
+# Default target
+all: bsdaily
+
+# Combined pre-commit/CI gate: lint, format check, then tests.
+check: lint fmt-check test
+
+# Run tests only.
+test:
+	go test -race -timeout 30s ./...
+
+# Check if code is formatted (read-only).
+fmt-check:
+	@test -z "$$(gofmt -l .)" || (echo "Files not formatted:" && gofmt -l . && exit 1)
+
+# Format code.
+fmt:
+	go fmt ./...
+
+# Run linter only.
+lint:
+	golangci-lint run ./...
+
+# Build binary (pure Go; no CGO required since we use modernc.org/sqlite).
+bsdaily: internal/*/*.go cmd/bsdaily/*.go
+	CGO_ENABLED=0 go build -o $@ ./cmd/bsdaily
+
+# Clean build artifacts.
+clean:
+	rm -f bsdaily
+	go clean
+
+# Install dependencies.
+deps:
+	go mod download
+	go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
+
+# Run tests with coverage.
+test-coverage:
+	go test -v -coverprofile=coverage.out ./...
+	go tool cover -html=coverage.out -o coverage.html
+
+# Run integration tests.
+test-integration:
+	go test -v -tags=integration ./...
+
+install: bsdaily
+	cp ./bsdaily $(HOME)/bin/
+
+# Build and publish release artifacts via goreleaser.
+release:
+	goreleaser release --clean
+
+# Dry-run a release build without publishing or tagging.
+release-snapshot:
+	goreleaser release --clean --snapshot
+
+# Build Docker image.
+docker:
+	docker build -t bsdaily .
+
+# Install pre-commit hook.
+hooks:
+	@printf '#!/bin/sh\nset -e\n' > .git/hooks/pre-commit
+	@printf 'go mod tidy\ngo fmt ./...\ngit diff --exit-code -- go.mod go.sum || { echo "go mod tidy changed files; please stage and retry"; exit 1; }\n' >> .git/hooks/pre-commit
+	@printf 'make check\n' >> .git/hooks/pre-commit
+	@chmod +x .git/hooks/pre-commit

README.md

@@ -0,0 +1,251 @@
+# bsdaily
+
+[bsdaily](https://git.eeqj.de/sneak/bsdaily) is a command-line utility
+written in [Go](https://golang.org) that carves a single day (or a range of
+days) of [Bluesky](https://bsky.app) firehose data out of a large,
+continuously-growing SQLite database and writes it out as a self-contained,
+[zstd](https://facebook.github.io/zstd/)-compressed SQL dump. The dumps are
+named by date (e.g. `2026-06-27.sql.zst`), organized into per-month
+directories, and are designed to be published, archived, mirrored, and later
+re-merged back into a single database.
+
+The source database is read from a read-only [ZFS](https://openzfs.org)
+snapshot, so extraction never contends with the live firehose ingester that
+is writing to the original database. The tool is operationally
+conservative: it checks free disk space before starting, copies the snapshot
+to fast scratch storage, processes one day at a time to avoid SQLite lock
+contention, verifies every compressed output before publishing it, and
+writes output atomically via a temp-file-and-rename so a partial run never
+leaves a corrupt `.sql.zst` behind.
+
+This project was written by [@sneak](https://sneak.berlin) to produce a
+daily, mergeable, publicly-mirrorable archive of the Bluesky firehose. It is
+currently a one-person effort. The current version is pre-1.0 and there has
+not yet been a versioned release; [SemVer](https://semver.org) will be used
+for releases.
+
+# Build Status
+
+CI runs the standard `make check` (formatting, linting, tests). The `main`
+branch must always be green.
+
+# Participation
+
+Primary development happens on a privately-run Gitea instance at
+[https://git.eeqj.de/sneak/bsdaily](https://git.eeqj.de/sneak/bsdaily) and
+issues are [tracked
+there](https://git.eeqj.de/sneak/bsdaily/issues).
+
+Changes must always be formatted with a standard `go fmt`, syntactically
+valid, and must pass the linting defined in the repository (presently the
+`golangci-lint` defaults), which can be run with a `make lint`. The `main`
+branch is protected and all changes must be made via [pull
+requests](https://git.eeqj.de/sneak/bsdaily/pulls) and pass CI to be merged.
+
+See [`REPO_POLICIES.md`](REPO_POLICIES.md) for detailed coding standards,
+tooling requirements, and workflow conventions.
+
+# Problem Statement
+
+A Bluesky firehose ingester writes every observed post (and associated
+users, hashtags, URLs, and media references) into a single ever-growing
+SQLite database, `firehose.db`. This database has several properties that
+make it awkward to publish or archive directly:
+
+- It is **large and always growing**, so re-publishing the whole thing every
+  day is wasteful.
+- It is **continuously written**, so reading from it directly risks lock
+  contention with the live ingester and inconsistent reads.
+- It is **monolithic**, so there is no natural unit at which to mirror,
+  share, or distribute "just yesterday's posts".
+
+What is wanted instead is a stable, immutable, per-day artifact: a small
+file containing exactly one calendar day of firehose data, cheap to publish,
+cheap to mirror, and trivially re-mergeable into a full database by anyone
+who collects a set of them.
+
+# Proposed Solution
+
+A tool, `bsdaily`, that:
+
+- locates the most recent read-only **ZFS daily snapshot** of the firehose
+  filesystem, so it reads from a consistent point-in-time copy that the live
+  ingester cannot be writing to;
+- copies the snapshot's database files to fast scratch storage;
+- **extracts** a single day's `posts` (and all rows reachable from them) into
+  a fresh, minimal per-day SQLite database;
+- **dumps** that per-day database to SQL and pipes it through multithreaded
+  zstd compression;
+- **verifies** the compressed output (zstd integrity check plus a sanity
+  check that the decompressed stream actually looks like SQL);
+- **publishes** the result atomically as
+  `DailiesBase/YYYY-MM/YYYY-MM-DD.sql.zst`.
+
+Each daily dump is emitted with `INSERT` statements over the full schema
+(including the deduplicated `users`, `hashtags`, and `urls` lookup tables),
+so any collection of daily dumps can be merged into a single database by
+rewriting `INSERT INTO` to `INSERT OR IGNORE INTO` and replaying them in
+sequence. Two helper scripts ([`merge_daily_dumps.sh`](merge_daily_dumps.sh)
+and [`regenerate_auxiliary_tables.sql`](regenerate_auxiliary_tables.sql)) are
+included to do exactly this and to rebuild the aggregate statistics
+(`use_count`, `first_seen`, user `resolved_at`/`updated_at`) afterward.
+
+# Design Goals
+
+- **Never disturb the live ingester.** All reads come from a ZFS snapshot,
+  never the live database.
+- **Crash-safe, idempotent runs.** Output is written to a temp file and
+  atomically renamed; a day whose final output already exists is skipped, so
+  re-running a range is safe and resumable.
+- **Mergeable output.** Daily dumps re-combine losslessly into a full
+  database via `INSERT OR IGNORE`.
+- **Operationally cautious.** Free-space preflight checks on both scratch and
+  output filesystems; explicit verification of every artifact before it is
+  published.
+- **Fast where it's free.** Large snapshot copies use a 256MiB buffer,
+  pre-allocate the destination, and (on Linux) issue `posix_fadvise`
+  sequential/willneed hints; extraction uses aggressive,
+  crash-unsafe-by-design SQLite pragmas because the working data lives only
+  in disposable scratch space.
+
+# Non-Goals
+
+- **Real-time export.** `bsdaily` operates on daily snapshots; the freshest
+  day it can produce is the snapshot date minus one.
+- **Schema ownership.** The schema is defined by the upstream firehose
+  ingester; [`schema.sql`](schema.sql) is included for reference only.
+  `bsdaily` copies whatever table and index DDL it finds in the source.
+- **Cross-platform deployment.** It is built and run on Linux (the
+  free-space check and fadvise hints use `golang.org/x/sys/unix`; a non-Linux
+  build compiles but is a no-op for the fadvise hints). The hard-coded paths
+  assume the production host's ZFS layout.
+
+# How It Works
+
+A single run proceeds as follows:
+
+1. **Find the snapshot.** Scan `SnapshotBase` for directories matching
+   `zfs-auto-snap_daily-YYYY-MM-DD-NNNN`, pick the most recent, and confirm
+   it contains `firehose.db`.
+2. **Determine target days.** Default to the snapshot date minus one day;
+   or use `--date`, or every day in the inclusive `--from`/`--to` range.
+3. **Preflight disk space.** Require at least 500GiB free on the scratch
+   filesystem and 20GiB free on the output filesystem.
+4. **Copy the database to scratch.** Copy `firehose.db`, its `-wal`, and (if
+   present) its `-shm` from the snapshot into a fresh temp directory under
+   `TmpBase`.
+5. **Per day**, processed strictly one at a time to avoid SQLite contention:
+   - skip the day if its final output file already exists;
+   - `ATTACH` the copied source DB to a new empty per-day DB, recreate the
+     table DDL, and `INSERT ... SELECT` the target day's `posts` plus all
+     rows reachable from them (`posts_hashtags`, `posts_urls`, `hashtags`,
+     `urls`, `users`, and `media` if that table exists);
+   - abort the day cleanly if there are zero posts (`ErrNoPosts`), rather
+     than emitting an empty dump;
+   - recreate indexes, detach the source, and verify the inserted row count;
+   - `sqlite3 .dump | zstdmt` into a hidden temp file;
+   - run a zstd integrity check and confirm the decompressed head looks like
+     SQL;
+   - atomically rename into place and delete the per-day scratch DB.
+6. **Clean up** the temp directory and log a processed/skipped/total summary.
+
+# Usage
+
+```
+bsdaily                          # extract the snapshot date minus one day
+bsdaily --date 2026-06-27        # extract a single specific day
+bsdaily --from 2026-06-01 --to 2026-06-27   # extract an inclusive range
+```
+
+Flags:
+
+- `-d`, `--date YYYY-MM-DD` — extract a single day. Mutually exclusive with
+  `--from`/`--to`.
+- `--from YYYY-MM-DD` — start of an inclusive range (requires `--to`).
+- `--to YYYY-MM-DD` — end of an inclusive range (requires `--from`).
+
+With no flags, the tool extracts the day before the latest snapshot. All
+progress is logged as structured `slog` text to stderr.
+
+## Merging dumps back into a database
+
+```
+# Using the helper script:
+./merge_daily_dumps.sh merged.db daily_dumps/*.sql.zst
+
+# Or by hand:
+zstdcat *.sql.zst | sed 's/INSERT INTO/INSERT OR IGNORE INTO/g' | sqlite3 merged.db
+sqlite3 merged.db < regenerate_auxiliary_tables.sql
+```
+
+# Requirements
+
+- **Go** (see [`go.mod`](go.mod) for the toolchain version) to build.
+- **Linux** for production use (ZFS snapshots, `statfs` free-space checks,
+  `posix_fadvise` hints).
+- The **`sqlite3`** and **`zstdmt`** (multithreaded zstd) binaries on
+  `PATH`; `zstdcat` is used for verification. SQLite reads/writes during
+  extraction use the pure-Go [`modernc.org/sqlite`](https://pkg.go.dev/modernc.org/sqlite)
+  driver, so no cgo is required for that part.
+
+# Configuration
+
+Operational parameters are compile-time constants in
+[`internal/bsdaily/config.go`](internal/bsdaily/config.go):
+
+- `SnapshotBase` — `/srv/berlin.sneak.fs.blueskyarchive/.zfs/snapshot`
+- `TmpBase` — `/srv/storage/tmp` (fast scratch space for copies + per-day DBs)
+- `DailiesBase` — `/srv/berlin.sneak.fs.bluesky-dailies` (output root)
+- `MinTmpFreeGB` / `MinDailiesFreeGB` — `500` / `20`
+- `zstdCompressionLevel` — `15`
+- `sqliteCacheSizeKB` — `200000` (≈200MB)
+
+These are tuned for one specific production host; adjust and rebuild to run
+elsewhere.
+
+# Data Model
+
+The firehose schema (reference copy in [`schema.sql`](schema.sql)) centers
+on a `posts` table, with `users` keyed by DID and many-to-many junction
+tables linking posts to deduplicated `hashtags` and `urls`. An optional
+`media` table tracks downloaded blobs by content hash. `bsdaily` does not
+own this schema; it reflects whatever DDL exists in the source snapshot and
+selects forward from `posts` along the foreign-key relationships to produce a
+referentially-complete per-day slice.
+
+# Use Cases
+
+## Daily public archive
+
+Publish one small, immutable file per day to static HTTP (or IPFS, or a
+mirror network) so that anyone can fetch exactly the day(s) they want and
+re-merge them locally.
+
+## Backfilling a range
+
+Run `--from`/`--to` over a span of dates to (re)generate any missing daily
+dumps; already-published days are skipped, so the operation is resumable and
+safe to re-run.
+
+## Reconstituting a full database
+
+Collect any set of daily dumps and merge them with `INSERT OR IGNORE` to
+rebuild a complete, queryable SQLite database, then regenerate the aggregate
+statistics tables.
+
+# See Also
+
+## Links
+
+- Repo: [https://git.eeqj.de/sneak/bsdaily](https://git.eeqj.de/sneak/bsdaily)
+- Issues: [https://git.eeqj.de/sneak/bsdaily/issues](https://git.eeqj.de/sneak/bsdaily/issues)
+- Bluesky: [https://bsky.app](https://bsky.app)
+- zstd: [https://facebook.github.io/zstd/](https://facebook.github.io/zstd/)
+
+# Authors
+
+- [@sneak &lt;sneak@sneak.berlin&gt;](mailto:sneak@sneak.berlin)
+
+# License
+
+- [WTFPL](https://wtfpl.net)

cmd/bsdaily/main.go

@@ -21,8 +21,8 @@ func main() {
 	var toFlag string
 
 	rootCmd := &cobra.Command{
-		Use:   "bsdaily",
+		Use:          "bsdaily",
-		Short: "Extract a single day's data from the latest daily snapshot",
+		Short:        "Extract a single day's data from the latest daily snapshot",
 		SilenceUsage: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			hasDate := dateFlag != ""

go.mod

@@ -1,6 +1,6 @@
 module git.eeqj.de/sneak/bsdaily
 
-go 1.25.5
+go 1.26.4
 
 require (
 	golang.org/x/sys v0.41.0

internal/bsdaily/copy.go

@@ -69,4 +69,4 @@ func CopyFile(src, dst string) (err error) {
 		"elapsed", elapsed.Round(time.Millisecond),
 		"throughput_mbps", fmt.Sprintf("%.1f", throughputMBps))
 	return nil
-}
+}

internal/bsdaily/copy_linux.go

@@ -3,8 +3,8 @@
 package bsdaily
 
 import (
-	"os"
 	"golang.org/x/sys/unix"
+	"os"
 )
 
 func applyFileAdvice(file *os.File, size int64) {
@@ -13,4 +13,4 @@ func applyFileAdvice(file *os.File, size int64) {
 	_ = unix.Fadvise(fd, 0, size, 2)
 	// POSIX_FADV_WILLNEED = 3 - prefetch file into cache
 	_ = unix.Fadvise(fd, 0, size, 3)
-}
+}

internal/bsdaily/copy_other.go

@@ -6,4 +6,4 @@ import "os"
 
 func applyFileAdvice(file *os.File, size int64) {
 	// Fadvise not available on this platform
-}
+}