bring repo into policy compliance; vendor assets; Gitea CI
Vendor the front-end assets and drop the third-party CDN dependencies (BootstrapCDN is being sunset): the bootstrap 4.0.0 css/js, jquery 3.2.1 slim, and popper 1.12.9 now live under static/ and are served from the app, byte-for-byte identical to the previous SRI-pinned files. Migrate CI from Drone to a Gitea Actions workflow that runs docker build . on push, with the checkout action pinned by SHA. Bring the repo up to standard: - add REPO_POLICIES.md, .editorconfig, .dockerignore, .golangci.yml, and a comprehensive root-anchored .gitignore - rewrite the Makefile with the required test/lint/fmt/fmt-check/check/ docker/hooks targets (golangci-lint, 30s test timeout, verbose rerun on failure, check modifies nothing) - rewrite the Dockerfile as a hash-pinned multistage build: a lint stage (golangci-lint), a glibc build+test stage (the legacy sqlite driver needs cgo+glibc), and a debian-slim runtime carrying the binary, templates, and static assets - add real tests for the hn package - bring the code into golangci-lint (default: all) compliance: fix the malformed gorm struct tags, check previously-ignored errors, dispatch the zerolog error event, avoid a uint->Duration overflow, split long functions, and add doc comments — all behaviour-preserving - expand the README with the required sections
This commit is contained in:
8
.dockerignore
Normal file
8
.dockerignore
Normal file
@@ -0,0 +1,8 @@
|
||||
.git
|
||||
.gitea
|
||||
.DS_Store
|
||||
|
||||
# local build artifacts / data (built inside the image instead)
|
||||
server
|
||||
storage.sqlite
|
||||
output/
|
||||
17
.drone.yml
17
.drone.yml
@@ -1,17 +0,0 @@
|
||||
kind: pipeline
|
||||
name: default
|
||||
|
||||
steps:
|
||||
- name: docker
|
||||
image: plugins/docker
|
||||
network_mode: bridge
|
||||
settings:
|
||||
repo: sneak/orangesite
|
||||
dry_run: true
|
||||
username:
|
||||
from_secret: docker_username
|
||||
password:
|
||||
from_secret: docker_password
|
||||
tags:
|
||||
- ${DRONE_COMMIT_SHA}
|
||||
- ${DRONE_BRANCH}
|
||||
12
.editorconfig
Normal file
12
.editorconfig
Normal file
@@ -0,0 +1,12 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
end_of_line = lf
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
9
.gitea/workflows/check.yml
Normal file
9
.gitea/workflows/check.yml
Normal file
@@ -0,0 +1,9 @@
|
||||
name: check
|
||||
on: [push]
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# actions/checkout v4.2.2, 2026-02-22
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
- run: docker build .
|
||||
34
.gitignore
vendored
34
.gitignore
vendored
@@ -1,16 +1,36 @@
|
||||
# ---> Go
|
||||
# Binaries for programs and plugins
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Editors
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
*.bak
|
||||
.idea/
|
||||
.vscode/
|
||||
*.sublime-*
|
||||
|
||||
# Environment / secrets
|
||||
.env
|
||||
.env.*
|
||||
*.pem
|
||||
*.key
|
||||
|
||||
# Go binaries and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Test binary, build with `go test -c`
|
||||
# Go test/coverage output
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
server
|
||||
storage.sqlite
|
||||
# build artifacts / local data (root-anchored so they don't shadow
|
||||
# the cmd/server package directory)
|
||||
/server
|
||||
/storage.sqlite
|
||||
/output/
|
||||
/.lintsetup
|
||||
|
||||
32
.golangci.yml
Normal file
32
.golangci.yml
Normal file
@@ -0,0 +1,32 @@
|
||||
version: "2"
|
||||
|
||||
run:
|
||||
timeout: 5m
|
||||
modules-download-mode: readonly
|
||||
|
||||
linters:
|
||||
default: all
|
||||
disable:
|
||||
# Genuinely incompatible with project patterns
|
||||
- exhaustruct # Requires all struct fields
|
||||
- depguard # Dependency allow/block lists
|
||||
- godot # Requires comments to end with periods
|
||||
- wsl # Deprecated, replaced by wsl_v5
|
||||
- wrapcheck # Too verbose for internal packages
|
||||
- varnamelen # Short names like db, id are idiomatic Go
|
||||
|
||||
linters-settings:
|
||||
lll:
|
||||
line-length: 88
|
||||
funlen:
|
||||
lines: 80
|
||||
statements: 50
|
||||
cyclop:
|
||||
max-complexity: 15
|
||||
dupl:
|
||||
threshold: 100
|
||||
|
||||
issues:
|
||||
exclude-use-default: false
|
||||
max-issues-per-linter: 0
|
||||
max-same-issues: 0
|
||||
69
Dockerfile
69
Dockerfile
@@ -1,32 +1,55 @@
|
||||
FROM golang:1.14 as builder
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
WORKDIR /go/src/git.eeqj.de/sneak/orangesite
|
||||
# Lint stage — fast feedback on formatting and lint issues.
|
||||
# golangci/golangci-lint:v2.12.1 (Debian; bundles go, make, gcc), 2026-07-26
|
||||
FROM golangci/golangci-lint@sha256:c9843d374ca80ecbac86081ec4dd7fe2bb6187b03224f59a0cc2f80759e1845b AS lint
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN make fmt-check
|
||||
RUN make lint
|
||||
|
||||
# Build stage — compiles and tests. CGO is required by mattn/go-sqlite3, and
|
||||
# the pinned (legacy) sqlite driver only builds against glibc, so this stage
|
||||
# is Debian-based rather than alpine.
|
||||
# golang:1.24-bookworm, 2026-07-26
|
||||
FROM golang@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS builder
|
||||
WORKDIR /src
|
||||
|
||||
# Force BuildKit to complete the lint stage before compiling/testing.
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
|
||||
#RUN make lint && make build
|
||||
RUN make build
|
||||
RUN make test
|
||||
|
||||
WORKDIR /go
|
||||
RUN tar cvfz go-src.tgz src && du -sh *
|
||||
ARG VERSION=dev
|
||||
ARG TARGETARCH=unknown
|
||||
RUN CGO_ENABLED=1 go build -trimpath \
|
||||
-ldflags="-s -w -X main.Version=${VERSION} -X main.Buildarch=${TARGETARCH}" \
|
||||
-o /server ./cmd/server
|
||||
|
||||
# this container doesn't do anything except hold the build artifact
|
||||
# and make sure it compiles.
|
||||
# Runtime stage — the (glibc-linked) binary plus the on-disk templates and
|
||||
# vendored assets. ca-certificates is needed for the outbound TLS calls to
|
||||
# the Hacker News API.
|
||||
# debian:bookworm-slim, 2026-07-26
|
||||
FROM debian@sha256:7b140f374b289a7c2befc338f42ebe6441b7ea838a042bbd5acbfca6ec875818
|
||||
|
||||
FROM alpine
|
||||
|
||||
RUN mkdir -p /app/bin
|
||||
|
||||
COPY --from=builder /go/src/git.eeqj.de/sneak/orangesite/server /app/bin/server
|
||||
# FIXME figure out how to embed these stupid templates
|
||||
COPY --from=builder /go/src/git.eeqj.de/sneak/orangesite/view /app/view
|
||||
|
||||
# put the source in there too for safekeeping
|
||||
COPY --from=builder /go/go-src.tgz /usr/local/src/go-src.tgz
|
||||
|
||||
# this is where the db gets stored:
|
||||
VOLUME /data
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
CMD /app/bin/server
|
||||
COPY --from=builder /server /usr/local/bin/server
|
||||
COPY view ./view
|
||||
COPY static ./static
|
||||
|
||||
# FIXME add testing
|
||||
# sqlite database lives on a mounted volume
|
||||
VOLUME /data
|
||||
ENV DATABASE_PATH=/data/storage.sqlite
|
||||
|
||||
EXPOSE 8080
|
||||
ENTRYPOINT ["server"]
|
||||
|
||||
84
Makefile
84
Makefile
@@ -2,81 +2,77 @@
|
||||
export DATABASE_PATH := ./storage.sqlite
|
||||
|
||||
VERSION := $(shell git rev-parse --short HEAD)
|
||||
BUILDTIME := $(shell date -u '+%Y-%m-%dT%H:%M:%SZ')
|
||||
BUILDTIMEFILENAME := $(shell date -u '+%Y%m%d-%H%M%SZ')
|
||||
BUILDTIMETAG := $(shell date -u '+%Y%m%d%H%M%S')
|
||||
BUILDUSER := $(shell whoami)
|
||||
BUILDHOST := $(shell hostname -s)
|
||||
BUILDARCH := $(shell uname -m)
|
||||
BUILDTIMETAG := $(shell date -u '+%Y%m%d%H%M%S')
|
||||
|
||||
FN := server
|
||||
IMAGENAME := sneak/orangesite
|
||||
|
||||
UNAME_S := $(shell uname -s)
|
||||
|
||||
GOLDFLAGS += -X main.Version=$(VERSION)
|
||||
GOLDFLAGS += -X main.Buildarch=$(BUILDARCH)
|
||||
|
||||
# osx can't statically link apparently?!
|
||||
ifeq ($(UNAME_S),Darwin)
|
||||
GOFLAGS := -ldflags "$(GOLDFLAGS)"
|
||||
endif
|
||||
|
||||
ifneq ($(UNAME_S),Darwin)
|
||||
GOFLAGS = -ldflags "-linkmode external -extldflags -static $(GOLDFLAGS)"
|
||||
endif
|
||||
.PHONY: default run debug build clean \
|
||||
test lint fmt fmt-check check docker hooks \
|
||||
docker-dist docker-push
|
||||
|
||||
default: run
|
||||
|
||||
debug: build
|
||||
GOTRACEBACK=all DEBUG=1 ./$(FN)
|
||||
# --- development ---------------------------------------------------------
|
||||
|
||||
run: build
|
||||
./$(FN)
|
||||
|
||||
debug: build
|
||||
GOTRACEBACK=all DEBUG=1 ./$(FN)
|
||||
|
||||
build:
|
||||
go build -o $(FN) $(GOFLAGS) ./cmd/$(FN)
|
||||
|
||||
clean:
|
||||
-rm ./$(FN)
|
||||
-rm -f ./$(FN)
|
||||
|
||||
build: ./$(FN)
|
||||
# --- required policy targets ---------------------------------------------
|
||||
|
||||
.lintsetup:
|
||||
go get -v -u golang.org/x/lint/golint
|
||||
go get -u github.com/GeertJohan/fgt
|
||||
touch .lintsetup
|
||||
# run tests quietly; on failure, rerun verbosely for diagnostics
|
||||
test:
|
||||
@go test -timeout 30s ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -timeout 30s -v ./...; exit 1; }
|
||||
|
||||
lint: fmt .lintsetup
|
||||
fgt golint ./...
|
||||
|
||||
go-get:
|
||||
cd cmd/$(FN) && go get -v
|
||||
|
||||
./$(FN): */*.go cmd/*/*.go go-get
|
||||
cd cmd/$(FN) && go build -o ../../$(FN) $(GOFLAGS) .
|
||||
lint:
|
||||
golangci-lint run ./...
|
||||
|
||||
fmt:
|
||||
gofmt -s -w .
|
||||
|
||||
test: lint build-docker-image
|
||||
fmt-check:
|
||||
@out="$$(gofmt -s -l .)"; \
|
||||
if [ -n "$$out" ]; then \
|
||||
echo "gofmt needed on:"; echo "$$out"; exit 1; \
|
||||
fi
|
||||
|
||||
is_uncommitted:
|
||||
git diff --exit-code >/dev/null 2>&1
|
||||
# check must not modify any files in the repo
|
||||
check: fmt-check lint test
|
||||
|
||||
build-docker-image: clean
|
||||
docker:
|
||||
docker build -t $(IMAGENAME) .
|
||||
|
||||
build-docker-image-dist: is_uncommitted clean
|
||||
docker build -t $(IMAGENAME):$(VERSION) -t $(IMAGENAME):latest -t $(IMAGENAME):$(BUILDTIMETAG) .
|
||||
hooks:
|
||||
@printf '#!/bin/sh\nset -e\n' > .git/hooks/pre-commit
|
||||
@printf 'go mod tidy\ngofmt -s -w .\n' >> .git/hooks/pre-commit
|
||||
@printf 'git diff --exit-code -- go.mod go.sum || { echo "go mod tidy changed files; please stage and retry"; exit 1; }\n' >> .git/hooks/pre-commit
|
||||
@printf 'make check\n' >> .git/hooks/pre-commit
|
||||
@chmod +x .git/hooks/pre-commit
|
||||
@echo "installed .git/hooks/pre-commit"
|
||||
|
||||
dist: lint build-docker-image
|
||||
-mkdir -p ./output
|
||||
docker run --rm --entrypoint cat $(IMAGENAME) /bin/$(FN) > output/$(FN)
|
||||
docker save $(IMAGENAME) | bzip2 > output/$(BUILDTIMEFILENAME).$(FN).tbz2
|
||||
# --- image distribution --------------------------------------------------
|
||||
|
||||
hub: upload-docker-image
|
||||
docker-dist: docker
|
||||
docker tag $(IMAGENAME) $(IMAGENAME):$(VERSION)
|
||||
docker tag $(IMAGENAME) $(IMAGENAME):$(BUILDTIMETAG)
|
||||
|
||||
upload-docker-image: build-docker-image
|
||||
docker-push: docker-dist
|
||||
docker push $(IMAGENAME):$(VERSION)
|
||||
docker push $(IMAGENAME):$(BUILDTIMETAG)
|
||||
docker push $(IMAGENAME):latest
|
||||
|
||||
.PHONY: build fmt test is_uncommitted build-docker-image dist hub upload-docker-image clean run rundebug default build-docker-image-dist
|
||||
|
||||
82
README.md
82
README.md
@@ -1,18 +1,80 @@
|
||||
# Orangesite Transparency Log
|
||||
|
||||
Live: https://orangesite.sneak.cloud
|
||||
Orangesite is a WTFPL-licensed Go web application by
|
||||
[@sneak](https://sneak.berlin) that records which stories drop off the Hacker
|
||||
News front page and how long they lasted, as a small transparency log.
|
||||
|
||||
Shows stories that were on the orangesite front page within the last 24
|
||||
hours, but aren't any more. Sorted by when they exited the frontpage, most
|
||||
recent first.
|
||||
Live: <https://orangesite.sneak.cloud>
|
||||
|
||||
Stories on the frontpage for less than a half hour (likely manually
|
||||
moderator nuked if rank > 25 or so) are marked red for convenience.
|
||||
It shows stories that were on the HN front page within the last 24 hours but
|
||||
are not any more, sorted by when they left the front page, most recent first.
|
||||
Stories that were on the front page for less than half an hour (a likely sign
|
||||
of moderator intervention) are marked in red.
|
||||
|
||||
# TODO
|
||||
## Getting Started
|
||||
|
||||
* continue to resist the urge to use the orange
|
||||
The service needs Go and a C compiler (the sqlite driver uses cgo). Run it
|
||||
locally:
|
||||
|
||||
# Author
|
||||
```sh
|
||||
git clone https://git.eeqj.de/sneak/orangesite
|
||||
cd orangesite
|
||||
make run # builds ./server and serves on :8080
|
||||
```
|
||||
|
||||
* [sneak@sneak.berlin](mailto:sneak@sneak.berlin)
|
||||
Or build and run the container image (which runs `make check` as part of the
|
||||
build):
|
||||
|
||||
```sh
|
||||
make docker
|
||||
docker run -p 8080:8080 -v orangesite-data:/data sneak/orangesite
|
||||
```
|
||||
|
||||
Configuration is via environment variables:
|
||||
|
||||
- `DATABASE_PATH` — path to the sqlite database file (default
|
||||
`/data/storage.sqlite`, or `./storage.sqlite` under `make run`).
|
||||
- `DEBUG` — set to any non-empty value to enable debug logging and gorm SQL
|
||||
logging.
|
||||
|
||||
## Rationale
|
||||
|
||||
The Hacker News front page turns over constantly, and stories sometimes
|
||||
disappear far faster than their score and age would predict. Orangesite keeps
|
||||
an independent record of front-page tenure so that departures — especially
|
||||
unusually fast ones — remain visible after the fact.
|
||||
|
||||
## Design
|
||||
|
||||
The program is a single Go binary:
|
||||
|
||||
- `cmd/server` — the entrypoint; wires build-time version info into the `hn`
|
||||
package and runs the server.
|
||||
- `hn` — the application package:
|
||||
- `fetcher.go` scrapes the HN top-stories API once a minute and records
|
||||
each story's first appearance, rank changes, best rank reached, and the
|
||||
moment it leaves the front page.
|
||||
- `db.go` defines the gorm models, persisted to sqlite.
|
||||
- `handlers.go` renders the index and about pages.
|
||||
- `server.go` sets up the echo HTTP server, logging, and routes.
|
||||
- `view/` — pongo2 HTML templates, rendered at request time.
|
||||
- `static/` — vendored front-end assets (bootstrap, jquery, popper), served
|
||||
under `/static` so the site depends on no third-party CDNs.
|
||||
|
||||
## TODO
|
||||
|
||||
- Continue to resist the urge to use the orange.
|
||||
- Embed the `view/` templates and `static/` assets into the binary rather than
|
||||
shipping them alongside it.
|
||||
- Honour a `PORT` environment variable instead of hard-coding `:8080`.
|
||||
- Add graceful shutdown instead of relying on `Fatal`.
|
||||
- Harden the HTTP surface (security headers, server timeouts, request size
|
||||
limits) before tagging a `1.0`.
|
||||
|
||||
## License
|
||||
|
||||
WTFPL — see the [`LICENSE`](LICENSE) file.
|
||||
|
||||
## Author
|
||||
|
||||
[@sneak](https://sneak.berlin) — <sneak@sneak.berlin>
|
||||
|
||||
368
REPO_POLICIES.md
Normal file
368
REPO_POLICIES.md
Normal file
@@ -0,0 +1,368 @@
|
||||
---
|
||||
title: Repository Policies
|
||||
last_modified: 2026-07-06
|
||||
---
|
||||
|
||||
This document covers repository structure, tooling, and workflow standards. Code
|
||||
style conventions are in separate documents:
|
||||
|
||||
- [Code Styleguide](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE.md)
|
||||
(general, bash, Docker)
|
||||
- [Go](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_GO.md)
|
||||
- [JavaScript](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_JS.md)
|
||||
- [Python](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_PYTHON.md)
|
||||
- [Go HTTP Server Conventions](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/GO_HTTP_SERVER_CONVENTIONS.md)
|
||||
|
||||
---
|
||||
|
||||
- Cross-project documentation (such as this file) must include
|
||||
`last_modified: YYYY-MM-DD` in the YAML front matter so it can be kept in sync
|
||||
with the authoritative source as policies evolve.
|
||||
|
||||
- **ALL external references must be pinned by cryptographic hash.** This
|
||||
includes Docker base images, Go modules, npm packages, GitHub Actions, and
|
||||
anything else fetched from a remote source. Version tags (`@v4`, `@latest`,
|
||||
`:3.21`, etc.) are server-mutable and therefore remote code execution
|
||||
vulnerabilities. The ONLY acceptable way to reference an external dependency
|
||||
is by its content hash (Docker `@sha256:...`, Go module hash in `go.sum`, npm
|
||||
integrity hash in lockfile, GitHub Actions `@<commit-sha>`). No exceptions.
|
||||
This also means never `curl | bash` to install tools like pyenv, nvm, rustup,
|
||||
etc. Instead, download a specific release archive from GitHub, verify its hash
|
||||
(hardcoded in the Dockerfile or script), and only then install. Unverified
|
||||
install scripts are arbitrary remote code execution. This is the single most
|
||||
important rule in this document. Double-check every external reference in
|
||||
every file before committing. There are zero exceptions to this rule.
|
||||
|
||||
- Every repo with software must have a root `Makefile` with these targets:
|
||||
`make test`, `make lint`, `make fmt` (writes), `make fmt-check` (read-only),
|
||||
`make check` (prereqs: `test`, `lint`, `fmt-check`), `make docker`, and
|
||||
`make hooks` (installs pre-commit hook). A model Makefile is at
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/Makefile`.
|
||||
|
||||
- Always use Makefile targets (`make fmt`, `make test`, `make lint`, etc.)
|
||||
instead of invoking the underlying tools directly. The Makefile is the single
|
||||
source of truth for how these operations are run.
|
||||
|
||||
- The Makefile is authoritative documentation for how the repo is used. Beyond
|
||||
the required targets above, it should have targets for every common operation:
|
||||
running a local development server (`make run`, `make dev`), re-initializing
|
||||
or migrating the database (`make db-reset`, `make migrate`), building
|
||||
artifacts (`make build`), generating code, seeding data, or anything else a
|
||||
developer would do regularly. If someone checks out the repo and types
|
||||
`make<tab>`, they should see every meaningful operation available. A new
|
||||
contributor should be able to understand the entire development workflow by
|
||||
reading the Makefile.
|
||||
|
||||
- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
|
||||
as a build step so the build fails if the branch is not green. For non-server
|
||||
repos, the Dockerfile should bring up a development environment and run
|
||||
`make check`. For server repos, `make check` should run as an early build
|
||||
stage before the final image is assembled.
|
||||
|
||||
- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
|
||||
repos use a multistage build where linting runs in an independent stage based
|
||||
on the `golangci/golangci-lint` image (pinned by hash). This stage runs
|
||||
`make fmt-check` and `make lint` before the full build begins. The build stage
|
||||
then declares an explicit dependency on the lint stage via
|
||||
`COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
|
||||
linting before proceeding to compilation and tests. This ensures lint failures
|
||||
surface in seconds rather than minutes, without blocking on dependency
|
||||
download or compilation in the build stage.
|
||||
|
||||
The standard pattern for a Go repo Dockerfile is:
|
||||
|
||||
```dockerfile
|
||||
# Lint stage — fast feedback on formatting and lint issues
|
||||
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
||||
FROM golangci/golangci-lint@sha256:... AS lint
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN make fmt-check
|
||||
RUN make lint
|
||||
|
||||
# Build stage
|
||||
# golang:1.x-alpine, YYYY-MM-DD
|
||||
FROM golang@sha256:... AS builder
|
||||
WORKDIR /src
|
||||
|
||||
# Force BuildKit to run the lint stage before proceeding
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN make test
|
||||
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
||||
-o /app ./cmd/app/
|
||||
|
||||
# Runtime stage
|
||||
FROM alpine@sha256:...
|
||||
COPY --from=builder /app /usr/local/bin/app
|
||||
ENTRYPOINT ["app"]
|
||||
```
|
||||
|
||||
Key points:
|
||||
- The lint stage uses the `golangci/golangci-lint` image directly (it
|
||||
includes both Go and the linter), so there is no need to install the
|
||||
linter separately.
|
||||
- `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
|
||||
a stage dependency. BuildKit runs stages in parallel by default; without
|
||||
this line, the build stage would not wait for lint to finish and a lint
|
||||
failure might not fail the overall build.
|
||||
- If the project uses `//go:embed` directives that reference build artifacts
|
||||
(e.g. a web frontend compiled in a separate stage), the lint stage must
|
||||
create placeholder files so the embed directives resolve. Example:
|
||||
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
|
||||
The lint stage should not depend on the actual build output — it exists to
|
||||
fail fast.
|
||||
- If the project requires CGO or system libraries for linting (e.g.
|
||||
`vips-dev`), install them in the lint stage with `apk add`.
|
||||
- The build stage runs `make test` after compilation setup. Tests run in the
|
||||
build stage, not the lint stage, because they may require compiled
|
||||
artifacts or heavier dependencies.
|
||||
|
||||
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
|
||||
runs `docker build .` on push. Since the Dockerfile already runs `make check`,
|
||||
a successful build implies all checks pass.
|
||||
|
||||
- Use platform-standard formatters: `black` for Python, `prettier` for
|
||||
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
||||
two exceptions: four-space indents (except Go), and `proseWrap: always` for
|
||||
Markdown (hard-wrap at 80 columns). Documentation and writing repos (Markdown,
|
||||
HTML, CSS) should also have `.prettierrc` and `.prettierignore`.
|
||||
|
||||
- Pre-commit hook: `make check` if local testing is possible, otherwise
|
||||
`make lint && make fmt-check`. The Makefile should provide a `make hooks`
|
||||
target to install the pre-commit hook.
|
||||
|
||||
- All repos with software must have tests that run via the platform-standard
|
||||
test framework (`go test`, `pytest`, `jest`/`vitest`, etc.). If no meaningful
|
||||
tests exist yet, add the most minimal test possible — e.g. importing the
|
||||
module under test to verify it compiles/parses. There is no excuse for
|
||||
`make test` to be a no-op.
|
||||
|
||||
- `make test` must complete in under 20 seconds. Add a 30-second timeout in the
|
||||
Makefile.
|
||||
|
||||
- **`make test` should use the conditional verbose rerun pattern.** Run tests
|
||||
without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
|
||||
show full output. This keeps CI logs and `docker build` output clean on
|
||||
success (just package/suite summaries) while providing full diagnostic detail
|
||||
on failure (every test case, every assertion). The general shell pattern:
|
||||
|
||||
```makefile
|
||||
test:
|
||||
@<test-command> || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
<test-command-with-v>; exit 1; }
|
||||
```
|
||||
|
||||
Go example:
|
||||
|
||||
```makefile
|
||||
test:
|
||||
@go test -timeout 30s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -timeout 30s -race -v ./...; exit 1; }
|
||||
```
|
||||
|
||||
Python example:
|
||||
|
||||
```makefile
|
||||
test:
|
||||
@python -m pytest || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
python -m pytest -v; exit 1; }
|
||||
```
|
||||
|
||||
The `exit 1` ensures the target always fails after a rerun — the first run
|
||||
already proved the tests are broken, so the build must not pass even if a
|
||||
flaky test happens to succeed on the second attempt. The rerun exists solely
|
||||
for diagnostic output.
|
||||
|
||||
- Docker builds must complete in under 5 minutes.
|
||||
|
||||
- `make check` must not modify any files in the repo. Tests may use temporary
|
||||
directories.
|
||||
|
||||
- `main` must always pass `make check`, no exceptions.
|
||||
|
||||
- Never commit secrets. `.env` files, credentials, API keys, and private keys
|
||||
must be in `.gitignore`. No exceptions.
|
||||
|
||||
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
|
||||
editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
|
||||
Fetch the standard `.gitignore` from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
|
||||
a new repo.
|
||||
|
||||
- **No build artifacts in version control.** Code-derived data (compiled
|
||||
bundles, minified output, generated assets) must never be committed to the
|
||||
repository if it can be avoided. The build process (e.g. Dockerfile, Makefile)
|
||||
should generate these at build time. Notable exception: Go protobuf generated
|
||||
files (`.pb.go`) ARE committed because repos need to work with `go get`, which
|
||||
downloads code but does not execute code generation.
|
||||
|
||||
- Never use `git add -A` or `git add .`. Always stage files explicitly by name.
|
||||
|
||||
- Never force-push to `main`.
|
||||
|
||||
- Make all changes on a feature branch. You can do whatever you want on a
|
||||
feature branch.
|
||||
|
||||
- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
|
||||
manually by the user. Fetch from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
|
||||
|
||||
- When pinning images or packages by hash, add a comment above the reference
|
||||
with the version and date (YYYY-MM-DD).
|
||||
|
||||
- Use `yarn`, not `npm`.
|
||||
|
||||
- Write all dates as YYYY-MM-DD (ISO 8601).
|
||||
|
||||
- Simple projects should be configured with environment variables.
|
||||
|
||||
- Dockerized web services listen on port 8080 by default, overridable with
|
||||
`PORT`.
|
||||
|
||||
- **HTTP/web services must be hardened for production internet exposure before
|
||||
tagging 1.0.** This means full compliance with security best practices
|
||||
including, without limitation, all of the following:
|
||||
- **Security headers** on every response:
|
||||
- `Strict-Transport-Security` (HSTS) with `max-age` of at least one year
|
||||
and `includeSubDomains`.
|
||||
- `Content-Security-Policy` (CSP) with a restrictive default policy
|
||||
(`default-src 'self'` as a baseline, tightened per-resource as
|
||||
needed). Never use `unsafe-inline` or `unsafe-eval` unless
|
||||
unavoidable, and document the reason.
|
||||
- `X-Frame-Options: DENY` (or `SAMEORIGIN` if framing is required).
|
||||
Prefer the `frame-ancestors` CSP directive as the primary control.
|
||||
- `X-Content-Type-Options: nosniff`.
|
||||
- `Referrer-Policy: strict-origin-when-cross-origin` (or stricter).
|
||||
- `Permissions-Policy` restricting access to browser features the
|
||||
application does not use (camera, microphone, geolocation, etc.).
|
||||
- **Request and response limits:**
|
||||
- Maximum request body size enforced on all endpoints (e.g. Go
|
||||
`http.MaxBytesReader`). Choose a sane default per-route; never accept
|
||||
unbounded input.
|
||||
- Maximum response body size where applicable (e.g. paginated APIs).
|
||||
- `ReadTimeout` and `ReadHeaderTimeout` on the `http.Server` to defend
|
||||
against slowloris attacks.
|
||||
- `WriteTimeout` on the `http.Server`.
|
||||
- `IdleTimeout` on the `http.Server`.
|
||||
- Per-handler execution time limits via `context.WithTimeout` or
|
||||
chi/stdlib `middleware.Timeout`.
|
||||
- **Authentication and session security:**
|
||||
- Rate limiting on password-based authentication endpoints. API keys are
|
||||
high-entropy and not susceptible to brute force, so they are exempt.
|
||||
- CSRF tokens on all state-mutating HTML forms. API endpoints
|
||||
authenticated via `Authorization` header (Bearer token, API key) are
|
||||
exempt because the browser does not attach these automatically.
|
||||
- Passwords stored using bcrypt, scrypt, or argon2 — never plain-text,
|
||||
MD5, or SHA.
|
||||
- Session cookies set with `HttpOnly`, `Secure`, and `SameSite=Lax` (or
|
||||
`Strict`) attributes.
|
||||
- **Reverse proxy awareness:**
|
||||
- True client IP detection when behind a reverse proxy
|
||||
(`X-Forwarded-For`, `X-Real-IP`). The application must accept
|
||||
forwarded headers only from a configured set of trusted proxy
|
||||
addresses — never trust `X-Forwarded-For` unconditionally.
|
||||
- **CORS:**
|
||||
- Authenticated endpoints must restrict `Access-Control-Allow-Origin` to
|
||||
an explicit allowlist of known origins. Wildcard (`*`) is acceptable
|
||||
only for public, unauthenticated read-only APIs.
|
||||
- **Error handling:**
|
||||
- Internal errors must never leak stack traces, SQL queries, file paths,
|
||||
or other implementation details to the client. Return generic error
|
||||
messages in production; detailed errors only when `DEBUG` is enabled.
|
||||
- **TLS:**
|
||||
- Services never terminate TLS directly. They are always deployed behind
|
||||
a TLS-terminating reverse proxy. The service itself listens on plain
|
||||
HTTP. However, HSTS headers and `Secure` cookie flags must still be
|
||||
set by the application so that the browser enforces HTTPS end-to-end.
|
||||
|
||||
This list is non-exhaustive. Apply defense-in-depth: if a standard security
|
||||
hardening measure exists for HTTP services and is not listed here, it is
|
||||
still expected. When in doubt, harden.
|
||||
|
||||
- `README.md` is the primary documentation. Required sections:
|
||||
- **Description**: First line must include the project name, purpose,
|
||||
category (web server, SPA, CLI tool, etc.), license, and author. Example:
|
||||
"µPaaS is an MIT-licensed Go web application by @sneak that receives
|
||||
git-frontend webhooks and deploys applications via Docker in realtime."
|
||||
- **Getting Started**: Copy-pasteable install/usage code block.
|
||||
- **Rationale**: Why does this exist?
|
||||
- **Design**: How is the program structured?
|
||||
- **TODO**: Update meticulously, even between commits. When planning, put
|
||||
the todo list in the README so a new agent can pick up where the last one
|
||||
left off.
|
||||
- **License**: MIT, GPL, or WTFPL. Ask the user for new projects. Include a
|
||||
`LICENSE` file in the repo root and a License section in the README.
|
||||
- **Author**: [@sneak](https://sneak.berlin).
|
||||
|
||||
- First commit of a new repo should contain only `README.md`.
|
||||
|
||||
- Go module root: `sneak.berlin/go/<name>`. Always run `go mod tidy` before
|
||||
committing.
|
||||
|
||||
- Use SemVer.
|
||||
|
||||
- Database migrations live in `internal/db/migrations/` and must be embedded in
|
||||
the binary.
|
||||
- `000_migration.sql` — contains ONLY the creation of the migrations
|
||||
tracking table itself. Nothing else.
|
||||
- `001_schema.sql` — the full application schema.
|
||||
- **Pre-1.0.0:** never add additional migration files (002, 003, etc.).
|
||||
There is no installed base to migrate. Edit `001_schema.sql` directly.
|
||||
- **Post-1.0.0:** add new numbered migration files for each schema change.
|
||||
Never edit existing migrations after release.
|
||||
|
||||
- All repos should have an `.editorconfig` enforcing the project's indentation
|
||||
settings.
|
||||
|
||||
- **Claude Code repo memory is versioned in the repo**, not left only in
|
||||
`~/.claude` on one machine. Each memory is one file at
|
||||
`.claude/memory/<memory>.md`, and every memory file must be `@`-imported
|
||||
from `.claude/CLAUDE.md` (one `- @memory/<memory>.md` list line per file;
|
||||
relative import paths resolve against `.claude/`, and Claude Code expands
|
||||
the imports into context at session launch). When adding a memory, add both
|
||||
the file and its import line. A root `MEMORY.md` is a violation — Claude
|
||||
Code never auto-loads it; split it into `.claude/memory/` files. Repos with
|
||||
no memories yet need no `.claude/` scaffolding.
|
||||
|
||||
- Avoid putting files in the repo root unless necessary. Root should contain
|
||||
only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
|
||||
`LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
|
||||
language-specific config). Everything else goes in a subdirectory. Canonical
|
||||
subdirectory names:
|
||||
- `bin/` — executable scripts and tools
|
||||
- `cmd/` — Go command entrypoints
|
||||
- `configs/` — configuration templates and examples
|
||||
- `deploy/` — deployment manifests (k8s, compose, terraform)
|
||||
- `docs/` — documentation and markdown (README.md stays in root)
|
||||
- `internal/` — Go internal packages
|
||||
- `internal/db/migrations/` — database migrations
|
||||
- `pkg/` — Go library packages
|
||||
- `share/` — systemd units, data files
|
||||
- `static/` — static assets (images, fonts, etc.)
|
||||
- `web/` — web frontend source
|
||||
|
||||
- When setting up a new repo, files from the `prompts` repo may be used as
|
||||
templates. Fetch them from
|
||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/<path>`.
|
||||
|
||||
- New repos must contain at minimum:
|
||||
- `README.md`, `.git`, `.gitignore`, `.editorconfig`
|
||||
- `LICENSE`, `REPO_POLICIES.md` (copy from the `prompts` repo)
|
||||
- `Makefile`
|
||||
- `Dockerfile`, `.dockerignore`
|
||||
- `.gitea/workflows/check.yml`
|
||||
- Go: `go.mod`, `go.sum`, `.golangci.yml`
|
||||
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
|
||||
- Python: `pyproject.toml`
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:generate go run github.com/UnnoTed/fileb0x b0x.yaml
|
||||
// Command server runs the orangesite transparency-log web application.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -7,8 +7,14 @@ import (
|
||||
"git.eeqj.de/sneak/orangesite/hn"
|
||||
)
|
||||
|
||||
var Version string
|
||||
var Buildarch string
|
||||
// Version and Buildarch are injected at build time via
|
||||
// -ldflags "-X main.Version=... -X main.Buildarch=...".
|
||||
//
|
||||
//nolint:gochecknoglobals // set at link time by the build, cannot be const
|
||||
var (
|
||||
Version string
|
||||
Buildarch string
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(hn.RunServer(Version, Buildarch))
|
||||
|
||||
1
go.mod
1
go.mod
@@ -5,7 +5,6 @@ go 1.14
|
||||
require (
|
||||
git.eeqj.de/sneak/goutil v0.0.0-20200330224956-7fad5dc142e5
|
||||
github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4
|
||||
github.com/hako/durafmt v0.0.0-20191009132224-3f39dc1ed9f4
|
||||
github.com/jinzhu/gorm v1.9.12
|
||||
github.com/juju/errors v0.0.0-20200330140219-3fe23663418f // indirect
|
||||
github.com/labstack/echo v3.3.10+incompatible
|
||||
|
||||
31
go.sum
31
go.sum
@@ -1,17 +1,21 @@
|
||||
git.eeqj.de/sneak/goutil v0.0.0-20200330224009-e54964f792cd h1:LlQYjhr5NA5WK9f/s0QnnxHZE3YbnAhSkqQ/sJCdHk8=
|
||||
git.eeqj.de/sneak/goutil v0.0.0-20200330224009-e54964f792cd/go.mod h1:eczIi5zp8IZnFLQbMF0Xufw6to+UMCbOxA4M4Hp7ORw=
|
||||
git.eeqj.de/sneak/goutil v0.0.0-20200330224956-7fad5dc142e5 h1:0hBq85ulrB0pfC+lxb/UAMfGRHkykDMis0E8i2Di3zI=
|
||||
git.eeqj.de/sneak/goutil v0.0.0-20200330224956-7fad5dc142e5/go.mod h1:eczIi5zp8IZnFLQbMF0Xufw6to+UMCbOxA4M4Hp7ORw=
|
||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM=
|
||||
github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
|
||||
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
|
||||
github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 h1:GY1+t5Dr9OKADM64SYnQjw/w99HMYvQ0A8/JoUkxVmc=
|
||||
github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA=
|
||||
github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI=
|
||||
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
|
||||
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/hako/durafmt v0.0.0-20191009132224-3f39dc1ed9f4 h1:60gBOooTSmNtrqNaRvrDbi8VAne0REaek2agjnITKSw=
|
||||
@@ -20,27 +24,29 @@ github.com/jinzhu/gorm v1.9.12 h1:Drgk1clyWT9t9ERbzHza6Mj/8FY/CqMyVzOiHviMo6Q=
|
||||
github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M=
|
||||
github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 h1:rhqTjzJlm7EbkELJDKMTU7udov+Se0xZkWmugr6zGok=
|
||||
github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
|
||||
github.com/juju/errors v0.0.0-20200330140219-3fe23663418f h1:MCOvExGLpaSIzLYB4iQXEHP4jYVU6vmzLNQPdMVrxnM=
|
||||
github.com/juju/errors v0.0.0-20200330140219-3fe23663418f/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
|
||||
github.com/juju/loggo v0.0.0-20180524022052-584905176618 h1:MK144iBQF9hTSwBW/9eJm034bVoG30IshVm688T2hi8=
|
||||
github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
|
||||
github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073 h1:WQM1NildKThwdP7qWrNAFGzp4ijNLw8RlgENkaI4MJs=
|
||||
github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/labstack/echo v1.4.4 h1:1bEiBNeGSUKxcPDGfZ/7IgdhJJZx8wV/pICJh4W2NJI=
|
||||
github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg=
|
||||
github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s=
|
||||
github.com/labstack/echo/v4 v4.1.10 h1:/yhIpO50CBInUbE/nHJtGIyhBv0dJe2cDAYxc3V3uMo=
|
||||
github.com/labstack/echo/v4 v4.1.10/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
|
||||
github.com/labstack/echo/v4 v4.1.16 h1:8swiwjE5Jkai3RPfZoahp8kjVCRNq+y7Q0hPji2Kz0o=
|
||||
github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI=
|
||||
github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
|
||||
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
|
||||
github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
|
||||
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
|
||||
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=
|
||||
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
@@ -48,7 +54,6 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
|
||||
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
|
||||
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw=
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
|
||||
github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||
@@ -58,27 +63,26 @@ github.com/mayowa/echo-pongo2 v0.0.0-20170410154925-661ce95e1767/go.mod h1:JCIHk
|
||||
github.com/peterhellberg/hn v0.0.0-20160106115829-a27cdd2ca854 h1:K6zV6rLnXlmq8H7gqclq/K8ZANQyqpvhrzSkq70qoZw=
|
||||
github.com/peterhellberg/hn v0.0.0-20160106115829-a27cdd2ca854/go.mod h1:4NUrlv14rntJHyfqrF46i63j+7lUU8yIx/y6ORuO32M=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
|
||||
github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8=
|
||||
github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
|
||||
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
|
||||
github.com/valyala/fasttemplate v1.1.0 h1:RZqt0yGBsps8NGvLSGW804QQqCUYYLsaOjTVHy1Ocw4=
|
||||
github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
|
||||
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
|
||||
github.com/ziflex/lecho v1.2.0 h1:/ykfd7V/aTsWUYNFimgbdhUiEMnWzvNaCxtbM/LX5F8=
|
||||
github.com/ziflex/lecho/v2 v2.0.0 h1:ggrWF5LaGAC+Y+WX71jFK7uYR7cUFbHjIgGqCyrYC5Q=
|
||||
github.com/ziflex/lecho/v2 v2.0.0/go.mod h1:s7dy9Fynjx6z+/7xE2BsK13vXIS3oQoo4ZaKXYG5xUs=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM=
|
||||
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM=
|
||||
@@ -86,7 +90,6 @@ golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPh
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
|
||||
@@ -96,13 +99,11 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 h1:TC0v2RSO1u2kn1ZugjrFXkRZAEaqMN/RW+OTZkBzmLE=
|
||||
golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
@@ -111,7 +112,11 @@ golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGm
|
||||
golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU=
|
||||
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
||||
23
hn/db.go
23
hn/db.go
@@ -1,3 +1,5 @@
|
||||
// Package hn implements the orangesite Hacker News front-page scraper, its
|
||||
// gorm-backed storage schema, and the HTTP handlers that render it.
|
||||
package hn
|
||||
|
||||
import (
|
||||
@@ -8,9 +10,16 @@ import (
|
||||
|
||||
// this schema is quite redundant, i know
|
||||
|
||||
// HNFrontPage records a story's tenure on the front page: when it appeared,
|
||||
// when it disappeared, and the best rank it reached. The HN-prefixed name is
|
||||
// retained deliberately: gorm derives the table name ("hn_front_pages") from
|
||||
// it, so renaming would orphan the existing production database.
|
||||
//
|
||||
//nolint:revive // see above: renaming changes the gorm table name
|
||||
type HNFrontPage struct {
|
||||
gorm.Model
|
||||
InternalID uint64 `gorm:"primary_key;auto_increment:true`
|
||||
|
||||
InternalID uint64
|
||||
HNID uint // HN integer id
|
||||
Appeared time.Time
|
||||
Disappeared time.Time
|
||||
@@ -21,9 +30,15 @@ type HNFrontPage struct {
|
||||
URL string // duh
|
||||
}
|
||||
|
||||
// HNStoryRank is a point-in-time snapshot of a single story's rank. The
|
||||
// HN-prefixed name is retained deliberately for the same gorm table-name
|
||||
// reason as HNFrontPage.
|
||||
//
|
||||
//nolint:revive // renaming changes the gorm table name
|
||||
type HNStoryRank struct {
|
||||
gorm.Model
|
||||
InternalStoryID uint64 `gorm:"primary_key;auto_increment:true`
|
||||
|
||||
InternalStoryID uint64
|
||||
HNID uint // HN integer id
|
||||
Title string // submission title
|
||||
URL string // duh
|
||||
@@ -32,9 +47,11 @@ type HNStoryRank struct {
|
||||
FetchedAt time.Time // identical within fetchid
|
||||
}
|
||||
|
||||
// FrontPageCache is a cached view of a story's front-page lifetime.
|
||||
type FrontPageCache struct {
|
||||
gorm.Model
|
||||
CacheID uint64 `gorm:"primary_key;auto_increment:true`
|
||||
|
||||
CacheID uint64
|
||||
HNID uint
|
||||
HighestRankReached uint
|
||||
URL string
|
||||
|
||||
202
hn/fetcher.go
202
hn/fetcher.go
@@ -6,111 +6,88 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/jinzhu/gorm"
|
||||
// sqlite3 dialect registered with gorm via import side effects.
|
||||
_ "github.com/jinzhu/gorm/dialects/sqlite"
|
||||
"github.com/peterhellberg/hn"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
func NewFetcher(db *gorm.DB) *Fetcher {
|
||||
f := new(Fetcher)
|
||||
f.db = db
|
||||
f.fetchIntervalSecs = 60
|
||||
f.hn = hn.NewClient(&http.Client{
|
||||
Timeout: time.Duration(5 * time.Second),
|
||||
})
|
||||
return f
|
||||
}
|
||||
const (
|
||||
// frontPageSize is the number of stories shown on the HN front page.
|
||||
frontPageSize = 30
|
||||
// defaultFetchInterval is how often the front page is scraped.
|
||||
defaultFetchInterval = 60 * time.Second
|
||||
// httpClientTimeout bounds each request to the HN API.
|
||||
httpClientTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
// Fetcher periodically scrapes the HN front page and records front-page
|
||||
// tenure in the database.
|
||||
type Fetcher struct {
|
||||
nextFetch time.Time
|
||||
fetchIntervalSecs uint
|
||||
fetchInterval time.Duration
|
||||
db *gorm.DB
|
||||
hn *hn.Client
|
||||
log *zerolog.Logger
|
||||
}
|
||||
|
||||
// NewFetcher builds a Fetcher that scrapes the HN front page once a minute.
|
||||
func NewFetcher(db *gorm.DB) *Fetcher {
|
||||
f := new(Fetcher)
|
||||
f.db = db
|
||||
f.fetchInterval = defaultFetchInterval
|
||||
f.hn = hn.NewClient(&http.Client{
|
||||
Timeout: httpClientTimeout,
|
||||
})
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
// AddLogger attaches a logger to the fetcher.
|
||||
func (f *Fetcher) AddLogger(l *zerolog.Logger) {
|
||||
f.log = l
|
||||
}
|
||||
|
||||
func (f *Fetcher) run() {
|
||||
|
||||
if os.Getenv("DEBUG") != "" {
|
||||
f.db.LogMode(true)
|
||||
}
|
||||
|
||||
f.db.AutoMigrate(&HNStoryRank{})
|
||||
f.db.AutoMigrate(&FrontPageCache{})
|
||||
f.db.AutoMigrate(&HNFrontPage{})
|
||||
|
||||
for {
|
||||
f.log.Info().
|
||||
Msg("fetching top stories from HN")
|
||||
f.nextFetch = time.Now().Add(time.Duration(f.fetchIntervalSecs) * time.Second)
|
||||
err := f.StoreFrontPage()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
until := time.Until(f.nextFetch)
|
||||
countdown := time.NewTimer(until)
|
||||
f.log.Info().Msgf("waiting %s until next fetch", until)
|
||||
<-countdown.C
|
||||
}
|
||||
}
|
||||
|
||||
// StoreFrontPage scrapes the current HN front page, records new and changed
|
||||
// stories, and marks stories that have left the front page.
|
||||
func (f *Fetcher) StoreFrontPage() error {
|
||||
|
||||
// FIXME set fetchid
|
||||
//r, err := f.db.Table("hn_story_rank").Select("MAX(FetchID)").Rows()
|
||||
|
||||
//pp.Print(r)
|
||||
//Select("max(FetchID)").Find(&HNStoryRank)
|
||||
|
||||
ids, err := f.hn.TopStories()
|
||||
t := time.Now()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 30 items on HN frontpage.
|
||||
for i, id := range ids[:30] {
|
||||
t := time.Now()
|
||||
|
||||
for i, id := range ids[:frontPageSize] {
|
||||
item, err := f.hn.Item(id)
|
||||
if err != nil {
|
||||
return (err)
|
||||
return err
|
||||
}
|
||||
/*
|
||||
s := HNStoryRank{
|
||||
HNID: uint(id),
|
||||
Rank: uint(i + 1),
|
||||
URL: item.URL,
|
||||
Title: item.Title,
|
||||
Score: item.Score,
|
||||
FetchedAt: t,
|
||||
}
|
||||
*/
|
||||
|
||||
//f.log.Debug().Msgf("storing story with rank %d in db", (i + 1))
|
||||
// FIXME this will grow unbounded and make the file too big if
|
||||
// I don't clean this up or otherwise limit the data in here
|
||||
// disabled for now
|
||||
//f.db.Create(&s)
|
||||
|
||||
//FIXME check to see if the same HNID was already on the frontpage
|
||||
//or not so we don't spam the db
|
||||
|
||||
// check to see if the item was on the frontpage already or not
|
||||
var c int
|
||||
f.db.Model(&HNFrontPage{}).Where("hn_id = ?", id).Count(&c)
|
||||
|
||||
if c == 0 {
|
||||
// first appearance on frontpage
|
||||
f.recordNewStory(id, i, t, item)
|
||||
} else {
|
||||
f.updateExistingStory(id, i, item)
|
||||
}
|
||||
}
|
||||
|
||||
return f.markDepartedStories(ids, t)
|
||||
}
|
||||
|
||||
// recordNewStory records a story appearing on the front page for the first
|
||||
// time.
|
||||
func (f *Fetcher) recordNewStory(id, rank int, t time.Time, item *hn.Item) {
|
||||
r := HNFrontPage{
|
||||
HNID: uint(id),
|
||||
Appeared: t,
|
||||
Disappeared: time.Time{},
|
||||
HighestRank: uint(i + 1),
|
||||
Rank: uint(i + 1),
|
||||
HighestRank: uint(rank + 1),
|
||||
Rank: uint(rank + 1),
|
||||
Title: item.Title,
|
||||
Score: uint(item.Score),
|
||||
URL: item.URL,
|
||||
@@ -118,36 +95,41 @@ func (f *Fetcher) StoreFrontPage() error {
|
||||
f.db.Create(&r)
|
||||
f.log.Info().
|
||||
Uint("hnid", uint(id)).
|
||||
Uint("rank", uint(i+1)).
|
||||
Uint("rank", uint(rank+1)).
|
||||
Str("title", item.Title).
|
||||
Int("score", item.Score).
|
||||
Str("url", item.URL).
|
||||
Msg("HN new story on frontpage")
|
||||
} else {
|
||||
// it's still here, (or back)
|
||||
}
|
||||
|
||||
// updateExistingStory updates the rank, record rank, and score of a story
|
||||
// still on (or returned to) the front page.
|
||||
func (f *Fetcher) updateExistingStory(id, rank int, item *hn.Item) {
|
||||
var old HNFrontPage
|
||||
f.db.Model(&HNFrontPage{}).Where("hn_id = ?", id).First(&old)
|
||||
|
||||
if old.Rank != uint(i+1) {
|
||||
if old.Rank != uint(rank+1) {
|
||||
f.log.Info().
|
||||
Uint("hnid", uint(id)).
|
||||
Uint("oldrank", old.Rank).
|
||||
Uint("newrank", uint(i+1)).
|
||||
Uint("newrank", uint(rank+1)).
|
||||
Int("score", item.Score).
|
||||
Str("title", item.Title).
|
||||
Str("url", item.URL).
|
||||
Msg("HN story rank changed, recording new rank")
|
||||
old.Rank = uint(i + 1)
|
||||
|
||||
old.Rank = uint(rank + 1)
|
||||
old.Score = uint(item.Score)
|
||||
}
|
||||
|
||||
if old.HighestRank > uint(i+1) {
|
||||
if old.HighestRank > uint(rank+1) {
|
||||
f.log.Info().
|
||||
Uint("hnid", uint(id)).
|
||||
Uint("oldrecord", old.HighestRank).
|
||||
Uint("newrecord", uint(i+1)).
|
||||
Uint("newrecord", uint(rank+1)).
|
||||
Msg("recording new record high rank for story")
|
||||
old.HighestRank = uint(i + 1)
|
||||
|
||||
old.HighestRank = uint(rank + 1)
|
||||
}
|
||||
|
||||
if old.Score != uint(item.Score) {
|
||||
@@ -159,31 +141,39 @@ func (f *Fetcher) StoreFrontPage() error {
|
||||
f.db.Save(&old)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// FIXME iterate over frontpage items still active in DB and note any
|
||||
// that are no longer on the scrape
|
||||
// markDepartedStories marks any active front-page rows whose story no longer
|
||||
// appears in the current scrape as departed at time t.
|
||||
func (f *Fetcher) markDepartedStories(ids []int, t time.Time) error {
|
||||
fpitems, err := f.db.Model(&HNFrontPage{}).Where("disappeared is ?", time.Time{}).Rows()
|
||||
if err != nil {
|
||||
f.log.Error().
|
||||
Err(err)
|
||||
f.log.Error().Err(err).Msg("querying active frontpage items")
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
var toupdate []uint
|
||||
|
||||
for fpitems.Next() {
|
||||
var item HNFrontPage
|
||||
f.db.ScanRows(fpitems, &item)
|
||||
//pp.Print(item)
|
||||
|
||||
err = f.db.ScanRows(fpitems, &item)
|
||||
if err != nil {
|
||||
f.log.Error().Err(err).Msg("scanning frontpage row")
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
exitedFrontPage := true
|
||||
for _, xd := range ids[:30] {
|
||||
|
||||
for _, xd := range ids[:frontPageSize] {
|
||||
if item.HNID == uint(xd) {
|
||||
exitedFrontPage = false
|
||||
}
|
||||
}
|
||||
|
||||
if exitedFrontPage {
|
||||
toupdate = append(toupdate, item.HNID)
|
||||
//item.Disappeared = t
|
||||
dur := t.Sub(item.Appeared).String()
|
||||
//f.db.Save(&item)
|
||||
f.log.Info().
|
||||
Uint("hnid", item.HNID).
|
||||
Uint("HighestRank", item.HighestRank).
|
||||
@@ -191,11 +181,41 @@ func (f *Fetcher) StoreFrontPage() error {
|
||||
Str("time_on_frontpage", dur).
|
||||
Str("url", item.URL).
|
||||
Msg("HN story exited frontpage")
|
||||
}
|
||||
}
|
||||
|
||||
// close result before we do the update
|
||||
_ = fpitems.Close()
|
||||
|
||||
}
|
||||
}
|
||||
fpitems.Close() // close result before we do the update
|
||||
f.db.Model(&HNFrontPage{}).Where("disappeared is ? and hn_id in (?)", time.Time{}, toupdate).Update("Disappeared", t)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// run migrates the schema and then scrapes the front page forever on the
|
||||
// configured interval.
|
||||
func (f *Fetcher) run() {
|
||||
if os.Getenv("DEBUG") != "" {
|
||||
f.db.LogMode(true)
|
||||
}
|
||||
|
||||
f.db.AutoMigrate(&HNStoryRank{})
|
||||
f.db.AutoMigrate(&FrontPageCache{})
|
||||
f.db.AutoMigrate(&HNFrontPage{})
|
||||
|
||||
for {
|
||||
f.log.Info().Msg("fetching top stories from HN")
|
||||
f.nextFetch = time.Now().Add(f.fetchInterval)
|
||||
|
||||
err := f.StoreFrontPage()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
until := time.Until(f.nextFetch)
|
||||
countdown := time.NewTimer(until)
|
||||
|
||||
f.log.Info().Msgf("waiting %s until next fetch", until)
|
||||
<-countdown.C
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,24 +10,24 @@ import (
|
||||
"github.com/labstack/echo"
|
||||
)
|
||||
|
||||
// RequestHandlerSet holds the dependencies shared by the HTTP handlers.
|
||||
type RequestHandlerSet struct {
|
||||
db *gorm.DB
|
||||
version string
|
||||
}
|
||||
|
||||
// NewRequestHandlerSet builds a RequestHandlerSet for the given version and
|
||||
// database handle.
|
||||
func NewRequestHandlerSet(version string, db *gorm.DB) *RequestHandlerSet {
|
||||
rhs := new(RequestHandlerSet)
|
||||
rhs.db = db
|
||||
rhs.version = version
|
||||
|
||||
return rhs
|
||||
}
|
||||
|
||||
func (r *RequestHandlerSet) indexHandler(c echo.Context) error {
|
||||
last24h := time.Now().Add(time.Second * 86400 * -1)
|
||||
var fpi []HNFrontPage
|
||||
r.db.Where("disappeared is not ? and disappeared > ?", time.Time{}, last24h).Order("disappeared desc").Find(&fpi)
|
||||
|
||||
type fprow struct {
|
||||
// exitRow is a story that has left the front page within the last 24h.
|
||||
type exitRow struct {
|
||||
Duration string
|
||||
DurationSecs uint
|
||||
URL string
|
||||
@@ -38,10 +38,30 @@ func (r *RequestHandlerSet) indexHandler(c echo.Context) error {
|
||||
TimeGone string
|
||||
TimeGoneSecs uint
|
||||
}
|
||||
var fprows []fprow
|
||||
|
||||
// currentRow is a story currently on the front page.
|
||||
type currentRow struct {
|
||||
Duration string
|
||||
DurationSecs uint
|
||||
URL string
|
||||
Title string
|
||||
Score uint
|
||||
HighestRank uint
|
||||
HNID uint
|
||||
Rank uint
|
||||
}
|
||||
|
||||
// exitedRows returns the stories that left the front page in the last 24h,
|
||||
// most recently departed first.
|
||||
func (r *RequestHandlerSet) exitedRows() []exitRow {
|
||||
last24h := time.Now().Add(time.Second * 86400 * -1)
|
||||
|
||||
var fpi []HNFrontPage
|
||||
r.db.Where("disappeared is not ? and disappeared > ?", time.Time{}, last24h).Order("disappeared desc").Find(&fpi)
|
||||
|
||||
rows := make([]exitRow, 0, len(fpi))
|
||||
for _, item := range fpi {
|
||||
fprows = append(fprows, fprow{
|
||||
rows = append(rows, exitRow{
|
||||
Duration: u.TimeDiffHuman(item.Disappeared, item.Appeared),
|
||||
DurationSecs: u.TimeDiffAbsSeconds(item.Disappeared, item.Appeared),
|
||||
URL: item.URL,
|
||||
@@ -54,23 +74,17 @@ func (r *RequestHandlerSet) indexHandler(c echo.Context) error {
|
||||
})
|
||||
}
|
||||
|
||||
type rowtwo struct {
|
||||
Duration string
|
||||
DurationSecs uint
|
||||
URL string
|
||||
Title string
|
||||
Score uint
|
||||
HighestRank uint
|
||||
HNID uint
|
||||
Rank uint
|
||||
return rows
|
||||
}
|
||||
var currentfp []rowtwo
|
||||
|
||||
// currentRows returns the stories currently on the front page, best rank first.
|
||||
func (r *RequestHandlerSet) currentRows() []currentRow {
|
||||
var cur []HNFrontPage
|
||||
r.db.Where("disappeared is ?", time.Time{}).Order("rank asc").Find(&cur)
|
||||
|
||||
rows := make([]currentRow, 0, len(cur))
|
||||
for _, item := range cur {
|
||||
currentfp = append(currentfp, rowtwo{
|
||||
rows = append(rows, currentRow{
|
||||
Duration: u.TimeDiffHuman(time.Now(), item.Appeared),
|
||||
DurationSecs: u.TimeDiffAbsSeconds(time.Now(), item.Appeared),
|
||||
URL: item.URL,
|
||||
@@ -82,12 +96,17 @@ func (r *RequestHandlerSet) indexHandler(c echo.Context) error {
|
||||
})
|
||||
}
|
||||
|
||||
return rows
|
||||
}
|
||||
|
||||
func (r *RequestHandlerSet) indexHandler(c echo.Context) error {
|
||||
tc := pongo2.Context{
|
||||
"time": time.Now().UTC().Format(time.RFC3339Nano),
|
||||
"exits": fprows,
|
||||
"current": currentfp,
|
||||
"exits": r.exitedRows(),
|
||||
"current": r.currentRows(),
|
||||
"gitrev": r.version,
|
||||
}
|
||||
|
||||
return c.Render(http.StatusOK, "index.html", tc)
|
||||
}
|
||||
|
||||
@@ -96,5 +115,6 @@ func (r *RequestHandlerSet) aboutHandler(c echo.Context) error {
|
||||
"time": time.Now().UTC().Format(time.RFC3339Nano),
|
||||
"gitrev": r.version,
|
||||
}
|
||||
|
||||
return c.Render(http.StatusOK, "about.html", tc)
|
||||
}
|
||||
|
||||
73
hn/hn_test.go
Normal file
73
hn/hn_test.go
Normal file
@@ -0,0 +1,73 @@
|
||||
package hn_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/orangesite/hn"
|
||||
"github.com/jinzhu/gorm"
|
||||
_ "github.com/jinzhu/gorm/dialects/sqlite"
|
||||
)
|
||||
|
||||
func TestNewFetcher(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
f := hn.NewFetcher(nil)
|
||||
if f == nil {
|
||||
t.Fatal("NewFetcher returned nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRequestHandlerSet(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
rhs := hn.NewRequestHandlerSet("v1.2.3", nil)
|
||||
if rhs == nil {
|
||||
t.Fatal("NewRequestHandlerSet returned nil")
|
||||
}
|
||||
}
|
||||
|
||||
// TestHNFrontPageRoundTrip exercises the gorm schema against an in-memory
|
||||
// sqlite database: migrate, insert, and read back a front-page row.
|
||||
func TestHNFrontPageRoundTrip(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, err := gorm.Open("sqlite3", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open in-memory db: %v", err)
|
||||
}
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
err = db.AutoMigrate(&hn.HNFrontPage{}).Error
|
||||
if err != nil {
|
||||
t.Fatalf("automigrate: %v", err)
|
||||
}
|
||||
|
||||
want := hn.HNFrontPage{
|
||||
HNID: 42,
|
||||
Appeared: time.Now(),
|
||||
Disappeared: time.Time{},
|
||||
HighestRank: 1,
|
||||
Rank: 3,
|
||||
Title: "hello world",
|
||||
Score: 100,
|
||||
URL: "https://example.com",
|
||||
}
|
||||
|
||||
err = db.Create(&want).Error
|
||||
if err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
|
||||
var got hn.HNFrontPage
|
||||
|
||||
err = db.Model(&hn.HNFrontPage{}).Where("hn_id = ?", 42).First(&got).Error
|
||||
if err != nil {
|
||||
t.Fatalf("query: %v", err)
|
||||
}
|
||||
|
||||
if got.Title != want.Title || got.Score != want.Score || got.URL != want.URL {
|
||||
t.Errorf("round-trip mismatch: got %+v, want title=%q score=%d url=%q",
|
||||
got, want.Title, want.Score, want.URL)
|
||||
}
|
||||
}
|
||||
30
hn/server.go
30
hn/server.go
@@ -5,6 +5,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/jinzhu/gorm"
|
||||
// sqlite3 dialect registered with gorm via import side effects.
|
||||
_ "github.com/jinzhu/gorm/dialects/sqlite"
|
||||
"github.com/labstack/echo"
|
||||
"github.com/labstack/echo/middleware"
|
||||
@@ -17,6 +18,8 @@ import (
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// App holds the running server's dependencies: the echo instance, the
|
||||
// logger, the database handle, the startup time, and the background fetcher.
|
||||
type App struct {
|
||||
version string
|
||||
buildarch string
|
||||
@@ -27,6 +30,8 @@ type App struct {
|
||||
fetcher *Fetcher
|
||||
}
|
||||
|
||||
// RunServer boots the application, starts the background fetcher, and blocks
|
||||
// serving HTTP until the server exits, returning a process exit code.
|
||||
func RunServer(version string, buildarch string) int {
|
||||
a := new(App)
|
||||
a.version = version
|
||||
@@ -34,7 +39,7 @@ func RunServer(version string, buildarch string) int {
|
||||
a.startup = time.Now()
|
||||
|
||||
a.init()
|
||||
defer a.db.Close()
|
||||
defer func() { _ = a.db.Close() }()
|
||||
|
||||
a.fetcher = NewFetcher(a.db)
|
||||
a.fetcher.AddLogger(a.log)
|
||||
@@ -48,6 +53,7 @@ func (a *App) init() {
|
||||
// setup logging
|
||||
l := log.With().Caller().Logger()
|
||||
log.Logger = l
|
||||
|
||||
tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
|
||||
if tty {
|
||||
out := zerolog.NewConsoleWriter(
|
||||
@@ -58,11 +64,14 @@ func (a *App) init() {
|
||||
)
|
||||
log.Logger = log.Output(out)
|
||||
}
|
||||
|
||||
// always log in UTC
|
||||
zerolog.TimestampFunc = func() time.Time {
|
||||
return time.Now().UTC()
|
||||
}
|
||||
|
||||
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
||||
|
||||
if os.Getenv("DEBUG") != "" {
|
||||
zerolog.SetGlobalLevel(zerolog.DebugLevel)
|
||||
}
|
||||
@@ -71,18 +80,18 @@ func (a *App) init() {
|
||||
|
||||
a.identify()
|
||||
|
||||
// open db
|
||||
// FIXME make configurable path
|
||||
DATABASE_PATH := "/data/storage.sqlite"
|
||||
|
||||
// database path defaults to the container data volume, overridable
|
||||
// with the DATABASE_PATH environment variable
|
||||
dbPath := "/data/storage.sqlite"
|
||||
if os.Getenv("DATABASE_PATH") != "" {
|
||||
DATABASE_PATH = os.Getenv("DATABASE_PATH")
|
||||
dbPath = os.Getenv("DATABASE_PATH")
|
||||
}
|
||||
|
||||
db, err := gorm.Open("sqlite3", DATABASE_PATH)
|
||||
db, err := gorm.Open("sqlite3", dbPath)
|
||||
if err != nil {
|
||||
panic("failed to open database: " + err.Error())
|
||||
}
|
||||
|
||||
a.db = db
|
||||
}
|
||||
|
||||
@@ -94,7 +103,6 @@ func (a *App) identify() {
|
||||
}
|
||||
|
||||
func (a *App) runForever() int {
|
||||
|
||||
// Echo instance
|
||||
a.e = echo.New()
|
||||
|
||||
@@ -120,16 +128,18 @@ func (a *App) runForever() int {
|
||||
if err != nil {
|
||||
a.e.Logger.Fatal(err)
|
||||
}
|
||||
|
||||
a.e.Renderer = r
|
||||
|
||||
rhs := NewRequestHandlerSet(a.version, a.db)
|
||||
|
||||
// Routes
|
||||
a.e.Static("/static", "static")
|
||||
a.e.GET("/", rhs.indexHandler)
|
||||
a.e.GET("/about", rhs.aboutHandler)
|
||||
|
||||
// Start server
|
||||
// Start server (blocks; Fatal exits the process on error)
|
||||
a.e.Logger.Fatal(a.e.Start(":8080"))
|
||||
|
||||
return 0 //FIXME setup graceful shutdown
|
||||
return 0
|
||||
}
|
||||
|
||||
7
static/css/bootstrap.min.css
vendored
Normal file
7
static/css/bootstrap.min.css
vendored
Normal file
File diff suppressed because one or more lines are too long
7
static/js/bootstrap.min.js
vendored
Normal file
7
static/js/bootstrap.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
4
static/js/jquery.slim.min.js
vendored
Normal file
4
static/js/jquery.slim.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
5
static/js/popper.min.js
vendored
Normal file
5
static/js/popper.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
@@ -10,7 +10,8 @@
|
||||
<meta name="author" content="">
|
||||
-->
|
||||
<title>{{ htmltitle }}</title>
|
||||
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
|
||||
<!-- vendored bootstrap 4.0.0 css (bootstrapcdn/maxcdn is being sunset) -->
|
||||
<link rel="stylesheet" href="/static/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
|
||||
|
||||
<style>
|
||||
{% include "style.css" %}
|
||||
@@ -22,9 +23,10 @@
|
||||
<body>
|
||||
{% block body %}
|
||||
{% endblock %}
|
||||
<script src="https://code.jquery.com/jquery-3.2.1.slim.min.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
|
||||
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
|
||||
<!-- vendored jquery 3.2.1 slim, popper 1.12.9, bootstrap 4.0.0 js -->
|
||||
<script src="/static/js/jquery.slim.min.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
|
||||
<script src="/static/js/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
|
||||
<script src="/static/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
||||
Reference in New Issue
Block a user