Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
99322710cb | ||
|
|
e69352606b | ||
|
|
ea0c484370 | ||
|
|
bce9d01142 | ||
|
|
1aa7d84401 | ||
|
|
b8c36689ae | ||
|
|
21bda80f88 | ||
|
|
fcb956124b | ||
|
|
cb18297d11 | ||
|
|
2fe5db60c2 |
No files matched your search
@@ -0,0 +1 @@
|
||||
/find-and-enflac
|
||||
@@ -0,0 +1,33 @@
|
||||
BIN := find-and-enflac
|
||||
PREFIX := $(HOME)
|
||||
|
||||
.PHONY: default install build check test lint fmt fmt-check clean
|
||||
|
||||
default: install
|
||||
|
||||
install: build
|
||||
install -d $(PREFIX)/bin
|
||||
install -m 0755 $(BIN) $(PREFIX)/bin/$(BIN)
|
||||
|
||||
build:
|
||||
go build -o $(BIN) .
|
||||
|
||||
check: fmt-check lint test
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
lint:
|
||||
go vet ./...
|
||||
|
||||
fmt:
|
||||
gofmt -s -w .
|
||||
|
||||
fmt-check:
|
||||
@out="$$(gofmt -s -l .)"; \
|
||||
if [ -n "$$out" ]; then \
|
||||
echo "unformatted files:"; echo "$$out"; exit 1; \
|
||||
fi
|
||||
|
||||
clean:
|
||||
rm -f $(BIN)
|
||||
@@ -0,0 +1,211 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Container extensions holding uncompressed PCM. Raw headerless .pcm is
|
||||
// excluded on purpose: sample rate, width and channel count cannot be
|
||||
// recovered from the file itself.
|
||||
var pcmExts = map[string]bool{
|
||||
".wav": true,
|
||||
".wave": true,
|
||||
".bwf": true,
|
||||
".rf64": true,
|
||||
".bw64": true,
|
||||
".w64": true,
|
||||
".aif": true,
|
||||
".aiff": true,
|
||||
".aifc": true,
|
||||
}
|
||||
|
||||
// findPCM returns every regular file under root whose extension names a
|
||||
// PCM container. Symlinks are not followed.
|
||||
func findPCM(root string) ([]string, error) {
|
||||
var found []string
|
||||
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.Type().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
if pcmExts[strings.ToLower(filepath.Ext(path))] {
|
||||
found = append(found, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sort.Strings(found)
|
||||
return found, nil
|
||||
}
|
||||
|
||||
type encoder struct {
|
||||
threads int
|
||||
dryRun bool
|
||||
}
|
||||
|
||||
// encodeOne compresses src, verifies the result decodes back to
|
||||
// identical audio, and only then removes src.
|
||||
func (e *encoder) encodeOne(ctx context.Context, src string) error {
|
||||
codecName, err := probeCodec(ctx, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var dst, codec, level string
|
||||
switch {
|
||||
case strings.HasPrefix(codecName, "pcm_f32"), strings.HasPrefix(codecName, "pcm_f64"):
|
||||
dst, codec, level = replaceExt(src, ".wv"), "wavpack", "3"
|
||||
case strings.HasPrefix(codecName, "pcm_"):
|
||||
dst, codec, level = replaceExt(src, ".flac"), "flac", "12"
|
||||
default:
|
||||
slog.Warn("skipping, not PCM", "src", src, "codec", codecName)
|
||||
return nil
|
||||
}
|
||||
|
||||
if _, err := os.Lstat(dst); err == nil {
|
||||
return fmt.Errorf("destination already exists: %s", dst)
|
||||
}
|
||||
|
||||
if e.dryRun {
|
||||
slog.Info("would encode", "src", src, "dst", dst, "codec", codec)
|
||||
return nil
|
||||
}
|
||||
|
||||
srcInfo, err := os.Stat(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ffmpeg(ctx,
|
||||
"-threads", fmt.Sprint(e.threads),
|
||||
"-i", src,
|
||||
"-map", "0:a:0",
|
||||
"-c:a", codec,
|
||||
"-compression_level", level,
|
||||
dst,
|
||||
); err != nil {
|
||||
os.Remove(dst)
|
||||
return err
|
||||
}
|
||||
|
||||
srcSum, err := audioMD5(ctx, src)
|
||||
if err != nil {
|
||||
os.Remove(dst)
|
||||
return err
|
||||
}
|
||||
dstSum, err := audioMD5(ctx, dst)
|
||||
if err != nil {
|
||||
os.Remove(dst)
|
||||
return err
|
||||
}
|
||||
if srcSum != dstSum {
|
||||
os.Remove(dst)
|
||||
return fmt.Errorf("verification failed, original kept: %s != %s", srcSum, dstSum)
|
||||
}
|
||||
|
||||
dstInfo, err := os.Stat(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.Remove(src); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
slog.Info("encoded",
|
||||
"src", src,
|
||||
"dst", dst,
|
||||
"ratio", fmt.Sprintf("%.3f", float64(dstInfo.Size())/float64(srcInfo.Size())),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeCodec reports the codec of the file's first audio stream, e.g.
|
||||
// pcm_s24le or pcm_f32le.
|
||||
func probeCodec(ctx context.Context, path string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, "ffprobe",
|
||||
"-v", "error",
|
||||
"-select_streams", "a:0",
|
||||
"-show_entries", "stream=codec_name",
|
||||
"-of", "default=nw=1:nokey=1",
|
||||
path,
|
||||
)
|
||||
cmd.Stdin = nil
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return "", fmt.Errorf("ffprobe: %w: %s", err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
|
||||
codec := strings.TrimSpace(stdout.String())
|
||||
if codec == "" {
|
||||
return "", fmt.Errorf("no audio stream")
|
||||
}
|
||||
return codec, nil
|
||||
}
|
||||
|
||||
// audioMD5 hashes the decoded audio, forced to 64-bit float so that
|
||||
// neither 32-bit integer nor 32-bit float samples lose anything on the
|
||||
// way to the hash. Without an explicit codec the md5 muxer would
|
||||
// default to pcm_s16le and hide real differences.
|
||||
func audioMD5(ctx context.Context, path string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, "ffmpeg",
|
||||
"-nostdin", "-v", "error",
|
||||
"-i", path,
|
||||
"-map", "0:a:0",
|
||||
"-c:a", "pcm_f64le",
|
||||
"-f", "md5", "-",
|
||||
)
|
||||
cmd.Stdin = nil
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return "", fmt.Errorf("ffmpeg md5 of %s: %w: %s", path, err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
|
||||
sum := strings.TrimSpace(stdout.String())
|
||||
if !strings.HasPrefix(sum, "MD5=") {
|
||||
return "", fmt.Errorf("unexpected md5 output for %s: %q", path, sum)
|
||||
}
|
||||
return strings.TrimPrefix(sum, "MD5="), nil
|
||||
}
|
||||
|
||||
func ffmpeg(ctx context.Context, args ...string) error {
|
||||
full := append([]string{"-nostdin", "-v", "error"}, args...)
|
||||
|
||||
cmd := exec.CommandContext(ctx, "ffmpeg", full...)
|
||||
cmd.Stdin = nil
|
||||
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return fmt.Errorf("ffmpeg: %w: %s", err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func replaceExt(path, ext string) string {
|
||||
return strings.TrimSuffix(path, filepath.Ext(path)) + ext
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.eeqj.de/sneak/hacks/find-and-enflac
|
||||
|
||||
go 1.26
|
||||
@@ -0,0 +1,121 @@
|
||||
// Command find-and-enflac walks a directory tree, finds every
|
||||
// uncompressed PCM audio file, and losslessly compresses them all in
|
||||
// parallel. An original is deleted only after its encoded replacement
|
||||
// has been verified to decode back to bit-identical audio.
|
||||
//
|
||||
// Integer PCM becomes FLAC. IEEE float PCM becomes WavPack, which is
|
||||
// the only common lossless codec that stores float samples exactly;
|
||||
// FLAC has no float sample type at all.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func main() {
|
||||
jobs := flag.Int("jobs", runtime.NumCPU(), "files to encode concurrently")
|
||||
threads := flag.Int("threads", runtime.NumCPU(), "threads per ffmpeg")
|
||||
dryRun := flag.Bool("n", false, "report what would be done, change nothing")
|
||||
|
||||
flag.Usage = func() {
|
||||
fmt.Fprintf(os.Stderr, "usage: %s [flags] [directory]\n\n", os.Args[0])
|
||||
fmt.Fprintf(os.Stderr, "Losslessly compress PCM audio under directory (default \".\").\n\n")
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
flag.Parse()
|
||||
|
||||
if flag.NArg() > 1 {
|
||||
flag.Usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
root := "."
|
||||
if flag.NArg() == 1 {
|
||||
root = flag.Arg(0)
|
||||
}
|
||||
|
||||
if err := run(root, *jobs, *threads, *dryRun); err != nil {
|
||||
slog.Error("fatal", "err", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(root string, jobs, threads int, dryRun bool) error {
|
||||
for _, tool := range []string{"ffmpeg", "ffprobe"} {
|
||||
if _, err := exec.LookPath(tool); err != nil {
|
||||
return fmt.Errorf("%s not found in PATH: %w", tool, err)
|
||||
}
|
||||
}
|
||||
|
||||
if jobs < 1 || threads < 1 {
|
||||
return fmt.Errorf("jobs and threads must be at least 1")
|
||||
}
|
||||
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("not a directory: %s", root)
|
||||
}
|
||||
|
||||
// Ctrl-C cancels in-flight ffmpeg runs; a killed encode fails
|
||||
// verification, so its partial output is removed and its source
|
||||
// is left alone.
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
srcs, err := findPCM(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(srcs) == 0 {
|
||||
slog.Info("no PCM audio found", "root", root)
|
||||
return nil
|
||||
}
|
||||
slog.Info("found PCM audio", "files", len(srcs), "jobs", jobs, "threads", threads)
|
||||
|
||||
enc := &encoder{threads: threads, dryRun: dryRun}
|
||||
|
||||
var failed atomic.Int64
|
||||
queue := make(chan string)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for range jobs {
|
||||
wg.Go(func() {
|
||||
for src := range queue {
|
||||
if err := enc.encodeOne(ctx, src); err != nil {
|
||||
slog.Error("failed", "src", src, "err", err)
|
||||
failed.Add(1)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
for _, src := range srcs {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
queue <- src
|
||||
}
|
||||
close(queue)
|
||||
wg.Wait()
|
||||
|
||||
if n := failed.Load(); n > 0 {
|
||||
return fmt.Errorf("%d of %d file(s) failed", n, len(srcs))
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
module flappy
|
||||
|
||||
go 1.25
|
||||
|
||||
require (
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
|
||||
github.com/charmbracelet/x/ansi v0.10.1 // indirect
|
||||
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect
|
||||
github.com/charmbracelet/x/term v0.2.1 // indirect
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
golang.org/x/sys v0.36.0 // indirect
|
||||
golang.org/x/text v0.3.8 // indirect
|
||||
)
|
||||
@@ -0,0 +1,43 @@
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
|
||||
github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
|
||||
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs=
|
||||
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk=
|
||||
github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
|
||||
github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
|
||||
github.com/charmbracelet/x/ansi v0.10.1 h1:rL3Koar5XvX0pHGfovN03f5cxLbCF2YvLeyz7D2jVDQ=
|
||||
github.com/charmbracelet/x/ansi v0.10.1/go.mod h1:3RQDQ6lDnROptfpWuUVIUG64bD2g2BgntdxH0Ya5TeE=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs=
|
||||
github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ=
|
||||
github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
|
||||
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
|
||||
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
|
||||
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
|
||||
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
|
||||
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E=
|
||||
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
|
||||
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
+382
@@ -0,0 +1,382 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
)
|
||||
|
||||
// Physics tuned so per-frame displacement stays around one row, which keeps
|
||||
// motion smooth and the 7-row gap threadable at 30fps.
|
||||
const (
|
||||
fps = 30
|
||||
gravity = 0.09 // rows/frame^2
|
||||
jumpVel = -0.95 // rows/frame; peak rise ≈ v²/2g ≈ 5 rows over ~0.35s
|
||||
maxFall = 1.15 // terminal velocity, rows/frame — never skip >1 row
|
||||
pipeSpeed = 0.5 // cells/frame → 15 cells/s
|
||||
pipeGap = 7
|
||||
pipeSpacing = 28 // cells between pipes → one pipe ≈ every 1.9s
|
||||
pipeWidth = 3
|
||||
groundH = 2
|
||||
)
|
||||
|
||||
// tickMsg advances the simulation. gen guards against stale tick chains:
|
||||
// without it, restarting can leave an old chain in flight and the game
|
||||
// silently runs at double speed.
|
||||
type tickMsg struct{ gen int }
|
||||
|
||||
func tick(gen int) tea.Cmd {
|
||||
return tea.Tick(time.Second/fps, func(time.Time) tea.Msg {
|
||||
return tickMsg{gen: gen}
|
||||
})
|
||||
}
|
||||
|
||||
type pipe struct {
|
||||
x float64
|
||||
gapY int
|
||||
scored bool
|
||||
}
|
||||
|
||||
type star struct{ x, y int }
|
||||
|
||||
type model struct {
|
||||
termW, termH int
|
||||
width int // field width in cells
|
||||
height int // field height in cells (terminal minus header+footer)
|
||||
birdX int
|
||||
birdY float64
|
||||
birdVel float64
|
||||
pipes []pipe
|
||||
stars []star
|
||||
score int
|
||||
best int
|
||||
gen int // tick generation
|
||||
gameOver bool
|
||||
started bool
|
||||
resized bool
|
||||
}
|
||||
|
||||
func newGame(best, gen int, termW, termH int) model {
|
||||
m := model{best: best, gen: gen}
|
||||
if termW > 0 && termH > 0 {
|
||||
m.applySize(termW, termH)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// applySize records the terminal size, sizes the field, and rebuilds the
|
||||
// static starfield. The starfield must be stable between frames — rolling
|
||||
// new random speckles every render is what caused full-screen flicker.
|
||||
func (m *model) applySize(w, h int) {
|
||||
m.termW, m.termH = w, h
|
||||
m.width = w
|
||||
fieldH := h - 2 // one row header, one row footer
|
||||
if fieldH < 8 {
|
||||
fieldH = 8
|
||||
}
|
||||
m.height = fieldH
|
||||
m.birdX = w / 4
|
||||
if m.birdX < 4 {
|
||||
m.birdX = 4
|
||||
}
|
||||
if !m.resized {
|
||||
m.birdY = float64(m.height-groundH) / 2
|
||||
m.resized = true
|
||||
}
|
||||
// keep the bird inside the new bounds after a resize
|
||||
floor := float64(m.height - groundH - 1)
|
||||
if m.birdY > floor {
|
||||
m.birdY = floor
|
||||
}
|
||||
// static starfield, ~1 speckle per 40 sky cells
|
||||
skyH := m.height - groundH
|
||||
m.stars = m.stars[:0]
|
||||
n := m.width * skyH / 40
|
||||
for i := 0; i < n; i++ {
|
||||
m.stars = append(m.stars, star{x: rand.Intn(m.width), y: rand.Intn(skyH)})
|
||||
}
|
||||
}
|
||||
|
||||
func (m model) Init() tea.Cmd { return nil }
|
||||
|
||||
func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
switch msg := msg.(type) {
|
||||
case tea.WindowSizeMsg:
|
||||
m.applySize(msg.Width, msg.Height)
|
||||
return m, nil
|
||||
|
||||
case tea.KeyMsg:
|
||||
switch msg.String() {
|
||||
case " ", "space", "enter", "up", "k":
|
||||
if m.gameOver {
|
||||
// restart: bump generation so any in-flight tick is ignored
|
||||
nm := newGame(m.best, m.gen+1, m.termW, m.termH)
|
||||
nm.started = true
|
||||
nm.birdVel = jumpVel // start with a flap, like the original
|
||||
return nm, tick(nm.gen)
|
||||
}
|
||||
if !m.started {
|
||||
m.started = true
|
||||
m.birdVel = jumpVel
|
||||
return m, tick(m.gen)
|
||||
}
|
||||
m.birdVel = jumpVel
|
||||
case "q", "esc", "ctrl+c":
|
||||
return m, tea.Quit
|
||||
}
|
||||
|
||||
case tickMsg:
|
||||
if msg.gen != m.gen {
|
||||
return m, nil // stale chain — drop it
|
||||
}
|
||||
if !m.started || m.gameOver {
|
||||
return m, nil // stop the loop; nothing animates on end screens
|
||||
}
|
||||
if m.resized {
|
||||
m.step()
|
||||
}
|
||||
return m, tick(m.gen)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *model) step() {
|
||||
m.birdVel += gravity
|
||||
if m.birdVel > maxFall {
|
||||
m.birdVel = maxFall
|
||||
}
|
||||
m.birdY += m.birdVel
|
||||
|
||||
// ceiling clamp
|
||||
if m.birdY < 0 {
|
||||
m.birdY = 0
|
||||
m.birdVel = 0
|
||||
}
|
||||
// ground collision
|
||||
floor := float64(m.height - groundH - 1)
|
||||
if m.birdY >= floor {
|
||||
m.birdY = floor
|
||||
m.gameOver = true
|
||||
}
|
||||
|
||||
// advance pipes
|
||||
for i := range m.pipes {
|
||||
m.pipes[i].x -= pipeSpeed
|
||||
}
|
||||
for len(m.pipes) > 0 && m.pipes[0].x+pipeWidth < 0 {
|
||||
m.pipes = m.pipes[1:]
|
||||
}
|
||||
// spawn pipes
|
||||
usableH := m.height - groundH - pipeGap - 4
|
||||
if usableH < 1 {
|
||||
usableH = 1
|
||||
}
|
||||
if len(m.pipes) == 0 || m.pipes[len(m.pipes)-1].x < float64(m.width-pipeSpacing) {
|
||||
gapY := rand.Intn(usableH) + 2
|
||||
m.pipes = append(m.pipes, pipe{x: float64(m.width), gapY: gapY})
|
||||
}
|
||||
|
||||
by := int(m.birdY)
|
||||
for i := range m.pipes {
|
||||
p := &m.pipes[i]
|
||||
px := int(p.x)
|
||||
// score once the pipe's trailing edge passes the bird
|
||||
if !p.scored && px+pipeWidth <= m.birdX {
|
||||
p.scored = true
|
||||
m.score++
|
||||
if m.score > m.best {
|
||||
m.best = m.score
|
||||
}
|
||||
}
|
||||
// collision with pipe body
|
||||
if m.birdX >= px && m.birdX < px+pipeWidth {
|
||||
if by < p.gapY || by > p.gapY+pipeGap {
|
||||
m.gameOver = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// styles ----------------------------------------------------------------------
|
||||
|
||||
var (
|
||||
birdStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("220")).Bold(true)
|
||||
birdUpStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("226")).Bold(true)
|
||||
birdDnStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("214")).Bold(true)
|
||||
pipeStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("34"))
|
||||
pipeHi = lipgloss.NewStyle().Foreground(lipgloss.Color("46"))
|
||||
capStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("36"))
|
||||
skyStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("33"))
|
||||
groundStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("142"))
|
||||
dirtStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("130"))
|
||||
scoreStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("15")).Bold(true)
|
||||
bestStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("214")).Bold(true)
|
||||
overStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("196")).Bold(true)
|
||||
hintStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("245"))
|
||||
titleStyle = lipgloss.NewStyle().
|
||||
Foreground(lipgloss.Color("220")).
|
||||
Background(lipgloss.Color("23")).
|
||||
Bold(true)
|
||||
)
|
||||
|
||||
func birdGlyph(vel float64) (string, lipgloss.Style) {
|
||||
switch {
|
||||
case vel < -0.3:
|
||||
return "➚", birdUpStyle
|
||||
case vel > 0.5:
|
||||
return "➘", birdDnStyle
|
||||
default:
|
||||
return "➙", birdStyle
|
||||
}
|
||||
}
|
||||
|
||||
func (m model) View() string {
|
||||
if !m.resized {
|
||||
return hintStyle.Render(" sizing terminal… ")
|
||||
}
|
||||
width, height := m.width, m.height
|
||||
grid := make([][]string, height)
|
||||
for i := range grid {
|
||||
grid[i] = make([]string, width)
|
||||
for j := range grid[i] {
|
||||
grid[i][j] = " "
|
||||
}
|
||||
}
|
||||
|
||||
// static sky speckles (precomputed — stable between frames)
|
||||
for _, s := range m.stars {
|
||||
if s.y < height-groundH && s.x < width {
|
||||
grid[s.y][s.x] = skyStyle.Render("·")
|
||||
}
|
||||
}
|
||||
|
||||
// pipes
|
||||
for _, p := range m.pipes {
|
||||
px := int(p.x)
|
||||
topCap := p.gapY - 1
|
||||
botCap := p.gapY + pipeGap + 1
|
||||
for x := px; x < px+pipeWidth && x < width; x++ {
|
||||
if x < 0 {
|
||||
continue
|
||||
}
|
||||
for y := 0; y < height-groundH; y++ {
|
||||
if y < p.gapY || y > p.gapY+pipeGap {
|
||||
if y == topCap || y == botCap {
|
||||
grid[y][x] = capStyle.Render("▓")
|
||||
} else if x == px {
|
||||
grid[y][x] = pipeHi.Render("█")
|
||||
} else {
|
||||
grid[y][x] = pipeStyle.Render("█")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ground
|
||||
for y := height - groundH; y < height; y++ {
|
||||
for x := 0; x < width; x++ {
|
||||
if y == height-groundH {
|
||||
grid[y][x] = groundStyle.Render("▔")
|
||||
} else {
|
||||
grid[y][x] = dirtStyle.Render("▓")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// bird
|
||||
by := int(m.birdY)
|
||||
if by >= 0 && by < height && m.birdX >= 0 && m.birdX < width {
|
||||
glyph, style := birdGlyph(m.birdVel)
|
||||
grid[by][m.birdX] = style.Render(glyph)
|
||||
}
|
||||
|
||||
// overlays are written directly into the grid, cell by cell, so styled
|
||||
// field content underneath can never be corrupted by string splicing.
|
||||
if !m.started && !m.gameOver {
|
||||
m.overlayGrid(grid, []overline{
|
||||
{" FLAPPY BIRD ", titleStyle},
|
||||
{},
|
||||
{"press SPACE to flap", hintStyle},
|
||||
})
|
||||
} else if m.gameOver {
|
||||
m.overlayGrid(grid, []overline{
|
||||
{"✖ GAME OVER ✖", overStyle},
|
||||
{},
|
||||
{fmt.Sprintf("score %d • SPACE to retry", m.score), hintStyle},
|
||||
})
|
||||
}
|
||||
|
||||
// compose field
|
||||
var b strings.Builder
|
||||
for _, row := range grid {
|
||||
for _, c := range row {
|
||||
b.WriteString(c)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
field := b.String()
|
||||
|
||||
header := scoreStyle.Render(fmt.Sprintf(" SCORE %d ", m.score)) +
|
||||
" " +
|
||||
bestStyle.Render(fmt.Sprintf("BEST %d ", m.best))
|
||||
footer := hintStyle.Render(" SPACE: flap • Q: quit ")
|
||||
|
||||
return header + "\n" + field + footer
|
||||
}
|
||||
|
||||
// overline is one centred line of overlay text with its style.
|
||||
type overline struct {
|
||||
text string
|
||||
style lipgloss.Style
|
||||
}
|
||||
|
||||
// overlayGrid writes overlay lines into the middle of the grid, one styled
|
||||
// cell per rune. Empty lines clear a centred band for readability.
|
||||
func (m model) overlayGrid(grid [][]string, lines []overline) {
|
||||
startY := (len(grid) - len(lines)) / 2
|
||||
if startY < 0 {
|
||||
startY = 0
|
||||
}
|
||||
// widest line determines the cleared band
|
||||
maxW := 0
|
||||
for _, ol := range lines {
|
||||
if n := len([]rune(ol.text)); n > maxW {
|
||||
maxW = n
|
||||
}
|
||||
}
|
||||
for i, ol := range lines {
|
||||
y := startY + i
|
||||
if y < 0 || y >= len(grid) {
|
||||
continue
|
||||
}
|
||||
// clear a band as wide as the widest line so text sits on a clean row
|
||||
bandX := (m.width - maxW - 2) / 2
|
||||
for j := 0; j < maxW+2; j++ {
|
||||
x := bandX + j
|
||||
if x >= 0 && x < m.width {
|
||||
grid[y][x] = " "
|
||||
}
|
||||
}
|
||||
runes := []rune(ol.text)
|
||||
startX := (m.width - len(runes)) / 2
|
||||
for j, r := range runes {
|
||||
x := startX + j
|
||||
if x < 0 || x >= m.width {
|
||||
continue
|
||||
}
|
||||
grid[y][x] = ol.style.Render(string(r))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
p := tea.NewProgram(newGame(0, 0, 0, 0), tea.WithAltScreen())
|
||||
if _, err := p.Run(); err != nil {
|
||||
fmt.Println("error running game:", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
// Command geolocate estimates the machine's physical location by scanning
|
||||
// nearby Wi-Fi access points and submitting their BSSIDs to a public
|
||||
// Wi-Fi-to-location service, then records the result under ~/.data/location.
|
||||
//
|
||||
// This is a macOS port of the original Linux script. Two things changed since
|
||||
// the original was written:
|
||||
//
|
||||
// - The original scanned access points with nmcli (NetworkManager). macOS
|
||||
// has no equivalent, and Apple removed the `airport` CLI in macOS 14.4.
|
||||
// On macOS 26, CoreWLAN only returns real BSSIDs to a Developer-ID-signed,
|
||||
// notarized binary that the user has granted Location Services access. We
|
||||
// get that via the macwifi package, which ships such a signed helper.
|
||||
//
|
||||
// - The original queried Mozilla Location Services, which shut down in 2024.
|
||||
// We use BeaconDB instead: a community-run successor that speaks the same
|
||||
// Ichnaea API (identical request/response shape).
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/jaisonerick/macwifi"
|
||||
)
|
||||
|
||||
// geolocateURL is BeaconDB's Ichnaea-compatible geolocate endpoint, the
|
||||
// drop-in replacement for the defunct location.services.mozilla.com.
|
||||
const geolocateURL = "https://api.beacondb.net/v1/geolocate"
|
||||
|
||||
// accessPoint is one observed Wi-Fi access point in the Ichnaea request
|
||||
// schema. Only macAddress is required; signal strength and channel improve
|
||||
// the position estimate when available.
|
||||
type accessPoint struct {
|
||||
MacAddress string `json:"macAddress"`
|
||||
SignalStrength int `json:"signalStrength,omitempty"`
|
||||
Channel int `json:"channel,omitempty"`
|
||||
}
|
||||
|
||||
type geolocateRequest struct {
|
||||
WifiAccessPoints []accessPoint `json:"wifiAccessPoints,omitempty"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "geolocate:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
ts := time.Now().Unix()
|
||||
|
||||
// The first scan triggers macOS's Location Services prompt for the
|
||||
// bundled helper; allow generous time for a user to approve it.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Wi-Fi scanning requires Location Services. If it is disabled (or the
|
||||
// scan otherwise fails), fall back to IP-based geolocation rather than
|
||||
// aborting — matching the original, which posted an empty request when
|
||||
// it couldn't find enough access points.
|
||||
aps, err := listAccessPoints(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"geolocate: wifi scan unavailable (%v); falling back to IP geolocation\n", err)
|
||||
aps = nil
|
||||
}
|
||||
|
||||
loc, err := geolocate(aps)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
loc["timestamp"] = ts
|
||||
|
||||
out, err := json.Marshal(loc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dir := filepath.Join(home, ".data", "location")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := writeFile(filepath.Join(dir, "latest.json"), out); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeFile(filepath.Join(dir, fmt.Sprintf("%d.json", ts)), out)
|
||||
}
|
||||
|
||||
func writeFile(path string, content []byte) error {
|
||||
return os.WriteFile(path, content, 0o644)
|
||||
}
|
||||
|
||||
// listAccessPoints scans nearby Wi-Fi networks via CoreWLAN (through the
|
||||
// macwifi helper) and returns their BSSIDs. It returns nil if fewer than two
|
||||
// usable access points are found, mirroring the original script: the
|
||||
// geolocation service needs at least two observations to triangulate.
|
||||
func listAccessPoints(ctx context.Context) ([]accessPoint, error) {
|
||||
networks, err := macwifi.Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scanning wifi: %w", err)
|
||||
}
|
||||
|
||||
seen := make(map[string]bool)
|
||||
var aps []accessPoint
|
||||
for _, n := range networks {
|
||||
// BSSID is empty when Location Services access has not been
|
||||
// granted, or for saved-but-not-visible networks.
|
||||
if n.BSSID == "" || seen[n.BSSID] {
|
||||
continue
|
||||
}
|
||||
seen[n.BSSID] = true
|
||||
aps = append(aps, accessPoint{
|
||||
MacAddress: n.BSSID,
|
||||
SignalStrength: n.RSSI,
|
||||
Channel: n.Channel,
|
||||
})
|
||||
}
|
||||
|
||||
if len(aps) < 2 {
|
||||
return nil, nil
|
||||
}
|
||||
return aps, nil
|
||||
}
|
||||
|
||||
// geolocate posts the observed access points to BeaconDB and returns the
|
||||
// decoded response (e.g. {"location":{"lat":..,"lng":..},"accuracy":..}).
|
||||
// With no access points it degrades to a GeoIP estimate, matching the
|
||||
// original behaviour of posting an empty request.
|
||||
func geolocate(aps []accessPoint) (map[string]any, error) {
|
||||
body, err := json.Marshal(geolocateRequest{WifiAccessPoints: aps})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := http.Post(geolocateURL, "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("geolocation service returned %s: %s",
|
||||
resp.Status, bytes.TrimSpace(data))
|
||||
}
|
||||
|
||||
var result map[string]any
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
module sneak.berlin/go/geolocate
|
||||
|
||||
go 1.26.1
|
||||
|
||||
require github.com/jaisonerick/macwifi v1.0.0
|
||||
@@ -0,0 +1,2 @@
|
||||
github.com/jaisonerick/macwifi v1.0.0 h1:Jy86C7BBR/J7u1cxFWOWi+YlwYjiMMjYLUt6RGFD1Mc=
|
||||
github.com/jaisonerick/macwifi v1.0.0/go.mod h1:AtCf/s2Y4izU7IYcBSPoGe2lp2Oj53T7rz9D08uqSRM=
|
||||
@@ -0,0 +1,6 @@
|
||||
.env
|
||||
.git
|
||||
*.csv
|
||||
/ipabusedb-docker-reporter
|
||||
deploy.sh
|
||||
example_parser_script.py
|
||||
@@ -0,0 +1,5 @@
|
||||
# Copy to .env (gitignored). Variables already set in your shell win.
|
||||
IPABUSEDB_API_KEY=
|
||||
DOCKER_CONTAINER_NAME=sshd
|
||||
# Point the docker CLI at a remote daemon; credentials above stay local.
|
||||
DOCKER_HOST=ssh://user@sshd-host.example.com
|
||||
@@ -0,0 +1,3 @@
|
||||
/ipabusedb-docker-reporter
|
||||
/reports.csv
|
||||
/.env
|
||||
@@ -0,0 +1,41 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
#
|
||||
# Runs ipabusedb-docker-reporter as a runit service under the baseimage's
|
||||
# runsvinit: sleep INTERVAL seconds, report the last INTERVAL seconds of
|
||||
# sshd brute-force attempts to AbuseIPDB, repeat.
|
||||
#
|
||||
# Run with the host's docker socket bind-mounted and the config in the
|
||||
# container environment:
|
||||
#
|
||||
# docker run -d --restart unless-stopped \
|
||||
# -v /var/run/docker.sock:/var/run/docker.sock \
|
||||
# -e DOCKER_CONTAINER_NAME=sshd \
|
||||
# -e IPABUSEDB_API_KEY=... \
|
||||
# -e INTERVAL=1200 \
|
||||
# git.eeqj.de/sneak/ipabusedb-docker-reporter:latest
|
||||
|
||||
FROM git.eeqj.de/sneak/baseimage:2026-11-05
|
||||
|
||||
# The reporter shells out to "docker logs", so it needs the docker CLI.
|
||||
# The static binary from the official CLI image avoids pulling the whole
|
||||
# docker.io package (and daemon) in via apt. Pinned to a release tag plus
|
||||
# the multi-arch manifest list digest.
|
||||
COPY --from=docker:29.9.3-cli@sha256:2357639ad1ea7d0194134bf3292622fd4e562483966ed674e5d7c945b5f3b28d \
|
||||
/usr/local/bin/docker /usr/local/bin/docker
|
||||
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN --mount=type=cache,target=/root/go/pkg/mod go mod download
|
||||
COPY *.go ./
|
||||
RUN --mount=type=cache,target=/root/go/pkg/mod \
|
||||
--mount=type=cache,target=/root/.cache/go-build \
|
||||
CGO_ENABLED=0 go build -o /usr/local/bin/ipabusedb-docker-reporter .
|
||||
|
||||
COPY sv/ipabusedb-docker-reporter/run /etc/sv/ipabusedb-docker-reporter/run
|
||||
RUN chmod +x /etc/sv/ipabusedb-docker-reporter/run \
|
||||
&& ln -s /etc/sv/ipabusedb-docker-reporter /etc/service/ipabusedb-docker-reporter
|
||||
|
||||
# Seconds to sleep between runs; also the --since window of each run.
|
||||
ENV INTERVAL=1200
|
||||
|
||||
WORKDIR /
|
||||
@@ -0,0 +1,54 @@
|
||||
BIN := ipabusedb-docker-reporter
|
||||
IMAGE := git.eeqj.de/sneak/$(BIN)
|
||||
TAG := $(shell date -u +%Y-%m-%d)
|
||||
|
||||
# Each arch is built natively on a daemon of that arch: arm64 on the local
|
||||
# daemon, amd64 on fsn1app1. Override on the command line if needed.
|
||||
ARCHES := amd64 arm64
|
||||
DOCKER_HOST_amd64 ?= ssh://root@fsn1app1
|
||||
DOCKER_HOST_arm64 ?=
|
||||
docker_for = $(if $(DOCKER_HOST_$(1)),DOCKER_HOST=$(DOCKER_HOST_$(1))) docker
|
||||
|
||||
.PHONY: build test lint fmt fmt-check check clean image push manifest \
|
||||
$(ARCHES:%=image-%) $(ARCHES:%=push-%)
|
||||
|
||||
build:
|
||||
go build -o $(BIN) .
|
||||
|
||||
# Local image for this machine's arch, for testing.
|
||||
image:
|
||||
docker build -t $(BIN) .
|
||||
|
||||
# image-ARCH builds on that arch's daemon; push-ARCH pushes the arch tag.
|
||||
define arch_rules
|
||||
image-$(1):
|
||||
$$(call docker_for,$(1)) build --platform linux/$(1) -t $$(IMAGE):$(1) .
|
||||
|
||||
push-$(1): image-$(1)
|
||||
$$(call docker_for,$(1)) push $$(IMAGE):$(1)
|
||||
endef
|
||||
$(foreach a,$(ARCHES),$(eval $(call arch_rules,$(a))))
|
||||
|
||||
# Join the per-arch tags into the dated and latest multi-arch manifests.
|
||||
manifest:
|
||||
docker buildx imagetools create -t $(IMAGE):$(TAG) -t $(IMAGE):latest \
|
||||
$(ARCHES:%=$(IMAGE):%)
|
||||
|
||||
push: $(ARCHES:%=push-%) manifest
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
fmt:
|
||||
gofmt -w *.go
|
||||
|
||||
fmt-check:
|
||||
@test -z "$$(gofmt -l .)" || { gofmt -l .; echo "gofmt: files need formatting"; exit 1; }
|
||||
|
||||
lint: fmt-check
|
||||
go vet ./...
|
||||
|
||||
check: lint test build
|
||||
|
||||
clean:
|
||||
rm -f $(BIN) *.csv
|
||||
@@ -0,0 +1,108 @@
|
||||
# ipabusedb-docker-reporter
|
||||
|
||||
Reads the logs of a dockerised sshd with `docker logs -t`, extracts
|
||||
brute-force / invalid-user login attempts, and emits a CSV in the format
|
||||
expected by the [AbuseIPDB bulk-report
|
||||
endpoint](https://www.abuseipdb.com/bulk-report). Optionally submits it.
|
||||
|
||||
What gets reported:
|
||||
|
||||
- `Invalid user X from IP port N`
|
||||
- `User X from IP not allowed because not listed in AllowUsers`
|
||||
- `Failed password for invalid user X from IP port N`
|
||||
- `Connection closed by invalid user X IP port N` and
|
||||
`Disconnected from invalid user X IP port N`
|
||||
|
||||
Attempts with the username `git` are excluded (so people poking a git
|
||||
host with `git@` are not reported). Private, loopback and link-local
|
||||
source addresses are skipped. Each IP appears once in the CSV, dated with
|
||||
its most recent attempt and annotated with the number of matching lines.
|
||||
Categories are `18,22` (Brute-Force, SSH).
|
||||
|
||||
## Configuration
|
||||
|
||||
A `.env` file in the working directory is loaded at startup (see
|
||||
`.env.example`; `.env` is gitignored). Variables already set in the
|
||||
shell take precedence over the file.
|
||||
|
||||
| Variable | Purpose |
|
||||
| ----------------------- | -------------------------------------------------------- |
|
||||
| `DOCKER_CONTAINER_NAME` | container whose logs are read (required) |
|
||||
| `IPABUSEDB_API_KEY` | AbuseIPDB API key (required only with `--submit`) |
|
||||
| `DOCKER_HOST` | passed through to the `docker` CLI to target a remote daemon |
|
||||
|
||||
Because the program shells out to the `docker` CLI with its own
|
||||
environment, `DOCKER_HOST` (and `DOCKER_CONTEXT`, `DOCKER_TLS_VERIFY`,
|
||||
`DOCKER_CERT_PATH`) work exactly as they do for `docker` itself. Set
|
||||
`DOCKER_HOST=ssh://user@host` in `.env` to read logs from a remote
|
||||
daemon while the AbuseIPDB key never leaves the local machine.
|
||||
|
||||
Flags:
|
||||
|
||||
| Flag | Purpose |
|
||||
| --------------- | --------------------------------------------------------- |
|
||||
| `--since 24h` | only consider lines newer than now minus this duration |
|
||||
| `-o FILE` | write the CSV to `FILE` instead of stdout |
|
||||
| `--input FILE` | parse `FILE` (or `-` for stdin) instead of `docker logs` |
|
||||
| `--submit` | POST the CSV to the AbuseIPDB bulk-report endpoint |
|
||||
| `--env FILE` | env file to autoload (default `.env`; missing is ignored) |
|
||||
|
||||
## Usage
|
||||
|
||||
```
|
||||
$ make build
|
||||
$ export DOCKER_CONTAINER_NAME=sshd IPABUSEDB_API_KEY=...
|
||||
$ ./ipabusedb-docker-reporter --since 24h -o reports.csv
|
||||
$ ./ipabusedb-docker-reporter --since 24h --submit
|
||||
```
|
||||
|
||||
Submitting manually with curl instead of `--submit`:
|
||||
|
||||
```
|
||||
$ ./ipabusedb-docker-reporter --since 24h > reports.csv && curl https://api.abuseipdb.com/api/v2/bulk-report -F csv=@reports.csv -H "Key: $IPABUSEDB_API_KEY"
|
||||
```
|
||||
|
||||
## Running as a container
|
||||
|
||||
The `Dockerfile` builds on `git.eeqj.de/sneak/baseimage` and installs the
|
||||
program as a runit service (`sv/ipabusedb-docker-reporter/run`) that
|
||||
sleeps `INTERVAL` seconds, then runs the reporter with
|
||||
`--since ${INTERVAL}s --submit`, and repeats. The docker CLI is copied in
|
||||
from the official `docker` CLI image (pinned by tag and digest in the
|
||||
`Dockerfile`) so the program can read logs through the host's docker
|
||||
socket.
|
||||
|
||||
| Variable | Purpose |
|
||||
| ----------------------- | -------------------------------------------------- |
|
||||
| `DOCKER_CONTAINER_NAME` | container whose logs are read (required) |
|
||||
| `IPABUSEDB_API_KEY` | AbuseIPDB API key (required) |
|
||||
| `INTERVAL` | seconds between runs, and the report window (1200) |
|
||||
|
||||
`make image` builds a local image tagged `ipabusedb-docker-reporter` for
|
||||
this machine's architecture. `make push` builds `linux/amd64` and
|
||||
`linux/arm64` natively, each on a daemon of that architecture (arm64 on
|
||||
the local daemon, amd64 on `fsn1app1` via `DOCKER_HOST`; override
|
||||
`DOCKER_HOST_amd64` and `DOCKER_HOST_arm64` on the command line), pushes
|
||||
the per-arch tags, and joins them into
|
||||
`git.eeqj.de/sneak/ipabusedb-docker-reporter` tagged with today's UTC date
|
||||
and `latest`. Log in once with `docker login git.eeqj.de`; the client
|
||||
forwards the credentials to the remote daemon.
|
||||
|
||||
```
|
||||
$ make push
|
||||
$ docker run -d --restart unless-stopped \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-e DOCKER_CONTAINER_NAME=sshd \
|
||||
-e IPABUSEDB_API_KEY=... \
|
||||
-e INTERVAL=1200 \
|
||||
git.eeqj.de/sneak/ipabusedb-docker-reporter:latest
|
||||
```
|
||||
|
||||
Each run's CSV and the AbuseIPDB response go to the container log. If a
|
||||
required variable is missing the service logs an error, sleeps a minute
|
||||
and exits so runit can retry.
|
||||
|
||||
`make check` runs vet, the tests and the build. `loglines_example.txt`
|
||||
holds sample `docker logs -t` output used by the tests;
|
||||
`example_parser_script.py` is the reference script from AbuseIPDB that
|
||||
this program replaces.
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
|
||||
BIN="ipabusedb-docker-reporter"
|
||||
|
||||
DH="root@fsn1app1"
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
|
||||
cd "$DIR"
|
||||
|
||||
rsync -avP --delete ./ "$DH":/tmp/reporterdeploy/
|
||||
|
||||
#ssh $DH "cd /tmp/reporterdeploy && go get -v && go build -o ./${BIN} . && ls -tla"
|
||||
ssh $DH "cd /tmp/reporterdeploy && go get -v && go build . && ls -tla && mv ./${BIN} /usr/local/sbin"
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import re
|
||||
import csv
|
||||
from datetime import datetime, timezone
|
||||
import pytz
|
||||
|
||||
def main(arguments):
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument('infile', help="Input file", type=argparse.FileType('r'))
|
||||
parser.add_argument('-o', '--outfile', help="Output file",
|
||||
default=sys.stdout, type=argparse.FileType('w'))
|
||||
|
||||
args = parser.parse_args(arguments)
|
||||
|
||||
# Define field names.
|
||||
fieldnames = ['IP', 'Categories', 'Comment', 'ReportDate']
|
||||
# Begin CSV output.
|
||||
writer = csv.DictWriter(args.outfile, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
|
||||
# Initialize empty list to hold addresses
|
||||
ipv4_addresses = list()
|
||||
|
||||
for line in args.infile:
|
||||
# !! Match this format to your system's format.
|
||||
timestamp = "([a-zA-Z]+\s+[0-9]+ [0-9]+:[0-9]+:[0-9]+)"
|
||||
ipv4 = "([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})"
|
||||
comment = "(Invalid user [a-zA-Z0-9]+ from " + ipv4 + " port [0-9]+)"
|
||||
|
||||
# The regex of the line we're looking for, built up from component regexps.
|
||||
combined_re = timestamp + " .* " + comment
|
||||
|
||||
# Run the regexp.
|
||||
matches = re.findall(combined_re, line)
|
||||
# If this line is in the format we're looking for,
|
||||
if matches:
|
||||
# Pull the tuple out of the list.
|
||||
matches_flat = matches[0]
|
||||
|
||||
# Remove duplicate addresses from the report.
|
||||
if matches_flat[2] not in ipv4_addresses:
|
||||
ipv4_addresses.append(matches_flat[2])
|
||||
else:
|
||||
continue
|
||||
|
||||
### !!! You may need to update this. ###
|
||||
# Parse log datetime to Python datetime object so we can update the timezone.
|
||||
# The format string should must your log files. Here we use the default in Debian/Redhat distros.
|
||||
attack_datetime = datetime.strptime(matches_flat[0], '%b %d %H:%M:%S')
|
||||
# Assume year is the current year.
|
||||
attack_datetime = attack_datetime.replace(datetime.now().year)
|
||||
# !! Set tzinfo to your system timezone using timezone.
|
||||
my_tz = pytz.timezone('America/New_York')
|
||||
attack_datetime = attack_datetime.replace(tzinfo=my_tz)
|
||||
|
||||
# Format to ISO 8601 to make it universal and portable.
|
||||
attack_datetime_iso = attack_datetime.isoformat()
|
||||
|
||||
# We'll add the categories column statically at this step.
|
||||
# Output as a CSV row.
|
||||
writer.writerow({
|
||||
'IP': matches_flat[2],
|
||||
'Categories': "18,22",
|
||||
'Comment': matches_flat[1],
|
||||
'ReportDate': attack_datetime_iso
|
||||
})
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,5 @@
|
||||
module sneak.berlin/go/hacks/ipabusedb-docker-reporter
|
||||
|
||||
go 1.26.5
|
||||
|
||||
require github.com/joho/godotenv v1.5.1
|
||||
@@ -0,0 +1,2 @@
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
@@ -0,0 +1,13 @@
|
||||
2026-10-06T12:04:24.554951297Z User root from 41.93.28.11 not allowed because not listed in AllowUsers
|
||||
2026-10-06T12:04:24.595412750Z Connection closed by invalid user root 41.93.28.11 port 51904 [preauth]
|
||||
2026-10-06T12:04:24.687263829Z Invalid user production from 45.191.4.252 port 38924
|
||||
2026-10-06T12:04:24.927218990Z Connection closed by invalid user production 45.191.4.252 port 38924 [preauth]
|
||||
2026-10-06T12:04:28.005424065Z Timeout before authentication for connection from 95.78.250.88 to 172.19.0.29, pid = 307434
|
||||
2026-10-06T12:04:29.009224164Z User root from 164.77.161.180 not allowed because not listed in AllowUsers
|
||||
2026-10-06T12:04:29.494569018Z Connection closed by invalid user root 164.77.161.180 port 33916 [preauth]
|
||||
2026-10-06T12:04:30.222573149Z Invalid user production from 212.128.3.152 port 60626
|
||||
2026-10-06T12:04:30.266239421Z Connection closed by invalid user production 212.128.3.152 port 60626 [preauth]
|
||||
2026-10-06T12:04:38.873013329Z Timeout before authentication for connection from 212.19.8.30 to 172.19.0.29, pid = 308210
|
||||
2026-10-06T12:04:39.622456198Z Received disconnect from 172.19.0.1 port 53534:11: disconnected by user
|
||||
2026-10-06T12:04:41.066933472Z Received disconnect from 172.19.0.1 port 53540:11: disconnected by user
|
||||
2026-10-06T12:04:41.413322464Z Received disconnect from 172.19.0.1 port 53552:11: disconnected by user
|
||||
@@ -0,0 +1,396 @@
|
||||
// ipabusedb-docker-reporter reads the logs of a dockerised sshd via
|
||||
// "docker logs -t", extracts brute-force / invalid-user attempts, and
|
||||
// emits a CSV suitable for the AbuseIPDB bulk-report endpoint.
|
||||
//
|
||||
// Environment (a .env file in the working directory is loaded first;
|
||||
// variables already set in the process environment take precedence):
|
||||
//
|
||||
// DOCKER_CONTAINER_NAME container whose logs are read (required)
|
||||
// IPABUSEDB_API_KEY AbuseIPDB API key (required with --submit)
|
||||
// DOCKER_HOST passed through to the docker CLI, so e.g.
|
||||
// ssh://user@host reads logs from a remote daemon
|
||||
//
|
||||
// Flags:
|
||||
//
|
||||
// --since 24h only consider log lines newer than now()-duration
|
||||
// -o FILE write CSV to FILE instead of stdout
|
||||
// --input FILE parse FILE (or "-" for stdin) instead of docker logs
|
||||
// --submit POST the CSV to the AbuseIPDB bulk-report endpoint
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/joho/godotenv"
|
||||
)
|
||||
|
||||
const (
|
||||
bulkReportURL = "https://api.abuseipdb.com/api/v2/bulk-report"
|
||||
// AbuseIPDB categories: 18 = Brute-Force, 22 = SSH.
|
||||
categories = "18,22"
|
||||
excludedUser = "git"
|
||||
// AbuseIPDB rejects bulk reports with more than this many rows.
|
||||
bulkReportMaxRows = 10000
|
||||
)
|
||||
|
||||
// patterns match sshd messages that evidence an unauthorised login
|
||||
// attempt. Group 1 is the username, group 2 the source address. Order
|
||||
// is priority: the earliest matching pattern supplies the report comment.
|
||||
var patterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`Invalid user (\S+) from (\S+) port \d+`),
|
||||
regexp.MustCompile(`User (\S+) from (\S+) not allowed because not listed in AllowUsers`),
|
||||
regexp.MustCompile(`Failed password for invalid user (\S+) from (\S+) port \d+`),
|
||||
regexp.MustCompile(`Connection closed by invalid user (\S+) (\S+) port \d+`),
|
||||
regexp.MustCompile(`Disconnected from invalid user (\S+) (\S+) port \d+`),
|
||||
}
|
||||
|
||||
type event struct {
|
||||
ip netip.Addr
|
||||
user string
|
||||
comment string
|
||||
prio int // index into patterns; lower is more descriptive
|
||||
when time.Time
|
||||
count int
|
||||
}
|
||||
|
||||
// parser accumulates one event per source IP, keeping the most recent
|
||||
// matching message and counting how many matched in total.
|
||||
type parser struct {
|
||||
cutoff time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
events map[netip.Addr]*event
|
||||
skipped int
|
||||
// unparsed keeps the last few lines without a valid timestamp; the
|
||||
// docker CLI writes its own errors to stderr without one.
|
||||
unparsed []string
|
||||
}
|
||||
|
||||
const keepUnparsed = 5
|
||||
|
||||
func newParser(cutoff time.Time) *parser {
|
||||
return &parser{cutoff: cutoff, events: map[netip.Addr]*event{}}
|
||||
}
|
||||
|
||||
// handleLine parses one "docker logs -t" line: an RFC3339Nano timestamp,
|
||||
// a space, then the message.
|
||||
func (p *parser) handleLine(line string) {
|
||||
line = strings.TrimRight(line, "\r\n")
|
||||
ts, msg, ok := strings.Cut(line, " ")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
when, err := time.Parse(time.RFC3339Nano, ts)
|
||||
if err != nil {
|
||||
p.mu.Lock()
|
||||
p.skipped++
|
||||
p.unparsed = append(p.unparsed, line)
|
||||
if len(p.unparsed) > keepUnparsed {
|
||||
p.unparsed = p.unparsed[1:]
|
||||
}
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if !p.cutoff.IsZero() && when.Before(p.cutoff) {
|
||||
return
|
||||
}
|
||||
for prio, re := range patterns {
|
||||
m := re.FindStringSubmatch(msg)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
user, addr := m[1], m[2]
|
||||
if user == excludedUser {
|
||||
return
|
||||
}
|
||||
ip, err := netip.ParseAddr(addr)
|
||||
if err != nil || !reportable(ip) {
|
||||
return
|
||||
}
|
||||
p.record(ip, user, strings.TrimSpace(m[0]), prio, when)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func reportable(ip netip.Addr) bool {
|
||||
return ip.IsGlobalUnicast() && !ip.IsPrivate()
|
||||
}
|
||||
|
||||
// record merges an observation into the per-IP event: the report date is
|
||||
// the latest observation, the comment comes from the most descriptive
|
||||
// pattern (and, within the same pattern, the latest observation).
|
||||
func (p *parser) record(ip netip.Addr, user, comment string, prio int, when time.Time) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
ev, ok := p.events[ip]
|
||||
if !ok {
|
||||
p.events[ip] = &event{ip: ip, user: user, comment: comment, prio: prio, when: when, count: 1}
|
||||
return
|
||||
}
|
||||
ev.count++
|
||||
newer := when.After(ev.when)
|
||||
if newer {
|
||||
ev.when = when
|
||||
}
|
||||
if prio < ev.prio || (prio == ev.prio && newer) {
|
||||
ev.prio, ev.user, ev.comment = prio, user, comment
|
||||
}
|
||||
}
|
||||
|
||||
func (p *parser) ingest(r io.Reader) error {
|
||||
sc := bufio.NewScanner(r)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
for sc.Scan() {
|
||||
p.handleLine(sc.Text())
|
||||
}
|
||||
return sc.Err()
|
||||
}
|
||||
|
||||
func (p *parser) sorted() []*event {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
out := make([]*event, 0, len(p.events))
|
||||
for _, ev := range p.events {
|
||||
out = append(out, ev)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].when.Before(out[j].when) })
|
||||
return out
|
||||
}
|
||||
|
||||
func writeCSV(w io.Writer, events []*event) error {
|
||||
cw := csv.NewWriter(w)
|
||||
if err := cw.Write([]string{"IP", "Categories", "Comment", "ReportDate"}); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, ev := range events {
|
||||
comment := ev.comment
|
||||
if ev.count > 1 {
|
||||
comment = fmt.Sprintf("%s (%d events)", ev.comment, ev.count)
|
||||
}
|
||||
row := []string{
|
||||
ev.ip.String(),
|
||||
categories,
|
||||
comment,
|
||||
ev.when.UTC().Format(time.RFC3339),
|
||||
}
|
||||
if err := cw.Write(row); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
cw.Flush()
|
||||
return cw.Error()
|
||||
}
|
||||
|
||||
// readDockerLogs runs "docker logs -t [--since D] NAME" and feeds both
|
||||
// stdout and stderr (sshd logs to stderr) through the parser.
|
||||
func readDockerLogs(ctx context.Context, p *parser, container string, since time.Duration) error {
|
||||
args := []string{"logs", "-t"}
|
||||
if since > 0 {
|
||||
args = append(args, "--since", since.String())
|
||||
}
|
||||
args = append(args, container)
|
||||
// cmd.Env is left nil so the docker CLI inherits the full process
|
||||
// environment, including DOCKER_HOST / DOCKER_CONTEXT / DOCKER_TLS_*
|
||||
// loaded from .env, and can therefore target a remote daemon.
|
||||
cmd := exec.CommandContext(ctx, "docker", args...)
|
||||
slog.Info("exec", "cmd", strings.Join(cmd.Args, " "), "DOCKER_HOST", os.Getenv("DOCKER_HOST"))
|
||||
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return fmt.Errorf("starting docker: %w", err)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 2)
|
||||
for _, r := range []io.Reader{stdout, stderr} {
|
||||
wg.Add(1)
|
||||
go func(r io.Reader) {
|
||||
defer wg.Done()
|
||||
if err := p.ingest(r); err != nil {
|
||||
errs <- err
|
||||
}
|
||||
}(r)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
if err := cmd.Wait(); err != nil {
|
||||
p.mu.Lock()
|
||||
detail := strings.Join(p.unparsed, "\n")
|
||||
p.mu.Unlock()
|
||||
if detail != "" {
|
||||
return fmt.Errorf("docker logs %s: %w\n%s", container, err, detail)
|
||||
}
|
||||
return fmt.Errorf("docker logs %s: %w", container, err)
|
||||
}
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readInput(p *parser, path string) error {
|
||||
if path == "-" {
|
||||
return p.ingest(os.Stdin)
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
return p.ingest(f)
|
||||
}
|
||||
|
||||
func submit(ctx context.Context, apiKey string, csvData []byte) ([]byte, error) {
|
||||
var body bytes.Buffer
|
||||
mw := multipart.NewWriter(&body)
|
||||
part, err := mw.CreateFormFile("csv", "reports.csv")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := part.Write(csvData); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := mw.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, bulkReportURL, &body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Key", apiKey)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
|
||||
client := &http.Client{Timeout: 2 * time.Minute}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
respBody, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return respBody, fmt.Errorf("abuseipdb returned %s", resp.Status)
|
||||
}
|
||||
return respBody, nil
|
||||
}
|
||||
|
||||
func run() error {
|
||||
since := flag.Duration("since", 0, "only report events newer than now()-duration (e.g. 24h)")
|
||||
outPath := flag.String("o", "", "write CSV to this file instead of stdout")
|
||||
inPath := flag.String("input", "", "parse this file (or - for stdin) instead of running docker logs")
|
||||
doSubmit := flag.Bool("submit", false, "POST the CSV to the AbuseIPDB bulk-report endpoint")
|
||||
envFile := flag.String("env", ".env", "env file to autoload (missing file is ignored)")
|
||||
flag.Parse()
|
||||
|
||||
// Load .env without overriding variables already in the environment.
|
||||
if err := godotenv.Load(*envFile); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("loading %s: %w", *envFile, err)
|
||||
}
|
||||
|
||||
apiKey := os.Getenv("IPABUSEDB_API_KEY")
|
||||
container := os.Getenv("DOCKER_CONTAINER_NAME")
|
||||
if *doSubmit && apiKey == "" {
|
||||
return errors.New("IPABUSEDB_API_KEY must be set in the environment to --submit")
|
||||
}
|
||||
if *inPath == "" && container == "" {
|
||||
return errors.New("DOCKER_CONTAINER_NAME must be set in the environment")
|
||||
}
|
||||
|
||||
var cutoff time.Time
|
||||
if *since > 0 {
|
||||
cutoff = time.Now().Add(-*since)
|
||||
}
|
||||
p := newParser(cutoff)
|
||||
|
||||
ctx := context.Background()
|
||||
var err error
|
||||
if *inPath != "" {
|
||||
err = readInput(p, *inPath)
|
||||
} else {
|
||||
err = readDockerLogs(ctx, p, container, *since)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
events := p.sorted()
|
||||
if len(events) > bulkReportMaxRows {
|
||||
slog.Warn("truncating to AbuseIPDB bulk report limit", "limit", bulkReportMaxRows, "found", len(events))
|
||||
events = events[len(events)-bulkReportMaxRows:]
|
||||
}
|
||||
slog.Info("parsed logs", "abusers", len(events), "unparseable_lines", p.skipped)
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := writeCSV(&buf, events); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
out := io.Writer(os.Stdout)
|
||||
if *outPath != "" {
|
||||
f, err := os.Create(*outPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
out = f
|
||||
}
|
||||
if _, err := out.Write(buf.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !*doSubmit {
|
||||
return nil
|
||||
}
|
||||
if len(events) == 0 {
|
||||
slog.Info("nothing to submit")
|
||||
return nil
|
||||
}
|
||||
resp, err := submit(ctx, apiKey, buf.Bytes())
|
||||
if len(resp) > 0 {
|
||||
fmt.Fprintln(os.Stderr, string(resp))
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
slog.Info("submitted bulk report", "rows", len(events))
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, nil)))
|
||||
if err := run(); err != nil {
|
||||
slog.Error(err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestParseExampleLines(t *testing.T) {
|
||||
data, err := os.ReadFile("loglines_example.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := newParser(time.Time{})
|
||||
if err := p.ingest(bytes.NewReader(data)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
want := map[string]int{
|
||||
"41.93.28.11": 2,
|
||||
"45.191.4.252": 2,
|
||||
"164.77.161.180": 2,
|
||||
"212.128.3.152": 2,
|
||||
}
|
||||
if len(p.events) != len(want) {
|
||||
t.Fatalf("got %d events, want %d", len(p.events), len(want))
|
||||
}
|
||||
for ipStr, count := range want {
|
||||
ev, ok := p.events[netip.MustParseAddr(ipStr)]
|
||||
if !ok {
|
||||
t.Errorf("missing event for %s", ipStr)
|
||||
continue
|
||||
}
|
||||
if ev.count != count {
|
||||
t.Errorf("%s: count %d, want %d", ipStr, ev.count, count)
|
||||
}
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := writeCSV(&out, p.sorted()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(out.String()), "\n")
|
||||
if lines[0] != "IP,Categories,Comment,ReportDate" {
|
||||
t.Errorf("bad header: %q", lines[0])
|
||||
}
|
||||
if len(lines) != 5 {
|
||||
t.Fatalf("got %d csv lines, want 5:\n%s", len(lines), out.String())
|
||||
}
|
||||
if !strings.Contains(lines[1], "41.93.28.11,\"18,22\",") || !strings.HasSuffix(lines[1], "2026-10-06T12:04:24Z") {
|
||||
t.Errorf("unexpected first row: %q", lines[1])
|
||||
}
|
||||
// The descriptive "Invalid user ... from" line should win over the
|
||||
// later "Connection closed by invalid user" line for the comment.
|
||||
if !strings.Contains(lines[2], ",Invalid user production from 45.191.4.252 port 38924 (2 events),") {
|
||||
t.Errorf("unexpected comment in row: %q", lines[2])
|
||||
}
|
||||
// AllowUsers rejections are reported with their own message.
|
||||
if !strings.Contains(lines[1], "User root from 41.93.28.11 not allowed because not listed in AllowUsers (2 events)") {
|
||||
t.Errorf("unexpected comment in row: %q", lines[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExclusions(t *testing.T) {
|
||||
p := newParser(time.Time{})
|
||||
for _, line := range []string{
|
||||
"2026-10-06T12:04:24.687263829Z Invalid user git from 1.2.3.4 port 38924",
|
||||
"2026-10-06T12:04:24.687263829Z User git from 1.2.3.5 not allowed because not listed in AllowUsers",
|
||||
"2026-10-06T12:04:24.687263829Z Invalid user bob from 172.19.0.1 port 38924",
|
||||
"2026-10-06T12:04:24.687263829Z Invalid user bob from 10.0.0.9 port 38924",
|
||||
"2026-10-06T12:04:24.687263829Z Invalid user bob from 127.0.0.1 port 38924",
|
||||
"2026-10-06T12:04:24.687263829Z Invalid user bob from fd00::1 port 38924",
|
||||
"2026-10-06T12:04:24.687263829Z Invalid user bob from not-an-ip port 38924",
|
||||
"2026-10-06T12:04:24.687263829Z Received disconnect from 8.8.8.8 port 53534:11: disconnected by user",
|
||||
"garbage line without timestamp",
|
||||
} {
|
||||
p.handleLine(line)
|
||||
}
|
||||
if len(p.events) != 0 {
|
||||
t.Errorf("expected no events, got %v", p.events)
|
||||
}
|
||||
if p.skipped != 1 {
|
||||
t.Errorf("skipped = %d, want 1", p.skipped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSinceCutoff(t *testing.T) {
|
||||
cutoff := time.Date(2026, 10, 6, 12, 4, 29, 0, time.UTC)
|
||||
p := newParser(cutoff)
|
||||
p.handleLine("2026-10-06T12:04:24.687263829Z Invalid user production from 45.191.4.252 port 38924")
|
||||
p.handleLine("2026-10-06T12:04:30.222573149Z Invalid user production from 212.128.3.152 port 60626")
|
||||
if len(p.events) != 1 {
|
||||
t.Fatalf("got %d events, want 1", len(p.events))
|
||||
}
|
||||
if _, ok := p.events[netip.MustParseAddr("212.128.3.152")]; !ok {
|
||||
t.Error("expected only the post-cutoff event to remain")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIPv6AndLatestWins(t *testing.T) {
|
||||
p := newParser(time.Time{})
|
||||
p.handleLine("2026-10-06T12:04:24Z Invalid user a from 2001:db8::1 port 1")
|
||||
p.handleLine("2026-10-06T12:04:26Z Invalid user b from 2001:db8::1 port 2")
|
||||
p.handleLine("2026-10-06T12:04:25Z Invalid user c from 2001:db8::1 port 3")
|
||||
ev := p.events[netip.MustParseAddr("2001:db8::1")]
|
||||
if ev == nil || ev.count != 3 || ev.user != "b" {
|
||||
t.Fatalf("unexpected event: %+v", ev)
|
||||
}
|
||||
var out bytes.Buffer
|
||||
if err := writeCSV(&out, p.sorted()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "(3 events)") {
|
||||
t.Errorf("expected event count in comment:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/bin/sh
|
||||
# runit service: sleep INTERVAL seconds (default 1200), then report the
|
||||
# last INTERVAL seconds of sshd brute-force attempts from the logs of
|
||||
# DOCKER_CONTAINER_NAME to AbuseIPDB. runit restarts this script when it
|
||||
# exits, so one invocation is one iteration of the loop.
|
||||
#
|
||||
# Configuration comes from the container environment:
|
||||
# DOCKER_CONTAINER_NAME container whose logs are read (required)
|
||||
# IPABUSEDB_API_KEY AbuseIPDB API key (required)
|
||||
# INTERVAL seconds between runs (default 1200)
|
||||
exec 2>&1
|
||||
|
||||
: "${INTERVAL:=1200}"
|
||||
|
||||
for var in DOCKER_CONTAINER_NAME IPABUSEDB_API_KEY; do
|
||||
eval "val=\${$var:-}"
|
||||
if [ -z "$val" ]; then
|
||||
echo "ipabusedb-docker-reporter: $var is not set in the environment" >&2
|
||||
# runit would otherwise retry every second and flood the log.
|
||||
sleep 60
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "ipabusedb-docker-reporter: starting up, next report in ${INTERVAL}s"
|
||||
|
||||
# Sleep in the background so that SIGTERM from "sv stop" (which runsvinit
|
||||
# sends on "docker stop") interrupts the wait instead of orphaning sleep.
|
||||
trap 'kill "$sleeper" 2>/dev/null; exit 0' TERM INT
|
||||
sleep "$INTERVAL" &
|
||||
sleeper=$!
|
||||
wait "$sleeper"
|
||||
trap - TERM INT
|
||||
|
||||
echo "ipabusedb-docker-reporter: exec /usr/local/bin/ipabusedb-docker-reporter --since ${INTERVAL}s --submit"
|
||||
exec /usr/local/bin/ipabusedb-docker-reporter --since "${INTERVAL}s" --submit
|
||||
@@ -26,6 +26,9 @@ OPTS+=" --links --metadata"
|
||||
# from the destination after the transfer completes.
|
||||
OPTS+=" --delete-excluded --delete-after"
|
||||
OPTS+=" --progress --transfers 25 --stats-unit bits --retries 10"
|
||||
OPTS+=" --check-first"
|
||||
OPTS+=" --max-backlog 10000000000"
|
||||
OPTS+=" --order-by modtime,mixed,10"
|
||||
|
||||
RE=""
|
||||
|
||||
@@ -91,6 +94,9 @@ RE+=" --exclude=/.npm/"
|
||||
RE+=" --exclude=/.opencode/node_modules/"
|
||||
RE+=" --exclude=/.walletwasabi/"
|
||||
|
||||
# new 2026-08
|
||||
RE+=" --exclude=/Library/Metadata/CoreSpotlight/"
|
||||
|
||||
MINRE=""
|
||||
MINRE+=" --exclude=/.fseventsd/"
|
||||
MINRE+=" --exclude=/.Spotlight-V100/"
|
||||
|
||||
Reference in new issue
Block a user