Rename blocklists to IP lists and add IP lookup flags (closes #233)
check / check (push) Failing after 1m7s
check / check (push) Failing after 1m7s
The answer field is now ip_lists, the package internal/iplist and the state directory iplists. No compatibility is kept for the old names. Every IP lookup result gains a flags object with is_tor and is_bogon. The largest list file accepted is now 200 MiB. Model: opus-5-5
This commit is contained in:
sneak committed
1 parent
e58c6635e7
commit
9e8f687511
12 files changed
+375
-176
No files matched your search
@@ -0,0 +1,379 @@
|
||||
package iplist
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"go4.org/netipx"
|
||||
)
|
||||
|
||||
// Limits of one download.
|
||||
const (
|
||||
// downloadTimeout is how long one download may take.
|
||||
downloadTimeout = time.Minute
|
||||
|
||||
// maxDownloadSize is the size of the largest list file accepted, in
|
||||
// bytes. The largest list, FireHOL abusers, is about 2.5 MiB.
|
||||
maxDownloadSize = 200 << 20
|
||||
)
|
||||
|
||||
// The request headers a download sets.
|
||||
const (
|
||||
userAgentHeader = "User-Agent"
|
||||
ifModifiedSinceHeader = "If-Modified-Since"
|
||||
ifNoneMatchHeader = "If-None-Match"
|
||||
|
||||
// userAgent names routewatch to the publishers.
|
||||
userAgent = "routewatch (+https://git.eeqj.de/sneak/routewatch)"
|
||||
)
|
||||
|
||||
// The response headers that identify the version of a list a publisher sent.
|
||||
const (
|
||||
lastModifiedHeader = "Last-Modified"
|
||||
etagHeader = "ETag"
|
||||
)
|
||||
|
||||
const (
|
||||
// downloadSuffix ends the name of the file a download is written to,
|
||||
// beside the list's copy, until it has parsed.
|
||||
downloadSuffix = ".download"
|
||||
|
||||
// validatorsSuffix ends the name of the file beside a list's copy that
|
||||
// keeps the copy's validators.
|
||||
validatorsSuffix = ".validators"
|
||||
|
||||
// dirPermissions are those of the directory that holds the copies.
|
||||
dirPermissions = 0o750
|
||||
|
||||
// validatorsPermissions are those of the file that keeps a copy's
|
||||
// validators.
|
||||
validatorsPermissions = 0o600
|
||||
)
|
||||
|
||||
// validators are the Last-Modified and ETag headers a publisher sent with a
|
||||
// version of a list. A download sends them back, as If-Modified-Since and
|
||||
// If-None-Match, so that the publisher sends the list only when it has
|
||||
// changed. They are kept as the publisher wrote them.
|
||||
//
|
||||
//nolint:tagliatelle // snake_case keys, as the Go style guide asks
|
||||
type validators struct {
|
||||
LastModified string `json:"last_modified,omitempty"`
|
||||
ETag string `json:"etag,omitempty"`
|
||||
}
|
||||
|
||||
var (
|
||||
errUnexpectedStatus = errors.New("unexpected HTTP status")
|
||||
errTooLarge = errors.New("list is over the size limit")
|
||||
)
|
||||
|
||||
// Start loads each list's copy from disk. It then starts the goroutine that
|
||||
// downloads each list once its copy is older than its Interval, and a list
|
||||
// with no copy at once. The goroutine runs until Stop: the end of ctx does
|
||||
// not stop it.
|
||||
func (b *IPLists) Start(ctx context.Context) {
|
||||
err := os.MkdirAll(b.dir, dirPermissions)
|
||||
if err != nil {
|
||||
b.logger.Error("Failed to create the IP lists directory",
|
||||
"dir", b.dir, "error", err)
|
||||
}
|
||||
|
||||
nextDownload := make([]time.Time, len(b.lists))
|
||||
|
||||
for i, list := range b.lists {
|
||||
loadErr := b.loadCopy(list)
|
||||
if loadErr != nil && !errors.Is(loadErr, fs.ErrNotExist) {
|
||||
b.logger.Error("Failed to load an IP list copy",
|
||||
"list", list.Name, "error", loadErr)
|
||||
}
|
||||
|
||||
// A list with no copy has the zero time, so it is due at once.
|
||||
nextDownload[i] = b.updated(list.Name).Add(list.Interval)
|
||||
}
|
||||
|
||||
ctx, b.stop = context.WithCancel(context.WithoutCancel(ctx))
|
||||
|
||||
go func() {
|
||||
b.run(ctx, nextDownload)
|
||||
}()
|
||||
|
||||
b.logger.Info("IP list downloads started", "dir", b.dir)
|
||||
}
|
||||
|
||||
// Stop stops the downloads, cancelling one in progress, and waits until the
|
||||
// goroutine has returned or ctx ends. The copies in memory stay, so Lookup
|
||||
// still answers.
|
||||
func (b *IPLists) Stop(ctx context.Context) {
|
||||
b.stop()
|
||||
|
||||
select {
|
||||
case <-b.done:
|
||||
b.logger.Info("IP list downloads stopped")
|
||||
case <-ctx.Done():
|
||||
b.logger.Warn("Stop deadline passed before the IP list downloads " +
|
||||
"stopped")
|
||||
}
|
||||
}
|
||||
|
||||
// run downloads each list when nextDownload says, until ctx ends. After each
|
||||
// try, whether it worked or not, the list's next download is an Interval
|
||||
// later.
|
||||
func (b *IPLists) run(ctx context.Context, nextDownload []time.Time) {
|
||||
defer close(b.done)
|
||||
|
||||
if len(nextDownload) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for {
|
||||
soonest := slices.MinFunc(nextDownload, time.Time.Compare)
|
||||
timer := time.NewTimer(time.Until(soonest))
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
|
||||
for i, list := range b.lists {
|
||||
if time.Now().Before(nextDownload[i]) {
|
||||
continue
|
||||
}
|
||||
|
||||
err := b.download(ctx, list)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
b.logger.Error("IP list download failed; keeping the "+
|
||||
"previous copy", "list", list.Name, "error", err)
|
||||
}
|
||||
|
||||
nextDownload[i] = time.Now().Add(list.Interval)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// download asks the publisher for list, sending the validators of the list's
|
||||
// copy when it has them. A new version that parses becomes the list's copy,
|
||||
// on disk and in memory. An answer that the copy is still current marks the
|
||||
// copy as updated now. Anything else leaves the copy as it is and returns an
|
||||
// error.
|
||||
func (b *IPLists) download(ctx context.Context, list List) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, downloadTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, list.URL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req.Header.Set(userAgentHeader, userAgent)
|
||||
|
||||
held := b.copyValidators(list.Name)
|
||||
if held.LastModified != "" {
|
||||
req.Header.Set(ifModifiedSinceHeader, held.LastModified)
|
||||
}
|
||||
|
||||
if held.ETag != "" {
|
||||
req.Header.Set(ifNoneMatchHeader, held.ETag)
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
return b.replaceCopy(list, resp.Body, validators{
|
||||
LastModified: resp.Header.Get(lastModifiedHeader),
|
||||
ETag: resp.Header.Get(etagHeader),
|
||||
})
|
||||
case http.StatusNotModified:
|
||||
return b.markCurrent(list)
|
||||
default:
|
||||
return fmt.Errorf("%w: %s", errUnexpectedStatus, resp.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// replaceCopy writes a new version of list, read from body, beside the list's
|
||||
// copy. Only when it parses does it replace the copy, on disk and in memory.
|
||||
// sent are the validators the publisher sent with the new version.
|
||||
func (b *IPLists) replaceCopy(list List, body io.Reader, sent validators) error {
|
||||
root, err := os.OpenRoot(b.dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = root.Close() }()
|
||||
|
||||
downloadName := list.Name + downloadSuffix
|
||||
|
||||
addresses, err := writeAndParse(root, downloadName, body, list.Format)
|
||||
if err != nil {
|
||||
_ = root.Remove(downloadName)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// The old copy's validators are removed before the copy is replaced, so
|
||||
// that they are never kept with the new copy.
|
||||
err = root.Remove(list.Name + validatorsSuffix)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
|
||||
err = root.Rename(downloadName, list.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b.setCopy(list.Name, listCopy{
|
||||
addresses: addresses, updated: time.Now().UTC(), validators: sent,
|
||||
})
|
||||
b.logger.Info("Downloaded a new version of an IP list", "list", list.Name)
|
||||
|
||||
// Without this file, the first download after a restart asks for the
|
||||
// whole list.
|
||||
err = writeValidators(root, list.Name, sent)
|
||||
if err != nil {
|
||||
b.logger.Error("Failed to keep the validators of an IP list copy",
|
||||
"list", list.Name, "error", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeAndParse writes body to the file name in root, up to maxDownloadSize
|
||||
// bytes, and parses the file as format.
|
||||
func writeAndParse(
|
||||
root *os.Root, name string, body io.Reader, format Format,
|
||||
) (*netipx.IPSet, error) {
|
||||
f, err := root.Create(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
written, err := io.Copy(f, io.LimitReader(body, maxDownloadSize+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if written > maxDownloadSize {
|
||||
return nil, errTooLarge
|
||||
}
|
||||
|
||||
_, err = f.Seek(0, io.SeekStart)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return parse(f, format)
|
||||
}
|
||||
|
||||
// markCurrent records that the publisher says the list's copy is still
|
||||
// current: the copy, on disk and in memory, is marked as updated now.
|
||||
func (b *IPLists) markCurrent(list List) error {
|
||||
root, err := os.OpenRoot(b.dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = root.Close() }()
|
||||
|
||||
now := time.Now().UTC()
|
||||
|
||||
err = root.Chtimes(list.Name, now, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b.setUpdated(list.Name, now)
|
||||
b.logger.Debug("IP list copy is still current", "list", list.Name)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadCopy makes the list's copy on disk, with the validators kept beside it,
|
||||
// its copy in memory. The file's modification time is when the copy was
|
||||
// updated.
|
||||
func (b *IPLists) loadCopy(list List) error {
|
||||
root, err := os.OpenRoot(b.dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = root.Close() }()
|
||||
|
||||
f, err := root.Open(list.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
addresses, err := parse(f, list.Format)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
held, err := readValidators(root, list.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b.setCopy(list.Name, listCopy{
|
||||
addresses: addresses, updated: info.ModTime().UTC(), validators: held,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeValidators keeps held in the file beside the named list's copy.
|
||||
func writeValidators(root *os.Root, name string, held validators) error {
|
||||
data, err := json.Marshal(held)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return root.WriteFile(name+validatorsSuffix, data, validatorsPermissions)
|
||||
}
|
||||
|
||||
// readValidators returns the validators kept beside the named list's copy.
|
||||
// They are empty when there is no such file.
|
||||
func readValidators(root *os.Root, name string) (validators, error) {
|
||||
var held validators
|
||||
|
||||
data, err := root.ReadFile(name + validatorsSuffix)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return held, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return held, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(data, &held)
|
||||
|
||||
return held, err
|
||||
}
|
||||
Reference in new issue
Block a user