routewatch/internal/iplist/download.go

package iplist

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"net/http"
	"os"
	"slices"
	"time"

	"go4.org/netipx"
)

// Limits of one download.
const (
	// downloadTimeout is how long one download may take.
	downloadTimeout = time.Minute

	// maxDownloadSize is the size of the largest list file accepted, in
	// bytes. The largest list, FireHOL abusers, is about 2.5 MiB.
	maxDownloadSize = 200 << 20
)

// The request headers a download sets.
const (
	userAgentHeader       = "User-Agent"
	ifModifiedSinceHeader = "If-Modified-Since"
	ifNoneMatchHeader     = "If-None-Match"

	// userAgent names routewatch to the publishers.
	userAgent = "routewatch (+https://git.eeqj.de/sneak/routewatch)"
)

// The response headers that identify the version of a list a publisher sent.
const (
	lastModifiedHeader = "Last-Modified"
	etagHeader         = "ETag"
)

const (
	// downloadSuffix ends the name of the file a download is written to,
	// beside the list's copy, until it has parsed.
	downloadSuffix = ".download"

	// validatorsSuffix ends the name of the file beside a list's copy that
	// keeps the copy's validators.
	validatorsSuffix = ".validators"

	// dirPermissions are those of the directory that holds the copies.
	dirPermissions = 0o750

	// validatorsPermissions are those of the file that keeps a copy's
	// validators.
	validatorsPermissions = 0o600
)

// validators are the Last-Modified and ETag headers a publisher sent with a
// version of a list. A download sends them back, as If-Modified-Since and
// If-None-Match, so that the publisher sends the list only when it has
// changed. They are kept as the publisher wrote them.
//
//nolint:tagliatelle // snake_case keys, as the Go style guide asks
type validators struct {
	LastModified string `json:"last_modified,omitempty"`
	ETag         string `json:"etag,omitempty"`
}

var (
	errUnexpectedStatus = errors.New("unexpected HTTP status")
	errTooLarge         = errors.New("list is over the size limit")
)

// Start loads each list's copy from disk. It then starts the goroutine that
// downloads each list once its copy is older than its Interval, and a list
// with no copy at once. The goroutine runs until Stop: the end of ctx does
// not stop it.
func (b *IPLists) Start(ctx context.Context) {
	err := os.MkdirAll(b.dir, dirPermissions)
	if err != nil {
		b.logger.Error("Failed to create the IP lists directory",
			"dir", b.dir, "error", err)
	}

	nextDownload := make([]time.Time, len(b.lists))

	for i, list := range b.lists {
		loadErr := b.loadCopy(list)
		if loadErr != nil && !errors.Is(loadErr, fs.ErrNotExist) {
			b.logger.Error("Failed to load an IP list copy",
				"list", list.Name, "error", loadErr)
		}

		// A list with no copy has the zero time, so it is due at once.
		nextDownload[i] = b.updated(list.Name).Add(list.Interval)
	}

	ctx, b.stop = context.WithCancel(context.WithoutCancel(ctx))

	go func() {
		b.run(ctx, nextDownload)
	}()

	b.logger.Info("IP list downloads started", "dir", b.dir)
}

// Stop stops the downloads, cancelling one in progress, and waits until the
// goroutine has returned or ctx ends. The copies in memory stay, so Lookup
// still answers.
func (b *IPLists) Stop(ctx context.Context) {
	b.stop()

	select {
	case <-b.done:
		b.logger.Info("IP list downloads stopped")
	case <-ctx.Done():
		b.logger.Warn("Stop deadline passed before the IP list downloads " +
			"stopped")
	}
}

// run downloads each list when nextDownload says, until ctx ends. After each
// try, whether it worked or not, the list's next download is an Interval
// later.
func (b *IPLists) run(ctx context.Context, nextDownload []time.Time) {
	defer close(b.done)

	if len(nextDownload) == 0 {
		return
	}

	for {
		soonest := slices.MinFunc(nextDownload, time.Time.Compare)
		timer := time.NewTimer(time.Until(soonest))

		select {
		case <-ctx.Done():
			timer.Stop()

			return
		case <-timer.C:
		}

		for i, list := range b.lists {
			if time.Now().Before(nextDownload[i]) {
				continue
			}

			err := b.download(ctx, list)
			if ctx.Err() != nil {
				return
			}

			if err != nil {
				b.logger.Error("IP list download failed; keeping the "+
					"previous copy", "list", list.Name, "error", err)
			}

			nextDownload[i] = time.Now().Add(list.Interval)
		}
	}
}

// download asks the publisher for list, sending the validators of the list's
// copy when it has them. A new version that parses becomes the list's copy,
// on disk and in memory. An answer that the copy is still current marks the
// copy as updated now. Anything else leaves the copy as it is and returns an
// error.
func (b *IPLists) download(ctx context.Context, list List) error {
	ctx, cancel := context.WithTimeout(ctx, downloadTimeout)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, list.URL, nil)
	if err != nil {
		return err
	}

	req.Header.Set(userAgentHeader, userAgent)

	held := b.copyValidators(list.Name)
	if held.LastModified != "" {
		req.Header.Set(ifModifiedSinceHeader, held.LastModified)
	}

	if held.ETag != "" {
		req.Header.Set(ifNoneMatchHeader, held.ETag)
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	defer func() { _ = resp.Body.Close() }()

	switch resp.StatusCode {
	case http.StatusOK:
		return b.replaceCopy(list, resp.Body, validators{
			LastModified: resp.Header.Get(lastModifiedHeader),
			ETag:         resp.Header.Get(etagHeader),
		})
	case http.StatusNotModified:
		return b.markCurrent(list)
	default:
		return fmt.Errorf("%w: %s", errUnexpectedStatus, resp.Status)
	}
}

// replaceCopy writes a new version of list, read from body, beside the list's
// copy. Only when it parses does it replace the copy, on disk and in memory.
// sent are the validators the publisher sent with the new version.
func (b *IPLists) replaceCopy(list List, body io.Reader, sent validators) error {
	root, err := os.OpenRoot(b.dir)
	if err != nil {
		return err
	}

	defer func() { _ = root.Close() }()

	downloadName := list.Name + downloadSuffix

	addresses, err := writeAndParse(root, downloadName, body, list.Format)
	if err != nil {
		_ = root.Remove(downloadName)

		return err
	}

	// The old copy's validators are removed before the copy is replaced, so
	// that they are never kept with the new copy.
	err = root.Remove(list.Name + validatorsSuffix)
	if err != nil && !errors.Is(err, fs.ErrNotExist) {
		return err
	}

	err = root.Rename(downloadName, list.Name)
	if err != nil {
		return err
	}

	b.setCopy(list.Name, listCopy{
		addresses: addresses, updated: time.Now().UTC(), validators: sent,
	})
	b.logger.Info("Downloaded a new version of an IP list", "list", list.Name)

	// Without this file, the first download after a restart asks for the
	// whole list.
	err = writeValidators(root, list.Name, sent)
	if err != nil {
		b.logger.Error("Failed to keep the validators of an IP list copy",
			"list", list.Name, "error", err)
	}

	return nil
}

// writeAndParse writes body to the file name in root, up to maxDownloadSize
// bytes, and parses the file as format.
func writeAndParse(
	root *os.Root, name string, body io.Reader, format Format,
) (*netipx.IPSet, error) {
	f, err := root.Create(name)
	if err != nil {
		return nil, err
	}

	defer func() { _ = f.Close() }()

	written, err := io.Copy(f, io.LimitReader(body, maxDownloadSize+1))
	if err != nil {
		return nil, err
	}

	if written > maxDownloadSize {
		return nil, errTooLarge
	}

	_, err = f.Seek(0, io.SeekStart)
	if err != nil {
		return nil, err
	}

	return parse(f, format)
}

// markCurrent records that the publisher says the list's copy is still
// current: the copy, on disk and in memory, is marked as updated now.
func (b *IPLists) markCurrent(list List) error {
	root, err := os.OpenRoot(b.dir)
	if err != nil {
		return err
	}

	defer func() { _ = root.Close() }()

	now := time.Now().UTC()

	err = root.Chtimes(list.Name, now, now)
	if err != nil {
		return err
	}

	b.setUpdated(list.Name, now)
	b.logger.Debug("IP list copy is still current", "list", list.Name)

	return nil
}

// loadCopy makes the list's copy on disk, with the validators kept beside it,
// its copy in memory. The file's modification time is when the copy was
// updated.
func (b *IPLists) loadCopy(list List) error {
	root, err := os.OpenRoot(b.dir)
	if err != nil {
		return err
	}

	defer func() { _ = root.Close() }()

	f, err := root.Open(list.Name)
	if err != nil {
		return err
	}

	defer func() { _ = f.Close() }()

	info, err := f.Stat()
	if err != nil {
		return err
	}

	addresses, err := parse(f, list.Format)
	if err != nil {
		return err
	}

	held, err := readValidators(root, list.Name)
	if err != nil {
		return err
	}

	b.setCopy(list.Name, listCopy{
		addresses: addresses, updated: info.ModTime().UTC(), validators: held,
	})

	return nil
}

// writeValidators keeps held in the file beside the named list's copy.
func writeValidators(root *os.Root, name string, held validators) error {
	data, err := json.Marshal(held)
	if err != nil {
		return err
	}

	return root.WriteFile(name+validatorsSuffix, data, validatorsPermissions)
}

// readValidators returns the validators kept beside the named list's copy.
// They are empty when there is no such file.
func readValidators(root *os.Root, name string) (validators, error) {
	var held validators

	data, err := root.ReadFile(name + validatorsSuffix)
	if errors.Is(err, fs.ErrNotExist) {
		return held, nil
	}

	if err != nil {
		return held, err
	}

	err = json.Unmarshal(data, &held)

	return held, err
}