routewatch/internal/blocklist/download.go

package blocklist

import (
	"context"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"net/http"
	"os"
	"slices"
	"time"

	"go4.org/netipx"
)

// Limits of one download.
const (
	// downloadTimeout is how long one download may take.
	downloadTimeout = time.Minute

	// maxDownloadSize is the size of the largest list file accepted, in
	// bytes. The largest list, FireHOL abusers, is about 2.5 MiB.
	maxDownloadSize = 16 << 20
)

// The request headers a download sets.
const (
	userAgentHeader       = "User-Agent"
	ifModifiedSinceHeader = "If-Modified-Since"

	// userAgent names routewatch to the publishers.
	userAgent = "routewatch (+https://git.eeqj.de/sneak/routewatch)"
)

const (
	// downloadSuffix ends the name of the file a download is written to,
	// beside the list's copy, until it has parsed.
	downloadSuffix = ".download"

	// dirPermissions are those of the directory that holds the copies.
	dirPermissions = 0o750
)

var (
	errUnexpectedStatus = errors.New("unexpected HTTP status")
	errTooLarge         = errors.New("list is over the size limit")
)

// Start loads each list's copy from disk. It then starts the goroutine that
// downloads each list once its copy is older than its Interval, and a list
// with no copy at once. The goroutine runs until Stop: the end of ctx does
// not stop it.
func (b *Blocklists) Start(ctx context.Context) {
	err := os.MkdirAll(b.dir, dirPermissions)
	if err != nil {
		b.logger.Error("Failed to create the blocklists directory",
			"dir", b.dir, "error", err)
	}

	nextDownload := make([]time.Time, len(b.lists))

	for i, list := range b.lists {
		loadErr := b.loadCopy(list)
		if loadErr != nil && !errors.Is(loadErr, fs.ErrNotExist) {
			b.logger.Error("Failed to load a blocklist copy",
				"list", list.Name, "error", loadErr)
		}

		// A list with no copy has the zero time, so it is due at once.
		nextDownload[i] = b.updated(list.Name).Add(list.Interval)
	}

	ctx, b.stop = context.WithCancel(context.WithoutCancel(ctx))

	go func() {
		b.run(ctx, nextDownload)
	}()

	b.logger.Info("Blocklist downloads started", "dir", b.dir)
}

// Stop stops the downloads, cancelling one in progress, and waits until the
// goroutine has returned or ctx ends. The copies in memory stay, so Lookup
// still answers.
func (b *Blocklists) Stop(ctx context.Context) {
	b.stop()

	select {
	case <-b.done:
		b.logger.Info("Blocklist downloads stopped")
	case <-ctx.Done():
		b.logger.Warn("Stop deadline passed before the blocklist downloads " +
			"stopped")
	}
}

// run downloads each list when nextDownload says, until ctx ends. After each
// try, whether it worked or not, the list's next download is an Interval
// later.
func (b *Blocklists) run(ctx context.Context, nextDownload []time.Time) {
	defer close(b.done)

	if len(nextDownload) == 0 {
		return
	}

	for {
		soonest := slices.MinFunc(nextDownload, time.Time.Compare)
		timer := time.NewTimer(time.Until(soonest))

		select {
		case <-ctx.Done():
			timer.Stop()

			return
		case <-timer.C:
		}

		for i, list := range b.lists {
			if time.Now().Before(nextDownload[i]) {
				continue
			}

			err := b.download(ctx, list)
			if ctx.Err() != nil {
				return
			}

			if err != nil {
				b.logger.Error("Blocklist download failed; keeping the "+
					"previous copy", "list", list.Name, "error", err)
			}

			nextDownload[i] = time.Now().Add(list.Interval)
		}
	}
}

// download asks the publisher for list. A new version that parses becomes
// the list's copy, on disk and in memory. An answer that the copy is still
// current marks the copy as updated now. Anything else leaves the copy as it
// is and returns an error.
func (b *Blocklists) download(ctx context.Context, list List) error {
	ctx, cancel := context.WithTimeout(ctx, downloadTimeout)
	defer cancel()

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, list.URL, nil)
	if err != nil {
		return err
	}

	req.Header.Set(userAgentHeader, userAgent)

	updated := b.updated(list.Name)
	if !updated.IsZero() {
		req.Header.Set(ifModifiedSinceHeader, updated.Format(http.TimeFormat))
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	defer func() { _ = resp.Body.Close() }()

	switch resp.StatusCode {
	case http.StatusOK:
		return b.replaceCopy(list, resp.Body)
	case http.StatusNotModified:
		return b.markCurrent(list)
	default:
		return fmt.Errorf("%w: %s", errUnexpectedStatus, resp.Status)
	}
}

// replaceCopy writes a new version of list, read from body, beside the list's
// copy. Only when it parses does it replace the copy, on disk and in memory.
func (b *Blocklists) replaceCopy(list List, body io.Reader) error {
	root, err := os.OpenRoot(b.dir)
	if err != nil {
		return err
	}

	defer func() { _ = root.Close() }()

	downloadName := list.Name + downloadSuffix

	addresses, err := writeAndParse(root, downloadName, body, list.Format)
	if err != nil {
		_ = root.Remove(downloadName)

		return err
	}

	err = root.Rename(downloadName, list.Name)
	if err != nil {
		return err
	}

	b.setCopy(list.Name, listCopy{addresses: addresses, updated: time.Now().UTC()})
	b.logger.Info("Downloaded a new version of a blocklist", "list", list.Name)

	return nil
}

// writeAndParse writes body to the file name in root, up to maxDownloadSize
// bytes, and parses the file as format.
func writeAndParse(
	root *os.Root, name string, body io.Reader, format Format,
) (*netipx.IPSet, error) {
	f, err := root.Create(name)
	if err != nil {
		return nil, err
	}

	defer func() { _ = f.Close() }()

	written, err := io.Copy(f, io.LimitReader(body, maxDownloadSize+1))
	if err != nil {
		return nil, err
	}

	if written > maxDownloadSize {
		return nil, errTooLarge
	}

	_, err = f.Seek(0, io.SeekStart)
	if err != nil {
		return nil, err
	}

	return parse(f, format)
}

// markCurrent records that the publisher says the list's copy is still
// current: the copy, on disk and in memory, is marked as updated now.
func (b *Blocklists) markCurrent(list List) error {
	root, err := os.OpenRoot(b.dir)
	if err != nil {
		return err
	}

	defer func() { _ = root.Close() }()

	now := time.Now().UTC()

	err = root.Chtimes(list.Name, now, now)
	if err != nil {
		return err
	}

	b.setUpdated(list.Name, now)
	b.logger.Debug("Blocklist copy is still current", "list", list.Name)

	return nil
}

// loadCopy makes the list's copy on disk its copy in memory. The file's
// modification time is when the copy was updated.
func (b *Blocklists) loadCopy(list List) error {
	f, err := os.OpenInRoot(b.dir, list.Name)
	if err != nil {
		return err
	}

	defer func() { _ = f.Close() }()

	info, err := f.Stat()
	if err != nil {
		return err
	}

	addresses, err := parse(f, list.Format)
	if err != nil {
		return err
	}

	b.setCopy(list.Name, listCopy{
		addresses: addresses, updated: info.ModTime().UTC(),
	})

	return nil
}