package blocklist import ( "context" "encoding/json" "errors" "fmt" "io" "io/fs" "net/http" "os" "slices" "time" "go4.org/netipx" ) // Limits of one download. const ( // downloadTimeout is how long one download may take. downloadTimeout = time.Minute // maxDownloadSize is the size of the largest list file accepted, in // bytes. The largest list, FireHOL abusers, is about 2.5 MiB. maxDownloadSize = 16 << 20 ) // The request headers a download sets. const ( userAgentHeader = "User-Agent" ifModifiedSinceHeader = "If-Modified-Since" ifNoneMatchHeader = "If-None-Match" // userAgent names routewatch to the publishers. userAgent = "routewatch (+https://git.eeqj.de/sneak/routewatch)" ) // The response headers that identify the version of a list a publisher sent. const ( lastModifiedHeader = "Last-Modified" etagHeader = "ETag" ) const ( // downloadSuffix ends the name of the file a download is written to, // beside the list's copy, until it has parsed. downloadSuffix = ".download" // validatorsSuffix ends the name of the file beside a list's copy that // keeps the copy's validators. validatorsSuffix = ".validators" // dirPermissions are those of the directory that holds the copies. dirPermissions = 0o750 // validatorsPermissions are those of the file that keeps a copy's // validators. validatorsPermissions = 0o600 ) // validators are the Last-Modified and ETag headers a publisher sent with a // version of a list. A download sends them back, as If-Modified-Since and // If-None-Match, so that the publisher sends the list only when it has // changed. They are kept as the publisher wrote them. // //nolint:tagliatelle // snake_case keys, as the Go style guide asks type validators struct { LastModified string `json:"last_modified,omitempty"` ETag string `json:"etag,omitempty"` } var ( errUnexpectedStatus = errors.New("unexpected HTTP status") errTooLarge = errors.New("list is over the size limit") ) // Start loads each list's copy from disk. It then starts the goroutine that // downloads each list once its copy is older than its Interval, and a list // with no copy at once. The goroutine runs until Stop: the end of ctx does // not stop it. func (b *Blocklists) Start(ctx context.Context) { err := os.MkdirAll(b.dir, dirPermissions) if err != nil { b.logger.Error("Failed to create the blocklists directory", "dir", b.dir, "error", err) } nextDownload := make([]time.Time, len(b.lists)) for i, list := range b.lists { loadErr := b.loadCopy(list) if loadErr != nil && !errors.Is(loadErr, fs.ErrNotExist) { b.logger.Error("Failed to load a blocklist copy", "list", list.Name, "error", loadErr) } // A list with no copy has the zero time, so it is due at once. nextDownload[i] = b.updated(list.Name).Add(list.Interval) } ctx, b.stop = context.WithCancel(context.WithoutCancel(ctx)) go func() { b.run(ctx, nextDownload) }() b.logger.Info("Blocklist downloads started", "dir", b.dir) } // Stop stops the downloads, cancelling one in progress, and waits until the // goroutine has returned or ctx ends. The copies in memory stay, so Lookup // still answers. func (b *Blocklists) Stop(ctx context.Context) { b.stop() select { case <-b.done: b.logger.Info("Blocklist downloads stopped") case <-ctx.Done(): b.logger.Warn("Stop deadline passed before the blocklist downloads " + "stopped") } } // run downloads each list when nextDownload says, until ctx ends. After each // try, whether it worked or not, the list's next download is an Interval // later. func (b *Blocklists) run(ctx context.Context, nextDownload []time.Time) { defer close(b.done) if len(nextDownload) == 0 { return } for { soonest := slices.MinFunc(nextDownload, time.Time.Compare) timer := time.NewTimer(time.Until(soonest)) select { case <-ctx.Done(): timer.Stop() return case <-timer.C: } for i, list := range b.lists { if time.Now().Before(nextDownload[i]) { continue } err := b.download(ctx, list) if ctx.Err() != nil { return } if err != nil { b.logger.Error("Blocklist download failed; keeping the "+ "previous copy", "list", list.Name, "error", err) } nextDownload[i] = time.Now().Add(list.Interval) } } } // download asks the publisher for list, sending the validators of the list's // copy when it has them. A new version that parses becomes the list's copy, // on disk and in memory. An answer that the copy is still current marks the // copy as updated now. Anything else leaves the copy as it is and returns an // error. func (b *Blocklists) download(ctx context.Context, list List) error { ctx, cancel := context.WithTimeout(ctx, downloadTimeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, list.URL, nil) if err != nil { return err } req.Header.Set(userAgentHeader, userAgent) held := b.copyValidators(list.Name) if held.LastModified != "" { req.Header.Set(ifModifiedSinceHeader, held.LastModified) } if held.ETag != "" { req.Header.Set(ifNoneMatchHeader, held.ETag) } resp, err := http.DefaultClient.Do(req) if err != nil { return err } defer func() { _ = resp.Body.Close() }() switch resp.StatusCode { case http.StatusOK: return b.replaceCopy(list, resp.Body, validators{ LastModified: resp.Header.Get(lastModifiedHeader), ETag: resp.Header.Get(etagHeader), }) case http.StatusNotModified: return b.markCurrent(list) default: return fmt.Errorf("%w: %s", errUnexpectedStatus, resp.Status) } } // replaceCopy writes a new version of list, read from body, beside the list's // copy. Only when it parses does it replace the copy, on disk and in memory. // sent are the validators the publisher sent with the new version. func (b *Blocklists) replaceCopy(list List, body io.Reader, sent validators) error { root, err := os.OpenRoot(b.dir) if err != nil { return err } defer func() { _ = root.Close() }() downloadName := list.Name + downloadSuffix addresses, err := writeAndParse(root, downloadName, body, list.Format) if err != nil { _ = root.Remove(downloadName) return err } // The old copy's validators are removed before the copy is replaced, so // that they are never kept with the new copy. err = root.Remove(list.Name + validatorsSuffix) if err != nil && !errors.Is(err, fs.ErrNotExist) { return err } err = root.Rename(downloadName, list.Name) if err != nil { return err } b.setCopy(list.Name, listCopy{ addresses: addresses, updated: time.Now().UTC(), validators: sent, }) b.logger.Info("Downloaded a new version of a blocklist", "list", list.Name) // Without this file, the first download after a restart asks for the // whole list. err = writeValidators(root, list.Name, sent) if err != nil { b.logger.Error("Failed to keep the validators of a blocklist copy", "list", list.Name, "error", err) } return nil } // writeAndParse writes body to the file name in root, up to maxDownloadSize // bytes, and parses the file as format. func writeAndParse( root *os.Root, name string, body io.Reader, format Format, ) (*netipx.IPSet, error) { f, err := root.Create(name) if err != nil { return nil, err } defer func() { _ = f.Close() }() written, err := io.Copy(f, io.LimitReader(body, maxDownloadSize+1)) if err != nil { return nil, err } if written > maxDownloadSize { return nil, errTooLarge } _, err = f.Seek(0, io.SeekStart) if err != nil { return nil, err } return parse(f, format) } // markCurrent records that the publisher says the list's copy is still // current: the copy, on disk and in memory, is marked as updated now. func (b *Blocklists) markCurrent(list List) error { root, err := os.OpenRoot(b.dir) if err != nil { return err } defer func() { _ = root.Close() }() now := time.Now().UTC() err = root.Chtimes(list.Name, now, now) if err != nil { return err } b.setUpdated(list.Name, now) b.logger.Debug("Blocklist copy is still current", "list", list.Name) return nil } // loadCopy makes the list's copy on disk, with the validators kept beside it, // its copy in memory. The file's modification time is when the copy was // updated. func (b *Blocklists) loadCopy(list List) error { root, err := os.OpenRoot(b.dir) if err != nil { return err } defer func() { _ = root.Close() }() f, err := root.Open(list.Name) if err != nil { return err } defer func() { _ = f.Close() }() info, err := f.Stat() if err != nil { return err } addresses, err := parse(f, list.Format) if err != nil { return err } held, err := readValidators(root, list.Name) if err != nil { return err } b.setCopy(list.Name, listCopy{ addresses: addresses, updated: info.ModTime().UTC(), validators: held, }) return nil } // writeValidators keeps held in the file beside the named list's copy. func writeValidators(root *os.Root, name string, held validators) error { data, err := json.Marshal(held) if err != nil { return err } return root.WriteFile(name+validatorsSuffix, data, validatorsPermissions) } // readValidators returns the validators kept beside the named list's copy. // They are empty when there is no such file. func readValidators(root *os.Root, name string) (validators, error) { var held validators data, err := root.ReadFile(name + validatorsSuffix) if errors.Is(err, fs.ErrNotExist) { return held, nil } if err != nil { return held, err } err = json.Unmarshal(data, &held) return held, err }