package blocklist import ( "context" "errors" "fmt" "io" "io/fs" "net/http" "os" "slices" "time" "go4.org/netipx" ) // Limits of one download. const ( // downloadTimeout is how long one download may take. downloadTimeout = time.Minute // maxDownloadSize is the size of the largest list file accepted, in // bytes. The largest list, FireHOL abusers, is about 2.5 MiB. maxDownloadSize = 16 << 20 ) // The request headers a download sets. const ( userAgentHeader = "User-Agent" ifModifiedSinceHeader = "If-Modified-Since" // userAgent names routewatch to the publishers. userAgent = "routewatch (+https://git.eeqj.de/sneak/routewatch)" ) const ( // downloadSuffix ends the name of the file a download is written to, // beside the list's copy, until it has parsed. downloadSuffix = ".download" // dirPermissions are those of the directory that holds the copies. dirPermissions = 0o750 ) var ( errUnexpectedStatus = errors.New("unexpected HTTP status") errTooLarge = errors.New("list is over the size limit") ) // Start loads each list's copy from disk. It then starts the goroutine that // downloads each list once its copy is older than its Interval, and a list // with no copy at once. The goroutine runs until Stop: the end of ctx does // not stop it. func (b *Blocklists) Start(ctx context.Context) { err := os.MkdirAll(b.dir, dirPermissions) if err != nil { b.logger.Error("Failed to create the blocklists directory", "dir", b.dir, "error", err) } nextDownload := make([]time.Time, len(b.lists)) for i, list := range b.lists { loadErr := b.loadCopy(list) if loadErr != nil && !errors.Is(loadErr, fs.ErrNotExist) { b.logger.Error("Failed to load a blocklist copy", "list", list.Name, "error", loadErr) } // A list with no copy has the zero time, so it is due at once. nextDownload[i] = b.updated(list.Name).Add(list.Interval) } ctx, b.stop = context.WithCancel(context.WithoutCancel(ctx)) go func() { b.run(ctx, nextDownload) }() b.logger.Info("Blocklist downloads started", "dir", b.dir) } // Stop stops the downloads, cancelling one in progress, and waits until the // goroutine has returned or ctx ends. The copies in memory stay, so Lookup // still answers. func (b *Blocklists) Stop(ctx context.Context) { b.stop() select { case <-b.done: b.logger.Info("Blocklist downloads stopped") case <-ctx.Done(): b.logger.Warn("Stop deadline passed before the blocklist downloads " + "stopped") } } // run downloads each list when nextDownload says, until ctx ends. After each // try, whether it worked or not, the list's next download is an Interval // later. func (b *Blocklists) run(ctx context.Context, nextDownload []time.Time) { defer close(b.done) if len(nextDownload) == 0 { return } for { soonest := slices.MinFunc(nextDownload, time.Time.Compare) timer := time.NewTimer(time.Until(soonest)) select { case <-ctx.Done(): timer.Stop() return case <-timer.C: } for i, list := range b.lists { if time.Now().Before(nextDownload[i]) { continue } err := b.download(ctx, list) if ctx.Err() != nil { return } if err != nil { b.logger.Error("Blocklist download failed; keeping the "+ "previous copy", "list", list.Name, "error", err) } nextDownload[i] = time.Now().Add(list.Interval) } } } // download asks the publisher for list. A new version that parses becomes // the list's copy, on disk and in memory. An answer that the copy is still // current marks the copy as updated now. Anything else leaves the copy as it // is and returns an error. func (b *Blocklists) download(ctx context.Context, list List) error { ctx, cancel := context.WithTimeout(ctx, downloadTimeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, list.URL, nil) if err != nil { return err } req.Header.Set(userAgentHeader, userAgent) updated := b.updated(list.Name) if !updated.IsZero() { req.Header.Set(ifModifiedSinceHeader, updated.Format(http.TimeFormat)) } resp, err := http.DefaultClient.Do(req) if err != nil { return err } defer func() { _ = resp.Body.Close() }() switch resp.StatusCode { case http.StatusOK: return b.replaceCopy(list, resp.Body) case http.StatusNotModified: return b.markCurrent(list) default: return fmt.Errorf("%w: %s", errUnexpectedStatus, resp.Status) } } // replaceCopy writes a new version of list, read from body, beside the list's // copy. Only when it parses does it replace the copy, on disk and in memory. func (b *Blocklists) replaceCopy(list List, body io.Reader) error { root, err := os.OpenRoot(b.dir) if err != nil { return err } defer func() { _ = root.Close() }() downloadName := list.Name + downloadSuffix addresses, err := writeAndParse(root, downloadName, body, list.Format) if err != nil { _ = root.Remove(downloadName) return err } err = root.Rename(downloadName, list.Name) if err != nil { return err } b.setCopy(list.Name, listCopy{addresses: addresses, updated: time.Now().UTC()}) b.logger.Info("Downloaded a new version of a blocklist", "list", list.Name) return nil } // writeAndParse writes body to the file name in root, up to maxDownloadSize // bytes, and parses the file as format. func writeAndParse( root *os.Root, name string, body io.Reader, format Format, ) (*netipx.IPSet, error) { f, err := root.Create(name) if err != nil { return nil, err } defer func() { _ = f.Close() }() written, err := io.Copy(f, io.LimitReader(body, maxDownloadSize+1)) if err != nil { return nil, err } if written > maxDownloadSize { return nil, errTooLarge } _, err = f.Seek(0, io.SeekStart) if err != nil { return nil, err } return parse(f, format) } // markCurrent records that the publisher says the list's copy is still // current: the copy, on disk and in memory, is marked as updated now. func (b *Blocklists) markCurrent(list List) error { root, err := os.OpenRoot(b.dir) if err != nil { return err } defer func() { _ = root.Close() }() now := time.Now().UTC() err = root.Chtimes(list.Name, now, now) if err != nil { return err } b.setUpdated(list.Name, now) b.logger.Debug("Blocklist copy is still current", "list", list.Name) return nil } // loadCopy makes the list's copy on disk its copy in memory. The file's // modification time is when the copy was updated. func (b *Blocklists) loadCopy(list List) error { f, err := os.OpenInRoot(b.dir, list.Name) if err != nil { return err } defer func() { _ = f.Close() }() info, err := f.Stat() if err != nil { return err } addresses, err := parse(f, list.Format) if err != nil { return err } b.setCopy(list.Name, listCopy{ addresses: addresses, updated: info.ModTime().UTC(), }) return nil }