dnswatcher/internal/watcher/watcher.go

package watcher

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"maps"
	"slices"
	"sort"
	"strings"
	"time"

	"go.uber.org/fx"

	"sneak.berlin/go/dnswatcher/internal/config"
	"sneak.berlin/go/dnswatcher/internal/logger"
	"sneak.berlin/go/dnswatcher/internal/resolver"
	"sneak.berlin/go/dnswatcher/internal/state"
	"sneak.berlin/go/dnswatcher/internal/tlscheck"
)

// monitoredPorts are the TCP ports checked for each IP address.
var monitoredPorts = []int{80, 443} //nolint:gochecknoglobals

// tlsPort is the port used for TLS certificate checks.
const tlsPort = 443

// hoursPerDay converts days to hours for duration calculations.
const hoursPerDay = 24

// Status values recorded for nameserver and certificate checks.
const (
	statusOK    = "ok"
	statusError = "error"
)

// Params contains dependencies for Watcher.
type Params struct {
	fx.In

	Logger    *logger.Logger
	Config    *config.Config
	State     *state.State
	Resolver  DNSResolver
	PortCheck PortChecker
	TLSCheck  TLSChecker
	Notify    Notifier
}

// Watcher orchestrates all monitoring checks on a schedule.
type Watcher struct {
	log       *slog.Logger
	config    *config.Config
	state     *state.State
	resolver  DNSResolver
	portCheck PortChecker
	tlsCheck  TLSChecker
	notify    Notifier
	cancel    context.CancelFunc
	done      chan struct{} // closed when Run returns
	firstRun  bool
}

// New creates a new Watcher instance wired into the fx lifecycle.
func New(
	lifecycle fx.Lifecycle,
	params Params,
) (*Watcher, error) {
	w := &Watcher{
		log:       params.Logger.Get(),
		config:    params.Config,
		state:     params.State,
		resolver:  params.Resolver,
		portCheck: params.PortCheck,
		tlsCheck:  params.TLSCheck,
		notify:    params.Notify,
		firstRun:  true,
	}

	lifecycle.Append(fx.Hook{
		OnStart: func(startCtx context.Context) error {
			// The fx startup context expires after startup
			// completes, so the watcher's context drops its
			// cancellation. The watcher's lifetime is controlled
			// by w.cancel in OnStop.
			ctx, cancel := context.WithCancel(
				context.WithoutCancel(startCtx),
			)
			w.cancel = cancel
			w.done = make(chan struct{})

			go func() {
				defer close(w.done)

				w.Run(ctx)
			}()

			return nil
		},
		OnStop: func(ctx context.Context) error {
			w.cancel()

			// Run saves state as it returns. Waiting for it here
			// means the save is done before shutdown goes on.
			select {
			case <-w.done:
				return nil
			case <-ctx.Done():
				return fmt.Errorf(
					"waiting for the watcher to stop: %w",
					ctx.Err(),
				)
			}
		},
	})

	return w, nil
}

// Run starts the monitoring loop with periodic scheduling. When ctx
// is cancelled, it saves state and returns.
func (w *Watcher) Run(ctx context.Context) {
	w.log.Info(
		"watcher starting",
		"domains", len(w.config.Domains),
		"hostnames", len(w.config.Hostnames),
		// As text: the JSON log writes a time.Duration as bare
		// nanoseconds.
		"dnsInterval", w.config.DNSInterval.String(),
		"tlsInterval", w.config.TLSInterval.String(),
	)

	w.RunOnce(ctx)
	w.maybeSendTestNotification(ctx)

	dnsTicker := time.NewTicker(w.config.DNSInterval)
	tlsTicker := time.NewTicker(w.config.TLSInterval)

	defer dnsTicker.Stop()
	defer tlsTicker.Stop()

	for {
		select {
		case <-ctx.Done():
			w.saveState()
			w.log.Info("watcher stopped")

			return
		case <-dnsTicker.C:
			w.runDNSChecks(ctx)

			w.checkAllPorts(ctx)
			w.saveState()
		case <-tlsTicker.C:
			// Run DNS first so TLS checks use freshly
			// resolved IP addresses, not stale ones from
			// a previous cycle.
			w.runDNSChecks(ctx)

			w.runTLSChecks(ctx)
			w.saveState()
		}
	}
}

// RunOnce performs a single complete monitoring cycle.
// DNS checks run first so that port and TLS checks use
// freshly resolved IP addresses. Port checks run before
// TLS because TLS checks only target IPs with an open
// port 443.
func (w *Watcher) RunOnce(ctx context.Context) {
	w.detectFirstRun()

	// Phase 1: DNS resolution must complete first so that
	// subsequent checks use fresh IP addresses.
	w.runDNSChecks(ctx)

	// Phase 2: Port checks populate port state that TLS
	// checks depend on (TLS only targets IPs where port
	// 443 is open).
	w.checkAllPorts(ctx)

	// Phase 3: TLS checks use fresh DNS IPs and current
	// port state.
	w.runTLSChecks(ctx)

	w.saveState()
	w.firstRun = false
}

func (w *Watcher) detectFirstRun() {
	snap := w.state.GetSnapshot()
	hasState := len(snap.Domains) > 0 ||
		len(snap.Hostnames) > 0 ||
		len(snap.Ports) > 0 ||
		len(snap.Certificates) > 0

	if hasState {
		w.firstRun = false
	}
}

// runDNSChecks performs DNS resolution for all configured domains
// and hostnames, updating state with freshly resolved records.
// This must complete before port or TLS checks run so those
// checks operate on current IP addresses.
func (w *Watcher) runDNSChecks(ctx context.Context) {
	for _, domain := range w.config.Domains {
		w.checkDomain(ctx, domain)
	}

	for _, hostname := range w.config.Hostnames {
		w.checkHostname(ctx, hostname)
	}
}

// logFailedLookup logs a failed DNS lookup at error level, unless ctx
// was cancelled: shutdown cancels it, and a lookup it cut short did not
// fail. A lookup that ran out of time did fail, so it is logged.
func (w *Watcher) logFailedLookup(
	ctx context.Context,
	msg string,
	args ...any,
) {
	if errors.Is(ctx.Err(), context.Canceled) {
		return
	}

	w.log.Error(msg, args...)
}

func (w *Watcher) checkDomain(
	ctx context.Context,
	domain string,
) {
	nameservers, err := w.resolver.LookupNS(ctx, domain)
	if err != nil {
		w.logFailedLookup(
			ctx,
			"failed to lookup NS",
			"domain", domain,
			"error", err,
		)

		return
	}

	sort.Strings(nameservers)

	now := time.Now().UTC()

	prev, hasPrev := w.state.GetDomainState(domain)

	var prevAddresses map[string][]string
	if hasPrev {
		prevAddresses = prev.NameserverAddresses
	}

	addresses := w.resolveNameserverAddresses(
		ctx, nameservers, prevAddresses,
	)

	if hasPrev && !w.firstRun {
		w.detectNSChanges(ctx, domain, prev.Nameservers, nameservers)
		w.detectNSAddressChanges(ctx, domain, prevAddresses, addresses)
	}

	w.state.SetDomainState(domain, &state.DomainState{
		Nameservers:         nameservers,
		NameserverAddresses: addresses,
		LastChecked:         now,
	})

	// The apex domain's records are also checked and saved as a
	// hostname's, so that the port and TLS checks find its addresses.
	// Notifications about them name it as a domain (see nameLine).
	w.checkHostname(ctx, domain)
}

func (w *Watcher) detectNSChanges(
	ctx context.Context,
	domain string,
	oldNS, newNS []string,
) {
	oldSet := toSet(oldNS)
	newSet := toSet(newNS)

	var added, removed []string

	for ns := range newSet {
		if !oldSet[ns] {
			added = append(added, ns)
		}
	}

	for ns := range oldSet {
		if !newSet[ns] {
			removed = append(removed, ns)
		}
	}

	if len(added) == 0 && len(removed) == 0 {
		return
	}

	msg := fmt.Sprintf(
		"Domain: %s\nAdded: %s\nRemoved: %s",
		domain,
		strings.Join(added, ", "),
		strings.Join(removed, ", "),
	)

	w.notify.SendNotification(
		ctx,
		"NS Change: "+domain,
		msg,
		"warning",
	)
}

// resolveNameserverAddresses returns the sorted addresses each
// nameserver's name resolves to. A nameserver whose lookup fails, as it
// does when no nameserver of the name's zone answers, or finds no
// address keeps its addresses from prev and is not an address change.
func (w *Watcher) resolveNameserverAddresses(
	ctx context.Context,
	nameservers []string,
	prev map[string][]string,
) map[string][]string {
	addresses := make(map[string][]string, len(nameservers))

	for _, ns := range nameservers {
		ips, err := w.resolver.ResolveIPAddresses(ctx, ns)
		if err == nil && len(ips) > 0 {
			sort.Strings(ips)
			addresses[ns] = ips

			continue
		}

		w.logFailedLookup(
			ctx,
			"no addresses found for nameserver",
			"nameserver", ns,
			"error", err,
		)

		if prevIPs, ok := prev[ns]; ok {
			addresses[ns] = prevIPs
		}
	}

	return addresses
}

// detectNSAddressChanges notifies when a nameserver in both checks
// resolves to different addresses. A nameserver added or removed is
// reported by detectNSChanges alone, and one with no addresses saved by
// the previous check, as in a state file from before they were saved,
// is not compared.
func (w *Watcher) detectNSAddressChanges(
	ctx context.Context,
	domain string,
	prev, current map[string][]string,
) {
	for ns, cur := range current {
		old, ok := prev[ns]
		if !ok || sliceEqual(old, cur) {
			continue
		}

		msg := fmt.Sprintf(
			"Domain: %s\nNameserver: %s\nOld: %s\nNew: %s",
			domain, ns,
			strings.Join(old, ", "),
			strings.Join(cur, ", "),
		)

		w.notify.SendNotification(
			ctx,
			"NS Address Change: "+domain,
			msg,
			"warning",
		)
	}
}

func (w *Watcher) checkHostname(
	ctx context.Context,
	hostname string,
) {
	results, err := w.resolver.LookupAllRecords(ctx, hostname)
	if err != nil {
		w.logFailedLookup(
			ctx,
			"failed to lookup records",
			"hostname", hostname,
			"error", err,
		)

		return
	}

	prev, _ := w.state.GetHostnameState(hostname)

	w.updateHostnameState(
		ctx, hostname, buildHostnameState(results, prev, time.Now().UTC()),
	)
}

// updateHostnameState finishes a check of hostname from newState, built
// from its nameservers' answers: it follows the CNAME in them, notifies
// what changed since the previous check, and saves newState.
func (w *Watcher) updateHostnameState(
	ctx context.Context,
	hostname string,
	newState *state.HostnameState,
) {
	prev, hasPrev := w.state.GetHostnameState(hostname)

	w.resolveCNAMEAddresses(ctx, hostname, newState, prev)

	if hasPrev && !w.firstRun {
		w.detectHostnameChanges(ctx, hostname, prev, newState)
	}

	w.state.SetHostnameState(hostname, newState)
}

// resolveCNAMEAddresses saves in current the addresses at the end of
// hostname's CNAME chain, when the nameservers' answers in current hold
// a CNAME and no address, and an empty list otherwise. Every CNAME
// target the nameservers gave is followed with ResolveIPAddresses and
// the addresses found for all of them are saved, so nameservers that
// disagree on the target do not change the result from check to check.
// The addresses saved in prev, which may be nil, are kept when none of
// the name's nameservers answered its queries for A, AAAA and CNAME,
// and when a target cannot be followed, as when no nameserver of a zone
// in its chain answers.
func (w *Watcher) resolveCNAMEAddresses(
	ctx context.Context,
	hostname string,
	current, prev *state.HostnameState,
) {
	var prevAddresses []string
	if prev != nil {
		prevAddresses = prev.CNAMEAddresses
	}

	// Empty, not nil: nil means the addresses are not known.
	current.CNAMEAddresses = []string{}

	answered := false
	targets := make(map[string]bool)

	for _, nsState := range current.RecordsByNameserver {
		if nsState.Status != statusOK ||
			slices.Contains(nsState.FailedTypes, "A") ||
			slices.Contains(nsState.FailedTypes, "AAAA") ||
			slices.Contains(nsState.FailedTypes, "CNAME") {
			continue
		}

		answered = true

		if len(nsState.Records["A"]) > 0 || len(nsState.Records["AAAA"]) > 0 {
			return
		}

		for _, target := range nsState.Records["CNAME"] {
			targets[target] = true
		}
	}

	if !answered {
		current.CNAMEAddresses = prevAddresses

		return
	}

	for target := range targets {
		ips, err := w.resolver.ResolveIPAddresses(ctx, target)
		if err != nil {
			w.logFailedLookup(
				ctx,
				"failed to follow CNAME",
				"hostname", hostname,
				"target", target,
				"error", err,
			)

			current.CNAMEAddresses = prevAddresses

			return
		}

		current.CNAMEAddresses = append(current.CNAMEAddresses, ips...)
	}

	// Still the empty list when every chain ends in no address.
	slices.Sort(current.CNAMEAddresses)
	current.CNAMEAddresses = slices.Compact(current.CNAMEAddresses)
}

// buildHostnameState saves each nameserver's response. A nameserver
// that answered, even with NXDOMAIN or no records, is saved as ok, with
// the record types whose query failed; one that timed out or failed is
// saved as error with the reason, and its empty record set is not an
// answer. prev is the hostname's state from the previous check, or nil.
func buildHostnameState(
	results map[string]*resolver.NameserverResponse,
	prev *state.HostnameState,
	now time.Time,
) *state.HostnameState {
	hs := &state.HostnameState{
		RecordsByNameserver: make(
			map[string]*state.NameserverRecordState,
		),
		LastChecked: now,
	}

	for ns, resp := range results {
		nsState := &state.NameserverRecordState{
			Records:     maps.Clone(resp.Records),
			Status:      statusOK,
			LastChecked: now,
		}

		if resp.Status == resolver.StatusTimeout ||
			resp.Status == resolver.StatusError {
			nsState.Status = statusError
			nsState.Error = resp.Error
		} else {
			nsState.FailedTypes = resp.FailedTypes

			var prevNS *state.NameserverRecordState
			if prev != nil {
				prevNS = prev.RecordsByNameserver[ns]
			}

			keepFailedTypes(nsState, prevNS)
		}

		hs.RecordsByNameserver[ns] = nsState
	}

	return hs
}

// keepFailedTypes copies into nsState, for each record type in its
// FailedTypes, the records prevNS, the nameserver's state from the
// previous check, holds for that type, which may be none. When prevNS
// does not know them either, because the nameserver was new or failing
// then or the type was in its UnknownTypes, the type goes in
// nsState.UnknownTypes instead.
func keepFailedTypes(nsState, prevNS *state.NameserverRecordState) {
	for _, rtype := range nsState.FailedTypes {
		if prevNS == nil || prevNS.Status != statusOK ||
			slices.Contains(prevNS.UnknownTypes, rtype) {
			nsState.UnknownTypes = append(nsState.UnknownTypes, rtype)

			continue
		}

		if records, ok := prevNS.Records[rtype]; ok {
			nsState.Records[rtype] = records
		}
	}
}

func (w *Watcher) detectHostnameChanges(
	ctx context.Context,
	hostname string,
	prev, current *state.HostnameState,
) {
	w.detectRecordChanges(ctx, hostname, prev, current)
	w.detectNSDisappearances(ctx, hostname, prev, current)
	w.detectNSFailures(ctx, hostname, prev, current)
	w.detectInconsistencies(ctx, hostname, prev, current)
	w.detectCNAMEAddressChanges(ctx, hostname, prev, current)
}

// isDomain reports whether name is a configured apex domain, whose own
// records are checked and saved as a hostname's are.
func (w *Watcher) isDomain(name string) bool {
	return slices.Contains(w.config.Domains, name)
}

// nameLine is the line a notification about name's records starts with:
// "Domain: " and the name for a configured apex domain, and
// "Hostname: " otherwise.
func (w *Watcher) nameLine(name string) string {
	if w.isDomain(name) {
		return "Domain: " + name
	}

	return "Hostname: " + name
}

// portNameLines lists the names that resolve to a port's address, the
// configured apex domains on one line and the hostnames on the next,
// leaving out a line that would name nothing.
func (w *Watcher) portNameLines(names []string) string {
	var domains, hostnames []string

	for _, name := range names {
		if w.isDomain(name) {
			domains = append(domains, name)
		} else {
			hostnames = append(hostnames, name)
		}
	}

	var lines []string

	if len(domains) > 0 {
		lines = append(lines, "Domains: "+strings.Join(domains, ", "))
	}

	if len(hostnames) > 0 {
		lines = append(lines, "Hostnames: "+strings.Join(hostnames, ", "))
	}

	return strings.Join(lines, "\n")
}

// detectCNAMEAddressChanges notifies when the addresses at the end of
// hostname's CNAME chain differ from those the previous check saved,
// including a change from or to none. When the previous addresses are
// not known (nil), as on the first check after loading a state file
// written before they were saved, nothing is compared.
func (w *Watcher) detectCNAMEAddressChanges(
	ctx context.Context,
	hostname string,
	prev, current *state.HostnameState,
) {
	old, cur := prev.CNAMEAddresses, current.CNAMEAddresses
	if old == nil || sliceEqual(old, cur) {
		return
	}

	msg := fmt.Sprintf(
		"%s\nOld: %s\nNew: %s",
		w.nameLine(hostname),
		strings.Join(old, ", "),
		strings.Join(cur, ", "),
	)

	w.notify.SendNotification(
		ctx,
		"CNAME Address Change: "+hostname,
		msg,
		"warning",
	)
}

// detectRecordChanges compares each nameserver's records with those of
// the previous check. Only answers are compared: a nameserver that
// failed on either check has no records to compare. The records kept
// for a record type whose query failed are compared too, but not those
// of a type in UnknownTypes on either check, which the message leaves
// out as well.
func (w *Watcher) detectRecordChanges(
	ctx context.Context,
	hostname string,
	prev, current *state.HostnameState,
) {
	for ns, cur := range current.RecordsByNameserver {
		prevNS, ok := prev.RecordsByNameserver[ns]
		if !ok || prevNS.Status != statusOK || cur.Status != statusOK {
			continue
		}

		unknown := slices.Concat(prevNS.UnknownTypes, cur.UnknownTypes)
		oldRecords := withoutTypes(prevNS.Records, unknown)
		newRecords := withoutTypes(cur.Records, unknown)

		if recordsEqual(oldRecords, newRecords) {
			continue
		}

		msg := fmt.Sprintf(
			"%s\nNameserver: %s\n%s",
			w.nameLine(hostname), ns,
			recordDifferences(
				"Old", oldRecords,
				"New", newRecords,
			),
		)

		w.notify.SendNotification(
			ctx,
			"Record Change: "+hostname,
			msg,
			"warning",
		)
	}
}

func (w *Watcher) detectNSDisappearances(
	ctx context.Context,
	hostname string,
	prev, current *state.HostnameState,
) {
	for ns, prevNS := range prev.RecordsByNameserver {
		if _, ok := current.RecordsByNameserver[ns]; ok || prevNS.Status != statusOK {
			continue
		}

		msg := fmt.Sprintf(
			"%s\nNameserver: %s disappeared",
			w.nameLine(hostname), ns,
		)

		w.notify.SendNotification(
			ctx,
			"NS Failure: "+hostname,
			msg,
			"error",
		)
	}
}

// detectNSFailures notifies when a nameserver that answered on the
// previous check fails, and when one that failed answers again. A
// nameserver missing from the previous check is not compared.
func (w *Watcher) detectNSFailures(
	ctx context.Context,
	hostname string,
	prev, current *state.HostnameState,
) {
	for ns, cur := range current.RecordsByNameserver {
		prevNS, ok := prev.RecordsByNameserver[ns]
		if !ok {
			continue
		}

		switch {
		case prevNS.Status == statusOK && cur.Status == statusError:
			msg := fmt.Sprintf(
				"%s\nNameserver: %s\nError: %s",
				w.nameLine(hostname), ns, cur.Error,
			)

			w.notify.SendNotification(
				ctx,
				"NS Failure: "+hostname,
				msg,
				"error",
			)
		case prevNS.Status == statusError && cur.Status == statusOK:
			msg := fmt.Sprintf(
				"%s\nNameserver: %s recovered",
				w.nameLine(hostname), ns,
			)

			w.notify.SendNotification(
				ctx,
				"NS Recovery: "+hostname,
				msg,
				"success",
			)
		}
	}
}

func (w *Watcher) detectInconsistencies(
	ctx context.Context,
	hostname string,
	prev, current *state.HostnameState,
) {
	for _, pair := range newlyDisagreeingPairs(prev, current) {
		ns1, ns2 := pair[0], pair[1]
		state1 := current.RecordsByNameserver[ns1]
		state2 := current.RecordsByNameserver[ns2]

		// The record types left out of the comparison are left out of
		// the message too.
		failed := slices.Concat(state1.FailedTypes, state2.FailedTypes)

		msg := fmt.Sprintf(
			"%s\n%s",
			w.nameLine(hostname),
			recordDifferences(
				ns1, withoutTypes(state1.Records, failed),
				ns2, withoutTypes(state2.Records, failed),
			),
		)

		w.notify.SendNotification(
			ctx,
			"Inconsistency: "+hostname,
			msg,
			"warning",
		)
	}
}

// newlyDisagreeingPairs returns every pair of nameservers that answered
// in current and whose records differ there, in sorted order of name,
// except pairs where both nameservers answered in prev and already
// differed there. A nameserver missing from prev, or that failed there,
// is paired with every nameserver it differs from. A nameserver that
// failed in current has no records to compare and is in no pair. In
// both checks, a record type whose query failed at either nameserver is
// not compared.
func newlyDisagreeingPairs(
	prev, current *state.HostnameState,
) [][2]string {
	nameservers := make([]string, 0, len(current.RecordsByNameserver))
	for ns, cur := range current.RecordsByNameserver {
		if cur.Status == statusOK {
			nameservers = append(nameservers, ns)
		}
	}

	sort.Strings(nameservers)

	var pairs [][2]string

	for i, ns1 := range nameservers {
		for _, ns2 := range nameservers[i+1:] {
			if nameserversAgree(
				current.RecordsByNameserver[ns1],
				current.RecordsByNameserver[ns2],
			) {
				continue
			}

			prev1, ok1 := prev.RecordsByNameserver[ns1]
			prev2, ok2 := prev.RecordsByNameserver[ns2]

			if ok1 && ok2 &&
				prev1.Status == statusOK && prev2.Status == statusOK &&
				!nameserversAgree(prev1, prev2) {
				continue
			}

			pairs = append(pairs, [2]string{ns1, ns2})
		}
	}

	return pairs
}

func (w *Watcher) checkAllPorts(ctx context.Context) {
	// Phase 1: Build current IP:port → hostname associations
	// from fresh DNS data.
	associations := w.buildPortAssociations()

	// Phase 2: Check each unique IP:port and update state
	// with the full set of associated hostnames.
	for key, hostnames := range associations {
		ip, port := parsePortKey(key)
		if port == 0 {
			continue
		}

		w.checkSinglePort(ctx, ip, port, hostnames)
	}

	// Phase 3: Remove port state entries that no longer have
	// any hostname referencing them.
	w.cleanupStalePorts(associations)
}

// buildPortAssociations constructs a map from IP:port keys to
// the sorted set of hostnames currently resolving to that IP.
func (w *Watcher) buildPortAssociations() map[string][]string {
	assoc := make(map[string]map[string]bool)

	allNames := make(
		[]string, 0,
		len(w.config.Hostnames)+len(w.config.Domains),
	)
	allNames = append(allNames, w.config.Hostnames...)
	allNames = append(allNames, w.config.Domains...)

	for _, name := range allNames {
		ips := w.collectIPs(name)
		for _, ip := range ips {
			for _, port := range monitoredPorts {
				key := fmt.Sprintf("%s:%d", ip, port)
				if assoc[key] == nil {
					assoc[key] = make(map[string]bool)
				}

				assoc[key][name] = true
			}
		}
	}

	result := make(map[string][]string, len(assoc))
	for key, set := range assoc {
		hostnames := make([]string, 0, len(set))
		for h := range set {
			hostnames = append(hostnames, h)
		}

		sort.Strings(hostnames)

		result[key] = hostnames
	}

	return result
}

// parsePortKey splits an "ip:port" key into its components.
func parsePortKey(key string) (string, int) {
	lastColon := strings.LastIndex(key, ":")
	if lastColon < 0 {
		return key, 0
	}

	ip := key[:lastColon]

	var p int

	_, err := fmt.Sscanf(key[lastColon+1:], "%d", &p)
	if err != nil {
		return ip, 0
	}

	return ip, p
}

// cleanupStalePorts removes port state entries that are no
// longer referenced by any hostname in the current DNS data. An
// entry saved for a configured name none of whose nameservers
// answered is kept: that name's addresses are not known, not gone.
func (w *Watcher) cleanupStalePorts(
	currentAssociations map[string][]string,
) {
	for _, key := range w.state.GetAllPortKeys() {
		if _, exists := currentAssociations[key]; exists {
			continue
		}

		ps, ok := w.state.GetPortState(key)
		if ok && slices.ContainsFunc(ps.Hostnames, w.noNameserverAnswered) {
			continue
		}

		w.state.DeletePortState(key)
	}
}

// noNameserverAnswered reports whether name is a configured domain or
// hostname and none of its nameservers answered on its last check.
func (w *Watcher) noNameserverAnswered(name string) bool {
	if !slices.Contains(w.config.Hostnames, name) &&
		!slices.Contains(w.config.Domains, name) {
		return false
	}

	hs, ok := w.state.GetHostnameState(name)
	if !ok {
		return false
	}

	for _, nsState := range hs.RecordsByNameserver {
		if nsState.Status == statusOK {
			return false
		}
	}

	return true
}

// collectIPs returns the addresses saved for hostname: those in its
// nameservers' A and AAAA records, and those at the end of its CNAME
// chain.
func (w *Watcher) collectIPs(hostname string) []string {
	hs, ok := w.state.GetHostnameState(hostname)
	if !ok {
		return nil
	}

	ipSet := make(map[string]bool)

	for _, nsState := range hs.RecordsByNameserver {
		for _, ip := range nsState.Records["A"] {
			ipSet[ip] = true
		}

		for _, ip := range nsState.Records["AAAA"] {
			ipSet[ip] = true
		}
	}

	for _, ip := range hs.CNAMEAddresses {
		ipSet[ip] = true
	}

	result := make([]string, 0, len(ipSet))
	for ip := range ipSet {
		result = append(result, ip)
	}

	sort.Strings(result)

	return result
}

func (w *Watcher) checkSinglePort(
	ctx context.Context,
	ip string,
	port int,
	hostnames []string,
) {
	result, err := w.portCheck.CheckPort(ctx, ip, port)

	// A check the context cut short says nothing about the port, so it
	// is neither saved nor notified.
	if ctx.Err() != nil {
		return
	}

	if err != nil {
		w.log.Error(
			"port check failed",
			"ip", ip,
			"port", port,
			"error", err,
		)

		return
	}

	key := fmt.Sprintf("%s:%d", ip, port)
	now := time.Now().UTC()
	prev, hasPrev := w.state.GetPortState(key)

	if hasPrev && !w.firstRun && prev.Open != result.Open {
		stateStr := "closed"
		if result.Open {
			stateStr = "open"
		}

		msg := fmt.Sprintf(
			"%s\nAddress: %s\nPort now %s",
			w.portNameLines(hostnames), key, stateStr,
		)

		w.notify.SendNotification(
			ctx,
			"Port Change: "+key,
			msg,
			"warning",
		)
	}

	// A configured name on the saved list none of whose nameservers
	// answered stays on it, so the entry is kept when the other names
	// stop resolving to this address.
	savedHostnames := slices.Clone(hostnames)

	if hasPrev {
		for _, name := range prev.Hostnames {
			if !slices.Contains(hostnames, name) && w.noNameserverAnswered(name) {
				savedHostnames = append(savedHostnames, name)
			}
		}

		sort.Strings(savedHostnames)
	}

	w.state.SetPortState(key, &state.PortState{
		Open:        result.Open,
		Hostnames:   savedHostnames,
		LastChecked: now,
	})
}

func (w *Watcher) runTLSChecks(ctx context.Context) {
	for _, hostname := range w.config.Hostnames {
		w.checkTLSForHostname(ctx, hostname)
	}

	for _, domain := range w.config.Domains {
		w.checkTLSForHostname(ctx, domain)
	}
}

func (w *Watcher) checkTLSForHostname(
	ctx context.Context,
	hostname string,
) {
	ips := w.collectIPs(hostname)

	for _, ip := range ips {
		portKey := fmt.Sprintf("%s:%d", ip, tlsPort)

		ps, ok := w.state.GetPortState(portKey)
		if !ok || !ps.Open {
			continue
		}

		w.checkTLSCert(ctx, ip, hostname)
	}
}

func (w *Watcher) checkTLSCert(
	ctx context.Context,
	ip string,
	hostname string,
) {
	cert, err := w.tlsCheck.CheckCertificate(ctx, ip, hostname)

	// A check the context cut short says nothing about the certificate,
	// so it is neither saved nor notified.
	if ctx.Err() != nil {
		return
	}

	certKey := fmt.Sprintf("%s:%d:%s", ip, tlsPort, hostname)
	now := time.Now().UTC()
	prev, hasPrev := w.state.GetCertificateState(certKey)

	if err != nil {
		w.handleTLSError(
			ctx, certKey, hostname, ip,
			hasPrev, prev, now, err,
		)

		return
	}

	w.handleTLSSuccess(
		ctx, certKey, hostname, ip,
		hasPrev, prev, now, cert,
	)
}

func (w *Watcher) handleTLSError(
	ctx context.Context,
	certKey, hostname, ip string,
	hasPrev bool,
	prev *state.CertificateState,
	now time.Time,
	err error,
) {
	if hasPrev && !w.firstRun && prev.Status == statusOK {
		msg := fmt.Sprintf(
			"Host: %s\nIP: %s\nError: %s",
			hostname, ip, err,
		)

		w.notify.SendNotification(
			ctx,
			"TLS Failure: "+hostname,
			msg,
			"error",
		)
	}

	w.state.SetCertificateState(
		certKey, &state.CertificateState{
			Status:      statusError,
			Error:       err.Error(),
			LastChecked: now,
		},
	)
}

func (w *Watcher) handleTLSSuccess(
	ctx context.Context,
	certKey, hostname, ip string,
	hasPrev bool,
	prev *state.CertificateState,
	now time.Time,
	cert *tlscheck.CertificateInfo,
) {
	if hasPrev && !w.firstRun {
		w.detectTLSChanges(ctx, hostname, ip, prev, cert)
	}

	w.checkTLSExpiry(ctx, hostname, ip, cert)

	w.state.SetCertificateState(
		certKey, &state.CertificateState{
			CommonName:              cert.CommonName,
			Issuer:                  cert.Issuer,
			NotAfter:                cert.NotAfter,
			SubjectAlternativeNames: cert.SubjectAlternativeNames,
			Status:                  statusOK,
			LastChecked:             now,
		},
	)
}

func (w *Watcher) detectTLSChanges(
	ctx context.Context,
	hostname, ip string,
	prev *state.CertificateState,
	cert *tlscheck.CertificateInfo,
) {
	if prev.Status == statusError {
		msg := fmt.Sprintf(
			"Host: %s\nIP: %s\nTLS recovered",
			hostname, ip,
		)

		w.notify.SendNotification(
			ctx,
			"TLS Recovery: "+hostname,
			msg,
			"success",
		)

		return
	}

	changed := prev.CommonName != cert.CommonName ||
		prev.Issuer != cert.Issuer ||
		!sliceEqual(
			prev.SubjectAlternativeNames,
			cert.SubjectAlternativeNames,
		)

	if !changed {
		return
	}

	msg := fmt.Sprintf(
		"Host: %s\nIP: %s\n"+
			"Old CN: %s, Issuer: %s\n"+
			"New CN: %s, Issuer: %s",
		hostname, ip,
		prev.CommonName, prev.Issuer,
		cert.CommonName, cert.Issuer,
	)

	w.notify.SendNotification(
		ctx,
		"TLS Certificate Change: "+hostname,
		msg,
		"warning",
	)
}

func (w *Watcher) checkTLSExpiry(
	ctx context.Context,
	hostname, ip string,
	cert *tlscheck.CertificateInfo,
) {
	daysLeft := time.Until(cert.NotAfter).Hours() / hoursPerDay
	warningDays := float64(w.config.TLSExpiryWarning)

	if daysLeft > warningDays {
		return
	}

	msg := fmt.Sprintf(
		"Host: %s\nIP: %s\nCN: %s\n"+
			"Expires: %s (%.0f days)",
		hostname, ip, cert.CommonName,
		cert.NotAfter.Format(time.RFC3339),
		daysLeft,
	)

	w.notify.SendNotification(
		ctx,
		"TLS Expiry Warning: "+hostname,
		msg,
		"warning",
	)
}

func (w *Watcher) saveState() {
	err := w.state.Save()
	if err != nil {
		w.log.Error("failed to save state", "error", err)
	}
}

// maybeSendTestNotification sends a startup status notification
// after the first full scan completes, if SEND_TEST_NOTIFICATION
// is enabled. The message is informational, not an error or anomaly
// alert. It is written before it reaches any endpoint, so it claims
// nothing about whether the endpoints work. Domains and hostnames are
// counted from the configuration: the state's hostnames also hold each
// apex domain's own records.
func (w *Watcher) maybeSendTestNotification(ctx context.Context) {
	if !w.config.SendTestNotification {
		return
	}

	snap := w.state.GetSnapshot()

	msg := fmt.Sprintf(
		"dnswatcher has started and completed its initial scan.\n"+
			"Monitoring %d domain(s) and %d hostname(s).\n"+
			"Tracking %d port endpoint(s) and %d TLS certificate(s).\n"+
			"This is a test notification, sent to every configured "+
			"notification endpoint.",
		len(w.config.Domains),
		len(w.config.Hostnames),
		len(snap.Ports),
		len(snap.Certificates),
	)

	w.log.Info("sending startup test notification")

	w.notify.SendNotification(
		ctx,
		"✅ dnswatcher startup complete",
		msg,
		"success",
	)
}

// --- Utility functions ---

func toSet(items []string) map[string]bool {
	set := make(map[string]bool, len(items))
	for _, item := range items {
		set[item] = true
	}

	return set
}

// nameserversAgree reports whether two nameservers' states from the same
// check hold the same records, leaving out the record types either lists
// in FailedTypes: the records held for those are kept from an earlier
// check, or not known.
func nameserversAgree(a, b *state.NameserverRecordState) bool {
	failed := slices.Concat(a.FailedTypes, b.FailedTypes)

	return recordsEqual(
		withoutTypes(a.Records, failed), withoutTypes(b.Records, failed),
	)
}

// withoutTypes returns a copy of records without the record types in
// types.
func withoutTypes(
	records map[string][]string,
	types []string,
) map[string][]string {
	records = maps.Clone(records)

	for _, rtype := range types {
		delete(records, rtype)
	}

	return records
}

func recordsEqual(
	a, b map[string][]string,
) bool {
	if len(a) != len(b) {
		return false
	}

	for k, av := range a {
		bv, ok := b[k]
		if !ok || !sliceEqual(av, bv) {
			return false
		}
	}

	return true
}

// recordDifferences describes, in sorted order of type, each record
// type whose values differ between a and b: a line naming the type,
// then a line with a's values after labelA and one with b's after
// labelB. Types with the same values in both are left out.
func recordDifferences(
	labelA string, a map[string][]string,
	labelB string, b map[string][]string,
) string {
	types := make([]string, 0, len(a)+len(b))

	for recordType := range a {
		types = append(types, recordType)
	}

	for recordType := range b {
		if _, ok := a[recordType]; !ok {
			types = append(types, recordType)
		}
	}

	sort.Strings(types)

	var lines []string

	for _, recordType := range types {
		if sliceEqual(a[recordType], b[recordType]) {
			continue
		}

		lines = append(lines,
			"Type: "+recordType,
			labelA+": "+joinValues(a[recordType]),
			labelB+": "+joinValues(b[recordType]),
		)
	}

	return strings.Join(lines, "\n")
}

// joinValues lists record values separated by commas, or says none.
func joinValues(values []string) string {
	if len(values) == 0 {
		return "none"
	}

	return strings.Join(values, ", ")
}

func sliceEqual(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}

	aSorted := make([]string, len(a))
	bSorted := make([]string, len(b))

	copy(aSorted, a)
	copy(bSorted, b)

	sort.Strings(aSorted)
	sort.Strings(bSorted)

	for i := range aSorted {
		if aSorted[i] != bSorted[i] {
			return false
		}
	}

	return true
}