dnswatcher/internal/watcher/watcher.go

package watcher

import (
	"context"
	"fmt"
	"log/slog"
	"slices"
	"sort"
	"strings"
	"time"

	"go.uber.org/fx"

	"sneak.berlin/go/dnswatcher/internal/config"
	"sneak.berlin/go/dnswatcher/internal/logger"
	"sneak.berlin/go/dnswatcher/internal/resolver"
	"sneak.berlin/go/dnswatcher/internal/state"
	"sneak.berlin/go/dnswatcher/internal/tlscheck"
)

// monitoredPorts are the TCP ports checked for each IP address.
var monitoredPorts = []int{80, 443} //nolint:gochecknoglobals

// tlsPort is the port used for TLS certificate checks.
const tlsPort = 443

// hoursPerDay converts days to hours for duration calculations.
const hoursPerDay = 24

// Status values recorded for nameserver and certificate checks.
const (
	statusOK    = "ok"
	statusError = "error"
)

// Params contains dependencies for Watcher.
type Params struct {
	fx.In

	Logger    *logger.Logger
	Config    *config.Config
	State     *state.State
	Resolver  DNSResolver
	PortCheck PortChecker
	TLSCheck  TLSChecker
	Notify    Notifier
}

// Watcher orchestrates all monitoring checks on a schedule.
type Watcher struct {
	log       *slog.Logger
	config    *config.Config
	state     *state.State
	resolver  DNSResolver
	portCheck PortChecker
	tlsCheck  TLSChecker
	notify    Notifier
	cancel    context.CancelFunc
	done      chan struct{} // closed when Run returns
	firstRun  bool
}

// New creates a new Watcher instance wired into the fx lifecycle.
func New(
	lifecycle fx.Lifecycle,
	params Params,
) (*Watcher, error) {
	w := &Watcher{
		log:       params.Logger.Get(),
		config:    params.Config,
		state:     params.State,
		resolver:  params.Resolver,
		portCheck: params.PortCheck,
		tlsCheck:  params.TLSCheck,
		notify:    params.Notify,
		firstRun:  true,
	}

	lifecycle.Append(fx.Hook{
		OnStart: func(startCtx context.Context) error {
			// The fx startup context expires after startup
			// completes, so the watcher's context drops its
			// cancellation. The watcher's lifetime is controlled
			// by w.cancel in OnStop.
			ctx, cancel := context.WithCancel(
				context.WithoutCancel(startCtx),
			)
			w.cancel = cancel
			w.done = make(chan struct{})

			go func() {
				defer close(w.done)

				w.Run(ctx)
			}()

			return nil
		},
		OnStop: func(ctx context.Context) error {
			w.cancel()

			// Run saves state as it returns. Waiting for it here
			// means the save is done before shutdown goes on.
			select {
			case <-w.done:
				return nil
			case <-ctx.Done():
				return fmt.Errorf(
					"waiting for the watcher to stop: %w",
					ctx.Err(),
				)
			}
		},
	})

	return w, nil
}

// Run starts the monitoring loop with periodic scheduling. When ctx
// is cancelled, it saves state and returns.
func (w *Watcher) Run(ctx context.Context) {
	w.log.Info(
		"watcher starting",
		"domains", len(w.config.Domains),
		"hostnames", len(w.config.Hostnames),
		"dnsInterval", w.config.DNSInterval,
		"tlsInterval", w.config.TLSInterval,
	)

	w.RunOnce(ctx)
	w.maybeSendTestNotification(ctx)

	dnsTicker := time.NewTicker(w.config.DNSInterval)
	tlsTicker := time.NewTicker(w.config.TLSInterval)

	defer dnsTicker.Stop()
	defer tlsTicker.Stop()

	for {
		select {
		case <-ctx.Done():
			w.saveState()
			w.log.Info("watcher stopped")

			return
		case <-dnsTicker.C:
			w.runDNSChecks(ctx)

			w.checkAllPorts(ctx)
			w.saveState()
		case <-tlsTicker.C:
			// Run DNS first so TLS checks use freshly
			// resolved IP addresses, not stale ones from
			// a previous cycle.
			w.runDNSChecks(ctx)

			w.runTLSChecks(ctx)
			w.saveState()
		}
	}
}

// RunOnce performs a single complete monitoring cycle.
// DNS checks run first so that port and TLS checks use
// freshly resolved IP addresses. Port checks run before
// TLS because TLS checks only target IPs with an open
// port 443.
func (w *Watcher) RunOnce(ctx context.Context) {
	w.detectFirstRun()

	// Phase 1: DNS resolution must complete first so that
	// subsequent checks use fresh IP addresses.
	w.runDNSChecks(ctx)

	// Phase 2: Port checks populate port state that TLS
	// checks depend on (TLS only targets IPs where port
	// 443 is open).
	w.checkAllPorts(ctx)

	// Phase 3: TLS checks use fresh DNS IPs and current
	// port state.
	w.runTLSChecks(ctx)

	w.saveState()
	w.firstRun = false
}

func (w *Watcher) detectFirstRun() {
	snap := w.state.GetSnapshot()
	hasState := len(snap.Domains) > 0 ||
		len(snap.Hostnames) > 0 ||
		len(snap.Ports) > 0 ||
		len(snap.Certificates) > 0

	if hasState {
		w.firstRun = false
	}
}

// runDNSChecks performs DNS resolution for all configured domains
// and hostnames, updating state with freshly resolved records.
// This must complete before port or TLS checks run so those
// checks operate on current IP addresses.
func (w *Watcher) runDNSChecks(ctx context.Context) {
	for _, domain := range w.config.Domains {
		w.checkDomain(ctx, domain)
	}

	for _, hostname := range w.config.Hostnames {
		w.checkHostname(ctx, hostname)
	}
}

func (w *Watcher) checkDomain(
	ctx context.Context,
	domain string,
) {
	nameservers, err := w.resolver.LookupNS(ctx, domain)
	if err != nil {
		w.log.Error(
			"failed to lookup NS",
			"domain", domain,
			"error", err,
		)

		return
	}

	sort.Strings(nameservers)

	now := time.Now().UTC()

	prev, hasPrev := w.state.GetDomainState(domain)

	var prevAddresses map[string][]string
	if hasPrev {
		prevAddresses = prev.NameserverAddresses
	}

	addresses := w.resolveNameserverAddresses(
		ctx, nameservers, prevAddresses,
	)

	if hasPrev && !w.firstRun {
		w.detectNSChanges(ctx, domain, prev.Nameservers, nameservers)
		w.detectNSAddressChanges(ctx, domain, prevAddresses, addresses)
	}

	w.state.SetDomainState(domain, &state.DomainState{
		Nameservers:         nameservers,
		NameserverAddresses: addresses,
		LastChecked:         now,
	})

	// The apex domain's records are also checked as a hostname's, so
	// that the port and TLS checks find its addresses.
	w.checkHostname(ctx, domain)
}

func (w *Watcher) detectNSChanges(
	ctx context.Context,
	domain string,
	oldNS, newNS []string,
) {
	oldSet := toSet(oldNS)
	newSet := toSet(newNS)

	var added, removed []string

	for ns := range newSet {
		if !oldSet[ns] {
			added = append(added, ns)
		}
	}

	for ns := range oldSet {
		if !newSet[ns] {
			removed = append(removed, ns)
		}
	}

	if len(added) == 0 && len(removed) == 0 {
		return
	}

	msg := fmt.Sprintf(
		"Domain: %s\nAdded: %s\nRemoved: %s",
		domain,
		strings.Join(added, ", "),
		strings.Join(removed, ", "),
	)

	w.notify.SendNotification(
		ctx,
		"NS Change: "+domain,
		msg,
		"warning",
	)
}

// resolveNameserverAddresses returns the sorted addresses each
// nameserver's name resolves to. A nameserver whose lookup fails, as it
// does when no nameserver of the name's zone answers, or finds no
// address keeps its addresses from prev and is not an address change.
func (w *Watcher) resolveNameserverAddresses(
	ctx context.Context,
	nameservers []string,
	prev map[string][]string,
) map[string][]string {
	addresses := make(map[string][]string, len(nameservers))

	for _, ns := range nameservers {
		ips, err := w.resolver.ResolveIPAddresses(ctx, ns)
		if err == nil && len(ips) > 0 {
			sort.Strings(ips)
			addresses[ns] = ips

			continue
		}

		w.log.Error(
			"no addresses found for nameserver",
			"nameserver", ns,
			"error", err,
		)

		if prevIPs, ok := prev[ns]; ok {
			addresses[ns] = prevIPs
		}
	}

	return addresses
}

// detectNSAddressChanges notifies when a nameserver in both checks
// resolves to different addresses. A nameserver added or removed is
// reported by detectNSChanges alone, and one with no addresses saved by
// the previous check, as in a state file from before they were saved,
// is not compared.
func (w *Watcher) detectNSAddressChanges(
	ctx context.Context,
	domain string,
	prev, current map[string][]string,
) {
	for ns, cur := range current {
		old, ok := prev[ns]
		if !ok || sliceEqual(old, cur) {
			continue
		}

		msg := fmt.Sprintf(
			"Domain: %s\nNameserver: %s\nOld: %s\nNew: %s",
			domain, ns,
			strings.Join(old, ", "),
			strings.Join(cur, ", "),
		)

		w.notify.SendNotification(
			ctx,
			"NS Address Change: "+domain,
			msg,
			"warning",
		)
	}
}

func (w *Watcher) checkHostname(
	ctx context.Context,
	hostname string,
) {
	results, err := w.resolver.LookupAllRecords(ctx, hostname)
	if err != nil {
		w.log.Error(
			"failed to lookup records",
			"hostname", hostname,
			"error", err,
		)

		return
	}

	newState := buildHostnameState(results, time.Now().UTC())

	prev, hasPrev := w.state.GetHostnameState(hostname)

	w.resolveCNAMEAddresses(ctx, hostname, newState, prev)

	if hasPrev && !w.firstRun {
		w.detectHostnameChanges(ctx, hostname, prev, newState)
	}

	w.state.SetHostnameState(hostname, newState)
}

// resolveCNAMEAddresses saves in current the addresses at the end of
// hostname's CNAME chain, when the nameservers' answers in current hold
// a CNAME and no address, and an empty list otherwise. Every CNAME
// target the nameservers gave is followed with ResolveIPAddresses and
// the addresses found for all of them are saved, so nameservers that
// disagree on the target do not change the result from check to check.
// The addresses saved in prev, which may be nil, are kept when none of
// the name's nameservers answered, and when a target cannot be
// followed, as when no nameserver of a zone in its chain answers.
func (w *Watcher) resolveCNAMEAddresses(
	ctx context.Context,
	hostname string,
	current, prev *state.HostnameState,
) {
	var prevAddresses []string
	if prev != nil {
		prevAddresses = prev.CNAMEAddresses
	}

	// Empty, not nil: nil means the addresses are not known.
	current.CNAMEAddresses = []string{}

	answered := false
	targets := make(map[string]bool)

	for _, nsState := range current.RecordsByNameserver {
		if nsState.Status != statusOK {
			continue
		}

		answered = true

		if len(nsState.Records["A"]) > 0 || len(nsState.Records["AAAA"]) > 0 {
			return
		}

		for _, target := range nsState.Records["CNAME"] {
			targets[target] = true
		}
	}

	if !answered {
		current.CNAMEAddresses = prevAddresses

		return
	}

	for target := range targets {
		ips, err := w.resolver.ResolveIPAddresses(ctx, target)
		if err != nil {
			w.log.Error(
				"failed to follow CNAME",
				"hostname", hostname,
				"target", target,
				"error", err,
			)

			current.CNAMEAddresses = prevAddresses

			return
		}

		current.CNAMEAddresses = append(current.CNAMEAddresses, ips...)
	}

	// Still the empty list when every chain ends in no address.
	slices.Sort(current.CNAMEAddresses)
	current.CNAMEAddresses = slices.Compact(current.CNAMEAddresses)
}

// buildHostnameState saves each nameserver's response. A nameserver
// that answered, even with NXDOMAIN or no records, is saved as ok; one
// that timed out or failed is saved as error with the reason, and its
// empty record set is not an answer.
func buildHostnameState(
	results map[string]*resolver.NameserverResponse,
	now time.Time,
) *state.HostnameState {
	hs := &state.HostnameState{
		RecordsByNameserver: make(
			map[string]*state.NameserverRecordState,
		),
		LastChecked: now,
	}

	for ns, resp := range results {
		nsState := &state.NameserverRecordState{
			Records:     resp.Records,
			Status:      statusOK,
			LastChecked: now,
		}

		if resp.Status == resolver.StatusTimeout ||
			resp.Status == resolver.StatusError {
			nsState.Status = statusError
			nsState.Error = resp.Error
		}

		hs.RecordsByNameserver[ns] = nsState
	}

	return hs
}

func (w *Watcher) detectHostnameChanges(
	ctx context.Context,
	hostname string,
	prev, current *state.HostnameState,
) {
	w.detectRecordChanges(ctx, hostname, prev, current)
	w.detectNSDisappearances(ctx, hostname, prev, current)
	w.detectNSFailures(ctx, hostname, prev, current)
	w.detectInconsistencies(ctx, hostname, prev, current)
	w.detectCNAMEAddressChanges(ctx, hostname, prev, current)
}

// detectCNAMEAddressChanges notifies when the addresses at the end of
// hostname's CNAME chain differ from those the previous check saved,
// including a change from or to none. When the previous addresses are
// not known (nil), as on the first check after loading a state file
// written before they were saved, nothing is compared.
func (w *Watcher) detectCNAMEAddressChanges(
	ctx context.Context,
	hostname string,
	prev, current *state.HostnameState,
) {
	old, cur := prev.CNAMEAddresses, current.CNAMEAddresses
	if old == nil || sliceEqual(old, cur) {
		return
	}

	msg := fmt.Sprintf(
		"Hostname: %s\nOld: %s\nNew: %s",
		hostname,
		strings.Join(old, ", "),
		strings.Join(cur, ", "),
	)

	w.notify.SendNotification(
		ctx,
		"CNAME Address Change: "+hostname,
		msg,
		"warning",
	)
}

// detectRecordChanges compares each nameserver's records with those of
// the previous check. Only answers are compared: a nameserver that
// failed on either check has no records to compare.
func (w *Watcher) detectRecordChanges(
	ctx context.Context,
	hostname string,
	prev, current *state.HostnameState,
) {
	for ns, cur := range current.RecordsByNameserver {
		prevNS, ok := prev.RecordsByNameserver[ns]
		if !ok || prevNS.Status != statusOK || cur.Status != statusOK {
			continue
		}

		if recordsEqual(prevNS.Records, cur.Records) {
			continue
		}

		msg := fmt.Sprintf(
			"Hostname: %s\nNameserver: %s\n"+
				"Old: %v\nNew: %v",
			hostname, ns,
			prevNS.Records, cur.Records,
		)

		w.notify.SendNotification(
			ctx,
			"Record Change: "+hostname,
			msg,
			"warning",
		)
	}
}

func (w *Watcher) detectNSDisappearances(
	ctx context.Context,
	hostname string,
	prev, current *state.HostnameState,
) {
	for ns, prevNS := range prev.RecordsByNameserver {
		if _, ok := current.RecordsByNameserver[ns]; ok || prevNS.Status != statusOK {
			continue
		}

		msg := fmt.Sprintf(
			"Hostname: %s\nNameserver: %s disappeared",
			hostname, ns,
		)

		w.notify.SendNotification(
			ctx,
			"NS Failure: "+hostname,
			msg,
			"error",
		)
	}
}

// detectNSFailures notifies when a nameserver that answered on the
// previous check fails, and when one that failed answers again. A
// nameserver missing from the previous check is not compared.
func (w *Watcher) detectNSFailures(
	ctx context.Context,
	hostname string,
	prev, current *state.HostnameState,
) {
	for ns, cur := range current.RecordsByNameserver {
		prevNS, ok := prev.RecordsByNameserver[ns]
		if !ok {
			continue
		}

		switch {
		case prevNS.Status == statusOK && cur.Status == statusError:
			msg := fmt.Sprintf(
				"Hostname: %s\nNameserver: %s\nError: %s",
				hostname, ns, cur.Error,
			)

			w.notify.SendNotification(
				ctx,
				"NS Failure: "+hostname,
				msg,
				"error",
			)
		case prevNS.Status == statusError && cur.Status == statusOK:
			msg := fmt.Sprintf(
				"Hostname: %s\nNameserver: %s recovered",
				hostname, ns,
			)

			w.notify.SendNotification(
				ctx,
				"NS Recovery: "+hostname,
				msg,
				"success",
			)
		}
	}
}

func (w *Watcher) detectInconsistencies(
	ctx context.Context,
	hostname string,
	prev, current *state.HostnameState,
) {
	for _, pair := range newlyDisagreeingPairs(prev, current) {
		ns1, ns2 := pair[0], pair[1]

		msg := fmt.Sprintf(
			"Hostname: %s\n%s: %v\n%s: %v",
			hostname,
			ns1, current.RecordsByNameserver[ns1].Records,
			ns2, current.RecordsByNameserver[ns2].Records,
		)

		w.notify.SendNotification(
			ctx,
			"Inconsistency: "+hostname,
			msg,
			"warning",
		)
	}
}

// newlyDisagreeingPairs returns every pair of nameservers that answered
// in current and whose records differ there, in sorted order of name,
// except pairs where both nameservers answered in prev and already
// differed there. A nameserver missing from prev, or that failed there,
// is paired with every nameserver it differs from. A nameserver that
// failed in current has no records to compare and is in no pair.
func newlyDisagreeingPairs(
	prev, current *state.HostnameState,
) [][2]string {
	nameservers := make([]string, 0, len(current.RecordsByNameserver))
	for ns, cur := range current.RecordsByNameserver {
		if cur.Status == statusOK {
			nameservers = append(nameservers, ns)
		}
	}

	sort.Strings(nameservers)

	var pairs [][2]string

	for i, ns1 := range nameservers {
		for _, ns2 := range nameservers[i+1:] {
			if recordsEqual(
				current.RecordsByNameserver[ns1].Records,
				current.RecordsByNameserver[ns2].Records,
			) {
				continue
			}

			prev1, ok1 := prev.RecordsByNameserver[ns1]
			prev2, ok2 := prev.RecordsByNameserver[ns2]

			if ok1 && ok2 &&
				prev1.Status == statusOK && prev2.Status == statusOK &&
				!recordsEqual(prev1.Records, prev2.Records) {
				continue
			}

			pairs = append(pairs, [2]string{ns1, ns2})
		}
	}

	return pairs
}

func (w *Watcher) checkAllPorts(ctx context.Context) {
	// Phase 1: Build current IP:port → hostname associations
	// from fresh DNS data.
	associations := w.buildPortAssociations()

	// Phase 2: Check each unique IP:port and update state
	// with the full set of associated hostnames.
	for key, hostnames := range associations {
		ip, port := parsePortKey(key)
		if port == 0 {
			continue
		}

		w.checkSinglePort(ctx, ip, port, hostnames)
	}

	// Phase 3: Remove port state entries that no longer have
	// any hostname referencing them.
	w.cleanupStalePorts(associations)
}

// buildPortAssociations constructs a map from IP:port keys to
// the sorted set of hostnames currently resolving to that IP.
func (w *Watcher) buildPortAssociations() map[string][]string {
	assoc := make(map[string]map[string]bool)

	allNames := make(
		[]string, 0,
		len(w.config.Hostnames)+len(w.config.Domains),
	)
	allNames = append(allNames, w.config.Hostnames...)
	allNames = append(allNames, w.config.Domains...)

	for _, name := range allNames {
		ips := w.collectIPs(name)
		for _, ip := range ips {
			for _, port := range monitoredPorts {
				key := fmt.Sprintf("%s:%d", ip, port)
				if assoc[key] == nil {
					assoc[key] = make(map[string]bool)
				}

				assoc[key][name] = true
			}
		}
	}

	result := make(map[string][]string, len(assoc))
	for key, set := range assoc {
		hostnames := make([]string, 0, len(set))
		for h := range set {
			hostnames = append(hostnames, h)
		}

		sort.Strings(hostnames)

		result[key] = hostnames
	}

	return result
}

// parsePortKey splits an "ip:port" key into its components.
func parsePortKey(key string) (string, int) {
	lastColon := strings.LastIndex(key, ":")
	if lastColon < 0 {
		return key, 0
	}

	ip := key[:lastColon]

	var p int

	_, err := fmt.Sscanf(key[lastColon+1:], "%d", &p)
	if err != nil {
		return ip, 0
	}

	return ip, p
}

// cleanupStalePorts removes port state entries that are no
// longer referenced by any hostname in the current DNS data. An
// entry saved for a configured name none of whose nameservers
// answered is kept: that name's addresses are not known, not gone.
func (w *Watcher) cleanupStalePorts(
	currentAssociations map[string][]string,
) {
	for _, key := range w.state.GetAllPortKeys() {
		if _, exists := currentAssociations[key]; exists {
			continue
		}

		ps, ok := w.state.GetPortState(key)
		if ok && slices.ContainsFunc(ps.Hostnames, w.noNameserverAnswered) {
			continue
		}

		w.state.DeletePortState(key)
	}
}

// noNameserverAnswered reports whether name is a configured domain or
// hostname and none of its nameservers answered on its last check.
func (w *Watcher) noNameserverAnswered(name string) bool {
	if !slices.Contains(w.config.Hostnames, name) &&
		!slices.Contains(w.config.Domains, name) {
		return false
	}

	hs, ok := w.state.GetHostnameState(name)
	if !ok {
		return false
	}

	for _, nsState := range hs.RecordsByNameserver {
		if nsState.Status == statusOK {
			return false
		}
	}

	return true
}

// collectIPs returns the addresses saved for hostname: those in its
// nameservers' A and AAAA records, and those at the end of its CNAME
// chain.
func (w *Watcher) collectIPs(hostname string) []string {
	hs, ok := w.state.GetHostnameState(hostname)
	if !ok {
		return nil
	}

	ipSet := make(map[string]bool)

	for _, nsState := range hs.RecordsByNameserver {
		for _, ip := range nsState.Records["A"] {
			ipSet[ip] = true
		}

		for _, ip := range nsState.Records["AAAA"] {
			ipSet[ip] = true
		}
	}

	for _, ip := range hs.CNAMEAddresses {
		ipSet[ip] = true
	}

	result := make([]string, 0, len(ipSet))
	for ip := range ipSet {
		result = append(result, ip)
	}

	sort.Strings(result)

	return result
}

func (w *Watcher) checkSinglePort(
	ctx context.Context,
	ip string,
	port int,
	hostnames []string,
) {
	result, err := w.portCheck.CheckPort(ctx, ip, port)

	// A check the context cut short says nothing about the port, so it
	// is neither saved nor notified.
	if ctx.Err() != nil {
		return
	}

	if err != nil {
		w.log.Error(
			"port check failed",
			"ip", ip,
			"port", port,
			"error", err,
		)

		return
	}

	key := fmt.Sprintf("%s:%d", ip, port)
	now := time.Now().UTC()
	prev, hasPrev := w.state.GetPortState(key)

	if hasPrev && !w.firstRun && prev.Open != result.Open {
		stateStr := "closed"
		if result.Open {
			stateStr = "open"
		}

		msg := fmt.Sprintf(
			"Hosts: %s\nAddress: %s\nPort now %s",
			strings.Join(hostnames, ", "), key, stateStr,
		)

		w.notify.SendNotification(
			ctx,
			"Port Change: "+key,
			msg,
			"warning",
		)
	}

	// A configured name on the saved list none of whose nameservers
	// answered stays on it, so the entry is kept when the other names
	// stop resolving to this address.
	savedHostnames := slices.Clone(hostnames)

	if hasPrev {
		for _, name := range prev.Hostnames {
			if !slices.Contains(hostnames, name) && w.noNameserverAnswered(name) {
				savedHostnames = append(savedHostnames, name)
			}
		}

		sort.Strings(savedHostnames)
	}

	w.state.SetPortState(key, &state.PortState{
		Open:        result.Open,
		Hostnames:   savedHostnames,
		LastChecked: now,
	})
}

func (w *Watcher) runTLSChecks(ctx context.Context) {
	for _, hostname := range w.config.Hostnames {
		w.checkTLSForHostname(ctx, hostname)
	}

	for _, domain := range w.config.Domains {
		w.checkTLSForHostname(ctx, domain)
	}
}

func (w *Watcher) checkTLSForHostname(
	ctx context.Context,
	hostname string,
) {
	ips := w.collectIPs(hostname)

	for _, ip := range ips {
		portKey := fmt.Sprintf("%s:%d", ip, tlsPort)

		ps, ok := w.state.GetPortState(portKey)
		if !ok || !ps.Open {
			continue
		}

		w.checkTLSCert(ctx, ip, hostname)
	}
}

func (w *Watcher) checkTLSCert(
	ctx context.Context,
	ip string,
	hostname string,
) {
	cert, err := w.tlsCheck.CheckCertificate(ctx, ip, hostname)

	// A check the context cut short says nothing about the certificate,
	// so it is neither saved nor notified.
	if ctx.Err() != nil {
		return
	}

	certKey := fmt.Sprintf("%s:%d:%s", ip, tlsPort, hostname)
	now := time.Now().UTC()
	prev, hasPrev := w.state.GetCertificateState(certKey)

	if err != nil {
		w.handleTLSError(
			ctx, certKey, hostname, ip,
			hasPrev, prev, now, err,
		)

		return
	}

	w.handleTLSSuccess(
		ctx, certKey, hostname, ip,
		hasPrev, prev, now, cert,
	)
}

func (w *Watcher) handleTLSError(
	ctx context.Context,
	certKey, hostname, ip string,
	hasPrev bool,
	prev *state.CertificateState,
	now time.Time,
	err error,
) {
	if hasPrev && !w.firstRun && prev.Status == statusOK {
		msg := fmt.Sprintf(
			"Host: %s\nIP: %s\nError: %s",
			hostname, ip, err,
		)

		w.notify.SendNotification(
			ctx,
			"TLS Failure: "+hostname,
			msg,
			"error",
		)
	}

	w.state.SetCertificateState(
		certKey, &state.CertificateState{
			Status:      statusError,
			Error:       err.Error(),
			LastChecked: now,
		},
	)
}

func (w *Watcher) handleTLSSuccess(
	ctx context.Context,
	certKey, hostname, ip string,
	hasPrev bool,
	prev *state.CertificateState,
	now time.Time,
	cert *tlscheck.CertificateInfo,
) {
	if hasPrev && !w.firstRun {
		w.detectTLSChanges(ctx, hostname, ip, prev, cert)
	}

	w.checkTLSExpiry(ctx, hostname, ip, cert)

	w.state.SetCertificateState(
		certKey, &state.CertificateState{
			CommonName:              cert.CommonName,
			Issuer:                  cert.Issuer,
			NotAfter:                cert.NotAfter,
			SubjectAlternativeNames: cert.SubjectAlternativeNames,
			Status:                  statusOK,
			LastChecked:             now,
		},
	)
}

func (w *Watcher) detectTLSChanges(
	ctx context.Context,
	hostname, ip string,
	prev *state.CertificateState,
	cert *tlscheck.CertificateInfo,
) {
	if prev.Status == statusError {
		msg := fmt.Sprintf(
			"Host: %s\nIP: %s\nTLS recovered",
			hostname, ip,
		)

		w.notify.SendNotification(
			ctx,
			"TLS Recovery: "+hostname,
			msg,
			"success",
		)

		return
	}

	changed := prev.CommonName != cert.CommonName ||
		prev.Issuer != cert.Issuer ||
		!sliceEqual(
			prev.SubjectAlternativeNames,
			cert.SubjectAlternativeNames,
		)

	if !changed {
		return
	}

	msg := fmt.Sprintf(
		"Host: %s\nIP: %s\n"+
			"Old CN: %s, Issuer: %s\n"+
			"New CN: %s, Issuer: %s",
		hostname, ip,
		prev.CommonName, prev.Issuer,
		cert.CommonName, cert.Issuer,
	)

	w.notify.SendNotification(
		ctx,
		"TLS Certificate Change: "+hostname,
		msg,
		"warning",
	)
}

func (w *Watcher) checkTLSExpiry(
	ctx context.Context,
	hostname, ip string,
	cert *tlscheck.CertificateInfo,
) {
	daysLeft := time.Until(cert.NotAfter).Hours() / hoursPerDay
	warningDays := float64(w.config.TLSExpiryWarning)

	if daysLeft > warningDays {
		return
	}

	msg := fmt.Sprintf(
		"Host: %s\nIP: %s\nCN: %s\n"+
			"Expires: %s (%.0f days)",
		hostname, ip, cert.CommonName,
		cert.NotAfter.Format(time.RFC3339),
		daysLeft,
	)

	w.notify.SendNotification(
		ctx,
		"TLS Expiry Warning: "+hostname,
		msg,
		"warning",
	)
}

func (w *Watcher) saveState() {
	err := w.state.Save()
	if err != nil {
		w.log.Error("failed to save state", "error", err)
	}
}

// maybeSendTestNotification sends a startup status notification
// after the first full scan completes, if SEND_TEST_NOTIFICATION
// is enabled. The message is clearly informational ("all ok")
// and not an error or anomaly alert.
func (w *Watcher) maybeSendTestNotification(ctx context.Context) {
	if !w.config.SendTestNotification {
		return
	}

	snap := w.state.GetSnapshot()

	msg := fmt.Sprintf(
		"dnswatcher has started and completed its initial scan.\n"+
			"Monitoring %d domain(s) and %d hostname(s).\n"+
			"Tracking %d port endpoint(s) and %d TLS certificate(s).\n"+
			"All notification channels are working.",
		len(snap.Domains),
		len(snap.Hostnames),
		len(snap.Ports),
		len(snap.Certificates),
	)

	w.log.Info("sending startup test notification")

	w.notify.SendNotification(
		ctx,
		"✅ dnswatcher startup complete",
		msg,
		"success",
	)
}

// --- Utility functions ---

func toSet(items []string) map[string]bool {
	set := make(map[string]bool, len(items))
	for _, item := range items {
		set[item] = true
	}

	return set
}

func recordsEqual(
	a, b map[string][]string,
) bool {
	if len(a) != len(b) {
		return false
	}

	for k, av := range a {
		bv, ok := b[k]
		if !ok || !sliceEqual(av, bv) {
			return false
		}
	}

	return true
}

func sliceEqual(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}

	aSorted := make([]string, len(a))
	bSorted := make([]string, len(b))

	copy(aSorted, a)
	copy(bSorted, b)

	sort.Strings(aSorted)
	sort.Strings(bSorted)

	for i := range aSorted {
		if aSorted[i] != bSorted[i] {
			return false
		}
	}

	return true
}