check / check (push) Successful in 1m44s
An expiry warning was skipped when the last one for that hostname and address was sent less than DNSWATCHER_TLS_INTERVAL ago. Each TLS check runs after a DNS pass of varying length, so two checks can be less than the interval apart, and a certificate about to expire was warned about on every check or every other check, at random. TLS checks already start once per interval, so the in-memory record of when each warning was sent is removed and every check warns, as the README says. The test that expected the second check to stay silent is replaced by one that runs TLS checks on state built in the test, with no DNS. Model: opus-5-5
1132 lines
24 KiB
Go
1132 lines
24 KiB
Go
package watcher
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.uber.org/fx"
|
|
|
|
"sneak.berlin/go/dnswatcher/internal/config"
|
|
"sneak.berlin/go/dnswatcher/internal/logger"
|
|
"sneak.berlin/go/dnswatcher/internal/resolver"
|
|
"sneak.berlin/go/dnswatcher/internal/state"
|
|
"sneak.berlin/go/dnswatcher/internal/tlscheck"
|
|
)
|
|
|
|
// monitoredPorts are the TCP ports checked for each IP address.
|
|
var monitoredPorts = []int{80, 443} //nolint:gochecknoglobals
|
|
|
|
// tlsPort is the port used for TLS certificate checks.
|
|
const tlsPort = 443
|
|
|
|
// hoursPerDay converts days to hours for duration calculations.
|
|
const hoursPerDay = 24
|
|
|
|
// Status values recorded for nameserver and certificate checks.
|
|
const (
|
|
statusOK = "ok"
|
|
statusError = "error"
|
|
)
|
|
|
|
// Params contains dependencies for Watcher.
|
|
type Params struct {
|
|
fx.In
|
|
|
|
Logger *logger.Logger
|
|
Config *config.Config
|
|
State *state.State
|
|
Resolver DNSResolver
|
|
PortCheck PortChecker
|
|
TLSCheck TLSChecker
|
|
Notify Notifier
|
|
}
|
|
|
|
// Watcher orchestrates all monitoring checks on a schedule.
|
|
type Watcher struct {
|
|
log *slog.Logger
|
|
config *config.Config
|
|
state *state.State
|
|
resolver DNSResolver
|
|
portCheck PortChecker
|
|
tlsCheck TLSChecker
|
|
notify Notifier
|
|
cancel context.CancelFunc
|
|
done chan struct{} // closed when Run returns
|
|
firstRun bool
|
|
}
|
|
|
|
// New creates a new Watcher instance wired into the fx lifecycle.
|
|
func New(
|
|
lifecycle fx.Lifecycle,
|
|
params Params,
|
|
) (*Watcher, error) {
|
|
w := &Watcher{
|
|
log: params.Logger.Get(),
|
|
config: params.Config,
|
|
state: params.State,
|
|
resolver: params.Resolver,
|
|
portCheck: params.PortCheck,
|
|
tlsCheck: params.TLSCheck,
|
|
notify: params.Notify,
|
|
firstRun: true,
|
|
}
|
|
|
|
lifecycle.Append(fx.Hook{
|
|
OnStart: func(startCtx context.Context) error {
|
|
// The fx startup context expires after startup
|
|
// completes, so the watcher's context drops its
|
|
// cancellation. The watcher's lifetime is controlled
|
|
// by w.cancel in OnStop.
|
|
ctx, cancel := context.WithCancel(
|
|
context.WithoutCancel(startCtx),
|
|
)
|
|
w.cancel = cancel
|
|
w.done = make(chan struct{})
|
|
|
|
go func() {
|
|
defer close(w.done)
|
|
|
|
w.Run(ctx)
|
|
}()
|
|
|
|
return nil
|
|
},
|
|
OnStop: func(ctx context.Context) error {
|
|
w.cancel()
|
|
|
|
// Run saves state as it returns. Waiting for it here
|
|
// means the save is done before shutdown goes on.
|
|
select {
|
|
case <-w.done:
|
|
return nil
|
|
case <-ctx.Done():
|
|
return fmt.Errorf(
|
|
"waiting for the watcher to stop: %w",
|
|
ctx.Err(),
|
|
)
|
|
}
|
|
},
|
|
})
|
|
|
|
return w, nil
|
|
}
|
|
|
|
// Run starts the monitoring loop with periodic scheduling. When ctx
|
|
// is cancelled, it saves state and returns.
|
|
func (w *Watcher) Run(ctx context.Context) {
|
|
w.log.Info(
|
|
"watcher starting",
|
|
"domains", len(w.config.Domains),
|
|
"hostnames", len(w.config.Hostnames),
|
|
"dnsInterval", w.config.DNSInterval,
|
|
"tlsInterval", w.config.TLSInterval,
|
|
)
|
|
|
|
w.RunOnce(ctx)
|
|
w.maybeSendTestNotification(ctx)
|
|
|
|
dnsTicker := time.NewTicker(w.config.DNSInterval)
|
|
tlsTicker := time.NewTicker(w.config.TLSInterval)
|
|
|
|
defer dnsTicker.Stop()
|
|
defer tlsTicker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
w.saveState()
|
|
w.log.Info("watcher stopped")
|
|
|
|
return
|
|
case <-dnsTicker.C:
|
|
w.runDNSChecks(ctx)
|
|
|
|
w.checkAllPorts(ctx)
|
|
w.saveState()
|
|
case <-tlsTicker.C:
|
|
// Run DNS first so TLS checks use freshly
|
|
// resolved IP addresses, not stale ones from
|
|
// a previous cycle.
|
|
w.runDNSChecks(ctx)
|
|
|
|
w.runTLSChecks(ctx)
|
|
w.saveState()
|
|
}
|
|
}
|
|
}
|
|
|
|
// RunOnce performs a single complete monitoring cycle.
|
|
// DNS checks run first so that port and TLS checks use
|
|
// freshly resolved IP addresses. Port checks run before
|
|
// TLS because TLS checks only target IPs with an open
|
|
// port 443.
|
|
func (w *Watcher) RunOnce(ctx context.Context) {
|
|
w.detectFirstRun()
|
|
|
|
// Phase 1: DNS resolution must complete first so that
|
|
// subsequent checks use fresh IP addresses.
|
|
w.runDNSChecks(ctx)
|
|
|
|
// Phase 2: Port checks populate port state that TLS
|
|
// checks depend on (TLS only targets IPs where port
|
|
// 443 is open).
|
|
w.checkAllPorts(ctx)
|
|
|
|
// Phase 3: TLS checks use fresh DNS IPs and current
|
|
// port state.
|
|
w.runTLSChecks(ctx)
|
|
|
|
w.saveState()
|
|
w.firstRun = false
|
|
}
|
|
|
|
func (w *Watcher) detectFirstRun() {
|
|
snap := w.state.GetSnapshot()
|
|
hasState := len(snap.Domains) > 0 ||
|
|
len(snap.Hostnames) > 0 ||
|
|
len(snap.Ports) > 0 ||
|
|
len(snap.Certificates) > 0
|
|
|
|
if hasState {
|
|
w.firstRun = false
|
|
}
|
|
}
|
|
|
|
// runDNSChecks performs DNS resolution for all configured domains
|
|
// and hostnames, updating state with freshly resolved records.
|
|
// This must complete before port or TLS checks run so those
|
|
// checks operate on current IP addresses.
|
|
func (w *Watcher) runDNSChecks(ctx context.Context) {
|
|
for _, domain := range w.config.Domains {
|
|
w.checkDomain(ctx, domain)
|
|
}
|
|
|
|
for _, hostname := range w.config.Hostnames {
|
|
w.checkHostname(ctx, hostname)
|
|
}
|
|
}
|
|
|
|
func (w *Watcher) checkDomain(
|
|
ctx context.Context,
|
|
domain string,
|
|
) {
|
|
nameservers, err := w.resolver.LookupNS(ctx, domain)
|
|
if err != nil {
|
|
w.log.Error(
|
|
"failed to lookup NS",
|
|
"domain", domain,
|
|
"error", err,
|
|
)
|
|
|
|
return
|
|
}
|
|
|
|
sort.Strings(nameservers)
|
|
|
|
now := time.Now().UTC()
|
|
|
|
prev, hasPrev := w.state.GetDomainState(domain)
|
|
|
|
var prevAddresses map[string][]string
|
|
if hasPrev {
|
|
prevAddresses = prev.NameserverAddresses
|
|
}
|
|
|
|
addresses := w.resolveNameserverAddresses(
|
|
ctx, nameservers, prevAddresses,
|
|
)
|
|
|
|
if hasPrev && !w.firstRun {
|
|
w.detectNSChanges(ctx, domain, prev.Nameservers, nameservers)
|
|
w.detectNSAddressChanges(ctx, domain, prevAddresses, addresses)
|
|
}
|
|
|
|
w.state.SetDomainState(domain, &state.DomainState{
|
|
Nameservers: nameservers,
|
|
NameserverAddresses: addresses,
|
|
LastChecked: now,
|
|
})
|
|
|
|
// Also look up A/AAAA records for the apex domain so that
|
|
// port and TLS checks (which read HostnameState) can find
|
|
// the domain's IP addresses.
|
|
results, err := w.resolver.LookupAllRecords(ctx, domain)
|
|
if err != nil {
|
|
w.log.Error(
|
|
"failed to lookup records for domain",
|
|
"domain", domain,
|
|
"error", err,
|
|
)
|
|
|
|
return
|
|
}
|
|
|
|
newState := buildHostnameState(results, now)
|
|
|
|
prevHS, hasPrevHS := w.state.GetHostnameState(domain)
|
|
if hasPrevHS && !w.firstRun {
|
|
w.detectHostnameChanges(ctx, domain, prevHS, newState)
|
|
}
|
|
|
|
w.state.SetHostnameState(domain, newState)
|
|
}
|
|
|
|
func (w *Watcher) detectNSChanges(
|
|
ctx context.Context,
|
|
domain string,
|
|
oldNS, newNS []string,
|
|
) {
|
|
oldSet := toSet(oldNS)
|
|
newSet := toSet(newNS)
|
|
|
|
var added, removed []string
|
|
|
|
for ns := range newSet {
|
|
if !oldSet[ns] {
|
|
added = append(added, ns)
|
|
}
|
|
}
|
|
|
|
for ns := range oldSet {
|
|
if !newSet[ns] {
|
|
removed = append(removed, ns)
|
|
}
|
|
}
|
|
|
|
if len(added) == 0 && len(removed) == 0 {
|
|
return
|
|
}
|
|
|
|
msg := fmt.Sprintf(
|
|
"Domain: %s\nAdded: %s\nRemoved: %s",
|
|
domain,
|
|
strings.Join(added, ", "),
|
|
strings.Join(removed, ", "),
|
|
)
|
|
|
|
w.notify.SendNotification(
|
|
ctx,
|
|
"NS Change: "+domain,
|
|
msg,
|
|
"warning",
|
|
)
|
|
}
|
|
|
|
// resolveNameserverAddresses returns the sorted addresses each
|
|
// nameserver's name resolves to. A nameserver whose lookup fails, as it
|
|
// does when no nameserver of the name's zone answers, or finds no
|
|
// address keeps its addresses from prev and is not an address change.
|
|
func (w *Watcher) resolveNameserverAddresses(
|
|
ctx context.Context,
|
|
nameservers []string,
|
|
prev map[string][]string,
|
|
) map[string][]string {
|
|
addresses := make(map[string][]string, len(nameservers))
|
|
|
|
for _, ns := range nameservers {
|
|
ips, err := w.resolver.ResolveIPAddresses(ctx, ns)
|
|
if err == nil && len(ips) > 0 {
|
|
sort.Strings(ips)
|
|
addresses[ns] = ips
|
|
|
|
continue
|
|
}
|
|
|
|
w.log.Error(
|
|
"no addresses found for nameserver",
|
|
"nameserver", ns,
|
|
"error", err,
|
|
)
|
|
|
|
if prevIPs, ok := prev[ns]; ok {
|
|
addresses[ns] = prevIPs
|
|
}
|
|
}
|
|
|
|
return addresses
|
|
}
|
|
|
|
// detectNSAddressChanges notifies when a nameserver in both checks
|
|
// resolves to different addresses. A nameserver added or removed is
|
|
// reported by detectNSChanges alone, and one with no addresses saved by
|
|
// the previous check, as in a state file from before they were saved,
|
|
// is not compared.
|
|
func (w *Watcher) detectNSAddressChanges(
|
|
ctx context.Context,
|
|
domain string,
|
|
prev, current map[string][]string,
|
|
) {
|
|
for ns, cur := range current {
|
|
old, ok := prev[ns]
|
|
if !ok || sliceEqual(old, cur) {
|
|
continue
|
|
}
|
|
|
|
msg := fmt.Sprintf(
|
|
"Domain: %s\nNameserver: %s\nOld: %s\nNew: %s",
|
|
domain, ns,
|
|
strings.Join(old, ", "),
|
|
strings.Join(cur, ", "),
|
|
)
|
|
|
|
w.notify.SendNotification(
|
|
ctx,
|
|
"NS Address Change: "+domain,
|
|
msg,
|
|
"warning",
|
|
)
|
|
}
|
|
}
|
|
|
|
func (w *Watcher) checkHostname(
|
|
ctx context.Context,
|
|
hostname string,
|
|
) {
|
|
results, err := w.resolver.LookupAllRecords(ctx, hostname)
|
|
if err != nil {
|
|
w.log.Error(
|
|
"failed to lookup records",
|
|
"hostname", hostname,
|
|
"error", err,
|
|
)
|
|
|
|
return
|
|
}
|
|
|
|
newState := buildHostnameState(results, time.Now().UTC())
|
|
|
|
prev, hasPrev := w.state.GetHostnameState(hostname)
|
|
if hasPrev && !w.firstRun {
|
|
w.detectHostnameChanges(ctx, hostname, prev, newState)
|
|
}
|
|
|
|
w.state.SetHostnameState(hostname, newState)
|
|
}
|
|
|
|
// buildHostnameState saves each nameserver's response. A nameserver
|
|
// that answered, even with NXDOMAIN or no records, is saved as ok; one
|
|
// that timed out or failed is saved as error with the reason, and its
|
|
// empty record set is not an answer.
|
|
func buildHostnameState(
|
|
results map[string]*resolver.NameserverResponse,
|
|
now time.Time,
|
|
) *state.HostnameState {
|
|
hs := &state.HostnameState{
|
|
RecordsByNameserver: make(
|
|
map[string]*state.NameserverRecordState,
|
|
),
|
|
LastChecked: now,
|
|
}
|
|
|
|
for ns, resp := range results {
|
|
nsState := &state.NameserverRecordState{
|
|
Records: resp.Records,
|
|
Status: statusOK,
|
|
LastChecked: now,
|
|
}
|
|
|
|
if resp.Status == resolver.StatusTimeout ||
|
|
resp.Status == resolver.StatusError {
|
|
nsState.Status = statusError
|
|
nsState.Error = resp.Error
|
|
}
|
|
|
|
hs.RecordsByNameserver[ns] = nsState
|
|
}
|
|
|
|
return hs
|
|
}
|
|
|
|
func (w *Watcher) detectHostnameChanges(
|
|
ctx context.Context,
|
|
hostname string,
|
|
prev, current *state.HostnameState,
|
|
) {
|
|
w.detectRecordChanges(ctx, hostname, prev, current)
|
|
w.detectNSDisappearances(ctx, hostname, prev, current)
|
|
w.detectNSFailures(ctx, hostname, prev, current)
|
|
w.detectInconsistencies(ctx, hostname, prev, current)
|
|
}
|
|
|
|
// detectRecordChanges compares each nameserver's records with those of
|
|
// the previous check. Only answers are compared: a nameserver that
|
|
// failed on either check has no records to compare.
|
|
func (w *Watcher) detectRecordChanges(
|
|
ctx context.Context,
|
|
hostname string,
|
|
prev, current *state.HostnameState,
|
|
) {
|
|
for ns, cur := range current.RecordsByNameserver {
|
|
prevNS, ok := prev.RecordsByNameserver[ns]
|
|
if !ok || prevNS.Status != statusOK || cur.Status != statusOK {
|
|
continue
|
|
}
|
|
|
|
if recordsEqual(prevNS.Records, cur.Records) {
|
|
continue
|
|
}
|
|
|
|
msg := fmt.Sprintf(
|
|
"Hostname: %s\nNameserver: %s\n"+
|
|
"Old: %v\nNew: %v",
|
|
hostname, ns,
|
|
prevNS.Records, cur.Records,
|
|
)
|
|
|
|
w.notify.SendNotification(
|
|
ctx,
|
|
"Record Change: "+hostname,
|
|
msg,
|
|
"warning",
|
|
)
|
|
}
|
|
}
|
|
|
|
func (w *Watcher) detectNSDisappearances(
|
|
ctx context.Context,
|
|
hostname string,
|
|
prev, current *state.HostnameState,
|
|
) {
|
|
for ns, prevNS := range prev.RecordsByNameserver {
|
|
if _, ok := current.RecordsByNameserver[ns]; ok || prevNS.Status != statusOK {
|
|
continue
|
|
}
|
|
|
|
msg := fmt.Sprintf(
|
|
"Hostname: %s\nNameserver: %s disappeared",
|
|
hostname, ns,
|
|
)
|
|
|
|
w.notify.SendNotification(
|
|
ctx,
|
|
"NS Failure: "+hostname,
|
|
msg,
|
|
"error",
|
|
)
|
|
}
|
|
}
|
|
|
|
// detectNSFailures notifies when a nameserver that answered on the
|
|
// previous check fails, and when one that failed answers again. A
|
|
// nameserver missing from the previous check is not compared.
|
|
func (w *Watcher) detectNSFailures(
|
|
ctx context.Context,
|
|
hostname string,
|
|
prev, current *state.HostnameState,
|
|
) {
|
|
for ns, cur := range current.RecordsByNameserver {
|
|
prevNS, ok := prev.RecordsByNameserver[ns]
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
switch {
|
|
case prevNS.Status == statusOK && cur.Status == statusError:
|
|
msg := fmt.Sprintf(
|
|
"Hostname: %s\nNameserver: %s\nError: %s",
|
|
hostname, ns, cur.Error,
|
|
)
|
|
|
|
w.notify.SendNotification(
|
|
ctx,
|
|
"NS Failure: "+hostname,
|
|
msg,
|
|
"error",
|
|
)
|
|
case prevNS.Status == statusError && cur.Status == statusOK:
|
|
msg := fmt.Sprintf(
|
|
"Hostname: %s\nNameserver: %s recovered",
|
|
hostname, ns,
|
|
)
|
|
|
|
w.notify.SendNotification(
|
|
ctx,
|
|
"NS Recovery: "+hostname,
|
|
msg,
|
|
"success",
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (w *Watcher) detectInconsistencies(
|
|
ctx context.Context,
|
|
hostname string,
|
|
prev, current *state.HostnameState,
|
|
) {
|
|
for _, pair := range newlyDisagreeingPairs(prev, current) {
|
|
ns1, ns2 := pair[0], pair[1]
|
|
|
|
msg := fmt.Sprintf(
|
|
"Hostname: %s\n%s: %v\n%s: %v",
|
|
hostname,
|
|
ns1, current.RecordsByNameserver[ns1].Records,
|
|
ns2, current.RecordsByNameserver[ns2].Records,
|
|
)
|
|
|
|
w.notify.SendNotification(
|
|
ctx,
|
|
"Inconsistency: "+hostname,
|
|
msg,
|
|
"warning",
|
|
)
|
|
}
|
|
}
|
|
|
|
// newlyDisagreeingPairs returns every pair of nameservers that answered
|
|
// in current and whose records differ there, in sorted order of name,
|
|
// except pairs where both nameservers answered in prev and already
|
|
// differed there. A nameserver missing from prev, or that failed there,
|
|
// is paired with every nameserver it differs from. A nameserver that
|
|
// failed in current has no records to compare and is in no pair.
|
|
func newlyDisagreeingPairs(
|
|
prev, current *state.HostnameState,
|
|
) [][2]string {
|
|
nameservers := make([]string, 0, len(current.RecordsByNameserver))
|
|
for ns, cur := range current.RecordsByNameserver {
|
|
if cur.Status == statusOK {
|
|
nameservers = append(nameservers, ns)
|
|
}
|
|
}
|
|
|
|
sort.Strings(nameservers)
|
|
|
|
var pairs [][2]string
|
|
|
|
for i, ns1 := range nameservers {
|
|
for _, ns2 := range nameservers[i+1:] {
|
|
if recordsEqual(
|
|
current.RecordsByNameserver[ns1].Records,
|
|
current.RecordsByNameserver[ns2].Records,
|
|
) {
|
|
continue
|
|
}
|
|
|
|
prev1, ok1 := prev.RecordsByNameserver[ns1]
|
|
prev2, ok2 := prev.RecordsByNameserver[ns2]
|
|
|
|
if ok1 && ok2 &&
|
|
prev1.Status == statusOK && prev2.Status == statusOK &&
|
|
!recordsEqual(prev1.Records, prev2.Records) {
|
|
continue
|
|
}
|
|
|
|
pairs = append(pairs, [2]string{ns1, ns2})
|
|
}
|
|
}
|
|
|
|
return pairs
|
|
}
|
|
|
|
func (w *Watcher) checkAllPorts(ctx context.Context) {
|
|
// Phase 1: Build current IP:port → hostname associations
|
|
// from fresh DNS data.
|
|
associations := w.buildPortAssociations()
|
|
|
|
// Phase 2: Check each unique IP:port and update state
|
|
// with the full set of associated hostnames.
|
|
for key, hostnames := range associations {
|
|
ip, port := parsePortKey(key)
|
|
if port == 0 {
|
|
continue
|
|
}
|
|
|
|
w.checkSinglePort(ctx, ip, port, hostnames)
|
|
}
|
|
|
|
// Phase 3: Remove port state entries that no longer have
|
|
// any hostname referencing them.
|
|
w.cleanupStalePorts(associations)
|
|
}
|
|
|
|
// buildPortAssociations constructs a map from IP:port keys to
|
|
// the sorted set of hostnames currently resolving to that IP.
|
|
func (w *Watcher) buildPortAssociations() map[string][]string {
|
|
assoc := make(map[string]map[string]bool)
|
|
|
|
allNames := make(
|
|
[]string, 0,
|
|
len(w.config.Hostnames)+len(w.config.Domains),
|
|
)
|
|
allNames = append(allNames, w.config.Hostnames...)
|
|
allNames = append(allNames, w.config.Domains...)
|
|
|
|
for _, name := range allNames {
|
|
ips := w.collectIPs(name)
|
|
for _, ip := range ips {
|
|
for _, port := range monitoredPorts {
|
|
key := fmt.Sprintf("%s:%d", ip, port)
|
|
if assoc[key] == nil {
|
|
assoc[key] = make(map[string]bool)
|
|
}
|
|
|
|
assoc[key][name] = true
|
|
}
|
|
}
|
|
}
|
|
|
|
result := make(map[string][]string, len(assoc))
|
|
for key, set := range assoc {
|
|
hostnames := make([]string, 0, len(set))
|
|
for h := range set {
|
|
hostnames = append(hostnames, h)
|
|
}
|
|
|
|
sort.Strings(hostnames)
|
|
|
|
result[key] = hostnames
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
// parsePortKey splits an "ip:port" key into its components.
|
|
func parsePortKey(key string) (string, int) {
|
|
lastColon := strings.LastIndex(key, ":")
|
|
if lastColon < 0 {
|
|
return key, 0
|
|
}
|
|
|
|
ip := key[:lastColon]
|
|
|
|
var p int
|
|
|
|
_, err := fmt.Sscanf(key[lastColon+1:], "%d", &p)
|
|
if err != nil {
|
|
return ip, 0
|
|
}
|
|
|
|
return ip, p
|
|
}
|
|
|
|
// cleanupStalePorts removes port state entries that are no
|
|
// longer referenced by any hostname in the current DNS data. An
|
|
// entry saved for a configured name none of whose nameservers
|
|
// answered is kept: that name's addresses are not known, not gone.
|
|
func (w *Watcher) cleanupStalePorts(
|
|
currentAssociations map[string][]string,
|
|
) {
|
|
for _, key := range w.state.GetAllPortKeys() {
|
|
if _, exists := currentAssociations[key]; exists {
|
|
continue
|
|
}
|
|
|
|
ps, ok := w.state.GetPortState(key)
|
|
if ok && slices.ContainsFunc(ps.Hostnames, w.noNameserverAnswered) {
|
|
continue
|
|
}
|
|
|
|
w.state.DeletePortState(key)
|
|
}
|
|
}
|
|
|
|
// noNameserverAnswered reports whether name is a configured domain or
|
|
// hostname and none of its nameservers answered on its last check.
|
|
func (w *Watcher) noNameserverAnswered(name string) bool {
|
|
if !slices.Contains(w.config.Hostnames, name) &&
|
|
!slices.Contains(w.config.Domains, name) {
|
|
return false
|
|
}
|
|
|
|
hs, ok := w.state.GetHostnameState(name)
|
|
if !ok {
|
|
return false
|
|
}
|
|
|
|
for _, nsState := range hs.RecordsByNameserver {
|
|
if nsState.Status == statusOK {
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func (w *Watcher) collectIPs(hostname string) []string {
|
|
hs, ok := w.state.GetHostnameState(hostname)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
|
|
ipSet := make(map[string]bool)
|
|
|
|
for _, nsState := range hs.RecordsByNameserver {
|
|
for _, ip := range nsState.Records["A"] {
|
|
ipSet[ip] = true
|
|
}
|
|
|
|
for _, ip := range nsState.Records["AAAA"] {
|
|
ipSet[ip] = true
|
|
}
|
|
}
|
|
|
|
result := make([]string, 0, len(ipSet))
|
|
for ip := range ipSet {
|
|
result = append(result, ip)
|
|
}
|
|
|
|
sort.Strings(result)
|
|
|
|
return result
|
|
}
|
|
|
|
func (w *Watcher) checkSinglePort(
|
|
ctx context.Context,
|
|
ip string,
|
|
port int,
|
|
hostnames []string,
|
|
) {
|
|
result, err := w.portCheck.CheckPort(ctx, ip, port)
|
|
|
|
// A check the context cut short says nothing about the port, so it
|
|
// is neither saved nor notified.
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
|
|
if err != nil {
|
|
w.log.Error(
|
|
"port check failed",
|
|
"ip", ip,
|
|
"port", port,
|
|
"error", err,
|
|
)
|
|
|
|
return
|
|
}
|
|
|
|
key := fmt.Sprintf("%s:%d", ip, port)
|
|
now := time.Now().UTC()
|
|
prev, hasPrev := w.state.GetPortState(key)
|
|
|
|
if hasPrev && !w.firstRun && prev.Open != result.Open {
|
|
stateStr := "closed"
|
|
if result.Open {
|
|
stateStr = "open"
|
|
}
|
|
|
|
msg := fmt.Sprintf(
|
|
"Hosts: %s\nAddress: %s\nPort now %s",
|
|
strings.Join(hostnames, ", "), key, stateStr,
|
|
)
|
|
|
|
w.notify.SendNotification(
|
|
ctx,
|
|
"Port Change: "+key,
|
|
msg,
|
|
"warning",
|
|
)
|
|
}
|
|
|
|
// A configured name on the saved list none of whose nameservers
|
|
// answered stays on it, so the entry is kept when the other names
|
|
// stop resolving to this address.
|
|
savedHostnames := slices.Clone(hostnames)
|
|
|
|
if hasPrev {
|
|
for _, name := range prev.Hostnames {
|
|
if !slices.Contains(hostnames, name) && w.noNameserverAnswered(name) {
|
|
savedHostnames = append(savedHostnames, name)
|
|
}
|
|
}
|
|
|
|
sort.Strings(savedHostnames)
|
|
}
|
|
|
|
w.state.SetPortState(key, &state.PortState{
|
|
Open: result.Open,
|
|
Hostnames: savedHostnames,
|
|
LastChecked: now,
|
|
})
|
|
}
|
|
|
|
func (w *Watcher) runTLSChecks(ctx context.Context) {
|
|
for _, hostname := range w.config.Hostnames {
|
|
w.checkTLSForHostname(ctx, hostname)
|
|
}
|
|
|
|
for _, domain := range w.config.Domains {
|
|
w.checkTLSForHostname(ctx, domain)
|
|
}
|
|
}
|
|
|
|
func (w *Watcher) checkTLSForHostname(
|
|
ctx context.Context,
|
|
hostname string,
|
|
) {
|
|
ips := w.collectIPs(hostname)
|
|
|
|
for _, ip := range ips {
|
|
portKey := fmt.Sprintf("%s:%d", ip, tlsPort)
|
|
|
|
ps, ok := w.state.GetPortState(portKey)
|
|
if !ok || !ps.Open {
|
|
continue
|
|
}
|
|
|
|
w.checkTLSCert(ctx, ip, hostname)
|
|
}
|
|
}
|
|
|
|
func (w *Watcher) checkTLSCert(
|
|
ctx context.Context,
|
|
ip string,
|
|
hostname string,
|
|
) {
|
|
cert, err := w.tlsCheck.CheckCertificate(ctx, ip, hostname)
|
|
|
|
// A check the context cut short says nothing about the certificate,
|
|
// so it is neither saved nor notified.
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
|
|
certKey := fmt.Sprintf("%s:%d:%s", ip, tlsPort, hostname)
|
|
now := time.Now().UTC()
|
|
prev, hasPrev := w.state.GetCertificateState(certKey)
|
|
|
|
if err != nil {
|
|
w.handleTLSError(
|
|
ctx, certKey, hostname, ip,
|
|
hasPrev, prev, now, err,
|
|
)
|
|
|
|
return
|
|
}
|
|
|
|
w.handleTLSSuccess(
|
|
ctx, certKey, hostname, ip,
|
|
hasPrev, prev, now, cert,
|
|
)
|
|
}
|
|
|
|
func (w *Watcher) handleTLSError(
|
|
ctx context.Context,
|
|
certKey, hostname, ip string,
|
|
hasPrev bool,
|
|
prev *state.CertificateState,
|
|
now time.Time,
|
|
err error,
|
|
) {
|
|
if hasPrev && !w.firstRun && prev.Status == statusOK {
|
|
msg := fmt.Sprintf(
|
|
"Host: %s\nIP: %s\nError: %s",
|
|
hostname, ip, err,
|
|
)
|
|
|
|
w.notify.SendNotification(
|
|
ctx,
|
|
"TLS Failure: "+hostname,
|
|
msg,
|
|
"error",
|
|
)
|
|
}
|
|
|
|
w.state.SetCertificateState(
|
|
certKey, &state.CertificateState{
|
|
Status: statusError,
|
|
Error: err.Error(),
|
|
LastChecked: now,
|
|
},
|
|
)
|
|
}
|
|
|
|
func (w *Watcher) handleTLSSuccess(
|
|
ctx context.Context,
|
|
certKey, hostname, ip string,
|
|
hasPrev bool,
|
|
prev *state.CertificateState,
|
|
now time.Time,
|
|
cert *tlscheck.CertificateInfo,
|
|
) {
|
|
if hasPrev && !w.firstRun {
|
|
w.detectTLSChanges(ctx, hostname, ip, prev, cert)
|
|
}
|
|
|
|
w.checkTLSExpiry(ctx, hostname, ip, cert)
|
|
|
|
w.state.SetCertificateState(
|
|
certKey, &state.CertificateState{
|
|
CommonName: cert.CommonName,
|
|
Issuer: cert.Issuer,
|
|
NotAfter: cert.NotAfter,
|
|
SubjectAlternativeNames: cert.SubjectAlternativeNames,
|
|
Status: statusOK,
|
|
LastChecked: now,
|
|
},
|
|
)
|
|
}
|
|
|
|
func (w *Watcher) detectTLSChanges(
|
|
ctx context.Context,
|
|
hostname, ip string,
|
|
prev *state.CertificateState,
|
|
cert *tlscheck.CertificateInfo,
|
|
) {
|
|
if prev.Status == statusError {
|
|
msg := fmt.Sprintf(
|
|
"Host: %s\nIP: %s\nTLS recovered",
|
|
hostname, ip,
|
|
)
|
|
|
|
w.notify.SendNotification(
|
|
ctx,
|
|
"TLS Recovery: "+hostname,
|
|
msg,
|
|
"success",
|
|
)
|
|
|
|
return
|
|
}
|
|
|
|
changed := prev.CommonName != cert.CommonName ||
|
|
prev.Issuer != cert.Issuer ||
|
|
!sliceEqual(
|
|
prev.SubjectAlternativeNames,
|
|
cert.SubjectAlternativeNames,
|
|
)
|
|
|
|
if !changed {
|
|
return
|
|
}
|
|
|
|
msg := fmt.Sprintf(
|
|
"Host: %s\nIP: %s\n"+
|
|
"Old CN: %s, Issuer: %s\n"+
|
|
"New CN: %s, Issuer: %s",
|
|
hostname, ip,
|
|
prev.CommonName, prev.Issuer,
|
|
cert.CommonName, cert.Issuer,
|
|
)
|
|
|
|
w.notify.SendNotification(
|
|
ctx,
|
|
"TLS Certificate Change: "+hostname,
|
|
msg,
|
|
"warning",
|
|
)
|
|
}
|
|
|
|
func (w *Watcher) checkTLSExpiry(
|
|
ctx context.Context,
|
|
hostname, ip string,
|
|
cert *tlscheck.CertificateInfo,
|
|
) {
|
|
daysLeft := time.Until(cert.NotAfter).Hours() / hoursPerDay
|
|
warningDays := float64(w.config.TLSExpiryWarning)
|
|
|
|
if daysLeft > warningDays {
|
|
return
|
|
}
|
|
|
|
msg := fmt.Sprintf(
|
|
"Host: %s\nIP: %s\nCN: %s\n"+
|
|
"Expires: %s (%.0f days)",
|
|
hostname, ip, cert.CommonName,
|
|
cert.NotAfter.Format(time.RFC3339),
|
|
daysLeft,
|
|
)
|
|
|
|
w.notify.SendNotification(
|
|
ctx,
|
|
"TLS Expiry Warning: "+hostname,
|
|
msg,
|
|
"warning",
|
|
)
|
|
}
|
|
|
|
func (w *Watcher) saveState() {
|
|
err := w.state.Save()
|
|
if err != nil {
|
|
w.log.Error("failed to save state", "error", err)
|
|
}
|
|
}
|
|
|
|
// maybeSendTestNotification sends a startup status notification
|
|
// after the first full scan completes, if SEND_TEST_NOTIFICATION
|
|
// is enabled. The message is clearly informational ("all ok")
|
|
// and not an error or anomaly alert.
|
|
func (w *Watcher) maybeSendTestNotification(ctx context.Context) {
|
|
if !w.config.SendTestNotification {
|
|
return
|
|
}
|
|
|
|
snap := w.state.GetSnapshot()
|
|
|
|
msg := fmt.Sprintf(
|
|
"dnswatcher has started and completed its initial scan.\n"+
|
|
"Monitoring %d domain(s) and %d hostname(s).\n"+
|
|
"Tracking %d port endpoint(s) and %d TLS certificate(s).\n"+
|
|
"All notification channels are working.",
|
|
len(snap.Domains),
|
|
len(snap.Hostnames),
|
|
len(snap.Ports),
|
|
len(snap.Certificates),
|
|
)
|
|
|
|
w.log.Info("sending startup test notification")
|
|
|
|
w.notify.SendNotification(
|
|
ctx,
|
|
"✅ dnswatcher startup complete",
|
|
msg,
|
|
"success",
|
|
)
|
|
}
|
|
|
|
// --- Utility functions ---
|
|
|
|
func toSet(items []string) map[string]bool {
|
|
set := make(map[string]bool, len(items))
|
|
for _, item := range items {
|
|
set[item] = true
|
|
}
|
|
|
|
return set
|
|
}
|
|
|
|
func recordsEqual(
|
|
a, b map[string][]string,
|
|
) bool {
|
|
if len(a) != len(b) {
|
|
return false
|
|
}
|
|
|
|
for k, av := range a {
|
|
bv, ok := b[k]
|
|
if !ok || !sliceEqual(av, bv) {
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func sliceEqual(a, b []string) bool {
|
|
if len(a) != len(b) {
|
|
return false
|
|
}
|
|
|
|
aSorted := make([]string, len(a))
|
|
bSorted := make([]string, len(b))
|
|
|
|
copy(aSorted, a)
|
|
copy(bSorted, b)
|
|
|
|
sort.Strings(aSorted)
|
|
sort.Strings(bSorted)
|
|
|
|
for i := range aSorted {
|
|
if aSorted[i] != bSorted[i] {
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|