mailinabox/tools/parse-nginx-log-bootstrap-accesses.py

#!/usr/bin/python3
#
# This is a tool Josh uses on his box serving mailinabox.email to parse the nginx
# access log to see how many people are installing Mail-in-a-Box each day, by
# looking at accesses to the bootstrap.sh script (which is currently at the URL
# .../setup.sh).

import re
import glob
import gzip
import os.path
import json
import dateutil.parser

outfn = "/home/user-data/www/mailinabox.email/install-stats.json"

# Make a unique list of (date, ip address) pairs so we don't double-count
# accesses that are for the same install.
accesses = set()

# Scan the current and rotated access logs.
for fn in glob.glob("/var/log/nginx/access.log*"):
    # Gunzip if necessary.
    if fn.endswith(".gz"):
        f = gzip.open(fn)
    else:
        f = open(fn, "rb")

    # Loop through the lines in the access log.
    with f:
        for line in f:
            # Find lines that are GETs on the bootstrap script by either curl or wget.
            # (Note that we purposely skip ...?ping=1 requests which is the admin
            # panel querying us for updates.)
            # (Also, the URL changed in January 2016, but we'll accept both.)
            m = re.match(rb"(?P<ip>\S+) - - \[(?P<date>.*?)\] \"GET /(bootstrap.sh|setup.sh) HTTP/.*\" 200 \d+ .* \"(?:curl|wget)", line, re.I)
            if m:
                date, time = m.group("date").decode("ascii").split(":", 1)
                date = dateutil.parser.parse(date).date().isoformat()
                ip = m.group("ip").decode("ascii")
                accesses.add((date, ip))

# Aggregate by date.
by_date = {}
for date, ip in accesses:
    by_date[date] = by_date.get(date, 0) + 1

# Since logs are rotated, store the statistics permanently in a JSON file.
# Load in the stats from an existing file.
if os.path.exists(outfn):
    existing_data = json.load(open(outfn))
    for date, count in existing_data:
        if date not in by_date:
            by_date[date] = count

# Turn into a list rather than a dict structure to make it ordered.
by_date = sorted(by_date.items())

# Pop the last one because today's stats are incomplete.
by_date.pop(-1)

# Write out.
with open(outfn, "w") as f:
    json.dump(by_date, f, sort_keys=True, indent=True)