65 lines
2.2 KiB
Python
Executable File
65 lines
2.2 KiB
Python
Executable File
#!/usr/bin/python3
|
|
#
|
|
# This is a tool Josh uses on his box serving mailinabox.email to parse the nginx
|
|
# access log to see how many people are installing Mail-in-a-Box each day, by
|
|
# looking at accesses to the bootstrap.sh script (which is currently at the URL
|
|
# .../setup.sh).
|
|
|
|
import re
|
|
import glob
|
|
import gzip
|
|
import os.path
|
|
import json
|
|
import dateutil.parser
|
|
|
|
outfn = "/home/user-data/www/mailinabox.email/install-stats.json"
|
|
|
|
# Make a unique list of (date, ip address) pairs so we don't double-count
|
|
# accesses that are for the same install.
|
|
accesses = set()
|
|
|
|
# Scan the current and rotated access logs.
|
|
for fn in glob.glob("/var/log/nginx/access.log*"):
|
|
# Gunzip if necessary.
|
|
if fn.endswith(".gz"):
|
|
f = gzip.open(fn)
|
|
else:
|
|
f = open(fn, "rb")
|
|
|
|
# Loop through the lines in the access log.
|
|
with f:
|
|
for line in f:
|
|
# Find lines that are GETs on the bootstrap script by either curl or wget.
|
|
# (Note that we purposely skip ...?ping=1 requests which is the admin
|
|
# panel querying us for updates.)
|
|
# (Also, the URL changed in January 2016, but we'll accept both.)
|
|
m = re.match(rb"(?P<ip>\S+) - - \[(?P<date>.*?)\] \"GET /(bootstrap.sh|setup.sh) HTTP/.*\" 200 \d+ .* \"(?:curl|wget)", line, re.I)
|
|
if m:
|
|
date, time = m.group("date").decode("ascii").split(":", 1)
|
|
date = dateutil.parser.parse(date).date().isoformat()
|
|
ip = m.group("ip").decode("ascii")
|
|
accesses.add((date, ip))
|
|
|
|
# Aggregate by date.
|
|
by_date = {}
|
|
for date, ip in accesses:
|
|
by_date[date] = by_date.get(date, 0) + 1
|
|
|
|
# Since logs are rotated, store the statistics permanently in a JSON file.
|
|
# Load in the stats from an existing file.
|
|
if os.path.exists(outfn):
|
|
existing_data = json.load(open(outfn))
|
|
for date, count in existing_data:
|
|
if date not in by_date:
|
|
by_date[date] = count
|
|
|
|
# Turn into a list rather than a dict structure to make it ordered.
|
|
by_date = sorted(by_date.items())
|
|
|
|
# Pop the last one because today's stats are incomplete.
|
|
by_date.pop(-1)
|
|
|
|
# Write out.
|
|
with open(outfn, "w") as f:
|
|
json.dump(by_date, f, sort_keys=True, indent=True)
|