diff --git a/conf/fail2ban/filter.d/miab-munin.conf b/conf/fail2ban/filter.d/miab-munin.conf index da257b3e..d25d3fd9 100644 --- a/conf/fail2ban/filter.d/miab-munin.conf +++ b/conf/fail2ban/filter.d/miab-munin.conf @@ -3,5 +3,5 @@ before = common.conf [Definition] -failregex= - .*GET /admin/munin/.* HTTP/\d+\.\d+\" 401.* +failregex=^.+?:\d+ - .*GET /admin/munin/.* HTTP/\d+\.\d+\" 401.* ignoreregex = diff --git a/conf/goaccess_persist b/conf/goaccess_persist new file mode 100644 index 00000000..6c0c1d72 --- /dev/null +++ b/conf/goaccess_persist @@ -0,0 +1,2 @@ +#!/usr/bin/env /bin/bash +/usr/bin/goaccess --process-and-exit diff --git a/conf/nginx-top.conf b/conf/nginx-top.conf index c3f4c0d6..ae7debac 100644 --- a/conf/nginx-top.conf +++ b/conf/nginx-top.conf @@ -10,3 +10,10 @@ upstream php-fpm { server unix:/var/run/php/php8.0-fpm.sock; } +# Reconfigure access log to include vhost to match goaccess VCOMBINED. +# Cancel default logging, re-enabled in servers. +access_log off; +# Log format to match goaccess. +log_format vcombined '$host:$server_port $remote_addr - $remote_user [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent"'; diff --git a/conf/nginx.conf b/conf/nginx.conf index fafd3409..4dbc3e38 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -10,6 +10,8 @@ server { server_name $HOSTNAME; root /tmp/invalid-path-nothing-here; + access_log /var/log/nginx/access.log vcombined; + # Improve privacy: Hide version an OS information on # error pages and in the "Server" HTTP-Header. server_tokens off; @@ -36,6 +38,8 @@ server { server_name $HOSTNAME; + access_log /var/log/nginx/access.log vcombined; + # Improve privacy: Hide version an OS information on # error pages and in the "Server" HTTP-Header. server_tokens off; diff --git a/management/daily_tasks.sh b/management/daily_tasks.sh index 24a97245..b5937052 100755 --- a/management/daily_tasks.sh +++ b/management/daily_tasks.sh @@ -15,6 +15,11 @@ if [ "$(date "+%u")" -eq 1 ]; then management/mail_log.py -t week | management/email_administrator.py "Mail-in-a-Box Usage Report" fi +# On Mondays, i.e. once a week, send the administrator a web analytics report. +if [ "$(date "+%u")" -eq 1 ]; then + goaccess -o html | management/email_administrator_attachment.py "MIAB Web Analytics Report" "Mail-in-a-Box Web analytics report is attached." "webstats.html" +fi + # Take a backup. management/backup.py 2>&1 | management/email_administrator.py "Backup Status" diff --git a/management/email_administrator_attachment.py b/management/email_administrator_attachment.py new file mode 100755 index 00000000..57e7c8b0 --- /dev/null +++ b/management/email_administrator_attachment.py @@ -0,0 +1,73 @@ +#!/usr/local/lib/mailinabox/env/bin/python + +# Reads in STDIN. If the stream is not empty, mail it to the system administrator. + +import sys + +import html +import smtplib +import email + +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.mime.base import MIMEBase +from email import encoders + +# In Python 3.6: +#from email.message import Message + +from utils import load_environment + +# Load system environment info. +env = load_environment() + +# Process command line args. +subject = sys.argv[1] or 'MIAB Administration' +body = sys.argv[2] or 'Please see the attachment. --Mail-in-a-Box' +attachmentname = sys.argv[3] or 'attachment.html' + +# Administrator's email address. +admin_addr = "administrator@" + env['PRIMARY_HOSTNAME'] + +# Read in STDIN. +attachment = sys.stdin.read().strip() + +# If there's nothing coming in, just exit. +if attachment == "": + sys.exit(0) + +# create MIME message +msg = MIMEMultipart('alternative') + +# In Python 3.6: +#msg = Message() + +msg['From'] = '"{}" <{}>'.format(env['PRIMARY_HOSTNAME'], admin_addr) +msg['To'] = admin_addr +msg['Subject'] = "[{}] {}".format(env['PRIMARY_HOSTNAME'], subject) + +body_html = f'
{html.escape(body)}
' + +msg.attach(MIMEText(body, 'plain')) +msg.attach(MIMEText(body_html, 'html')) + +# Attach content as file +part = MIMEBase('application', 'octet-stream') +part.set_payload(attachment); +encoders.encode_base64(part); +part.add_header('Content-Disposition', f"attachment; filename={attachmentname}") + +msg.attach(part); + +# In Python 3.6: +#msg.set_content(content) +#msg.add_alternative(content_html, "html") + +# send +smtpclient = smtplib.SMTP('127.0.0.1', 25) +smtpclient.ehlo() +smtpclient.sendmail( + admin_addr, # MAIL FROM + admin_addr, # RCPT TO + msg.as_string()) +smtpclient.quit() diff --git a/setup/web.sh b/setup/web.sh index 3aafcd88..2cffb1be 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -19,7 +19,7 @@ fi echo "Installing Nginx (web server)..." -apt_install nginx php"${PHP_VER}"-cli php"${PHP_VER}"-fpm idn2 +apt_install nginx php"${PHP_VER}"-cli php"${PHP_VER}"-fpm idn2 goaccess rm -f /etc/nginx/sites-enabled/default @@ -145,6 +145,46 @@ if [ ! -f "$STORAGE_ROOT/www/default/index.html" ]; then fi chown -R "$STORAGE_USER" "$STORAGE_ROOT/www" + + + +echo "Setting up goaccess web analytics..." + +# Set default configuration for goaccess web stats. +mkdir -p "/var/lib/mailinabox/goaccess_db" +tools/editconf.py /etc/goaccess/goaccess.conf -c '#' -s \ + persist=true \ + restore=true \ + keep-last=7 \ + db-path=/var/lib/mailinabox/goaccess_db \ + html-report-title=Mailinabox \ + log-file=/var/log/nginx/access.log \ + log-format=VCOMBINED + + +# Create a pre-rotate action to preserve log info. +PREROT="/etc/logrotate.d/httpd-prerotate" +if [ -d "$PREROT" ] ; then + NOPREROT=1; # false, there is a prerotate +else + NOPREROT=0; # true, there is no prerotate +fi +mkdir -p "$PREROT" + +# If the prerotate doesn't exist, configure. +if [ "$NOPREROT" -eq 0 ]; then + echo "- Configuring log prerotate action." + chown root:root "$PREROT" + chmod 755 "$PREROT"; +else # There is a prerotate, no change. + echo "- No change to $PREROT"; +fi +# Create action. +cp conf/goaccess_persist "$PREROT" +chown root:root "$PREROT/goaccess_persist" +chmod a+x "$PREROT/goaccess_persist" + + # Start services. restart_service nginx restart_service php"$PHP_VER"-fpm diff --git a/tools/parse-nginx-log-bootstrap-accesses.py b/tools/parse-nginx-log-bootstrap-accesses.py index 8eb74dec..3a0c6cc5 100755 --- a/tools/parse-nginx-log-bootstrap-accesses.py +++ b/tools/parse-nginx-log-bootstrap-accesses.py @@ -23,7 +23,7 @@ for fn in glob.glob("/var/log/nginx/access.log*"): # Find lines that are GETs on the bootstrap script by either curl or wget. # (Note that we purposely skip ...?ping=1 requests which is the admin panel querying us for updates.) # (Also, the URL changed in January 2016, but we'll accept both.) - m = re.match(rb"(?P\S+) - - \[(?P.*?)\] \"GET /(bootstrap.sh|setup.sh) HTTP/.*\" 200 \d+ .* \"(?:curl|wget)", line, re.I) + m = re.match(rb"(?P\S+) (?P\S+) - - \[(?P.*?)\] \"GET /(bootstrap.sh|setup.sh) HTTP/.*\" 200 \d+ .* \"(?:curl|wget)", line, re.I) if m: date, time = m.group("date").decode("ascii").split(":", 1) date = dateutil.parser.parse(date).date().isoformat()