Initial commit of a log capture and reporting feature

This adds a new section to the admin panel called "Activity", that
supplies charts, graphs and details about messages entering and leaving
the host.

A new daemon captures details of system mail activity by monitoring
the /var/log/mail.log file, summarizing it into a sqllite database
that's kept in user-data.

.gitignore

@@ -1,7 +1,5 @@
 *~
-tests/__pycache__/
-management/__pycache__/
-tools/__pycache__/
+**/__pycache__/
 externals/
 .env
 .vagrant

conf/miabldap-capture.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=MIAB-LDAP log capture daemon
+ConditionPathExists=/etc/mailinabox.conf
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/python3 %BIN%/management/reporting/capture/capture.py
+ExecReload=/bin/kill -HUP $MAINPID
+
+[Install]
+WantedBy=multi-user.target

ehdd/postinstall.sh

@@ -14,6 +14,7 @@ if [ -e "$EHDD_IMG" ]; then
         systemctl disable php7.2-fpm
         systemctl disable mailinabox
         systemctl disable fail2ban
+        systemctl disable miabldap-capture
         #systemctl disable nsd
         [ -x /usr/sbin/slapd ] && systemctl disable slapd
 

ehdd/startup.sh

@@ -12,5 +12,6 @@ if [ -s /etc/mailinabox.conf ]; then
     systemctl link -f $(pwd)/conf/mailinabox.service
     systemctl start fail2ban
     systemctl restart mailinabox
+    systemctl start miabldap-capture
 fi
 

management/backup.py

@@ -252,6 +252,7 @@ def perform_backup(full_backup):
 	service_command("postfix", "stop", quit=True)
 	service_command("dovecot", "stop", quit=True)
 	service_command("slapd", "stop", quit=True)
+	service_command("miabldap-capture", "stop", quit=True)
 
 	# Execute a pre-backup script that copies files outside the homedir.
 	# Run as the STORAGE_USER user, not as root. Pass our settings in
@@ -281,6 +282,7 @@ def perform_backup(full_backup):
 			get_env(env))
 	finally:
 		# Start services again.
+		service_command("miabldap-capture", "start", quit=False)
 		service_command("slapd", "start", quit=False)
 		service_command("dovecot", "start", quit=False)
 		service_command("postfix", "start", quit=False)

management/daemon.py

@@ -696,6 +696,16 @@ def log_failed_login(request):
 
 # APP
 
+from daemon_logger import add_python_logging
+add_python_logging(app)
+
+from daemon_ui_common import add_ui_common
+add_ui_common(app)
+
+from daemon_reports import add_reports
+add_reports(app, env, authorized_personnel_only)
+
+
 if __name__ == '__main__':
 	if "DEBUG" in os.environ:
 		# Turn on Flask debugging.

management/daemon_logger.py

@@ -0,0 +1,111 @@
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
+import logging
+import flask
+
+# setup our root logger
+
+# keep a separate logger from app.logger, which only logs WARNING and
+# above, doesn't include the module name, or authentication
+# details
+
+class textcolor:
+	DANGER = '\033[31m'
+	WARN = '\033[93m'
+	SUCCESS = '\033[32m'
+	BOLD = '\033[1m'
+	FADED= '\033[37m'
+	RESET = '\033[0m'
+
+
+
+class AuthLogFormatter(logging.Formatter):
+	def __init__(self):
+		fmt='%(name)s:%(lineno)d(%(username)s/%(client)s): %(levelname)s[%(thread)d]: %(color)s%(message)s%(color_reset)s'
+		super(AuthLogFormatter, self).__init__(fmt=fmt)
+
+
+#
+# when logging, a "client" (oauth client) and/or an explicit
+# "username" (in the case the user in question is not logged in but
+# you want the username to appear in the logs) may be provided in the
+# log.xxx() call as the last argument. eg:
+#
+# log.warning('login attempt failed', { 'username': email })
+#
+
+class AuthLogFilter(logging.Filter):
+	def __init__(self, color_output, get_session_username_function):
+		self.color_output = color_output
+		self.get_session_username = get_session_username_function
+		super(AuthLogFilter, self).__init__()
+	
+	''' add `username` and `client` context info the the LogRecord '''
+	def filter(self, record):
+		record.color = ''
+		if self.color_output:
+			if record.levelno == logging.DEBUG:
+				record.color=textcolor.FADED
+			elif record.levelno == logging.INFO:
+				record.color=textcolor.BOLD
+			elif record.levelno == logging.WARNING:
+				record.color=textcolor.WARN
+			elif record.levelno in [logging.ERROR, logging.CRITICAL]:
+				record.color=textcolor.DANGER
+				
+		record.color_reset = textcolor.RESET if record.color else ''
+		record.client = '-'
+		record.username = '-'
+		record.thread = record.thread % 10000
+
+		opts = None
+		args_len = len(record.args)
+		if type(record.args) == dict:
+			opts = record.args
+			record.args = ()
+		elif args_len>0 and type(record.args[args_len-1]) == dict:
+			opts = record.args[args_len-1]
+			record.args = record.args[0:args_len-1]
+
+		if opts:
+			record.client = opts.get('client', '-')
+			record.username = opts.get('username', '-')
+
+		if record.username == '-':
+			try:
+				record.username = self.get_session_username()
+			except (RuntimeError, KeyError):
+				# not in an HTTP request context or not logged in
+				pass
+
+		return True
+
+def get_session_username():
+	if flask.request and hasattr(flask.request, 'user_email'):
+		# this is an admin panel login via "authorized_personnel_only"
+		return flask.request.user_email
+
+	# otherwise, this may be a user session login
+	return flask.session['user_id']
+	
+
+def add_python_logging(app):
+	# log to stdout in development mode
+	if app.debug:
+		log_level = logging.DEBUG
+		log_handler = logging.StreamHandler()
+
+	# log to syslog in production mode
+	else:
+		import utils
+		log_level = logging.INFO
+		log_handler = utils.create_syslog_handler()
+		
+	logging.basicConfig(level=log_level, handlers=[])
+	log_handler.setLevel(log_level)
+	log_handler.addFilter(AuthLogFilter(
+		app.debug,
+		get_session_username
+	))
+	log_handler.setFormatter(AuthLogFormatter())
+	log = logging.getLogger('')
+	log.addHandler(log_handler)

management/daemon_reports.py

@@ -0,0 +1,216 @@
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
+
+import os
+import logging
+import json
+import datetime
+import time
+import subprocess
+
+log = logging.getLogger(__name__)
+
+from flask import request, Response, session, redirect, jsonify, send_from_directory
+from functools import wraps
+from reporting.capture.db.SqliteConnFactory import SqliteConnFactory
+import reporting.uidata as uidata
+
+from mailconfig import get_mail_users
+
+		
+def add_reports(app, env, authorized_personnel_only):
+	'''call this function to add reporting/sem endpoints
+
+	`app` is a Flask instance
+	`env` is the Mail-in-a-Box LDAP environment
+	`authorized_personnel_only` is a flask wrapper from daemon.py
+	ensuring only authenticated admins can access the endpoint
+
+	'''
+
+	CAPTURE_STORAGE_ROOT = os.environ.get(
+		'CAPTURE_STORAGE_ROOT',
+		os.path.join(env['STORAGE_ROOT'], 'reporting')
+	)
+	sqlite_file = os.path.join(CAPTURE_STORAGE_ROOT, 'capture.sqlite')
+	db_conn_factory = SqliteConnFactory(sqlite_file)
+
+	# UI support
+	ui_dir = os.path.join(os.path.dirname(app.template_folder), 'reporting/ui')
+	def send_ui_file(filename):
+		return send_from_directory(ui_dir, filename)
+
+	@app.route("/reports/ui/<path:filename>", methods=['GET'])
+	def get_reporting_ui_file(filename):
+		return send_ui_file(filename)
+	
+	@app.route('/reports')
+	def reporting_redir():
+		return redirect('/reports/')
+
+	@app.route('/reports/', methods=['GET'])
+	def reporting_main():
+		return send_ui_file('index.html')
+
+
+	# Decorator to unwrap json payloads. It returns the json as a dict
+	# in named argument 'payload'
+	def json_payload(func):
+		@wraps(func)
+		def wrapper(*args, **kwargs):
+			try:
+				log.debug('payload:%s', request.data)
+				payload = json.loads(request.data)
+				return func(*args, payload=payload, **kwargs)
+			except json.decoder.JSONDecodeError as e:
+				log.warning('Bad request: data:%s ex:%s', request.data, e)
+				return ("Bad request", 400)
+		return wrapper
+
+	@app.route('/reports/uidata/messages-sent', methods=['POST'])
+	@authorized_personnel_only
+	@json_payload
+	def get_data_chart_messages_sent(payload):
+		conn = db_conn_factory.connect()
+		try:
+			return jsonify(uidata.messages_sent(conn, payload))
+		except uidata.InvalidArgsError as e:
+			return ('invalid request', 400)
+		finally:
+			db_conn_factory.close(conn)
+			
+	@app.route('/reports/uidata/messages-received', methods=['POST'])
+	@authorized_personnel_only
+	@json_payload
+	def get_data_chart_messages_received(payload):
+		conn = db_conn_factory.connect()
+		try:
+			return jsonify(uidata.messages_received(conn, payload))
+		except uidata.InvalidArgsError as e:
+			return ('invalid request', 400)
+		finally:
+			db_conn_factory.close(conn)
+			
+	@app.route('/reports/uidata/user-activity', methods=['POST'])
+	@authorized_personnel_only
+	@json_payload
+	def get_data_user_activity(payload):
+		conn = db_conn_factory.connect()
+		try:
+			return jsonify(uidata.user_activity(conn, payload))
+		except uidata.InvalidArgsError as e:
+			return ('invalid request', 400)
+		finally:
+			db_conn_factory.close(conn)
+
+	@app.route('/reports/uidata/flagged-connections', methods=['POST'])
+	@authorized_personnel_only
+	@json_payload
+	def get_data_flagged_connections(payload):
+		conn = db_conn_factory.connect()
+		try:
+			return jsonify(uidata.flagged_connections(conn, payload))
+		except uidata.InvalidArgsError as e:
+			return ('invalid request', 400)
+		finally:
+			db_conn_factory.close(conn)
+
+	@app.route('/reports/uidata/remote-sender-activity', methods=['POST'])
+	@authorized_personnel_only
+	@json_payload
+	def get_data_remote_sender_activity(payload):
+		conn = db_conn_factory.connect()
+		try:
+			return jsonify(uidata.remote_sender_activity(conn, payload))
+		except uidata.InvalidArgsError as e:
+			return ('invalid request', 400)
+		finally:
+			db_conn_factory.close(conn)
+
+	@app.route('/reports/uidata/user-list', methods=['GET'])
+	@authorized_personnel_only
+	def get_data_user_list():
+		return jsonify(get_mail_users(env, as_map=False))
+
+	@app.route('/reports/uidata/select-list-suggestions', methods=['POST'])
+	@authorized_personnel_only
+	@json_payload
+	def suggest(payload):
+		conn = db_conn_factory.connect()
+		try:
+			return jsonify(uidata.select_list_suggestions(conn, payload))
+		except uidata.InvalidArgsError as e:
+			return ('invalid request', 400)
+		finally:
+			db_conn_factory.close(conn)
+	
+	@app.route('/reports/capture/config', methods=['GET'])
+	@authorized_personnel_only
+	def get_capture_config():
+		try:
+			with open("/var/run/mailinabox/runtime_config.json") as fp:
+				return Response(fp.read(), mimetype="text/json")
+		except FileNotFoundError:
+			return jsonify({ 'status':'error', 'reason':'not running' })
+
+	@app.route('/reports/capture/config', methods=['POST'])
+	@authorized_personnel_only
+	@json_payload
+	def save_capture_config(payload):
+		try:
+			with open("/var/run/mailinabox/runtime_config.json") as fp:
+				loc = json.loads(fp.read()).get('from', { 'type':'unknown' })
+		except FileNotFoundError:
+			return ('service is not running', 403)
+
+		# loc: { type:'file', location:'<path>' }
+		if loc.get('type') != 'file':
+			return ('storage type is %s' % loc.get('type'), 403)
+
+		log.warning('overwriting config file %s', loc['location'])
+
+		# remove runtime-config extra fields that don't belong in
+		# the user config
+		if 'from' in payload:
+			del payload['from']
+			
+		with open(loc['location'], "w") as fp:
+			fp.write(json.dumps(payload, indent=4))
+
+		r = subprocess.run(["systemctl", "reload", "miabldap-capture"])
+		if r.returncode != 0:
+			log.warning('systemctl reload faild for miabldap-capture: code=%s', r.returncode)
+		else:
+			# wait a sec for daemon to pick up new config
+			# TODO: monitor runtime config for mtime change
+			time.sleep(1)
+			# invalidate stats cache. if prune policy changed, the stats
+			# may be invalid
+			uidata.clear_cache()
+			
+		return ("ok", 200)
+	
+
+	@app.route('/reports/capture/service/status', methods=['GET'])
+	@authorized_personnel_only
+	def get_capture_status():
+		service = "miabldap-capture.service"
+
+		if not os.path.exists("/etc/systemd/system/" + service):
+			return jsonify([ 'not installed', 'not installed' ])
+			
+		r1 = subprocess.run(["systemctl", "is-active", "--quiet", service ])
+		r2 = subprocess.run(["systemctl", "is-enabled", "--quiet", service ])
+		
+		return jsonify([
+			'running' if r1.returncode == 0 else 'stopped',
+			'enabled' if r2.returncode == 0 else 'disabled'
+		])
+
+	@app.route('/reports/capture/db/stats', methods=['GET'])
+	@authorized_personnel_only
+	def get_db_stats():
+		conn = db_conn_factory.connect()
+		try:
+			return jsonify(uidata.capture_db_stats(conn))
+		finally:
+			db_conn_factory.close(conn)

management/daemon_ui_common.py

@@ -0,0 +1,31 @@
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
+
+import os
+import logging
+import json
+import datetime
+
+# setup our root logger - named so oauth/*.py files become children
+log = logging.getLogger(__name__)
+
+from flask import request, session, redirect, jsonify, send_from_directory
+
+
+		
+def add_ui_common(app):
+	'''
+	call this function to add an endpoint that delivers common ui files
+
+	`app` is a Flask instance
+	'''
+
+	# UI support
+	ui_dir = os.path.join(os.path.dirname(app.template_folder), 'ui-common')
+	def send_ui_file(filename):
+		return send_from_directory(ui_dir, filename)
+
+	@app.route("/ui-common/<path:filename>", methods=['GET'])
+	def get_common_ui_file(filename):
+		return send_ui_file(filename)
+	
+		

management/reporting/capture/.gitignore

@@ -0,0 +1,2 @@
+tests/
+run.sh

management/reporting/capture/capture.py

@@ -0,0 +1,300 @@
+#!/usr/bin/env python3
+from logs.TailFile import TailFile
+from mail.InboundMailLogHandler import InboundMailLogHandler
+from logs.ReadPositionStoreInFile import ReadPositionStoreInFile
+from db.SqliteConnFactory import SqliteConnFactory
+from db.SqliteEventStore import SqliteEventStore
+from db.Pruner import Pruner
+from util.env import load_env_vars_from_file
+
+import time
+import logging, logging.handlers
+import json
+import re
+import os
+import sys
+import signal
+
+log = logging.getLogger(__name__)
+
+if os.path.exists('/etc/mailinabox.conf'):
+    env = load_env_vars_from_file('/etc/mailinabox.conf')
+else:
+    env = { 'STORAGE_ROOT': os.environ.get('STORAGE_ROOT', os.getcwd()) }
+
+CAPTURE_STORAGE_ROOT = os.path.join(env['STORAGE_ROOT'], 'reporting')
+
+
+# default configuration, if not specified or
+# CAPTURE_STORAGE_ROOT/config.json does not exist
+config_default = {
+    'default_config': True,
+    'capture': True,
+    'prune_policy': {
+        'frequency_min': 2400,
+        'older_than_days': 30
+    },
+    'drop_disposition': {
+        'failed_login_attempt': True,
+        'suspected_scanner': False,
+        'reject': True
+    }
+}
+
+config_default_file = os.path.join(CAPTURE_STORAGE_ROOT, 'config.json')
+
+
+# options
+    
+options = {
+    'daemon': True,
+    'log_level': logging.WARNING,
+    'log_file': "/var/log/mail.log",
+    'pos_file': "/var/lib/mailinabox/capture-pos.json",
+    'sqlite_file': os.path.join(CAPTURE_STORAGE_ROOT, 'capture.sqlite'),
+    'working_dir': "/var/run/mailinabox",
+    'config': config_default,
+    '_config_file': False,  # absolute path if "config" was from a file
+    '_runtime_config_file': "runtime_config.json" # relative to working_dir
+}
+
+
+def read_config_file(file, throw=False):
+    try:
+        with open(file) as fp:
+            newconfig = json.loads(fp.read())
+        newconfig['from'] = { 'type':'file', 'location':file }
+        return newconfig
+    except FileNotFoundError as e:
+        if not throw:
+            return False
+        raise e
+
+def write_config(tofile, config):
+    d = os.path.dirname(tofile)
+    if d and not os.path.exists(d):
+        os.mkdir(d, mode=0o770)
+    with open(tofile, "w") as fp:
+        fp.write(json.dumps(config))
+
+def usage():
+    print('usage: %s [options]' % sys.argv[0])
+    sys.exit(1)
+
+def process_cmdline(options):
+    argi = 1
+    marg = 0
+    while argi < len(sys.argv):
+        arg = sys.argv[argi]
+        have_next = ( argi+1 < len(sys.argv) )
+        
+        if arg=='-d':
+            options['daemon'] = False
+                
+        elif arg=='-loglevel' and have_next:
+            argi += 1
+            arg = sys.argv[argi].lower()
+            if arg=='info':
+                options['log_level'] = logging.INFO
+            elif arg=='warning':
+                options['log_level'] = logging.WARNING
+            elif arg=='debug':
+                options['log_level'] = logging.DEBUG
+            elif arg=='error':
+                options['log_level'] = logging.ERROR
+            else:
+                sys.stderr.write('unknown log level "%s"\n' % sys.argv[argi])
+                
+        elif arg=='-config' and have_next:
+            argi += 1
+            arg = sys.argv[argi]
+            try:
+                if arg.startswith('{'):
+                    options['config'] = json.loads(arg)
+                else:
+                    newconfig = read_config_file(arg, throw=True)
+                    options['config'] = newconfig
+                    options['_config_file'] = arg
+            except Exception as e:
+                if options['daemon']: log.exception(e)
+                raise e
+
+        elif arg=='-logfile' and have_next:
+            argi+=1
+            options['log_file'] = sys.argv[argi]
+            
+        elif arg=='-posfile' and have_next:
+            argi+=1
+            options['pos_file'] = sys.argv[argi]
+            
+        elif arg=='-sqlitefile' and have_next:
+            argi+=1
+            options['sqlite_file'] = sys.argv[argi]            
+
+        elif arg.startswith('-'):
+            usage()
+
+        else:
+            if marg==0:
+                options['log_file'] = arg
+            elif marg==1:
+                options['pos_file'] = arg
+            elif marg==2:
+                options['sqlite_file'] = arg
+            else:
+                usage()
+            marg += 1
+        argi += 1
+
+
+def set_working_dir(working_dir):
+    try:
+        if not os.path.exists(working_dir):
+            os.mkdir(working_dir, mode=0o770)
+        os.chdir(working_dir)
+    except Exception as e:
+        log.exception(e)
+        raise e
+
+def close_stdio():
+    sys.stdout.close()
+    sys.stderr.close()
+    sys.stdin.close()
+
+    
+# if config.json exists in the default location start with that
+# instead of `config_default`. config can still be changed with the
+# command line argument "-config"
+
+newconfig = read_config_file(config_default_file, throw=False)
+if newconfig:
+    options['config'] = newconfig
+    options['_config_file'] = config_default_file
+    
+# process command line
+process_cmdline(options)
+
+
+# init logging & set working directory
+if options['daemon']:
+    logging.basicConfig(
+        level = options['log_level'],
+        handlers= [
+            logging.handlers.SysLogHandler(address='/dev/log')
+        ],
+        format = 'miabldap/capture %(message)s'
+    )
+    log.warning('MIAB-LDAP capture/SEM daemon starting: wd=%s; log_file=%s; pos_file=%s; db=%s',
+                options['working_dir'],
+                options['log_file'],
+                options['pos_file'],
+                options['sqlite_file']
+    )
+    close_stdio()
+    set_working_dir(options['working_dir'])
+    
+else:
+    logging.basicConfig(level=options['log_level'])
+    log.info('starting: log_file=%s; pos_file=%s; db=%s',
+             options['log_file'],
+             options['pos_file'],
+             options['sqlite_file']
+    )
+
+
+# save runtime config
+write_config(options['_runtime_config_file'], options['config'])
+
+
+# start modules
+log.info('config: %s', options['config'])
+try:
+    db_conn_factory = SqliteConnFactory(
+        options['sqlite_file']
+    )
+    event_store = SqliteEventStore(
+        db_conn_factory
+    )
+    position_store = ReadPositionStoreInFile(
+        options['pos_file']
+    )
+    mail_tail = TailFile(
+        options['log_file'],
+        position_store
+    )
+    inbound_mail_handler = InboundMailLogHandler(
+        event_store,
+        capture_enabled = options['config'].get('capture',True),
+        drop_disposition = options['config'].get('drop_disposition')
+    )
+    mail_tail.add_handler(inbound_mail_handler)
+    pruner = Pruner(
+        db_conn_factory,
+        policy=options['config']['prune_policy']
+    )
+    pruner.add_prunable(event_store)
+    
+except Exception as e:
+    if options['daemon']: log.exception(e)
+    raise e
+
+
+# termination handler for graceful shutdowns
+def terminate(sig, stack):
+    if sig == signal.SIGTERM:
+        log.warning("shutting down due to SIGTERM")
+    log.debug("stopping mail_tail")
+    mail_tail.stop()
+    log.debug("stopping pruner")
+    pruner.stop()
+    log.debug("stopping position_store")
+    position_store.stop()
+    log.debug("stopping event_store")
+    event_store.stop()
+    try:
+        os.remove(options['_runtime_config_file'])
+    except Exception:
+        pass
+    log.info("stopped")
+
+# reload settings handler
+def reload(sig, stack):
+    # if the default config (`config_default`) is in use, check to see
+    # if a default config.json (`config_default_file`) now exists, and
+    # if so, use that
+    if options['config'].get('default_config', False) and os.path.exists(config_default_file):
+        options['config']['default_config'] = False
+        options['_config_file'] = config_default_file
+
+    log.info('%s records are in-progress',
+             inbound_mail_handler.get_inprogress_count())
+
+    if options['_config_file']:
+        log.info('reloading %s', options['_config_file'])
+        try:
+            newconfig = read_config_file(options['_config_file'], throw=True)
+            pruner.set_policy(
+                newconfig['prune_policy']
+            )
+            inbound_mail_handler.set_capture_enabled(
+                newconfig.get('capture', True)
+            )
+            inbound_mail_handler.update_drop_disposition(
+                newconfig.get('drop_disposition', {})
+            )
+            write_config(options['_runtime_config_file'], newconfig)
+        except Exception as e:
+            if options['daemon']:
+                log.exception(e)
+            else:
+                raise e
+
+signal.signal(signal.SIGTERM, terminate)
+signal.signal(signal.SIGINT, terminate)
+signal.signal(signal.SIGHUP, reload)
+
+
+# monitor and capture
+mail_tail.start()
+mail_tail.join()
+

management/reporting/capture/db/DatabaseConnectionFactory.py

@@ -0,0 +1,10 @@
+
+class DatabaseConnectionFactory(object):
+    def connect(self):
+        raise NotImplementedError()
+
+    def close(self, conn):
+        raise NotImplementedError()
+    
+
+    

management/reporting/capture/db/EventStore.py

@@ -0,0 +1,122 @@
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
+
+import threading
+import queue
+import logging
+from .Prunable import Prunable
+
+log = logging.getLogger(__name__)
+
+
+'''subclass this and override:
+
+    write_rec()
+    read_rec()
+
+to provide storage for event "records"
+
+EventStore is thread safe and uses a single thread to write all
+records.
+
+'''
+
+class EventStore(Prunable):
+	def __init__(self, db_conn_factory):
+		self.db_conn_factory = db_conn_factory
+		# we'll have a single thread do all the writing to the database
+		#self.queue = queue.SimpleQueue()  # available in Python 3.7+
+		self.queue = queue.Queue()
+		self.interrupt = threading.Event()
+		self.rec_added = threading.Event()
+		self.have_event = threading.Event()
+		self.t = threading.Thread(
+			target=self._bg_writer,
+			name="EventStore",
+			daemon=True
+		)
+		self.max_queue_size = 100000
+		self.t.start()
+
+	def connect(self):
+		return self.db_conn_factory.connect()
+
+	def close(self, conn):
+		self.db_conn_factory.close(conn)
+
+	def write_rec(self, conn, type, rec):
+		'''write a "rec" of the given "type" to the database. The subclass
+		must know how to do that. "type" is a string identifier of the
+		subclass's choosing. Users of this class should call store()
+		and not this function, which will queue the request and a
+		thread managed by this class will call this function.
+
+		'''
+		raise NotImplementedError()
+		
+	def read_rec(self, conn, type, args):
+		'''read from the database'''
+		raise NotImplementedError()
+
+	def prune(self, conn):
+		raise NotImplementedError()
+	
+	def store(self, type, rec):
+		self.queue.put({
+			'type': type,
+			'rec': rec
+		})
+		self.rec_added.set()
+		self.have_event.set()
+
+	def stop(self):
+		self.interrupt.set()
+		self.have_event.set()
+		self.t.join()
+
+	def __del__(self):
+		log.debug('EventStore __del__')
+		self.interrupt.set()
+		self.have_event.set()
+
+	def _pop(self):
+		try:
+			return self.queue.get(block=False)
+		except queue.Empty:
+			return None
+		
+	def _bg_writer(self):
+		log.debug('start EventStore thread')
+		conn = self.connect()
+		try:
+			while not self.interrupt.is_set() or not self.queue.empty():
+				item = self._pop()
+				if item:
+					try:
+						self.write_rec(conn, item['type'], item['rec'])
+					except Exception as e:
+						log.exception(e)
+						retry_count = item.get('retry_count', 0)
+						if self.interrupt.is_set():
+							log.warning('interrupted, dropping record: %s',item)
+						elif retry_count > 2:
+							log.warning('giving up after %s attempts, dropping record: %s', retry_count, item)
+						elif self.queue.qsize() >= self.max_queue_size:
+							log.warning('queue full, dropping record: %s', item)
+						else:
+							item['retry_count'] = retry_count + 1
+							self.queue.put(item)
+							# wait for another record to prevent immediate retry
+							if not self.interrupt.is_set():
+								self.have_event.wait()
+								self.rec_added.clear()
+								self.have_event.clear()
+					self.queue.task_done()   # remove for SimpleQueue
+							
+				else:
+					self.have_event.wait()
+					self.rec_added.clear()
+					self.have_event.clear()
+				
+		finally:
+			self.close(conn)			
+

management/reporting/capture/db/Prunable.py

@@ -0,0 +1,7 @@
+
+
+class Prunable(object):
+    def prune(self, conn, policy):
+        raise NotImplementedError()
+
+    

management/reporting/capture/db/Pruner.py

@@ -0,0 +1,79 @@
+
+import threading
+import logging
+
+log = logging.getLogger(__name__)
+
+
+class Pruner(object):
+    '''periodically calls the prune() method of registered Prunable
+    objects
+
+    '''
+    def __init__(self, db_conn_factory, policy={
+            'older_than_days': 7,
+            'frequency_min': 60
+    }):
+        self.db_conn_factory = db_conn_factory
+        self.policy = policy
+        self.prunables = []
+        self.interrupt = threading.Event()
+        self._new_thread()
+        self.t.start()
+
+
+    def _new_thread(self):
+        self.interrupt.clear()
+        self.t = threading.Thread(
+            target=self._bg_pruner,
+	    name="Pruner",
+	    daemon=True
+        )
+        
+    def add_prunable(self, inst):
+        self.prunables.append(inst)
+
+    def set_policy(self, policy):
+        self.stop()
+        self.policy = policy
+        # a new thread object must be created or Python(<3.8?) throws
+        # RuntimeError("threads can only be started once")
+        self._new_thread()
+        self.t.start()
+
+    def stop(self, do_join=True):
+        self.interrupt.set()
+        if do_join:
+            self.t.join()
+
+    def connect(self):
+        return self.db_conn_factory.connect()
+
+    def close(self, conn):
+        self.db_conn_factory.close(conn)
+                
+    def __del__(self):
+        self.stop(do_join=False)
+    
+    def _bg_pruner(self):
+        conn = self.connect()
+
+        def do_prune():
+            for prunable in self.prunables:
+                if not self.interrupt.is_set():
+                    try:
+                        prunable.prune(conn, self.policy)
+                    except Exception as e:
+                        log.exception(e)
+                        
+        try:
+            # prune right-off
+            do_prune()
+            
+            while not self.interrupt.is_set():
+                # wait until interrupted or it's time to prune
+                if self.interrupt.wait(self.policy['frequency_min'] * 60) is not True:
+                    do_prune()
+
+        finally:
+            self.close(conn)

management/reporting/capture/db/SqliteConnFactory.py

@@ -0,0 +1,52 @@
+import os, stat
+import sqlite3
+import logging
+import threading
+
+from .DatabaseConnectionFactory import DatabaseConnectionFactory
+
+log = logging.getLogger(__name__)
+
+
+class SqliteConnFactory(DatabaseConnectionFactory):
+    def __init__(self, db_path):
+        super(SqliteConnFactory, self).__init__()
+        log.debug('factory for %s', db_path)
+        self.db_path = db_path
+        self.db_basename = os.path.basename(db_path)
+        self.ensure_exists()
+
+    def ensure_exists(self):
+        # create the parent directory and set its permissions
+        parent = os.path.dirname(self.db_path)
+        if parent != '' and not os.path.exists(parent):
+            os.makedirs(parent)
+            os.chmod(parent,
+                     stat.S_IRWXU |
+                     stat.S_IRGRP |
+                     stat.S_IXGRP |
+                     stat.S_IROTH |
+                     stat.S_IXOTH
+            )
+
+        # if the database is new, create an empty file and set file
+        # permissions
+        if not os.path.exists(self.db_path):
+            log.debug('creating empty database: %s', self.db_basename)
+            with open(self.db_path, 'w') as fp:
+                pass
+
+            os.chmod(self.db_path,
+                     stat.S_IRUSR |
+                     stat.S_IWUSR
+            )
+
+    def connect(self):
+        log.debug('opening database %s', self.db_basename)
+        conn = sqlite3.connect(self.db_path)
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    def close(self, conn):
+        log.debug('closing database %s', self.db_basename)
+        conn.close()

management/reporting/capture/db/SqliteEventStore.py

@@ -0,0 +1,368 @@
+# -*- indent-tabs-mode: t; tab-width: 4; python-indent-offset: 4; -*-
+
+import sqlite3
+import os, stat
+import logging
+import json
+import datetime
+from .EventStore import EventStore
+
+log = logging.getLogger(__name__)
+
+#
+# schema
+#
+mta_conn_fields = [
+	'service',
+	'service_tid',
+	'connect_time',
+	'disconnect_time',
+	'remote_host',
+	'remote_ip',
+	'sasl_method',
+	'sasl_username',
+	'remote_auth_success',
+	'remote_auth_attempts',
+	'remote_used_starttls',
+	'disposition',
+]
+
+mta_accept_fields = [
+	'mta_conn_id',
+	'queue_time',
+	'queue_remove_time',
+	'subsystems',
+#	'spf_tid',
+	'spf_result',
+	'spf_reason',
+	'postfix_msg_id',
+	'message_id',
+	'dkim_result',
+	'dkim_reason',
+	'dmarc_result',
+	'dmarc_reason',
+	'envelope_from',
+	'message_size',
+	'message_nrcpt',
+    'accept_status',
+    'failure_info',
+	'failure_category',
+]
+
+mta_delivery_fields = [
+	'mta_accept_id',
+	'service',
+#	'service_tid',
+	'rcpt_to',
+#	'postgrey_tid',
+	'postgrey_result',
+	'postgrey_reason',
+	'postgrey_delay',
+#	'spam_tid',
+	'spam_result',
+	'spam_score',
+	'relay',
+    'status',
+	'delay',
+	'delivery_connection',
+	'delivery_connection_info',
+	'delivery_info',
+	'failure_category',
+]
+
+
+db_info_create_table_stmt = "CREATE TABLE IF NOT EXISTS db_info(id INTEGER PRIMARY KEY AUTOINCREMENT, key TEXT NOT NULL, value TEXT NOT NULL)"
+
+schema_updates = [
+	# update 0
+	[
+		# three "mta" tables having a one-to-many-to-many relationship:
+		#    mta_connection(1) -> mta_accept(0:N) -> mta_delivery(0:N)
+		#
+
+		"CREATE TABLE mta_connection(\
+			mta_conn_id INTEGER PRIMARY KEY AUTOINCREMENT,\
+			service TEXT NOT NULL,  /* 'smtpd', 'submission' or 'pickup' */\
+			service_tid TEXT NOT NULL,\
+			connect_time TEXT NOT NULL,\
+			disconnect_time TEXT,\
+			remote_host TEXT COLLATE NOCASE,\
+			remote_ip TEXT COLLATE NOCASE,\
+			sasl_method TEXT,             /* sasl: submission service only */\
+			sasl_username TEXT COLLATE NOCASE,\
+			remote_auth_success INTEGER,  /* count of successes */\
+			remote_auth_attempts INTEGER, /* count of attempts */\
+			remote_used_starttls INTEGER, /* 1 if STARTTLS used */\
+			disposition TEXT     /* 'normal','scanner','login_attempt',etc */\
+		)",
+
+		"CREATE INDEX idx_mta_connection_connect_time ON mta_connection(connect_time, sasl_username COLLATE NOCASE)",
+
+
+		"CREATE TABLE mta_accept(\
+			mta_accept_id INTEGER PRIMARY KEY AUTOINCREMENT,\
+			mta_conn_id INTEGER,\
+			queue_time TEXT,\
+			queue_remove_time TEXT,\
+			subsystems TEXT,\
+			/*spf_tid TEXT,*/\
+			spf_result TEXT,\
+			spf_reason TEXT,\
+			postfix_msg_id TEXT,\
+			message_id TEXT,\
+			dkim_result TEXT,\
+			dkim_reason TEXT,\
+			dmarc_result TEXT,\
+			dmarc_reason TEXT,\
+			envelope_from TEXT COLLATE NOCASE,\
+			message_size INTEGER,\
+			message_nrcpt INTEGER,\
+            accept_status TEXT,       /* 'accept','greylist','spf-reject',others... */\
+            failure_info TEXT,        /* details from mta or subsystems */\
+			failure_category TEXT,\
+            FOREIGN KEY(mta_conn_id) REFERENCES mta_connection(mta_conn_id) ON DELETE RESTRICT\
+		)",
+
+		"CREATE TABLE mta_delivery(\
+			mta_delivery_id INTEGER PRIMARY KEY AUTOINCREMENT,\
+			mta_accept_id INTEGER,\
+			service TEXT,           /* 'lmtp' or 'smtp' */\
+		    /*service_tid TEXT,*/\
+			rcpt_to TEXT COLLATE NOCASE,  /* email addr */\
+			/*postgrey_tid TEXT,*/\
+			postgrey_result TEXT,\
+			postgrey_reason TEXT,\
+			postgrey_delay NUMBER,\
+			/*spam_tid TEXT,*/         /* spam: lmtp only */\
+			spam_result TEXT,      /* 'clean' or 'spam' */\
+			spam_score NUMBER,     /* eg: 2.10 */\
+			relay TEXT,            /* hostname[IP]:port */\
+            status TEXT,           /* 'sent', 'bounce', 'reject', etc */\
+			delay NUMBER,          /* fractional seconds, 'sent' status only */\
+		    delivery_connection TEXT, /* 'trusted' or 'untrusted' */\
+		    delivery_connection_info TEXT, /* details on TLS connection */\
+			delivery_info TEXT,    /* details from the remote mta */\
+			failure_category TEXT,\
+            FOREIGN KEY(mta_accept_id) REFERENCES mta_accept(mta_accept_id) ON DELETE RESTRICT\
+		)",
+
+		"CREATE INDEX idx_mta_delivery_rcpt_to ON mta_delivery(rcpt_to COLLATE NOCASE)",
+
+		"CREATE TABLE state_cache(\
+		    state_cache_id INTEGER PRIMARY KEY AUTOINCREMENT,\
+		    owner_id INTEGER NOT NULL,\
+            state TEXT\
+		)",
+
+		"INSERT INTO db_info (key,value) VALUES ('schema_version', '0')"
+	]
+]
+
+
+
+class SqliteEventStore(EventStore):
+	
+	def __init__(self, db_conn_factory):
+		super(SqliteEventStore, self).__init__(db_conn_factory)
+		self.update_schema()
+		
+	def update_schema(self):
+		''' update the schema to the latest version
+
+		'''
+		c = None
+		conn = None
+		try:
+			conn = self.connect()
+			c = conn.cursor()
+			c.execute(db_info_create_table_stmt)
+			conn.commit()
+			c.execute("SELECT value from db_info WHERE key='schema_version'")
+			v = c.fetchone()
+			if v is None:
+				v = -1
+			else:
+				v = int(v[0])
+			for idx in range(v+1, len(schema_updates)):
+				log.info('updating database to v%s', idx)
+				for stmt in schema_updates[idx]:
+					try:
+						c.execute(stmt)
+					except Exception as e:
+						log.error('problem with sql statement at version=%s error="%s" stmt="%s"' % (idx, e, stmt))
+						raise e
+					
+			conn.commit()
+			
+		finally:
+			if c: c.close(); c=None
+			if conn: self.close(conn); conn=None
+		
+
+
+	def write_rec(self, conn, type, rec):
+		if type=='inbound_mail':
+			#log.debug('wrote inbound_mail record')
+			self.write_inbound_mail(conn, rec)
+		elif type=='state':
+			''' rec: {
+			        owner_id: int,
+			        state: list
+			    }
+			'''
+			self.write_state(conn, rec)
+		else:
+			raise ValueError('type "%s" not implemented' % type)
+
+
+	def _insert(self, table, fields):
+		insert = 'INSERT INTO ' + table + ' (' + \
+			",".join(fields) + \
+			') VALUES (' + \
+			"?,"*(len(fields)-1) + \
+			'?)'
+		return insert
+
+	def _values(self, fields, data_dict):
+		values = []
+		for field in fields:
+			if field in data_dict:
+				values.append(data_dict[field])
+				data_dict.pop(field)
+			else:
+				values.append(None)
+
+		for field in data_dict:
+			if type(data_dict[field]) != list and not field.startswith('_') and not field.endswith('_tid'):
+				log.warning('unused field: %s', field)
+		return values
+
+			
+	def write_inbound_mail(self, conn, rec):
+		c = None
+		try:
+			c = conn.cursor()
+			
+			# mta_connection
+			insert = self._insert('mta_connection', mta_conn_fields)
+			values = self._values(mta_conn_fields, rec)
+			#log.debug('INSERT: %s VALUES: %s REC=%s', insert, values, rec)
+			c.execute(insert, values)
+			conn_id = c.lastrowid
+
+			accept_insert = self._insert('mta_accept', mta_accept_fields)
+			delivery_insert = self._insert('mta_delivery', mta_delivery_fields)
+			for accept in rec.get('mta_accept', []):
+				accept['mta_conn_id'] = conn_id
+				values = self._values(mta_accept_fields, accept)
+				c.execute(accept_insert, values)
+				accept_id = c.lastrowid
+				
+				for delivery in accept.get('mta_delivery', []):
+					delivery['mta_accept_id'] = accept_id
+					values = self._values(mta_delivery_fields, delivery)
+					c.execute(delivery_insert, values)
+
+			conn.commit()
+
+		except sqlite3.Error as e:
+			conn.rollback()
+			raise e
+			
+		finally:
+			if c: c.close(); c=None
+			
+
+			
+	def write_state(self, conn, rec):
+		c = None
+		try:
+			c = conn.cursor()
+			
+			owner_id = rec['owner_id']
+			insert = 'INSERT INTO state_cache (owner_id, state) VALUES (?, ?)'
+			for item in rec['state']:
+				item_json = json.dumps(item)
+				c.execute(insert, (owner_id, item_json))
+				
+			conn.commit()
+
+		except sqlite3.Error as e:
+			conn.rollback()
+			raise e
+			
+		finally:
+			if c: c.close(); c=None
+
+			
+	def read_rec(self, conn, type, args):
+		if type=='state':
+			return self.read_state(
+				conn,
+				args['owner_id'],
+				args.get('clear',False)
+			)
+		else:
+			raise ValueError('type "%s" not implemented' % type)
+		
+	def read_state(self, conn, owner_id, clear):
+		c = None
+		state = []
+		try:
+			c = conn.cursor()
+			select = 'SELECT state FROM state_cache WHERE owner_id=? ORDER BY state_cache_id'
+			for row in c.execute(select, (owner_id,)):
+				state.append(json.loads(row[0]))
+
+			if clear:
+				delete = 'DELETE FROM state_cache WHERE owner_id=?'
+				c.execute(delete, (owner_id,))
+				conn.commit()
+
+		finally:
+			if c: c.close(); c=None
+
+		return state
+	
+	def prune(self, conn, policy):
+		older_than_days = datetime.timedelta(days=policy['older_than_days'])
+		if older_than_days.days <= 0:
+			return
+		now = datetime.datetime.now(datetime.timezone.utc)
+		d = (now - older_than_days)
+		dstr = d.isoformat(sep=' ', timespec='seconds')
+		
+		c = None
+		try:
+			c = conn.cursor()
+			deletes = [
+				'DELETE FROM mta_delivery WHERE mta_accept_id IN (\
+				  SELECT mta_accept.mta_accept_id FROM mta_accept\
+				  JOIN mta_connection ON mta_connection.mta_conn_id = mta_accept.mta_conn_id\
+				  WHERE connect_time < ?)',
+
+				'DELETE FROM mta_accept WHERE mta_accept_id IN (\
+				  SELECT mta_accept.mta_accept_id FROM mta_accept\
+				  JOIN mta_connection ON mta_connection.mta_conn_id = mta_accept.mta_conn_id\
+				  WHERE connect_time < ?)',
+
+				'DELETE FROM mta_connection WHERE connect_time < ?'
+			]
+
+			counts = []
+			for delete in deletes:
+				c.execute(delete, (dstr,))
+				counts.append(str(c.rowcount))
+			conn.commit()
+			counts.reverse()
+			log.info("pruned %s rows", "/".join(counts))
+
+		except sqlite3.Error as e:
+			conn.rollback()
+			raise e
+				
+		finally:
+			if c: c.close()
+
+
+	

management/reporting/capture/logs/DateParser.py

@@ -0,0 +1,33 @@
+import datetime
+import pytz
+
+rsyslog_traditional_regexp = '^(.{15})'
+
+with open('/etc/timezone') as fp:
+    timezone_id = fp.read().strip()
+
+
+def rsyslog_traditional(str):
+    # Handles the default timestamp in rsyslog
+    # (RSYSLOG_TraditionalFileFormat)
+    #
+    # eg: "Dec 6 06:25:04"  (always 15 characters)
+    #
+    # the date string is in local time
+    #
+    d = datetime.datetime.strptime(str, '%b %d %H:%M:%S')
+
+    # since the log date has no year, use the current year
+    today = datetime.date.today()
+    year = today.year
+    if d.month == 12 and today.month == 1:
+        year -= 1
+    d = d.replace(year=year)
+
+    # convert to UTC
+    if timezone_id == 'Etc/UTC':
+        return d
+    local_tz = pytz.timezone(timezone_id)
+    return local_tz.localize(d, is_dst=None).astimezone(pytz.utc)
+
+    

management/reporting/capture/logs/ReadLineHandler.py

@@ -0,0 +1,15 @@
+
+'''subclass this and override methods to handle log output'''
+class ReadLineHandler(object):
+    def handle(self, line):
+        ''' handle a single line of output '''
+        raise NotImplementedError()
+    
+    def end_of_callbacks(self, thread):
+        '''called when no more output will be sent to handle(). override this
+        method to save state, or perform cleanup during this
+        callback
+
+        '''
+        pass
+    

management/reporting/capture/logs/ReadPositionStore.py

@@ -0,0 +1,26 @@
+'''subclass this and override all methods to persist the position of
+the log file that has been processed so far.
+
+this enables the log monitor to pick up where it left off
+
+a single FilePositionStore can safely be used with multiple
+LogMonitor instances
+
+'''
+class ReadPositionStore(object):
+    def get(self, log_file, inode):
+        '''return the offset from the start of the file of the last
+        position saved for log_file having the given inode, or zero if
+        no position is currently saved
+
+        '''
+        raise NotImplementedError()
+
+    def save(self, log_file, inode, offset):
+        '''save the current position'''
+        raise NotImplementedError()
+
+    def clear(self, log_file):
+        '''remove all entries for `log_file`'''
+        raise NotImplementedError()
+    

management/reporting/capture/logs/ReadPositionStoreInFile.py

@@ -0,0 +1,84 @@
+from .ReadPositionStore import ReadPositionStore
+
+import threading
+import json
+import os
+import logging
+
+log = logging.getLogger(__name__)
+
+
+class ReadPositionStoreInFile(ReadPositionStore):
+    def __init__(self, output_file):
+        self.output_file = output_file
+        self.changed = False
+        self.lock = threading.Lock()
+        self.interrupt = threading.Event()
+
+        if os.path.exists(output_file):
+            with open(output_file, "r", encoding="utf-8") as fp:
+                self.db = json.loads(fp.read())
+        else:
+            self.db = {}
+
+        self.t = threading.Thread(
+            target=self._persist_bg,
+            name="ReadPositionStoreInFile",
+            daemon=True
+        )
+        self.t.start()
+
+    def __del__(self):
+        log.debug('ReadPositionStoreInFile __del__')
+        self.interrupt.set()
+
+    def stop(self):
+        self.interrupt.set()
+        self.t.join()
+        
+    def get(self, file, inode):
+        with self.lock:
+            if file in self.db and str(inode) in self.db[file]:
+                return self.db[file][str(inode)]
+        return 0            
+    
+    def save(self, file, inode, pos):
+        with self.lock:
+            if not file in self.db:
+                self.db[file] = { str(inode):pos }
+            else:
+                self.db[file][str(inode)] = pos
+            self.changed = True
+    
+    def clear(self, file):
+        with self.lock:
+            self.db[file] = {}
+            self.changed = True
+
+
+    def persist(self):
+        if self.changed:
+            try:
+                with open(self.output_file, "w") as fp:
+                    with self.lock:
+                        json_str = json.dumps(self.db)
+                        self.changed = False
+                        
+                    try:
+                        fp.write(json_str)
+                    except Exception as e:
+                        with self.lock:
+                            self.changed = True
+                        log.error(e)
+                        
+            except Exception as e:
+                log.error(e)
+
+        
+    def _persist_bg(self):
+        while not self.interrupt.is_set():
+            # wait 60 seconds before persisting
+            self.interrupt.wait(60)
+            # even if interrupted, persist one final time
+            self.persist()
+                        

management/reporting/capture/logs/TailFile.py

@@ -0,0 +1,160 @@
+import threading
+import os
+import logging
+import stat
+
+from .ReadLineHandler import ReadLineHandler
+
+log = logging.getLogger(__name__)
+
+'''Spawn a thread to "tail" a log file. For each line read, provided
+callbacks do something with the output. Callbacks must be a subclass
+of ReadLineHandler.
+
+'''
+
+class TailFile(threading.Thread):
+    def __init__(self, log_file, store=None):
+        ''' log_file - the log file to monitor
+            store - a ReadPositionStore instance
+        '''
+        self.log_file = log_file
+        self.store = store
+
+        self.fp = None
+        self.inode = None
+        self.callbacks = []
+        self.interrupt = threading.Event()
+
+        name=f'{__name__}-{os.path.basename(log_file)}'
+        log.debug('init thread: %s', name)
+        super(TailFile, self).__init__(name=name, daemon=True)
+        
+    def stop(self, do_join=True):
+        log.debug('TailFile stopping')
+        self.interrupt.set()
+        # close must be called to unblock the thread fp.readline() call
+        self._close()
+        if do_join:
+            self.join()
+
+    def __del__(self):
+        self.stop(do_join=False)
+
+    def add_handler(self, fn):
+        assert self.is_alive() == False
+        self.callbacks.append(fn)
+
+    def clear_callbacks(self):
+        assert self.is_alive() == False
+        self.callbacks = []
+
+    def _open(self):
+        self._close()
+        self.inode = os.stat(self.log_file)[stat.ST_INO]
+        self.fp = open(
+            self.log_file,
+            "r",
+            encoding="utf-8",
+            errors="backslashreplace"
+        )
+
+    def _close(self):        
+        if self.fp is not None:
+            self.fp.close()
+        self.fp = None
+
+    def _is_rotated(self):
+        try:
+            return os.stat(self.log_file)[stat.ST_INO] != self.inode
+        except FileNotFoundError:
+            return False
+
+    def _issue_callbacks(self, line):
+        for cb in self.callbacks:
+            if isinstance(cb, ReadLineHandler):
+                cb.handle(line)
+            else:
+                cb(line)
+
+    def _notify_end_of_callbacks(self):
+        for cb in self.callbacks:
+            if isinstance(cb, ReadLineHandler):
+                cb.end_of_callbacks(self)
+
+    def _restore_read_position(self):
+        if self.fp is None:
+            return
+        
+        if self.store is None:
+            self.fp.seek(
+                0,
+                os.SEEK_END
+            )
+        else:
+            pos = self.store.get(self.log_file, self.inode)
+            size = os.stat(self.log_file)[stat.ST_SIZE]
+            if size < pos:
+                log.debug("truncated: %s" % self.log_file)
+                self.fp.seek(0, os.SEEK_SET)
+            else:
+                # if pos>size here, the seek call succeeds and returns
+                # 'pos', but future reads will fail
+                self.fp.seek(pos, os.SEEK_SET)
+        
+    def run(self):
+        self.interrupt.clear()
+
+        # initial open - wait until file exists
+        while not self.interrupt.is_set() and self.fp is None:
+            try:
+                self._open()
+            except FileNotFoundError:
+                log.debug('log file "%s" not found, waiting...', self.log_file)
+                self.interrupt.wait(2)
+                continue
+
+        # restore reading position
+        self._restore_read_position()
+        
+        while not self.interrupt.is_set():                
+            try:
+                line = self.fp.readline() # blocking
+                if line=='':
+                    log.debug('got EOF')
+                    # EOF - check if file was rotated
+                    if self._is_rotated():
+                        log.debug('rotated')
+                        self._open()
+                        if self.store is not None:
+                            self.store.clear(self.log_file)
+                            
+                    # if not rotated, sleep
+                    else:
+                        self.interrupt.wait(1)
+                        
+                else:
+                    # save position and call all callbacks
+                    if self.store is not None:
+                        self.store.save(
+                            self.log_file,
+                            self.inode,
+                            self.fp.tell()
+                        )
+                    self._issue_callbacks(line)
+
+            except Exception as e:
+                log.exception(e)
+                if self.interrupt.wait(1) is not True:
+                    if self._is_rotated():
+                        self._open()
+
+
+        self._close()
+
+        try:
+            self._notify_end_of_callbacks()
+        except Exception as e:
+            log.exception(e)
+            
+        

management/reporting/capture/mail/PostfixLogParser.py

@@ -0,0 +1,125 @@
+
+class PostfixLogParser(object):
+
+    @staticmethod
+    def split_host(str):
+        ''' split string in form HOST[IP] and return HOST and IP '''
+        ip_start = str.find('[')
+        ip_end = -1
+        if ip_start>=0:
+            ip_end = str.find(']', ip_start)
+            if ip_start<0 or ip_end<0:
+                return str, str
+            return str[0:ip_start], str[ip_start+1:ip_end]
+
+    @staticmethod
+    def strip_brackets(str, bracket_l='<', bracket_r='>'):
+        # strip enclosing '<>'
+        if len(str)>=2 and str[0]==bracket_l and str[-1]==bracket_r:
+            return str[1:-1]
+        return str
+
+
+    class SplitList(object):
+        ''' split a postfix name=value list. For example:
+
+        "delay=4.7, to=<alice@post.com>, status=sent (250 2.0.0 <user@domain.tld> YB5nM1eS01+lSgAAlWWVsw Saved)"
+
+        returns: {
+           "delay": {
+                "name": "delay",
+                "value": "4.7"
+           },
+           "to": {
+                "name": "to",
+                "value": "alice@post.com"
+           },
+           "status": {
+                "name": "status",
+                "value": "sent",
+                "comment": "250 2.0.0 <user@domain.tld> YB5nM1eS01+lSgAAlWWVsw Saved"
+           }
+        }
+
+        '''
+        def __init__(self, str, delim=',', strip_brackets=True):
+            self.str = str
+            self.delim = delim
+            self.strip_brackets = True
+            self.pos = 0
+
+        def asDict(self):
+            d = {}
+            for pair in self:
+                d[pair['name']] = pair
+            return d
+
+        def __iter__(self):
+            self.pos = 0
+            return self
+
+        def __next__(self):
+            if self.pos >= len(self.str):
+                raise StopIteration
+
+            # name
+            eq = self.str.find('=', self.pos)
+            if eq<0:
+                self.pos = len(self.str)
+                raise StopIteration
+
+            name = self.str[self.pos:eq].strip()
+
+            # value and comment
+            self.pos = eq+1
+            value = []
+            comment = []
+
+            while self.pos < len(self.str):
+                c = self.str[self.pos]
+                self.pos += 1
+
+                if c=='<':
+                    idx = self.str.find('>', self.pos)
+                    if idx>=0:
+                        value.append(self.str[self.pos-1:idx+1])
+                        self.pos = idx+1
+                        continue
+
+                if c=='(':
+                    # parens may be nested...
+                    open_count = 1
+                    begin = self.pos
+                    while self.pos < len(self.str) and open_count>0:
+                        c = self.str[self.pos]
+                        self.pos += 1
+                        if c=='(':
+                            open_count += 1
+                        elif c==')':
+                            open_count -= 1
+                    if open_count == 0:
+                        comment.append(self.str[begin:self.pos-1])
+                    else:
+                        comment.append(self.str[begin:len(self.str)])
+                    continue
+
+                if c==self.delim:
+                    break
+
+                begin = self.pos-1
+                while self.pos < len(self.str):
+                    lookahead = self.str[self.pos]
+                    if lookahead in [self.delim,'<','(']:
+                        break
+                    self.pos += 1
+
+                value.append(self.str[begin:self.pos])
+
+            if self.strip_brackets and len(value)==1:
+                value[0] = PostfixLogParser.strip_brackets(value[0])
+
+            return {
+                'name': name,
+                'value': ''.join(value),
+                'comment': None if len(comment)==0 else '; '.join(comment)
+            }

management/reporting/capture/util/DictQuery.py

@@ -0,0 +1,108 @@
+
+
+class DictQuery(object):
+
+    @staticmethod
+    def find(data_list, q_list, return_first_exact=False, reverse=False):
+        '''find items in list `data_list` using the query specified in
+           `q_list` (a list of dicts).
+
+            side-effects:
+               q_list is modified ('_val' is added)
+
+        '''
+        if data_list is None:
+            if return_first_exact:
+                return None
+            else:
+                return []
+
+        if type(q_list) is not list:
+            q_list = [ q_list ]
+
+        # set _val to value.lower() if ignorecase is True
+        for q in q_list:
+            if q=='*': continue
+            ignorecase = q.get('ignorecase', False)
+            match_val = q['value']
+            if ignorecase and match_val is not None:
+                match_val = match_val.lower()
+            q['_val'] = match_val
+
+        # find all matches
+        matches = []
+        direction = -1 if reverse else 1
+        idx = len(data_list)-1 if reverse else 0
+        while (reverse and idx>=0) or (not reverse and idx<len(data_list)):
+            item = data_list[idx]
+            if 'rank' in item and 'item' in item:
+                # for re-querying... 
+                item=item['item']
+            count_mismatch = 0
+            autoset_list = []
+            optional_list = []
+
+            for q in q_list:
+                if q=='*': continue
+                cmp_val = item.get(q['key'])
+                if cmp_val is not None and q.get('ignorecase'):
+                    cmp_val = cmp_val.lower()
+
+                op = q.get('op', '=')
+                mismatch = False
+                if op == '=':
+                    mismatch = q['_val'] != cmp_val
+                elif op == '!=':
+                    mismatch = q['_val'] == cmp_val
+                else:
+                    raise TypeError('No such op:  ' + op)
+                    
+                if mismatch:
+                    count_mismatch += 1                        
+                    if cmp_val is None:
+                        if q.get('autoset'):
+                            autoset_list.append(q)
+                        elif q.get('optional'):
+                            optional_list.append(q)
+                    if return_first_exact:
+                        break
+
+            if return_first_exact:
+                if count_mismatch == 0:
+                    return item
+            else:            
+                optional_count = len(autoset_list) + len(optional_list)
+                if count_mismatch - optional_count == 0:
+                    rank = '{0:05d}.{1:08d}'.format(
+                        optional_count,
+                        len(data_list) - idx if reverse else idx
+                    )
+                    matches.append({
+                        'exact': ( optional_count == 0 ),
+                        'rank': rank,
+                        'autoset_list': autoset_list,
+                        'optional_list': optional_list,
+                        'item': item
+                    })
+
+            idx += direction
+
+        if not return_first_exact:
+            # return the list sorted so the items with the fewest
+            # number of required autoset/optional's appear first
+            matches.sort(key=lambda x: x['rank'])
+            return matches
+
+
+    @staticmethod
+    def autoset(match, incl_optional=False):
+        item = match['item']
+        for q in match['autoset_list']:
+            assert item.get(q['key']) is None
+            item[q['key']] = q['value']
+        if incl_optional:
+            for q in match['optional_list']:
+                item[q['key']] = q['value']
+
+
+

management/reporting/capture/util/env.py

@@ -0,0 +1,9 @@
+def load_env_vars_from_file(fn):
+    # Load settings from a KEY=VALUE file.
+    env = {}
+    for line in open(fn):
+        env.setdefault(*line.strip().split("=", 1))
+    # strip_quotes:
+    for k in env: env[k]=env[k].strip('"')
+    return env
+

management/reporting/capture/util/safe.py

@@ -0,0 +1,18 @@
+def safe_int(str, default_value=0):
+    try:
+        return int(str)
+    except ValueError:
+        return default_value
+
+def safe_append(d, key, value):
+    if key not in d:
+        d[key] = [ value ]
+    else:
+        d[key].append(value)
+    return d
+
+def safe_del(d, key):
+    if key in d:
+        del d[key]
+    return d
+

management/reporting/ui/capture-db-stats.js

@@ -0,0 +1,95 @@
+
+
+Vue.component('capture-db-stats', {
+    props: {
+    },
+
+    template:'<div>'+
+        '<template v-if="stats">'+
+           '<caption class="text-nowrap">Database date range</caption><div class="ml-2">First: {{stats.mta_connect.connect_time.min_str}}</div><div class="ml-2">Last: {{stats.mta_connect.connect_time.max_str}}</div>'+
+           '<div class="mt-2">'+
+           '  <b-table-lite small caption="Connections by disposition" caption-top :fields="row_counts.fields" :items=row_counts.items></b-table-lite>'+
+           '</div>'+
+        '</template>'+
+        '<spinner v-else></spinner>'+
+        '</div>'
+    ,
+
+    data: function() {
+        return {
+            stats: null,
+            stats_time: null,
+            row_counts: {}
+        };
+    },
+
+    created: function() {
+        this.getStats();
+    },
+
+    methods: {
+        getStats: function() {
+            axios.get('/reports/capture/db/stats')
+                .then(response => {
+                    this.stats = response.data;
+                    this.stats_time = Date.now();
+
+                    // convert dates
+                    var parser = d3.utcParse(this.stats.date_parse_format);
+                    [ 'min', 'max' ].forEach( k => {
+                        var d = parser(this.stats.mta_connect.connect_time[k]);
+                        this.stats.mta_connect.connect_time[k] = d;
+                        this.stats.mta_connect.connect_time[k+'_str'] =
+                            d==null ? '-' : DateFormatter.dt_long(d);
+                    });
+
+                    // make a small bvTable of row counts
+                    this.row_counts = {
+                        items: [],
+                        fields: [ 'name', 'count', 'percent' ],
+                        field_types: [
+                            { type:'text/plain', label:'Disposition' },
+                            'number/plain',
+                            { type: 'number/percent', label:'Pct', places:1 },
+                        ],
+                    };
+                    BvTable.setFieldDefinitions(
+                        this.row_counts.fields,
+                        this.row_counts.field_types
+                    );
+                    this.row_counts.fields[0].formatter = (v, key, item) => {
+                        return new ConnectionDisposition(v).short_desc
+                    };
+                    this.row_counts.fields[0].tdClass = 'text-capitalize';
+
+
+                    const total = this.stats.mta_connect.count;
+                    for (var name in this.stats.mta_connect.disposition)
+                    {
+                        const count =
+                              this.stats.mta_connect.disposition[name].count;
+                        this.row_counts.items.push({
+                            name: name,
+                            count: count,
+                            percent: count / total
+                        });
+                    }
+                    this.row_counts.items.sort((a,b) => {
+                        return a.count > b.count ? -1 :
+                            a.count < b.count ? 1 : 0;
+                    })
+                    this.row_counts.items.push({
+                        name:'Total',
+                        count:this.stats.mta_connect.count,
+                        percent:1,
+                        '_rowVariant': 'primary'
+                    });
+
+                    
+                })
+                .catch(error => {
+                    this.$root.handleError(error);
+                });
+        },
+    }
+});

management/reporting/ui/chart-multi-line-timeseries.js

@@ -0,0 +1,206 @@
+Vue.component('chart-multi-line-timeseries', {
+    props: {
+        chart_data: { type:Object, required:false }, /* TimeseriesData */
+        width: { type:Number, default: ChartPrefs.default_width },
+        height: { type:Number, default: ChartPrefs.default_height },
+    },
+    
+    render: function(ce) {
+        return ChartVue.create_svg(ce, [0, 0, this.width, this.height]);
+    },
+
+    data: function() {
+        return {
+            tsdata: this.chart_data,
+            margin: {
+                top: ChartPrefs.axis_font_size,
+                bottom: ChartPrefs.axis_font_size * 2,
+                left: ChartPrefs.axis_font_size *3,
+                right: ChartPrefs.axis_font_size
+            },
+            xscale: null,
+            yscale: null,
+            colors: ChartPrefs.line_colors
+        };
+    },
+
+    watch: {
+        'chart_data': function(newval) {            
+            this.tsdata = newval;
+            this.draw();
+        }
+    },
+
+    mounted: function() {
+        this.draw();
+    },
+        
+    methods: {
+
+        draw: function() {
+            if (! this.tsdata) {
+                return;
+            }
+            
+            const svg = d3.select(this.$el);
+            svg.selectAll("g").remove();
+
+            if (this.tsdata.dates.length == 0) {
+                // no data ...
+                svg.append("g")
+                    .append("text")
+                    .attr("font-family", ChartPrefs.default_font_family)
+                    .attr("font-size", ChartPrefs.label_font_size)
+                    .attr("text-anchor", "middle")
+                    .attr("x", this.width/2)
+                    .attr("y", this.height/2)
+                    .text("no data");
+            }
+
+            this.xscale = d3.scaleUtc()
+                .domain(d3.extent(this.tsdata.dates))
+                .nice()
+                .range([this.margin.left, this.width - this.margin.right])
+
+            this.yscale = d3.scaleLinear()
+                .domain([
+                    d3.min(this.tsdata.series, d => d3.min(d.values)),
+                    d3.max(this.tsdata.series, d => d3.max(d.values))
+                ])
+                .nice()
+                .range([this.height - this.margin.bottom, this.margin.top])
+            
+            svg.append("g")
+                .call(this.xAxis.bind(this))
+                .attr("font-size", ChartPrefs.axis_font_size);
+
+            svg.append("g")
+                .call(this.yAxis.bind(this))
+                .attr("font-size", ChartPrefs.axis_font_size);
+
+            const line = d3.line()
+                  .defined(d => !isNaN(d))
+                  .x((d, i) => this.xscale(this.tsdata.dates[i]))
+                  .y(d => this.yscale(d));
+
+            const path = svg.append("g")
+                  .attr("fill", "none")
+                  .attr("stroke-width", 1.5)
+                  .attr("stroke-linejoin", "round")
+                  .attr("stroke-linecap", "round")
+                  .selectAll("path")
+                  .data(this.tsdata.series)
+                  .join("path")
+                  .style("mix-blend-mode", "multiply")
+                  .style("stroke", (d, i) => this.colors[i])
+                  .attr("d", d => line(d.values))
+            ;
+            
+            svg.call(this.hover.bind(this), path);
+        },
+
+        xAxis: function(g) {
+            var x = g.attr(
+                'transform',
+                `translate(0, ${this.height - this.margin.bottom})`
+            ).call(
+                d3.axisBottom(this.xscale)
+                    .ticks(this.width / 80)
+                    .tickSizeOuter(0)
+            );
+            return x;
+        },
+        
+        yAxis: function(g) {
+            var y = g.attr(
+                "transform",
+                `translate(${this.margin.left},0)`
+            ).call(
+                d3.axisLeft(this.yscale)
+                    .ticks(this.height/50)
+            ).call(
+                g => g.select(".domain").remove()
+            ).call( 
+                g => ChartVue.add_yAxisLegend(g, this.tsdata, this.colors)
+            );
+            return y;
+        },
+
+        hover: function(svg, path) {
+            if ("ontouchstart" in document) svg
+                .style("-webkit-tap-highlight-color", "transparent")
+                .on("touchmove", moved.bind(this))
+                .on("touchstart", entered)
+                .on("touchend", left)
+            else svg
+                .on("mousemove", moved.bind(this))
+                .on("mouseenter", entered)
+                .on("mouseleave", left);
+
+            const dot = svg.append("g")
+                  .attr("display", "none");
+
+            dot.append("circle")
+                .attr("r", 2.5);
+
+            dot.append("text")
+                .attr("font-family", ChartPrefs.default_font_family)
+                .attr("font-size", ChartPrefs.default_font_size)
+                .attr("text-anchor", "middle")
+                .attr("y", -8);
+
+            function moved(event) {
+                if (!event) event = d3.event;
+                event.preventDefault();
+                var pointer;
+                if (d3.pointer)
+                    pointer = d3.pointer(event, svg.node());
+                else
+                    pointer = d3.mouse(svg.node());
+                const xvalue = this.xscale.invert(pointer[0]); // date
+                const yvalue = this.yscale.invert(pointer[1]); // number
+                //const i = d3.bisectCenter(this.tsdata.dates, xvalue); // index
+                const i = d3.bisect(this.tsdata.dates, xvalue); // index
+                if (i >= this.tsdata.dates.length) return;
+                // closest series
+                var closest = null;
+                for (var sidx=0; sidx<this.tsdata.series.length; sidx++) {
+                    var v = Math.abs(this.tsdata.series[sidx].values[i] - yvalue);
+                    if (closest === null || v<closest.val) {
+                        closest = {
+                            sidx: sidx,
+                            val: v
+                        };
+                    }
+                }
+                const s = this.tsdata.series[closest.sidx];
+                if (i>= s.values.length) {
+                    dot.attr("display", "none");
+                    return;
+                }
+                else {
+                    dot.attr("display", null);
+                    path.attr("stroke", d => d === s ? null : "#ddd")
+                        .filter(d => d === s).raise();
+                    dot.attr(
+                        "transform",
+                        `translate(${this.xscale(this.tsdata.dates[i])},${this.yscale(s.values[i])})`
+                    );
+                    dot.select("text").text(`${this.tsdata.formatDateTimeShort(this.tsdata.dates[i])} (${NumberFormatter.format(s.values[i])})`);
+                }
+            }
+
+            function entered() {
+                path.style("mix-blend-mode", null).attr("stroke", "#ddd");
+                dot.attr("display", null);
+            }
+            
+            function left() {
+                path.style("mix-blend-mode", "multiply").attr("stroke", null);
+                dot.attr("display", "none");
+            }
+        }
+    }
+});
+
+

management/reporting/ui/chart-pie.js

@@ -0,0 +1,165 @@
+Vue.component('chart-pie', {
+    /*
+     * chart_data: [
+     *   { name: 'name', value: value },
+     *   ...
+     * ]
+     *
+     * if prop `labels` is false, a legend is shown instead of
+     * labeling each pie slice
+     */
+    props: {
+        chart_data: Array,
+        formatter: { type: Function, default: NumberFormatter.format },
+        name_formatter: Function,
+        labels: { type:Boolean, default: true },
+        width: { type:Number, default: ChartPrefs.default_width },
+        height: { type:Number, default: ChartPrefs.default_height },
+    },
+    
+    render: function(ce) {
+        var svg = ChartVue.create_svg(ce, [
+            -this.width/2, -this.height/2, this.width, this.height
+        ]);
+        if (this.labels) {
+            return svg;
+        }
+
+        /* 
+      <div class="ml-1">
+        <div v-for="d in legend">
+          <span class="d-inline-block text-right pr-1 rounded" :style="{width:'5em','background-color':d.color}">{{d.value_str}}</span> {{d.name}}
+        </div>
+      </div>
+        */
+        var legend_children = [];
+        this.legend.forEach(d => {
+            var span = ce('span', { attrs: {
+                'class': 'd-inline-block text-right pr-1 mr-1 rounded',
+                'style': `width:5em; background-color:${d.color}`
+            }}, this.formatter(d.value));
+
+            legend_children.push(ce('div', [ span, d.name ]));
+        });
+
+        var div_legend = ce('div', { attrs: {
+            'class': 'ml-1 mt-2'
+        }}, legend_children);
+        
+        return ce('div', { attrs: {
+            'class': "d-flex align-items-start"
+        }}, [ svg, div_legend ]);
+
+    },
+
+    computed: {
+        legend: function() {
+            if (this.labels) {
+                return null;
+            }
+            
+            var legend = [];
+            if (this.chart_data) {
+                this.chart_data.forEach((d,i) => {
+                    legend.push({
+                        name: this.name_formatter ?
+                            this.name_formatter(d.name) : d.name,
+                        value: d.value,
+                        color: this.colors[i % this.colors.length]
+                    });
+                });
+            }
+            legend.sort((a,b) => {
+                return a.value > b.value ? -11 :
+                    a.value < b.value ? 1 : 0;
+            });
+            return legend;
+        }
+    },
+
+    data: function() {
+        return {
+            chdata: this.chart_data,
+            colors: this.colors || ChartPrefs.colors,
+        };
+    },
+
+    watch: {
+        'chart_data': function(newval) {            
+            this.chdata = newval;
+            this.draw();
+        }
+    },
+
+    mounted: function() {
+        this.draw();
+    },
+        
+    methods: {
+
+        draw: function() {
+            if (! this.chdata) return;
+            
+            var svg = d3.select(this.$el);
+            if (! this.labels) svg = svg.select('svg');
+            svg.selectAll("g").remove();
+
+            var chdata = this.chdata;
+            var nodata = false;
+            if (d3.sum(this.chdata, d => d.value) == 0) {
+                // no data
+                chdata = [{ name:'no data', value:100 }]
+                nodata = true;
+            }
+
+            const pie = d3.pie().sort(null).value(d => d.value);
+            const arcs = pie(chdata);
+            const arc = d3.arc()
+                  .innerRadius(0)
+                  .outerRadius(Math.min(this.width, this.height) / 2 - 1);
+
+            var radius = Math.min(this.width, this.height) / 2;
+            if (chdata.length == 1)
+                radius *= 0.1;
+            else if (chdata.length <= 3)
+                radius *= 0.65;
+            else if (chdata.length <= 6)
+                radius *= 0.7;
+            else
+                radius *= 0.8;
+            arcLabel = d3.arc().innerRadius(radius).outerRadius(radius);
+            
+            svg.append("g")
+                .attr("stroke", "white")
+                .selectAll("path")
+                .data(arcs)
+                .join("path")
+                .attr("fill", (d,i) => this.colors[i % this.colors.length])
+                .attr("d", arc)
+                .append("title")
+                .text(d => `${d.data.name}: ${this.formatter(d.data.value)}`);
+
+            if (this.labels) {
+                svg.append("g")
+                    .attr("font-family", ChartPrefs.default_font_family)
+                    .attr("font-size", ChartPrefs.label_font_size)
+                    .attr("text-anchor", "middle")
+                    .selectAll("text")
+                    .data(arcs)
+                    .join("text")
+                    .attr("transform", d => `translate(${arcLabel.centroid(d)})`)
+                    .call(text => text.append("tspan")
+                          .attr("y", "-0.4em")
+                          .attr("font-weight", "bold")
+                          .text(d => d.data.name))
+                    .call(text => text.filter(d => (d.endAngle - d.startAngle) > 0.25).append("tspan")
+                          .attr("x", 0)
+                          .attr("y", "0.7em")
+                          .attr("fill-opacity", 0.7)
+                          .text(d => nodata ? null : this.formatter(d.data.value)));
+            }
+        }
+
+    },
+
+});

management/reporting/ui/chart-stacked-bar-timeseries.js

@@ -0,0 +1,225 @@
+/*
+ stacked bar chart
+*/
+
+Vue.component('chart-stacked-bar-timeseries', {
+    props: {
+        chart_data: { type:Object, required:false }, /* TimeseriesData */
+        width: { type:Number, default: ChartPrefs.default_width },
+        height: { type:Number, default: ChartPrefs.default_height },
+    },
+    
+    render: function(ce) {
+        return ChartVue.create_svg(ce, [0, 0, this.width, this.height]);
+    },
+
+    data: function() {
+        return {
+            tsdata: null,
+            stacked: null,
+            margin: {
+                top: ChartPrefs.axis_font_size,
+                bottom: ChartPrefs.axis_font_size*2,
+                left: ChartPrefs.axis_font_size*3,
+                right: ChartPrefs.axis_font_size
+            },
+            xscale: null,
+            yscale: null,
+            colors: ChartPrefs.colors, /* array of colors */
+        };
+    },
+
+    mounted: function() {
+        if (this.chart_data) {
+            this.stack(this.chart_data);
+            this.draw();
+        }
+    },
+
+    watch: {
+        'chart_data': function(newv, oldv) {
+            this.stack(newv);
+            this.draw();
+        }
+    },
+
+    methods: {
+        stack: function(data) {
+            /* "stack" the data using d3.stack() */
+            // 1. reorganize into the format stack() wants -- an
+            // array of objects, with each object having a key of
+            // 'date', plus one for each series
+            var stacker_input = data.dates.map((d, i) => {
+                var array_el = { date: d }
+                data.series.forEach(s => {
+                    array_el[s.name] = s.values[i];
+                })
+                return array_el;
+            });
+            
+            // 2. call d3.stack() to get the stacking function, which
+            // creates yet another version of the series data,
+            // reformatted for more easily creating stacked bars.
+            //
+            // It returns a new array (see the d3 docs):
+            // [
+            //   [  /* series 1 */
+            //      [ Number, Number, data: { date: Date } ],
+            //      [ ... ], ...
+            //   ],
+            //   [  /* series 2 */
+            //      [ Number, Number, data: { date: Date } ],
+            //      [ ... ], ...
+            //   ],
+            //   ...
+            // ]
+            //
+            var stacker = d3.stack()
+                .keys(data.series.map(s => s.name))
+                .order(d3.stackOrderNone)
+                .offset(d3.stackOffsetNone);
+
+            // 3. store the data
+            this.tsdata = data;
+            this.stacked = stacker(stacker_input);
+        },
+        
+
+        draw: function() {
+            const svg = d3.select(this.$el);
+            svg.selectAll("g").remove();
+
+            if (this.tsdata.dates.length == 0) {
+                // no data ...
+                svg.append("g")
+                    .append("text")
+                    .attr("font-family", ChartPrefs.default_font_family)
+                    .attr("font-size", ChartPrefs.label_font_size)
+                    .attr("text-anchor", "middle")
+                    .attr("x", this.width/2)
+                    .attr("y", this.height/2)
+                    .text("no data");
+            }
+            
+            this.xscale = d3.scaleUtc()
+                .domain(d3.extent(this.tsdata.dates))
+                .nice()
+                .range([this.margin.left, this.width - this.margin.right])
+            
+            var barwidth = this.tsdata.barwidth(this.xscale, 1);
+            var padding = barwidth / 2;
+            
+            this.yscale = d3.scaleLinear()
+                .domain([
+                    0,
+                    d3.sum(this.tsdata.series, s => d3.max(s.values))
+                ])
+                .range([
+                    this.height - this.margin.bottom,
+                    this.margin.top,
+                ]);
+            
+            svg.append("g")
+                .call(this.xAxis.bind(this, padding))
+                .attr("font-size", ChartPrefs.axis_font_size);
+            
+            svg.append("g")
+                .call(this.yAxis.bind(this))
+                .attr("font-size", ChartPrefs.axis_font_size);
+
+            
+            for (var s_idx=0; s_idx<this.tsdata.series.length; s_idx++) {
+                svg.append("g")
+                    .datum(s_idx)
+                    .attr("fill", this.colors[s_idx])
+                    .selectAll("rect")
+                    .data(this.stacked[s_idx])
+                    .join("rect")
+                    .attr("x", d => this.xscale(d.data.date) - barwidth/2 + padding)
+                    .attr("y", d => this.yscale(d[1]))
+                    .attr("height", d => this.yscale(d[0]) - this.yscale(d[1]))
+                    .attr("width", barwidth)
+                    .call( hover.bind(this) )
+                         
+                    // .append("title")
+                    // .text(d => `${this.tsdata.series[s_idx].name}: ${NumberFormatter.format(d.data[this.tsdata.series[s_idx].name])}`)
+                ;
+            }
+
+            var hovinfo = svg.append("g");
+
+            function hover(rect) {
+                if ("ontouchstart" in document) rect
+                    .style("-webkit-tap-highlight-color", "transparent")
+                    .on("touchstart", entered.bind(this))
+                    .on("touchend", left)
+                else rect
+                    .on("mouseenter", entered.bind(this))
+                    .on("mouseleave", left);
+
+                function entered(event, d) {
+                    var rect = d3.select(event.target)
+                        .attr("fill", "#ccc");
+                    var d = rect.datum();
+                    var s_idx = d3.select(rect.node().parentNode).datum();
+                    var s_name = this.tsdata.series[s_idx].name;
+                    var v = d.data[s_name];
+                    var x = Number(rect.attr('x')) + barwidth/2;
+                    
+                    hovinfo.attr(
+                        "transform",
+                        `translate( ${x}, ${rect.attr('y')} )`)
+                        .append('text')
+                        .attr("font-family", ChartPrefs.default_font_family)
+                        .attr("font-size", ChartPrefs.default_font_size)
+                        .attr("text-anchor", "middle")
+                        .attr("y", -3)
+                        .text(`${this.tsdata.formatDateTimeShort(d.data.date)}`);
+                    hovinfo.append("text")
+                        .attr("font-family", ChartPrefs.default_font_family)
+                        .attr("font-size", ChartPrefs.default_font_size)
+                        .attr("text-anchor", "middle")
+                        .attr("y", -3 - ChartPrefs.default_font_size)
+                        .text(`${s_name} (${NumberFormatter.format(v)})`);
+
+                }
+
+                function left(event) {
+                    d3.select(event.target).attr("fill", null);
+                    hovinfo.selectAll("text").remove();
+                }
+            }
+        },
+
+        xAxis: function(padding, g) {
+            var x = g.attr(
+                'transform',
+                `translate(${padding}, ${this.height - this.margin.bottom})`
+            ).call(
+                d3.axisBottom(this.xscale)
+                    .ticks(this.width / 80)
+                    .tickSizeOuter(0)
+            );
+            return x;
+        },
+        
+        yAxis: function(g) {
+            var y = g.attr(
+                "transform",
+                `translate(${this.margin.left},0)`
+            ).call(
+                d3.axisLeft(this.yscale)
+                    .ticks(this.height/50)
+            ).call(g =>
+                   g.select(".domain").remove()
+            ).call(g => {
+                ChartVue.add_yAxisLegend(g, this.tsdata, this.colors);
+            });
+            
+            return y;
+        },
+
+    }
+});
+
+

management/reporting/ui/chart-table.js

@@ -0,0 +1,45 @@
+Vue.component('chart-table', {
+    props: {
+        items: Array,
+        fields: Array,
+        caption: String
+    },
+
+    /* <b-table-lite striped small :fields="fields_x" :items="items" caption-top><template #table-caption><span class="text-nowrap">{{caption}}</span></template></b-table>*/
+    render: function(ce) {
+        var scopedSlots= {
+            'table-caption': props =>
+                ce('span', { class: { 'text-nowrap':true }}, this.caption)
+        };
+        if (this.$scopedSlots) {
+            for (var slotName in this.$scopedSlots) {
+                scopedSlots[slotName] = this.$scopedSlots[slotName];
+            }
+        }
+        var table = ce('b-table-lite', {
+            props: {
+                'striped': true,
+                'small': true,
+                'fields': this.fields_x,
+                'items': this.items,
+                'caption-top': true
+            },
+            scopedSlots: scopedSlots
+        });
+        
+        return table;
+    },
+
+    computed: {
+        fields_x: function() {
+            if (this.items.length == 0) {
+                return [{
+                    key: 'no data',
+                    thClass: 'text-nowrap'
+                }];
+            }
+            return this.fields;
+        }
+    }
+
+});

management/reporting/ui/chart.readme.txt

@@ -0,0 +1,20 @@
+/*
+most charts have been adapted from ones at Obserable,
+which had the following license:
+
+https://observablehq.com/@d3/multi-line-chart
+
+Copyright 2018–2020 Observable, Inc.
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/

management/reporting/ui/charting.js

@@ -0,0 +1,972 @@
+
+class ChartPrefs {
+    static get colors() {
+        // see: https://github.com/d3/d3-scale-chromatic
+        return d3.schemeSet2;
+    }
+
+    static get line_colors() {
+        // see: https://github.com/d3/d3-scale-chromatic
+        return d3.schemeCategory10;
+    }
+
+    static get default_width() {
+        return 600;
+    }
+
+    static get default_height() {
+        return 400;
+    }
+
+    static get axis_font_size() {
+        return 12;
+    }
+
+    static get default_font_size() {
+        return 10;
+    }
+
+    static get label_font_size() {
+        return 12;
+    }
+
+    static get default_font_family() {
+        return "sans-serif";
+    }
+
+    static get locales() {
+        return "en";
+    }
+};
+
+
+class DateFormatter {
+    /*
+     * date and time
+     */
+    static dt_long(d, options) {
+        let opt = {
+            weekday: 'long',
+            year: 'numeric',
+            month: 'long',
+            day: 'numeric'
+        };
+        Object.assign(opt, options);
+        return d.toLocaleString(ChartPrefs.locales, opt);
+    }
+    static dt_short(d, options) {
+        return d.toLocaleString(ChartPrefs.locales, options);
+    }
+
+    /*
+     * date
+     */
+    static d_long(d, options) {
+        let opt = {
+            weekday: 'long',
+            year: 'numeric',
+            month: 'long',
+            day: 'numeric'
+        };
+        Object.assign(opt, options);
+        return d.toLocaleDateString(ChartPrefs.locales, opt);
+    }
+    static d_short(d, options) {
+        return d.toLocaleDateString(ChartPrefs.locales, options);
+    }
+
+    /*
+     * time
+     */
+    static t_long(d, options) {
+        return d.toLocaleTimeString(ChartPrefs.locales, options);
+    }
+    static t_short(d, options) {
+        return d.toLocaleTimeString(ChartPrefs.locales, options);
+    }
+    static t_span(d, unit) {
+        // `d` is milliseconds
+        // `unit` is desired max precision output unit (eg 's')
+        unit = unit || 's';
+        const cvt = [{
+            ms: (24 * 60 * 60 * 1000),
+            ushort: 'd',
+            ulong: 'day'
+        }, {
+            ms: (60 * 60 * 1000),
+            ushort: 'h',
+            ulong: 'hour'
+        }, {
+            ms: (60 * 1000),
+            ushort: 'm',
+            ulong: 'minute'
+        }, {
+            ms: 1000,
+            ushort: 's',
+            ulong: 'second'
+        }, {
+            ms: 1,
+            ushort: 'ms',
+            ulong: 'milliseconds'
+        }];
+
+        var first = false;
+        var remainder = d;
+        var out = [];
+        var done = false;
+        cvt.forEach( c => {
+            if (done) return;
+            var amt = Math.floor( remainder / c.ms );
+            remainder = remainder % c.ms;
+            if (first || amt > 0) {
+                first = true;
+                out.push(amt + c.ushort);
+            }
+            if (unit == c.ushort || unit == c.ulong) {
+                done = true;
+            }
+        });        
+        return out.join(' ');
+    }
+
+    /*
+     * universal "YYYY-MM-DD HH:MM:SS" formats
+     */
+    static ymd(d) {
+        const ye = d.getFullYear();
+        const mo = '0'+(d.getMonth() + 1);
+        const da = '0'+d.getDate();
+        return `${ye}-${mo.substr(mo.length-2)}-${da.substr(da.length-2)}`;
+    }
+
+    static ymd_utc(d) {
+        const ye = d.getUTCFullYear();
+        const mo = '0'+(d.getUTCMonth() + 1);
+        const da = '0'+d.getUTCDate();
+        return `${ye}-${mo.substr(mo.length-2)}-${da.substr(da.length-2)}`;
+    }
+
+    static ymdhms(d) {
+        const ho = '0'+d.getHours();
+        const mi = '0'+d.getMinutes();
+        const se = '0'+d.getSeconds();
+        return `${DateFormatter.ymd(d)} ${ho.substr(ho.length-2)}:${mi.substr(mi.length-2)}:${se.substr(se.length-2)}`;
+    }
+
+    static ymdhms_utc(d) {
+        const ho = '0'+d.getUTCHours();
+        const mi = '0'+d.getUTCMinutes();
+        const se = '0'+d.getUTCSeconds();
+        return `${DateFormatter.ymd_utc(d)} ${ho.substr(ho.length-2)}:${mi.substr(mi.length-2)}:${se.substr(se.length-2)}`;
+    }
+};
+
+
+class DateRange {
+    /*
+     * ranges
+     */
+    static ytd() {
+        var s = new Date();
+        s.setMonth(0);
+        s.setDate(1);
+        s.setHours(0);
+        s.setMinutes(0);
+        s.setSeconds(0);
+        s.setMilliseconds(0);
+        return [ s, new Date() ];
+    }
+    static ytd_as_ymd() {
+        return DateRange.ytd().map(d => DateFormatter.ymd(d));
+    }
+
+    static mtd() {
+        var s = new Date();
+        s.setDate(1);
+        s.setHours(0);
+        s.setMinutes(0);
+        s.setSeconds(0);
+        s.setMilliseconds(0);
+        return [ s, new Date() ];
+    }
+    static mtd_as_ymd() {
+        return DateRange.mtd().map(d => DateFormatter.ymd(d));
+    }
+
+    static wtd() {
+        var s = new Date();
+        var offset = s.getDay() * (24 * 60 * 60 * 1000);
+        s.setTime(s.getTime() - offset);
+        s.setHours(0);
+        s.setMinutes(0);
+        s.setSeconds(0);
+        s.setMilliseconds(0);
+        return [ s, new Date() ];
+    }
+    static wtd_as_ymd() {
+        return DateRange.wtd().map(d => DateFormatter.ymd(d));
+    }
+
+    static rangeFromType(type) {
+        if (type == 'wtd')
+            return DateRange.wtd();
+        else if (type == 'mtd')
+            return DateRange.mtd();
+        else if (type == 'ytd')
+            return DateRange.ytd();
+        return null;
+    }
+};
+
+
+class NumberFormatter {
+    static format(v) {
+        return isNaN(v) || v===null ? "N/A" : v.toLocaleString(ChartPrefs.locales);
+    }
+
+    static decimalFormat(v, places, style) {
+        if (isNaN(v) || v===null) return "N/A";
+        if (places === undefined || isNaN(places)) places = 1;
+        if (style === undefined || typeof style != 'string') style = 'decimal';
+        var options = {
+            style: style,
+            minimumFractionDigits: places
+        };
+        v = v.toLocaleString(ChartPrefs.locales, options);
+        return v;        
+    }
+
+    static percentFormat(v, places) {
+        if (places === undefined || isNaN(places)) places = 0;
+        return NumberFormatter.decimalFormat(v, places, 'percent');
+    }
+
+    static humanFormat(v, places) {
+        if (isNaN(v) || v===null) return "N/A";
+        if (places === undefined || isNaN(places)) places = 1;
+        const options = {
+            style: 'unit',
+            minimumFractionDigits: places,
+            unit: 'byte'
+        };
+        var xunit = '';
+        const f = Math.pow(10, places);
+        if (v >= NumberFormatter.tb) {
+            v = Math.round(v / NumberFormatter.tb * f) / f;
+            options.unit='terabyte';
+            xunit = 'T';
+        }
+        else if (v >= NumberFormatter.gb) {
+            v = Math.round(v / NumberFormatter.gb * f) / f;
+            options.unit='gigabyte';
+            xunit = 'G';
+        }
+        else if (v >= NumberFormatter.mb) {
+            v = Math.round(v / NumberFormatter.mb * f) / f;
+            options.unit='megabyte';
+            xunit = 'M';
+        }
+        else if (v >= NumberFormatter.kb) {
+            v = Math.round(v / NumberFormatter.kb * f) / f;
+            options.unit='kilobyte';
+            xunit = 'K';
+        }
+        else {
+            options.minimumFractionDigits = 0;
+            places = 0;
+        }
+        try {
+            return v.toLocaleString(ChartPrefs.locales, options);
+        } catch(e) {
+            if (e instanceof RangeError) {
+                // probably "invalid unit"
+                return NumberFormatter.decimalFormat(v, places) + xunit;
+            }
+        }
+    }
+};
+
+// define static constants in NumberFormatter
+['kb','mb','gb','tb'].forEach((unit,idx) => {
+    Object.defineProperty(NumberFormatter, unit, {
+        value: Math.pow(1024, idx+1),
+        writable: false,
+        enumerable: false,
+        configurable: false
+    });
+});
+
+
+class BvTable {
+    constructor(data, opt) {
+        opt = opt || {};
+        Object.assign(this, data);
+        if (!this.items || !this.fields || !this.field_types) {
+            throw new AssertionError();
+        }
+
+        BvTable.arraysToObjects(this.items, this.fields);
+        BvTable.setFieldDefinitions(this.fields, this.field_types);
+
+        if (opt._showDetails) {
+            // _showDetails must be set to make it reactive
+            this.items.forEach(item => {
+                item._showDetails = false;
+            })
+        }
+    }
+
+    field_index_of(key) {
+        for (var i=0; i<this.fields.length; i++) {
+            if (this.fields[i].key == key) return i;
+        }
+        return -1;
+    }
+
+    get_field(key, only_showing) {
+        var i = this.field_index_of(key);
+        if (i>=0) return this.fields[i];
+        return this.x_fields && !only_showing ? this.x_fields[key] : null;
+    }
+
+    combine_fields(names, name2, formatter) {
+        // combine field(s) `names` into `name2`, then remove
+        // `names`. use `formatter` as the formatter function
+        // for the new combined field.
+        //
+        // if name2 is not given, just remove all `names` fields
+        //
+        // removed fields are placed into this.x_fields array
+        if (typeof names == 'string') names = [ names ]
+        var idx2 = name2 ? this.field_index_of(name2) : -1;
+        if (! this.x_fields) this.x_fields = {};
+                
+        names.forEach(name1 => {
+            var idx1 = this.field_index_of(name1);
+            if (idx1 < 0) return;
+            this.x_fields[name1] = this.fields[idx1];
+            this.fields.splice(idx1, 1);
+            if (idx2>idx1) --idx2;
+        });
+        
+        if (idx2 < 0) return null;
+        
+        this.fields[idx2].formatter = formatter;
+        return this.fields[idx2];
+    }
+
+    
+    static arraysToObjects(items, fields) {
+        /*
+         * convert array-of-arrays `items` to an array of objects
+         * suitable for a <b-table> items (rows of the table).
+         *
+         * `items` is modified in-place
+         *
+         * `fields` is an array of strings, which will become the keys
+         * of each new object. the length of each array in `items`
+         * must match the length of `fields` and the indexes must
+         * correspond.
+         *
+         * the primary purpose is to allow the data provider (server)
+         * to send something like:
+         *
+         *   { "items": [ 
+         *        [ "alice", 10.6, 200, "top-10" ],
+         *         .... 
+         *     ], 
+         *     "fields": [ "name", "x", "y", "label" ] 
+         *   }
+         *
+         * instead of:
+         *
+         *   { "items": [ 
+         *        { "name":"a", "x":10.6, "y":200, "label":"top-10" },
+         *        ... 
+         *     ],
+         *     "fields": [ "name", "x", "y", "label" ]
+         *   }
+         *
+         * which requires much more bandwidth
+         * 
+         */
+        if (items.length > 0 && !Array.isArray(items[0]))
+        {
+            // already converted
+            return;
+        }
+        for (var i=0; i<items.length; i++) {
+            var o = {};
+            fields.forEach((field, idx) => {
+                o[field] = items[i][idx];
+            });
+            items[i] = o;
+        }
+    }
+    
+    static setFieldDefinitions(fields, types) {
+        /* 
+         * change elements of array `fields` to bootstrap-vue table
+         * field (column) definitions
+         *
+         * `fields` is an array of field names or existing field
+         * definitions to update. `types` is a correponding array
+         * having the type of each field which will cause one or more
+         * of the following properties to be set on each field:
+         * 'tdClass', 'thClass', 'label', and 'formatter'
+         */
+        for (var i=0; i<fields.length && i<types.length; i++) {
+            var field = fields[i];
+            fields[i] = new BvTableField(field, types[i]);
+        }
+    }
+    
+};
+
+
+class BvTableField {
+    constructor(field, field_type) {
+        // this:
+        //    key    - required
+        //    label
+        //    tdClass
+        //    thClass
+        //    formatter
+        // .. etc (see bootstrap-vue Table component docs)
+
+        if (typeof field == 'string') {
+            this.key = field;
+        }
+        else {
+            Object.assign(this, field);
+        }
+
+        var ft = field_type;
+        var field = this;
+        
+        if (typeof ft == 'string') {
+            ft = { type: ft };
+        }
+
+        if (! ft.subtype && ft.type.indexOf('/') >0) {
+            // optional format, eg "text/email"
+            var s = ft.type.split('/');
+            ft.type = s[0];
+            ft.subtype = s.length > 1 ? s[1] : null;
+        }
+        
+        if (ft.label !== undefined) {
+            field.label = ft.label;
+        }
+        
+        if (ft.type == 'decimal') {
+            Object.assign(ft, {
+                type: 'number',
+                subtype: 'decimal'
+            });
+        }
+        
+        if (ft.type == 'text')  {
+            // as-is
+        }
+        else if (ft.type == 'number') {
+            if (ft.subtype == 'plain' ||
+                ft.subtype == 'decimal' && isNaN(ft.places)
+               )
+            {
+                Object.assign(
+                    field,
+                    BvTableField.numberFieldDefinition()
+                );
+            }
+        
+            else if (ft.subtype == 'size') {
+                Object.assign(
+                    field,
+                    BvTableField.sizeFieldDefinition()
+                );
+            }
+        
+            else if (ft.subtype == 'decimal') {
+                Object.assign(
+                    field,
+                    BvTableField.decimalFieldDefinition(ft.places)
+                );
+            }
+        
+            else if (ft.subtype == 'percent') {
+                Object.assign(
+                    field,
+                    BvTableField.percentFieldDefinition(ft.places)
+                );
+            }
+        }
+        else if (ft.type == 'datetime') {
+            Object.assign(
+                field,
+                BvTableField.datetimeFieldDefinition(ft.showas || 'short', ft.format)
+            );
+        }
+        else if (ft.type == 'time' && ft.subtype == 'span') {
+            Object.assign(
+                field,
+                BvTableField.timespanFieldDefinition(ft.unit || 'ms')
+            );
+        }
+        
+    }
+
+
+    static numberFieldDefinition() {
+        // <b-table> field definition for a localized numeric value.
+        // eg: "5,001". for additional attributes, see:
+        // https://bootstrap-vue.org/docs/components/table#field-definition-reference
+        return {
+            formatter: NumberFormatter.format,
+            tdClass: 'text-right',
+            thClass: 'text-right'
+        };
+    }
+
+    static sizeFieldDefinition(decimal_places) {
+        // <b-table> field definition for a localized numeric value in
+        // human readable format. eg: "5.1K".  `decimal_places` is
+        // optional, which defaults to 1
+        return {
+            formatter: value =>
+                NumberFormatter.humanFormat(value, decimal_places),
+            tdClass: 'text-right text-nowrap',
+            thClass: 'text-right'
+        };
+    }
+
+    static datetimeFieldDefinition(variant, format) {
+        // if the formatter is passed string (utc) dates, convert them
+        // to a native Date objects using the format in `format`.  eg:
+        // "%Y-%m-%d %H:%M:%S".
+        //
+        // `variant` can be "long" (default) or "short"
+        var parser = (format ? d3.utcParse(format) : null);        
+        if (variant === 'short') {
+            return {
+                formatter: v =>
+                    DateFormatter.dt_short(parser ? parser(v) : v)
+            };
+        }
+        else {
+            return {
+                formatter: v =>
+                    DateFormatter.dt_long(parser ? parser(v) : v)
+            };
+        }
+    }
+
+    static timespanFieldDefinition(unit, output_unit) {
+        var factor = 1;
+        if (unit == 's') factor = 1000;
+        return {
+            formatter: v => DateFormatter.t_span(v * factor, output_unit)
+        };
+    }
+
+    static decimalFieldDefinition(decimal_places) {
+        return {
+            formatter: value =>
+                NumberFormatter.decimalFormat(value, decimal_places),
+            tdClass: 'text-right',
+            thClass: 'text-right'
+        };
+    }
+
+    static percentFieldDefinition(decimal_places) {
+        return {
+            formatter: value =>
+                NumberFormatter.percentFormat(value, decimal_places),
+            tdClass: 'text-right',
+            thClass: 'text-right'
+        };
+    }
+
+    add_cls(cls, to_what) {
+        if (Array.isArray(this[to_what])) {
+            this[to_what].push(cls);
+        }
+        else if (this[to_what] !== undefined) {
+            this[to_what] = [ this[to_what], cls ];
+        }
+        else {
+            this[to_what] = cls;
+        }
+    }
+
+    add_tdClass(cls) {
+        this.add_cls(cls, 'tdClass');
+    }
+
+};
+
+
+class MailBvTable extends BvTable {
+    flag(key, fn) {
+        var field = this.get_field(key, true);
+        if (!field) return;
+        field.add_tdClass(fn);
+    }
+    
+    flag_fields(tdClass) {
+        // flag on certain cell values by setting _flagged in each
+        // "flagged" item during its tdClass callback function. add
+        // `tdClass` to the rendered value
+        
+        tdClass = tdClass || 'text-danger';
+        
+        this.flag('accept_status', (v, key, item) => {
+            if (v === 'reject') {
+                item._flagged = true;
+                return tdClass;
+            }
+        });
+                
+        this.flag('relay', (v, key, item) => {
+            if (item.delivery_connection == 'untrusted') {
+                item._flagged = true;
+                return tdClass;
+            }
+        });
+
+        this.flag('status', (v, key, item) => {
+            if (v != 'sent') {
+                item._flagged = true;
+                return tdClass;
+            }
+        });
+
+        this.flag('spam_result', (v, key, item) => {
+            if (item.spam_result && v != 'clean') {
+                item._flagged = true;
+                return tdClass;
+            }
+        });
+
+        this.flag('spf_result', (v, key, item) => {
+            if (v == 'Fail' ||v == 'Softfail') {
+                item._flagged = true;
+                return tdClass;
+            }
+        });
+        
+        this.flag('dkim_result', (v, key, item) => {
+            if (item.dkim_result && v != 'pass') {
+                item._flagged = true;
+                return tdClass;
+            }
+        });
+        
+        this.flag('dmarc_result', (v, key, item) => {
+            if (v == 'fail') {
+                item._flagged = true;
+                return tdClass;
+            }
+        });
+
+        this.flag('postgrey_result', (v, key, item) => {
+            if (item.postgrey_result && v != 'pass') {
+                item._flagged = true;
+                return tdClass;
+            }
+        });
+
+        this.flag('disposition', (v, key, item) => {
+            if (item.disposition != 'ok') {
+                item._flagged = true;
+                return tdClass;
+            }
+        });
+
+        return this;
+    }
+
+    apply_rowVariant_grouping(variant, group_fn) {
+        // there is 1 row for each recipient of a message
+        // - give all rows of the same message the same
+        // color
+        //
+        // variant is a bootstrap variant like "primary"
+        //
+        // group_fn is a callback receiving an item (row of data) and
+        // the item index and should return null if the item is not
+        // showing or return the group value
+        var last_group = -1;
+        var count = 0;
+        for (var idx=0; idx < this.items.length; idx++)
+        {
+            const item = this.items[idx];
+            const group = group_fn(item, idx);
+            if (group === null || group === undefined) continue
+            
+            if (group != last_group) {
+                ++count;
+                last_group = group;
+            }
+            item._rowVariant = count % 2 == 0 ? variant : '';
+        }
+    }
+}
+
+
+class ChartVue {
+    
+    static svg_attrs(viewBox) {
+        var attrs = {
+            width: viewBox[2],
+            height: viewBox[3],
+            viewBox: viewBox.join(' '),
+            style: 'overflow: visible',
+            xmlns: 'http://www.w3.org/2000/svg'
+        };
+        return attrs;
+    }
+
+    static create_svg(create_fn, viewBox, children) {
+        var svg = create_fn('svg', {
+            attrs: ChartVue.svg_attrs(viewBox),
+            children
+        });
+        return svg;
+    }
+
+    static add_yAxisLegend(g, data, colors) {
+        //var gtick = g.select(".tick:last-of-type").append("g");
+        const h = ChartPrefs.axis_font_size;
+        var gtick = g.append("g")
+            .attr('transform',
+                  `translate(0, ${h * data.series.length})`);
+        
+        gtick.selectAll('rect')
+            .data(data.series)
+            .join('rect')
+            .attr('x', 3)
+            .attr('y', (d, i) => -h + i*h)
+            .attr('width', h)
+            .attr('height', h)
+            .attr('fill', (d, i) => colors[i]);                
+        gtick.selectAll('text')
+            .data(data.series)
+            .join('text')
+            .attr('x', h + 6)
+            .attr('y', (d, i) => i*h )
+            .attr("text-anchor", "start")
+            .attr("font-weight", "bold")
+            .attr("fill", 'currentColor')
+            .text(d => d.name);
+        return g;
+    }
+};
+
+
+
+/*
+ * Timeseries data layout: {
+ *   y: 'description',
+ *   binsize: Number, // size in minutes,
+ *   date_parse_format: '%Y-%m-%d',
+ *   dates: [ 'YYYY-MM-DD HH:MM:SS', ... ],
+ *   series: [
+ *      {
+ *         id: 'id',
+ *         name: 'series 1 desc',
+ *         values: [ Number, .... ]
+ *      },
+ *      {
+ *         id: 'id',
+ *         name: 'series 2 desc'
+ *         values: [ ... ],
+ *      }, 
+ *      ...
+ *   ]
+ * }
+ */    
+
+class TimeseriesData {
+    constructor(data) {
+        Object.assign(this, data);
+        this.convert_dates();
+    }
+
+    get_series(id) {
+        for (var i=0; i<this.series.length; i++) {
+            if (this.series[i].id == id) return this.series[i];
+        }
+    }
+
+    dataView(desired_series_ids) {
+        var dataview = Object.assign({}, this);
+        dataview.series = [];
+        
+        var desired = {}
+        desired_series_ids.forEach(id => desired[id] = true);
+        this.series.forEach(s => {
+            if (desired[s.id]) dataview.series.push(s);
+        });
+        return new TimeseriesData(dataview);
+    }
+
+    binsizeWithUnit() {
+        // normalize binsize (which is a time span in minutes)
+        const days = Math.floor(this.binsize / (24 * 60));
+        const hours = Math.floor( (this.binsize - days*24*60) / 60 );
+        const mins = this.binsize - days*24*60 - hours*60;
+        if (days == 0 && hours == 0) {
+            return {
+                unit: 'minute',
+                value: mins
+            };
+        }
+        if (days == 0) {
+            return {
+                unit: 'hour',
+                value: hours
+            };
+        }
+        return {
+            unit: 'day',
+            value: days
+        };
+    }
+    
+    binsizeTimespan() {
+        /* return the binsize timespan in seconds */
+        return this.binsize * 60;
+    }
+
+    static binsizeOfRange(range) {
+        // target 100-120 datapoints
+        const target = 100;
+        const tolerance = 0.2; // 20%
+        
+        if (typeof range[0] == 'string') {
+            var parser = d3.utcParse('%Y-%m-%d %H:%M:%S');
+            range = range.map(parser);
+        }
+
+        const span_min = Math.ceil(
+            (range[1].getTime() - range[0].getTime()) / (1000*60*target)
+        );
+        const bin_days = Math.floor(span_min / (24*60));
+        const bin_hours = Math.floor((span_min - bin_days*24*60) / 60);
+        if (bin_days >= 1) {
+            return bin_days * 24 * 60 +
+                (bin_hours > (24 * tolerance) ? bin_hours*60: 0);
+        }
+        
+        const bin_mins = span_min - bin_days*24*60 - bin_hours*60;
+        if (bin_hours >= 1) {
+            return bin_hours * 60 +
+                (bin_mins > (60 * tolerance) ? bin_mins: 0 );
+        }
+        return bin_mins;
+    }
+
+    barwidth(xscale, barspacing) {
+        /* get the width of a bar in a bar chart */
+        var start = this.range[0];
+        var end = this.range[1];
+        var bins = (end.getTime() - start.getTime()) / (1000 * this.binsizeTimespan());
+        return Math.max(1, (xscale.range()[1] - xscale.range()[0])/bins - (barspacing || 0));
+    }
+
+    formatDateTimeLong(d) {
+        var options = { hour: 'numeric' };
+        var b = this.binsizeWithUnit();
+        if (b.unit === 'minute') {
+            options.minute = 'numeric';
+            return DateFormatter.dt_long(d, options);
+        }
+        if (b.unit === 'hour') {
+            return DateFormatter.dt_long(d, options);
+        }
+        if (b.unit === 'day') {
+            return DateFormatter.d_long(d);
+        }
+        throw new Error(`Unknown binsize unit: ${b.unit}`);
+    }
+
+    formatDateTimeShort(d) {
+        var options = {
+            year: 'numeric',
+            month: 'numeric',
+            day: 'numeric',
+            weekday: undefined
+        };
+        var b = this.binsizeWithUnit();
+        if (b.unit === 'minute') {
+            Object.assign(options, {
+                hour: 'numeric',
+                minute: 'numeric'
+            });
+            return DateFormatter.dt_long(d, options);
+        }
+        if (b.unit === 'hour') {
+            options.hour = 'numeric';
+            return DateFormatter.dt_long(d, options);
+        }
+        if (b.unit === 'day') {
+            return DateFormatter.d_short(d);
+        }
+        throw new Error(`Unknown binsize unit: ${b.unit}`);
+    }
+
+        
+    convert_dates() {
+        // all dates from the server are UTC strings
+        // convert to Date objects
+        if (this.dates.length > 0 && typeof this.dates[0] == 'string')
+        {
+            var parser = d3.utcParse(this.date_parse_format);
+            this.dates = this.dates.map(parser);
+        }
+        if (this.range.length > 0 && typeof this.range[0] == 'string')
+        {
+            var parser = d3.utcParse(this.range_parse_format);
+            this.range = this.range.map(parser);
+        }        
+    }
+};
+
+
+class ConnectionDisposition {
+    constructor(disposition) {
+        const data = {
+            'failed_login_attempt': {
+                short_desc: 'failed login attempt',
+            },
+            'insecure': {
+                short_desc: 'insecure connection'
+            },
+            'ok': {
+                short_desc: 'normal, secure connection'
+            },
+            'reject': {
+                short_desc: 'mail attempt rejected'
+            },
+            'suspected_scanner': {
+                short_desc: 'suspected scanner'
+            }
+        };        
+        this.disposition = disposition;
+        this.info = data[disposition];        
+        if (! this.info) {
+            this.info = {
+                short_desc: disposition.replace('_',' ')
+            }
+        }
+    }
+
+    get short_desc() {
+        return this.info.short_desc;
+    }
+
+    static formatter(disposition) {
+        return new ConnectionDisposition(disposition).short_desc;
+    }
+};

management/reporting/ui/date-range-picker.js

@@ -0,0 +1,165 @@
+Vue.component('date-range-picker', {
+    props: {
+        start_range: [ String, Array ],  // "ytd", "mtd", "wtd", or [start, end] where start and end are strings in format YYYY-MM-DD in localti
+        recall_id: String, // save / recall from localStorage
+    },
+    template: '<div class="d-flex align-items-center flex-wrap">'+
+        '<div>Date range:<br><b-form-select v-model="range_type" :options="options" size="sm" @change="range_type_change"></b-form-select></div>'+
+        '<div class="ml-2">From:<br><b-form-datepicker v-model="range[0]" style="max-width:20rem" :disabled="range_type != \'custom\'"></b-form-datepicker></div>' +
+        '<div class="ml-2">To:<br><b-form-datepicker v-model="range[1]" style="max-width:20rem" :min="range[0]" :disabled="range_type != \'custom\'"></b-form-datepicker></div>' +
+        '</div>'
+    ,
+    data: function() {
+        var range_type = null;
+        var range = null;
+        var default_range_type = 'mtd';
+        const recall_id_prefix = 'date-range-picker/';
+
+        var v = null;
+        if (typeof this.start_range === 'string') {
+            if (this.start_range.substring(0,1) == '-') {
+                default_range_type = this.start_range.substring(1);
+            }
+            else {
+                v = this.validate_input_range(this.start_range);
+            }
+        }
+        else {
+            v = this.validate_input_range(this.start_range);
+        }
+        
+        if (v) {
+            // handles explicit valid "range-type", [ start, end ]
+            range_type = v.range_type;
+            range = v.range;
+        }
+        else if (this.recall_id) {
+            const id = recall_id_prefix+this.recall_id;
+            try {
+                var v = JSON.parse(localStorage.getItem(id));
+                range = v.range;
+                range_type = v.range_type;
+            } catch(e) {
+                // pass
+                console.error(e);
+                console.log(localStorage.getItem(id));
+            }
+        }
+
+        if (!range) {
+            range_type = default_range_type;
+            range = DateRange.rangeFromType(range_type)
+                .map(DateFormatter.ymd);
+        }
+        
+        return {
+            recall_id_full:(this.recall_id ?
+                            recall_id_prefix + this.recall_id :
+                            null),
+            range: range,
+            range_type: range_type,
+            options: [
+                { value:'wtd', text:'Week-to-date' },
+                { value:'mtd', text:'Month-to-date' },
+                { value:'ytd', text:'Year-to-date' },
+                { value:'custom', text:'Custom' }
+            ],
+        }
+    },
+
+    created: function() {
+        this.notify_change(true);
+    },
+
+    watch: {
+        'range': function() {
+            this.notify_change();
+        }
+    },
+
+    methods: {
+
+        validate_input_range: function(range) {
+            // if range is a string it's a range_type (eg "ytd")
+            // othersize its an 2-element array [start, end] of dates
+            // in YYYY-MM-DD (localtime) format
+            if (typeof range == 'string') {
+                var dates = DateRange.rangeFromType(range)
+                    .map(DateFormatter.ymd);
+                if (! range) return null;
+                return { range:dates, range_type:range };
+            }
+            else if (range.length == 2) {
+                var parser = d3.timeParse('%Y-%m-%d');
+                if (! parser(range[0]) || !parser(range[1]))
+                    return null;
+                return { range, range_type:'custom' };
+            }
+            else {
+                return null;
+            }
+        },
+
+        set_range: function(range) {
+            // if range is a string it's a range_type (eg "ytd")
+            // othersize its an 2-element array [start, end] of dates
+            // in YYYY-MM-DD (localtime) format
+            var v = this.validate_input_range(range);
+            if (!v) return false;
+            this.range = v.range;
+            this.range_type = v.range_type;
+            this.notify_change();
+            return true;
+        },
+        
+        notify_change: function(init) {
+            var parser = d3.timeParse('%Y-%m-%d');
+
+            var end_utc = new Date();
+            end_utc.setTime(
+                parser(this.range[1]).getTime() + (24 * 60 * 60 * 1000)
+            );
+            var range_utc = [
+                DateFormatter.ymdhms_utc(parser(this.range[0])),
+                DateFormatter.ymdhms_utc(end_utc)
+            ];
+            
+            this.$emit('change', {
+                // localtime "YYYY-MM-DD" format - exactly what came
+                // from the ui
+                range: this.range,
+
+                // convert localtime to utc, include hours. add 1 day
+                // to end so that range_utc encompasses times >=start
+                // and <end. save in the format "YYYYY-MM-DD HH:MM:SS"
+                range_utc: range_utc,
+
+                // 'custom', 'ytd', 'mtd', etc
+                range_type: this.range_type,
+                
+                // just created, if true
+                init: init || false,
+            });
+            
+            if (this.recall_id_full) {
+                localStorage.setItem(this.recall_id_full, JSON.stringify({
+                    range: this.range,
+                    range_type: this.range_type
+                }));
+            }
+        },
+        
+        range_type_change: function(evt) {
+            // ui select callback
+            if (this.range_type == 'wtd')
+                this.range = DateRange.wtd_as_ymd();
+            else if (this.range_type == 'mtd')
+                this.range = DateRange.mtd_as_ymd();
+            else if (this.range_type == 'ytd')
+                this.range = DateRange.ytd_as_ymd();
+        },
+
+    }
+    
+});
+

management/reporting/ui/index.html

@@ -0,0 +1,58 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title> Activity - MiaB-LDAP </title>
+    <base href="/admin/">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <meta name="robots" content="noindex, nofollow">
+
+    <link rel="stylesheet" href="ui-common/ui-bootstrap.css"/>
+    <link rel="stylesheet" href="ui-common/ui.css"/>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-vue@2/dist/bootstrap-vue.min.css"/>
+    
+    <!-- vue, vue-router, bootstrap-vue, axios, d3 -->
+    <script src="https://cdn.jsdelivr.net/npm/vue@2"></script>
+    <!--script src="https://cdn.jsdelivr.net/npm/vue@2/dist/vue.js"></script-->
+    <script src="https://cdn.jsdelivr.net/npm/vue-router@3"></script>
+    <script src="https://cdn.jsdelivr.net/npm/bootstrap-vue@2/dist/bootstrap-vue.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/bootstrap-vue@2/dist/bootstrap-vue-icons.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/axios"></script>
+    <script src="https://d3js.org/d3.v6.min.js"></script>
+
+    <script>axios.defaults.baseURL="/admin"</script>
+
+    <!-- our code -->
+    <script src="ui-common/exceptions.js"></script>
+    <script src="ui-common/authentication.js"></script>
+    <script src="ui-common/page-layout.js"></script>
+    <script src="ui-common/page-header.js"></script>
+    <script src="reports/ui/settings.js"></script>
+    <script src="reports/ui/capture-db-stats.js"></script>
+    <script src="reports/ui/charting.js"></script>
+    <script src="reports/ui/wbr-text.js"></script>
+    <script src="reports/ui/date-range-picker.js"></script>
+    <script src="reports/ui/chart-pie.js"></script>
+    <script src="reports/ui/chart-table.js"></script>
+    <script src="reports/ui/chart-stacked-bar-timeseries.js"></script>
+    <script src="reports/ui/chart-multi-line-timeseries.js"></script>
+    <script src="reports/ui/panel-messages-sent.js"></script>
+    <script src="reports/ui/panel-messages-received.js"></script>
+    <script src="reports/ui/panel-flagged-connections.js"></script>
+    <script src="reports/ui/panel-user-activity.js"></script>
+    <script src="reports/ui/panel-remote-sender-activity.js"></script>
+
+    <script src="reports/ui/reports-page-header.js"></script>
+    <script src="reports/ui/page-settings.js"></script>
+    <script src="reports/ui/page-reports-main.js"></script>
+    <script src="reports/ui/index.js"></script>
+  </head>
+    
+  <body onload="init_app()">
+
+    <router-view id="app"></router-view>
+    
+  </body>
+</html>
+

management/reporting/ui/index.js

@@ -0,0 +1,75 @@
+/*
+ * reports index page
+ */ 
+
+
+
+const app = {
+    router: new VueRouter({
+        routes: [
+            { path: '/', component: Vue.component('page-reports-main') },
+            { path: '/settings', component: Vue.component('page-settings') },
+            { path: '/:panel', component: Vue.component('page-reports-main') },
+        ],
+        scrollBehavior: function(to, from, savedPosition) {
+            if (savedPosition) {
+                return savedPosition
+            }
+        },
+    }),
+    
+    components: {
+        'page-settings': Vue.component('page-settings'),
+        'page-reports-main': Vue.component('page-reports-main'),
+    },
+        
+    data: {
+        me: null,
+    },
+
+    mounted: function() {
+        this.getMe();
+    },
+        
+    methods: {
+        getMe: function() {
+            axios.get('me').then(response => {
+                this.me = new Me(response.data);
+            }).catch(error => {
+                this.handleError(error);
+            });
+        },
+
+        handleError: function(error) {
+            if (error instanceof AuthenticationError) {
+                console.log(error);
+                window.location = '/admin';
+                return;
+            }
+            
+            console.error(error);
+            if (error instanceof ReferenceError) {
+                // uncaught coding bug, ignore
+                return;
+            }
+            if (error.status && error.reason)
+            {
+                // axios
+                error = error.reason;
+            }
+            this.$nextTick(() => {alert(''+error) });
+        }
+    }
+};
+
+
+
+function init_app() {
+    init_axios_interceptors();
+    
+    UserSettings.load().then(settings => {
+        new Vue(app).$mount('#app');
+    }).catch(error => {
+        alert('' + error);
+    });
+}

management/reporting/ui/page-reports-main.html

@@ -0,0 +1,94 @@
+<page-layout>
+  
+  <template v-slot:header>    
+    <reports-page-header :loading_counter="loading"></reports-page-header>
+  </template>
+  
+  <!-- div -->
+    
+    <b-modal ref="stats" hide-header no-fade ok-only no-close-on-backdrop>
+      <capture-db-stats></capture-db-stats>
+    </b-modal>
+    
+    <div class="d-flex align-items-end">
+      <date-range-picker ref="date_picker" :start_range="get_start_range($route, '-mtd')" recall_id="reports-main" @change="date_change($event)"></date-range-picker>
+      <div class="ml-auto mr-1" title="Database stats" role="button" @click="$refs.stats.show()"><b-icon icon="server" scale="1.5" aria-label="Database stats" variant="primary"></b-icon><b-icon icon="info" scale="1.5"></b-icon></div>
+    </div>
+    
+    <b-navbar type="dark" variant="secondary" class="mt-1">
+      <b-navbar-brand v-if="panel==''">Choose</b-navbar-brand>
+      <b-navbar-nav style="font-size:1.2em">
+        <b-nav-item
+          :active="panel=='messages-sent'"
+          :to="get_route('messages-sent')">Messages sent
+        </b-nav-item>
+        <b-nav-item
+          :active="panel=='messages-received'"
+          :to="get_route('messages-received')">Messages received
+        </b-nav-item>
+        <b-nav-item
+          :active="panel=='user-activity'"
+          :to="get_route('user-activity')">User activity
+        </b-nav-item>
+        <b-nav-item
+          :active="panel=='remote-sender-activity'"
+          :to="get_route('remote-sender-activity')">Remote sender activity
+        </b-nav-item>
+        <b-nav-item
+          :active="panel=='flagged-connections'"
+          :to="get_route('flagged-connections')">Notable connections
+        </b-nav-item>
+      </b-navbar-nav>
+    </b-navbar>
+
+    <keep-alive>
+      <panel-messages-sent
+        v-if="panel=='messages-sent'"
+        :date_range="range_utc"
+        :binsize="get_binsize()"
+        @loading="loading += $event"
+        :user_link="get_route('user-activity')"
+        class="mt-3">
+      </panel-messages-sent>
+      
+      <panel-messages-received
+        v-if="panel=='messages-received'"
+        :date_range="range_utc"
+        :binsize="get_binsize()"
+        @loading="loading += $event"
+        :user_link="get_route('user-activity', {tab:1})"
+        :remote_sender_email_link="get_route('remote-sender-activity')"
+        :remote_sender_server_link="get_route('remote-sender-activity')"
+        class="mt-3">
+      </panel-messages-received>
+      
+      <panel-user-activity
+        v-if="panel=='user-activity'"
+        :date_range="range_utc"
+        @loading="loading += $event"
+        class="mt-3">
+      </panel-user-activity>
+      
+      <panel-remote-sender-activity
+        v-if="panel=='remote-sender-activity'"
+        :date_range="range_utc"
+        @loading="loading += $event"
+        class="mt-3">
+      </panel-remote-sender-activity>
+
+      <panel-flagged-connections
+        v-if="panel=='flagged-connections'"
+        :date_range="range_utc"
+        :binsize="get_binsize()"
+        @loading="loading += $event"
+        :user_link="get_route('user-activity')"
+        :remote_sender_email_link="get_route('remote-sender-activity')"
+        :remote_sender_server_link="get_route('remote-sender-activity')"
+        class="mt-3">
+      </panel-flagged-connections>
+    </keep-alive>
+    
+  <!-- /div -->
+  
+  
+</page-layout>

management/reporting/ui/page-reports-main.js

@@ -0,0 +1,110 @@
+Vue.component('page-reports-main', function(resolve, reject) {
+    axios.get('reports/ui/page-reports-main.html').then((response) => { resolve({
+
+        template: response.data,
+
+        components: {
+            'page-layout': Vue.component('page-layout'),
+            'reports-page-header': Vue.component('reports-page-header'),
+            'date-range-picker': Vue.component('date-range-picker'),
+            'panel-messages-sent': Vue.component('panel-messages-sent'),
+            'panel-messages-received': Vue.component('panel-messages-received'),
+            'panel-flagged-connections': Vue.component('panel-flagged-connections'),
+            'panel-user-activity': Vue.component('panel-user-activity'),
+        },
+        
+        data: function() {
+            return {
+                // page-header loading spinner
+                loading: 0,
+                
+                // panels
+                panel: this.$route.params.panel || '',
+
+                // date picker - the range, if in the route, is set
+                // via date_change(), called during date picker
+                // create()
+                range_utc: null,  // Array(2): Date objects (UTC)
+                range: null, // Array(2): YYYY-MM-DD (localtime)
+                range_type: null, // String: "custom","ytd","mtd", etc
+            };
+        },
+
+        beforeRouteUpdate: function(to, from, next) {
+            //console.log(`page route update: to=${JSON.stringify({path:to.path, query:to.query})}  ....  from=${JSON.stringify({path:from.path, query:from.query})}`);
+            this.panel = to.params.panel;
+            // note: the range is already set in the class data
+            //
+            //  1. at component start - the current range is extracted
+            //     from $route by get_route_range() and passed to the
+            //     date picker as a prop, which subsequently
+            //     $emits('change') during creation where date_change()
+            //     updates the class data.
+            //
+            //  2. during user interaction - the date picker
+            //     $emits('change') where date_change() updates the
+            //     class data.
+            next();
+        },
+        
+        methods: {
+            get_start_range: function(to, default_range) {
+                if (to.query.range_type) {
+                    return to.query.range_type;
+                }
+                else if (to.query.start && to.query.end) {
+                    // start and end must be YYYY-MM-DD (localtime)
+                    return [
+                        to.query.start,
+                        to.query.end
+                    ];
+                }
+                else {
+                    return default_range;
+                }
+            },
+            
+            get_binsize: function() {
+                if (! this.range_utc) return 0;
+                return TimeseriesData.binsizeOfRange(this.range_utc);
+            },
+
+            date_change: function(evt) {
+                // date picker 'change' event
+                this.range_type = evt.range_type;
+                this.range_utc = evt.range_utc;
+                this.range = evt.range;
+                var route = this.get_route(this.panel);
+                if (! evt.init) {
+                    this.$router.replace(route);
+                }
+            },
+
+            get_route: function(panel, ex_query) {
+                // return vue-router route to `panel`
+                // eg: "/<panel>?start=YYYY-MM-DD&end=YYYY-MM-DD"
+                //
+                // additional panel query elements should be set in
+                // the panel's activate method
+                var route = { path: panel };
+                if (this.range_type != 'custom') {
+                    route.query = {
+                        range_type: this.range_type
+                    };
+                }
+                else {
+                    route.query = {
+                        start: this.range[0],
+                        end: this.range[1]
+                    };
+                }
+                Object.assign(route.query, ex_query);
+                return route;
+            },
+            
+        }
+    })}).catch((e) => {
+        reject(e);
+    });
+    
+});

management/reporting/ui/page-settings.html

@@ -0,0 +1,72 @@
+<page-layout>
+  
+  <template v-slot:header>
+    <reports-page-header :loading_counter="loading"></reports-page-header>
+  </template>
+
+  <div>
+    <div class="d-flex">
+      <div>Settings</div>
+      <router-link :to="from_route || '/'" class="ml-auto">Back to reports</router-link>
+    </div>
+
+    
+    <b-card class="mt-2">
+      <b-card-title>
+        UI settings
+      </b-card-title>
+      
+      <b-card-body>
+        <div class="d-flex align-items-baseline">
+          <div class="mr-1">Table data row limit</div>
+          <input type="number" min="5" v-model="row_limit" style="max-width:8em" v-on:keyup="update_user_settings"></input>
+          <div class="text-danger ml-2">
+            <em>{{row_limit_error}}</em>
+          </div>
+        </div>
+      </b-card-body>
+    </b-card>
+
+    
+    <b-card class="mt-2" v-if="capture_config && status">
+      <b-card-title>
+        Capture daemon
+      </b-card-title>
+      
+      <b-card-body>
+        <h4 class="d-flex">
+          <b-badge :variant="status_variant(status[0])">{{status[0]}}</b-badge>
+          <b-badge class="ml-2" :variant="status_variant(status[1])">{{status[1]}}</b-badge>
+          <b-badge class="ml-2" v-if="is_running()" :variant="status_variant(capture_config.capture)"> {{ capture_config.capture ? 'capturing' : 'paused' }}</b-badge>
+        </h4>
+        <p><i>(systemd service "miabldap-capture")</i></p>
+        
+        <b-form @submit.prevent class="mt-3" v-if="is_running()">
+          <b-form-checkbox v-model="capture" @change="config_changed=true">
+            Capture enabled
+          </b-form-checkbox> <em class="text-danger">Warning: when capture is disabled, the daemon will no longer record log activity</em>
+          <div class="d-flex align-items-baseline">
+            <div class="mr-1">Delete database records older than </div>
+            <input type="number" min="0" v-model="older_than_days" style="max-width:6em" v-on:keyup="config_changed_if(older_than_days, 0, null, capture_config.prune_policy.older_than_days)"></input>
+            <div class="ml-1">days</div>
+          </div>
+          <div class="mb-3 ml-2">
+            <em>(a value of zero preserves all records)</em>
+          </div>
+          
+          <b-form-checkbox v-model="capture_config.drop_disposition.faild_login_attempt" @change="config_changed=true">Ignore failed login attempts</b-form-checkbox>
+          <b-form-checkbox v-model="capture_config.drop_disposition.suspected_scanner" @change="config_changed=true">Ignore suspected scanner activity</b-form-checkbox>
+          <b-form-checkbox v-model="capture_config.drop_disposition.reject" @change="config_changed=true">Ignore rejected mail attempts</b-form-checkbox>
+
+        </b-form>
+
+        <div v-if="config_changed" class="mt-3">
+          <b-button variant="danger" @click="save_capture_config()">Commit changes and update server</b-button>
+        </div>
+
+      </b-card-body>
+    </b-card>
+    
+  </div>
+  
+</page-layout>

management/reporting/ui/page-settings.js

@@ -0,0 +1,123 @@
+Vue.component('page-settings', function(resolve, reject) {
+    axios.get('reports/ui/page-settings.html').then((response) => { resolve({
+
+        template: response.data,
+
+        components: {
+            'page-layout': Vue.component('page-layout'),
+            'reports-page-header': Vue.component('reports-page-header'),
+        },
+
+        data: function() {
+            return {
+                from_route: null,
+                loading: 0,
+
+                // server status and config
+                capture_config: null,
+                config_changed: false,
+                status: null,
+
+                // capture config models that require processing
+                // before the value is valid for `capture_config`, or
+                // the `capture_config` value is used by multiple
+                // elements (eg. one showing current state)
+                capture: true,
+                older_than_days: '',
+                
+                // user settings
+                row_limit: '' + UserSettings.get().row_limit,
+                row_limit_error: ''
+            };
+        },
+
+        beforeRouteEnter: function(to, from, next) {
+            next(vm => {
+                vm.from_route = from;
+            });
+        },
+
+        created: function() {
+            this.loadData();
+        },
+
+        methods: {
+            is_running: function() {
+                return this.status[0] == 'running';
+            },
+
+            status_variant: function(status) {
+                if (status == 'running') return 'success';
+                if (status == 'enabled') return 'success';
+                if (status == 'disabled') return 'warning';
+                if (status === true) return 'success';
+                return 'danger'
+            },
+
+            loadData: function() {
+                this.loading += 1;
+                Promise.all([
+                    CaptureConfig.get(),
+                    axios.get('/reports/capture/service/status')
+                ]).then(responses => {
+                    this.capture_config = responses[0];
+                    if (this.capture_config.status !== 'error') {
+                        this.older_than_days = '' +
+                            this.capture_config.prune_policy.older_than_days;
+                        this.capture = this.capture_config.capture;
+                    }
+                    this.status = responses[1].data;
+                    this.config_changed = false;
+                }).catch(error => {
+                    this.$root.handleError(error);
+                }).finally(() => {
+                    this.loading -= 1;
+                });
+            },
+
+            update_user_settings: function() {
+                if (this.row_limit == '') {
+                    this.row_limit_error = 'not valid';
+                    return;
+                }
+                try {
+                    const s = UserSettings.get();
+                    s.row_limit = Number(this.row_limit);
+                } catch(e) {
+                    this.row_limit_error = e.message;
+                }
+            },
+
+            config_changed_if: function(v, range_min, range_max, current_value) {
+                v = Number(v);
+                if (range_min !== null && v < range_min) return;
+                if (range_max !== null && v > range_max) return;
+                if (current_value !== null && v == current_value) return;
+                this.config_changed = true;
+            },
+
+            save_capture_config: function() {
+                this.loading+=1;
+                var newconfig = Object.assign({}, this.capture_config);
+                this.capture_config.prune_policy.older_than_days =
+                    Number(this.older_than_days);
+                newconfig.capture = this.capture;
+                axios.post('/reports/capture/config', newconfig)
+                    .then(response => {
+                        this.loadData();
+                    })
+                    .catch(error => {
+                        this.$root.handleError(error);
+                    })
+                    .finally(() => {
+                        this.loading-=1;
+                    });
+            }
+        }
+
+    })}).catch((e) => {
+        reject(e);
+    });
+});
+
+    

management/reporting/ui/panel-flagged-connections.html

@@ -0,0 +1,87 @@
+<div>
+  <div class="d-flex flex-wrap align-items-start">
+    <div class="p-2">
+      <strong>Connections by disposition</strong>
+      <chart-pie
+        :chart_data="connections_by_disposition"
+        :name_formatter="disposition_formatter"
+        :labels="false"
+        :width="radius_pie *2"
+        :height="radius_pie *2">
+      </chart-pie>
+    </div>
+
+    <chart-multi-line-timeseries
+      class="p-2"
+      :chart_data="failed_logins"
+      :width="width"
+      :height="linechart_height">
+    </chart-multi-line-timeseries>
+    
+    <chart-multi-line-timeseries
+      class="p-2"
+      :chart_data="suspected_scanners"
+      :width="width"
+      :height="linechart_height">
+    </chart-multi-line-timeseries>
+
+    <div class="d-flex flex-wrap align-items-start">
+      <chart-table
+        v-if="insecure_inbound"
+        :items="insecure_inbound.items"
+        :fields="insecure_inbound.fields"
+        :caption="insecure_inbound.y"
+        class="ml-4 mt-2"
+        style="max-width:50px">
+        <template #cell(sasl_username)="data">
+          <router-link class="text-dark" :to='link_to_user(data.value, 1)'>{{ data.value }}</router-link>
+        </template>
+        <template #cell(envelope_from)="data">
+          <router-link class="text-dark" :to='link_to_remote_sender_email(data.value)'>{{ data.value }}</router-link>
+        </template>
+        <template #cell(rcpt_to)="data">
+          <router-link class="text-dark" :to='link_to_user(data.value, 1)'>{{ data.value }}</router-link>
+        </template>
+      </chart-table>
+      
+      <chart-table
+        v-if="insecure_outbound"
+        :items="insecure_outbound.items"
+        :fields="insecure_outbound.fields"
+        :caption="insecure_outbound.y"
+        class="ml-4 mt-2"
+        style="max-width:50px">
+        <template #cell(sasl_username)="data">
+          <router-link class="text-dark" :to='link_to_user(data.value)'>{{ data.value }}</router-link>
+        </template>
+      </chart-table>
+    </div>
+
+    <div class="d-flex flex-wrap align-items-center">
+      <div class="p-2">
+        <strong>Mail delivery rejects by category</strong>
+        <chart-pie
+          :chart_data="reject_by_failure_category"
+          :labels="false"
+          :width="radius_pie *2"
+          :height="radius_pie *2">
+        </chart-pie>
+      </div>
+      
+      <chart-table
+        v-if="top_hosts_rejected"
+        :items="top_hosts_rejected.items"
+        :fields="top_hosts_rejected.fields"
+        :caption="top_hosts_rejected.y"
+        class="ml-4 mt-2"
+        style="max-width:50px">
+        <template #cell(remote_host)="data">
+          <router-link class="text-dark" :to='link_to_remote_sender_server(data.value)'>{{ data.value }}</router-link>
+        </template>
+      </chart-table>
+    </div>
+    
+  </div>
+
+</div>
+

management/reporting/ui/panel-flagged-connections.js

@@ -0,0 +1,133 @@
+Vue.component('panel-flagged-connections', function(resolve, reject) {
+    axios.get('reports/ui/panel-flagged-connections.html').then((response) => { resolve({
+
+        template: response.data,
+        
+        props: {
+            date_range: Array,  // YYYY-MM-DD strings (UTC)
+            binsize: Number,    // for timeseries charts, in minutes
+            user_link: Object,   // a route
+            remote_sender_email_link: Object, // a route
+            remote_sender_server_link: Object, // a route
+            width: { type:Number, default: ChartPrefs.default_width },
+            height: { type:Number, default: ChartPrefs.default_height },
+        },
+
+        components: {
+            'chart-multi-line-timeseries': Vue.component('chart-multi-line-timeseries'),
+            'chart-stacked-bar-timeseries': Vue.component('chart-stacked-bar-timeseries'),
+            'chart-pie': Vue.component('chart-pie'),
+            'chart-table': Vue.component('chart-table'),
+        },
+
+        computed: {
+            radius_pie: function() {
+                return this.height / 5;
+            },
+            linechart_height: function() {
+                return this.height / 2;
+            }
+        },
+        
+        data: function() {
+            return {
+                data_date_range: null,
+                colors: ChartPrefs.colors,
+                failed_logins: null,  // TimeseriesData
+                suspected_scanners: null, // TimeseriesData
+                connections_by_disposition: null,  // pie chart data
+                disposition_formatter: ConnectionDisposition.formatter,
+                reject_by_failure_category: null, // pie chart data
+                top_hosts_rejected: null,  // table
+                insecure_inbound: null,  // table
+                insecure_outbound: null, // table
+            };
+        },
+                
+        activated: function() {
+            // see if props changed when deactive
+            if (this.date_range && this.date_range !== this.data_date_range)
+                this.getChartData();            
+        },
+        
+        watch: {
+            // watch props for changes
+            'date_range': function() {
+                this.getChartData();
+            }
+        },
+        
+        methods: {
+            link_to_user: function(user_id, tab) {
+                // add user=user_id to the user_link route
+                var r = Object.assign({}, this.user_link);
+                r.query = Object.assign({}, this.user_link.query);
+                r.query.user = user_id;
+                r.query.tab = tab;
+                return r;
+            },
+            link_to_remote_sender_email: function(email) {
+                // add email=email to the remote_sender_email route
+                var r = Object.assign({}, this.remote_sender_email_link);
+                r.query = Object.assign({}, this.remote_sender_email_link.query);
+                r.query.email = email;
+                return r;
+            },            
+            link_to_remote_sender_server: function(server) {
+                // add server=server to the remote_sender_server route
+                var r = Object.assign({}, this.remote_sender_server_link);
+                r.query = Object.assign({}, this.remote_sender_server_link.query);
+                r.query.server = server;
+                return r;
+            },
+            
+            getChartData: function() {
+                this.$emit('loading', 1);                
+                axios.post('reports/uidata/flagged-connections', {
+                    'start': this.date_range[0],
+                    'end': this.date_range[1],
+                    'binsize': this.binsize,
+                }).then(response => {
+                    this.data_date_range = this.date_range;
+
+                    // line charts
+                    var ts = new TimeseriesData(response.data.flagged);
+                    this.failed_logins =
+                        ts.dataView(['failed_login_attempt']);
+                    this.suspected_scanners =
+                        ts.dataView(['suspected_scanner']);
+                    
+                    // pie chart for connections by disposition
+                    this.connections_by_disposition =
+                        response.data.connections_by_disposition;
+
+                    // pie chart for reject by failure_category
+                    this.reject_by_failure_category =
+                        response.data.reject_by_failure_category;
+
+                    // table of top 10 hosts rejected by failure_category
+                    this.top_hosts_rejected =
+                        new BvTable(response.data.top_hosts_rejected);
+
+                    // insecure connections tables
+                    this.insecure_inbound
+                        = new BvTable(response.data.insecure_inbound);
+                    this.insecure_outbound
+                        = new BvTable(response.data.insecure_outbound);
+                    
+                }).catch(error => {
+                    this.$root.handleError(error);
+                }).finally(() => {
+                    this.$emit('loading', -1);
+                });
+            },
+            
+        }
+        
+        
+    })}).catch((e) => {
+        reject(e);
+    });
+    
+});
+

management/reporting/ui/panel-messages-received.html

@@ -0,0 +1,64 @@
+<div>
+<div class="d-flex flex-wrap align-items-top">
+  
+  <chart-multi-line-timeseries
+    class="pt-1"
+    :chart_data="data_received"
+    :width="width"
+    :height="height/2">
+  </chart-multi-line-timeseries>
+
+  <div class="d-flex flex-wrap align-items-top">
+    <chart-table
+      v-if="top_senders_by_count"
+      :items="top_senders_by_count.items"
+      :fields="top_senders_by_count.fields"
+      :caption="top_senders_by_count.y"
+      class="ml-4 mt-2"
+      style="max-width:50px">
+      <template #cell(email)="data">
+        <router-link class="text-dark" :to='link_to_remote_sender_email(data.value)'>{{ data.value }}</router-link>
+      </template>
+    </chart-table>
+    
+    <chart-table
+      v-if="top_senders_by_size"
+      :items="top_senders_by_size.items"
+      :fields="top_senders_by_size.fields"
+      :caption="top_senders_by_size.y"
+      class="ml-4 mt-2"
+      style="max-width:50px">
+      <template #cell(email)="data">
+        <router-link class="text-dark" :to='link_to_remote_sender_email(data.value)'>{{ data.value }}</router-link>
+      </template>
+    </chart-table>
+  </div>
+
+  <div class="d-flex flex-wrap align-items-top">
+    <chart-table
+      v-if="top_hosts_by_spam_score"
+      :items="top_hosts_by_spam_score.items"
+      :fields="top_hosts_by_spam_score.fields"
+      :caption="top_hosts_by_spam_score.y"
+      class="ml-4 mt-2"
+      style="max-width:50px">
+      <template #cell(remote_host)="data">
+        <router-link class="text-dark" :to='link_to_remote_sender_server(data.value)'>{{ data.value }}</router-link>
+      </template>
+    </chart-table>
+    
+    <chart-table
+      v-if="top_user_receiving_spam"
+      :items="top_user_receiving_spam.items"
+      :fields="top_user_receiving_spam.fields"
+      :caption="top_user_receiving_spam.y"
+      class="ml-4 mt-2"
+      style="max-width:50px">
+      <template #cell(rcpt_to)="data">
+        <router-link class="text-dark" :to='link_to_user(data.value)'>{{ data.value }}</router-link>
+      </template>
+    </chart-table>
+  </div>
+  
+</div>
+</div>

management/reporting/ui/panel-messages-received.js

@@ -0,0 +1,116 @@
+/*
+  set of charts/tables showing messages received from internet servers
+
+*/
+
+Vue.component('panel-messages-received', function(resolve, reject) {
+    axios.get('reports/ui/panel-messages-received.html').then((response) => { resolve({
+
+        template: response.data,
+        
+        props: {
+            date_range: Array,
+            binsize: Number,
+            user_link: Object,
+            remote_sender_email_link: Object,
+            remote_sender_server_link: Object,
+            width: { type:Number, default: ChartPrefs.default_width },
+            height: { type:Number, default: ChartPrefs.default_height },
+        },
+
+        components: {
+            'chart-multi-line-timeseries': Vue.component('chart-multi-line-timeseries'),
+            // 'chart-stacked-bar-timeseries': Vue.component('chart-stacked-bar-timeseries'),
+            // 'chart-pie': Vue.component('chart-pie'),
+            'chart-table': Vue.component('chart-table'),
+        },
+        
+        data: function() {
+            return {
+                data_date_range: null,
+                data_received: null,
+                top_senders_by_count: null,
+                top_senders_by_size: null,
+                top_hosts_by_spam_score: null,
+                top_user_receiving_spam: null,
+            };
+        },
+        
+        computed: {
+        },
+        
+        activated: function() {
+            // see if props changed when deactive
+            if (this.date_range && this.date_range !== this.data_date_range)
+                this.getChartData();            
+        },
+        
+        watch: {
+            // watch props for changes
+            'date_range': function() {
+                this.getChartData();
+            }
+        },
+        
+        methods: {
+            link_to_user: function(user_id) {
+                // add user=user_id to the user_link route
+                var r = Object.assign({}, this.user_link);
+                r.query = Object.assign({}, this.user_link.query);
+                r.query.user = user_id;
+                return r;
+            },
+            link_to_remote_sender_email: function(email) {
+                // add email=email to the remote_sender_email route
+                var r = Object.assign({}, this.remote_sender_email_link);
+                r.query = Object.assign({}, this.remote_sender_email_link.query);
+                r.query.email = email;
+                return r;
+            },
+            link_to_remote_sender_server: function(server) {
+                // add server=server to the remote_sender_server route
+                var r = Object.assign({}, this.remote_sender_server_link);
+                r.query = Object.assign({}, this.remote_sender_server_link.query);
+                r.query.server = server;
+                return r;
+            },
+
+            getChartData: function() {
+                this.$emit('loading', 1);
+                axios.post('reports/uidata/messages-received', {
+                    'start': this.date_range[0],
+                    'end': this.date_range[1],
+                    'binsize': this.binsize,
+                }).then(response => {
+                    this.data_date_range = this.date_range;
+                    var ts = new TimeseriesData(response.data.ts_received);
+                    this.data_received = ts;
+
+                    [ 'top_senders_by_count',
+                      'top_senders_by_size',
+                      'top_hosts_by_spam_score',
+                      'top_user_receiving_spam'
+                    ].forEach(item => {
+                        this[item] = response.data[item];
+                        BvTable.setFieldDefinitions(
+                            this[item].fields,
+                            this[item].field_types
+                        );
+                    });
+                    
+                }).catch(error => {
+                    this.$root.handleError(error);
+                }).finally(() => {
+                    this.$emit('loading', -1);
+                });
+            },
+            
+        }
+        
+        
+    })}).catch((e) => {
+        reject(e);
+    });
+    
+});
+

management/reporting/ui/panel-messages-sent.html

@@ -0,0 +1,50 @@
+<div>
+  <div class="d-flex flex-wrap align-items-center">
+  <chart-multi-line-timeseries
+    class="pt-1"
+    :chart_data="data_sent"
+    :width="width"
+    :height="height_sent">
+  </chart-multi-line-timeseries>
+  
+  <div class="d-flex flex-wrap align-items-top">
+    <chart-table
+      v-if="top_senders_by_count"
+      :items="top_senders_by_count.items"
+      :fields="top_senders_by_count.fields"
+      :caption="top_senders_by_count.y"
+      class="ml-4 mt-2"
+      style="max-width:50px">
+      <template #cell(user)="data">
+        <router-link class="text-dark" :to="link_to_user(data.value)">{{ data.value }}</router-link>
+      </template>
+    </chart-table>
+    
+    <chart-table
+      v-if="top_senders_by_size"
+      :items="top_senders_by_size.items"
+      :fields="top_senders_by_size.fields"
+      :caption="top_senders_by_size.y"
+      class="ml-4 mt-2"
+      style="max-width:50px">
+      <template #cell(user)="data">
+        <router-link class="text-dark" :to="link_to_user(data.value)">{{ data.value }}</router-link>
+      </template>
+    </chart-table>
+  </div>
+  
+  <chart-stacked-bar-timeseries
+    class="pt-1"
+    :chart_data="data_recip"
+    :width="width"
+    :height="height_recip">
+  </chart-stacked-bar-timeseries>
+  
+  <chart-pie
+    class="ml-4"
+    :chart_data="data_recip_pie"
+    :width="radius_recip_pie *2"
+    :height="radius_recip_pie *2">
+  </chart-pie>  
+</div>
+</div>

management/reporting/ui/panel-messages-sent.js

@@ -0,0 +1,132 @@
+/*
+  set of charts/tables showing messages sent by local users
+  - number of messages sent over time
+    - delivered locally
+    - delivered remotely
+  - top senders
+
+  emits:
+     'loading'    event=number
+*/
+
+Vue.component('panel-messages-sent', function(resolve, reject) {
+    axios.get('reports/ui/panel-messages-sent.html').then((response) => { resolve({
+
+        template: response.data,
+        
+        props: {
+            date_range: Array,  // YYYY-MM-DD strings (UTC)
+            binsize: Number,    // for timeseries charts, in minutes
+            // to enable clickable users, specify the route in
+            // user_link. the user_id will be added to the
+            // route.query as user=user_id. If set to 'true', the
+            // current route will be used.
+            user_link: Object,
+            width: { type:Number, default: ChartPrefs.default_width },
+            height: { type:Number, default: ChartPrefs.default_height },
+        },
+
+        components: {
+            'chart-multi-line-timeseries': Vue.component('chart-multi-line-timeseries'),
+            'chart-stacked-bar-timeseries': Vue.component('chart-stacked-bar-timeseries'),
+            'chart-pie': Vue.component('chart-pie'),
+            'chart-table': Vue.component('chart-table'),
+        },
+        
+        data: function() {
+            return {
+                data_date_range: null,
+                data_sent: null,
+                data_recip: null,
+                data_recip_pie: null,
+                top_senders_by_count: null,
+                top_senders_by_size: null,
+            };
+        },
+        
+        computed: {
+            height_sent: function() {
+                return this.height / 2;
+            },
+            
+            height_recip: function() {
+                return this.height / 2;
+            },
+            
+            radius_recip_pie: function() {
+                return this.height /5;
+            },
+        },
+        
+        activated: function() {
+            // see if props changed when deactive
+            if (this.date_range && this.date_range !== this.data_date_range)
+                this.getChartData();            
+        },
+        
+        watch: {
+            // watch props for changes
+            'date_range': function() {
+                this.getChartData();
+            }
+        },
+        
+        methods: {
+            link_to_user: function(user_id) {
+                // add user=user_id to the user_link route
+                var r = Object.assign({}, this.user_link);
+                r.query = Object.assign({}, this.user_link.query);
+                r.query.user = user_id;
+                return r;
+            },
+            
+            getChartData: function() {
+                this.$emit('loading', 1);                
+                axios.post('reports/uidata/messages-sent', {
+                    'start': this.date_range[0],
+                    'end': this.date_range[1],
+                    'binsize': this.binsize,
+                }).then(response => {
+                    this.data_date_range = this.date_range;
+                    var ts = new TimeseriesData(response.data.ts_sent);
+                    this.data_sent = ts.dataView(['sent']);
+                    this.data_recip = ts.dataView(['local','remote'])
+                    
+                    this.data_recip_pie = [{
+                        name:'local',
+                        value:d3.sum(ts.get_series('local').values)
+                    }, {
+                        name:'remote',
+                        value:d3.sum(ts.get_series('remote').values)
+                    }];
+                    
+                    this.top_senders_by_count =
+                        response.data.top_senders_by_count;
+                    BvTable.setFieldDefinitions(
+                        this.top_senders_by_count.fields,
+                        this.top_senders_by_count.field_types
+                    );
+                    
+                    this.top_senders_by_size =
+                        response.data.top_senders_by_size;
+                    BvTable.setFieldDefinitions(
+                        this.top_senders_by_size.fields,
+                        this.top_senders_by_size.field_types
+                    );
+                    
+                }).catch(error => {
+                    this.$root.handleError(error);
+                }).finally(() => {
+                    this.$emit('loading', -1);
+                });
+            },
+            
+        }
+        
+        
+    })}).catch((e) => {
+        reject(e);
+    });
+    
+});
+

management/reporting/ui/panel-remote-sender-activity.html

@@ -0,0 +1,74 @@
+<div>
+
+  <b-modal ref="suggest_modal" scrollable header-bg-variant="dark" header-text-variant="light" ok-only ok-title="close" no-close-on-backdrop>
+    <template #modal-title>
+      {{ select_list.suggestions.length }} suggestions found
+    </template>
+    <div v-if="select_list.limited" class="text-danger">Too many results - the server returned only a limited set.</div>
+    <template v-if="select_list.suggestions.length>0">
+      <div>Choose one:</div>
+      <div v-for="suggestion in select_list.suggestions" class="text-nowrap">
+        <a href="" @click.prevent="choose_suggestion(suggestion)">
+          {{ suggestion }}
+        </a>
+      </div>
+    </template>
+    <template v-else>
+      <div>nothing matched</div>
+    </template>
+  </b-modal>
+
+  <datalist id="panel-rsa-recent">
+    <option v-if="recent_senders" v-for="s in recent_senders">{{ s }}</option>
+  </datalist>
+  
+  <b-form @submit.prevent.stop="change_sender" class="d-flex mb-3">
+    <div class="d-flex mr-2" style="margin-top:0.25rem" title="Sender type">
+      <b-form-radio v-model="sender_type" value="email" @change="update_recent_list()">Email</b-form-radio>
+      <b-form-radio class="ml-1" v-model="sender_type" value="server" @change="update_recent_list()">Server</b-form-radio>
+    </div>
+    <b-input-group style="width:40em">
+      <b-form-input v-if="sender_type=='email'" class="h-auto" :autofocus="data_sender===null" list="panel-rsa-recent" v-model="email" placeholder="Enter an email address (envelope FROM)"></b-form-input>
+      <b-form-input v-else class="h-auto" :autofocus="data_sender===null" list="panel-rsa-recent" v-model="server" placeholder="Enter a hostname or ip address"></b-form-input>
+      <b-input-group-append>
+        <b-button variant="primary" @click="change_sender" :disabled="sender_type=='email' && (email == '' || email==data_sender) || sender_type=='server' && (server =='' || server==data_sender)">Search</b-button>
+      </b-input-group-append>
+    </b-input-group>
+    <b-alert variant="warning" class="ml-2" :show="activity && activity.items.length>=get_row_limit()"><sup>*</sup> Tables limited to {{ get_row_limit() }} rows <router-link to="/settings"><b-icon icon="gear-fill"></b-icon></router-link></b-alert>
+    <b-form-checkbox class="ml-auto" v-model="show_only_flagged" @change="show_only_flagged_change()">Flagged only</b-form-checkbox>
+  </b-form>
+
+  <b-tabs content-class="mt2" v-model="tab_index" v-if="activity">
+    <b-tab>
+      <template #title>
+        {{ data_sender || ''}}<sup v-if="activity.items.length >= get_row_limit()">*</sup> ({{activity.unique_sends}} &rarr; {{activity.items.length}})
+      </template>
+      <b-table
+        class="sticky-table-header-0 bg-light"
+        small
+        :filter="show_only_flagged_filter"
+        :filter-function="table_filter_cb"
+        tbody-tr-class="cursor-pointer"
+        details-td-class="cursor-default"
+        @row-clicked="row_clicked"
+        :items="activity.items"
+        :fields="activity.fields">
+        <template #row-details="row">
+          <b-card>
+            <div><strong>Connection disposition</strong>: {{ disposition_formatter(row.item.disposition) }}</div>
+            <div v-if="row.item.sasl_username"><strong>Sasl username</strong>: {{row.item.sasl_username}}</div>
+            <div v-if="row.item.category"><strong>Failure category</strong>: {{row.item.category}}</div>
+            <div v-if="row.item.failure_info"><strong>Failure info</strong>: {{row.item.failure_info}}</div>
+            <div v-if="row.item.dkim_reason"><strong>Dkim reason</strong>: {{row.item.dkim_reason}}</div>
+            <div v-if="row.item.dmarc_reason"><strong>Dmarc reason</strong>: {{row.item.dmarc_reason}}</div>
+            <div v-if="row.item.postgrey_reason"><strong>Postgrey reason</strong>: {{row.item.postgrey_reason}}</div>
+            <div v-if="row.item.postgrey_delay"><strong>Postgrey delay</strong>: {{activity.x_fields.postgrey_delay.formatter(row.item.postgrey_delay)}}</div>
+            <div v-if="row.item.spam_result"><strong>Spam score</strong>: {{activity.x_fields.spam_score.formatter(row.item.spam_score)}}</div>
+          </b-card>
+        </template>
+        
+      </b-table>
+    </b-tab>    
+  </b-tabs>
+  
+</div>

management/reporting/ui/panel-remote-sender-activity.js

@@ -0,0 +1,247 @@
+/*
+  details on the activity of a remote sender (envelope from)
+*/
+
+Vue.component('panel-remote-sender-activity', function(resolve, reject) {
+    axios.get('reports/ui/panel-remote-sender-activity.html').then((response) => { resolve({
+
+        template: response.data,
+        
+        props: {
+            date_range: Array,  // YYYY-MM-DD strings (UTC)
+        },
+
+        data: function() {
+            const usersetting_prefix = 'panel-rsa-';
+            const sender_type = this.$route.query.email ? 'email' :
+                  ( this.$route.query.server ? 'server' : 'email' );
+
+            return {
+                email: this.$route.query.email || '', /* v-model */
+                server: this.$route.query.server || '', /* v-model */
+                sender_type: sender_type, /* "email" or "server" only */
+                
+                tab_index: 0, /* v-model */
+                
+                show_only_flagged: false,
+                show_only_flagged_filter: null,
+                
+                data_sender: null, /* sender for active table data */
+                data_sender_type: null, /* "email" or "server" */
+                data_date_range: null, /* date range for active table data */
+                
+                activity: null, /* table data */
+                disposition_formatter: ConnectionDisposition.formatter,
+
+                /* recent list */
+                set_prefix: usersetting_prefix,
+                recent_senders: UserSettings.get()
+                    .get_recent_list(usersetting_prefix + sender_type),
+
+                /* suggestions (from server) */
+                select_list: { suggestions: [] }
+            };
+        },
+
+        activated: function() {
+            const new_email = this.$route.query.email;
+            const new_server = this.$route.query.server;
+            const new_sender_type = new_email ? 'email' :
+                  ( new_server ? 'server' : null );
+
+            var load = false;
+            if (new_sender_type &&
+                new_sender_type != this.sender_type)
+            {
+                this.sender_type = new_sender_type;
+                load = true;
+            }            
+            if (this.sender_type == 'email' &&
+                new_email &&
+                new_email != this.email)
+            {
+                this.email = new_email;
+                this.getChartData();
+                return;
+            }
+            if (this.sender_type == 'server' &&
+                new_server &&
+                new_server != this.server)
+            {
+                this.server = new_server;
+                this.getChartData();
+                return;
+            }
+            
+            // see if props changed when deactive
+            if (load || this.date_range &&
+                this.date_range !== this.data_date_range)
+            {
+                this.getChartData();
+            }
+            else
+            {
+                // ensure the route query contains the sender
+                this.update_route();
+            }
+        },
+
+        watch: {
+            // watch props for changes
+            'date_range': function() {
+                this.getChartData();
+            }
+        },
+        
+        methods: {
+            update_recent_list: function() {
+                this.recent_senders = UserSettings.get()
+                    .get_recent_list(this.set_prefix + this.sender_type);
+            },
+            
+            update_route: function() {
+                // ensure the route contains query element
+                // "email=<data_sender>" or "server=<data_sender>"
+                // for the loaded data
+                if (this.data_sender && this.data_sender !== this.$route.query[this.sender_type]) {
+                    var route = Object.assign({}, this.$route);
+                    route.query = Object.assign({}, this.$route.query);
+                    delete route.query.sender;
+                    delete route.query.email;
+                    route.query[this.sender_type] = this.data_sender;
+                    this.$router.replace(route);
+                }
+            },
+            
+            change_sender: function() {
+                axios.post('/reports/uidata/select-list-suggestions', {
+                    type: this.sender_type == 'email' ?
+                        'envelope_from' : 'remote_host',
+                    query: this.sender_type == 'email' ?
+                        this.email.trim() : this.server.trim(),
+                    start_date: this.date_range[0],
+                    end_date: this.date_range[1]
+                }).then(response => {
+                    if (response.data.exact) {
+                        this.getChartData();
+                    }
+                    else {
+                        this.select_list = response.data;
+                        this.$refs.suggest_modal.show()
+                    }
+                }).catch(error => {
+                    this.$root.handleError(error);
+                });
+            },
+
+            choose_suggestion: function(suggestion) {
+                this[this.sender_type] = suggestion;
+                this.getChartData();
+                this.$refs.suggest_modal.hide();
+            },
+
+            combine_fields: function() {
+                // remove these fields...
+                this.activity
+                    .combine_fields([
+                        'sent_id',
+                        'sasl_username',
+                        'spam_score',
+                        'dkim_reason',
+                        'dmarc_reason',
+                        'postgrey_reason',
+                        'postgrey_delay',
+                        'category',
+                        'failure_info',
+                    ]);
+            },
+
+            get_row_limit: function() {
+                return UserSettings.get().row_limit;
+            },
+
+            update_activity_rowVariant: function() {
+                // there is 1 row for each recipient of a message
+                // - give all rows of the same message the same
+                // color
+                this.activity.apply_rowVariant_grouping('info', (item, idx) => {
+                    if (this.show_only_flagged && !item._flagged)
+                        return null;
+                    return item.sent_id;
+                });                
+            },
+            
+            show_only_flagged_change: function() {
+                // 'change' event callback for checkbox
+                this.update_activity_rowVariant();
+                // trigger BV to filter or not filter via
+                // reactive `show_only_flagged_filter`
+                this.show_only_flagged_filter=
+                    (this.show_only_flagged ? 'yes' : null );                
+            },
+                
+            table_filter_cb: function(item, filter) {
+                // when filter is non-null, this is called by BV for
+                // each row to determine whether it will be filtered
+                // (false) or included in the output (true)
+                return item._flagged;
+            },
+            
+            getChartData: function() {
+                if (!this.date_range || !this[this.sender_type]) {
+                    return;
+                }
+
+                this.$emit('loading', 1);
+                axios.post('reports/uidata/remote-sender-activity', {
+                    row_limit: this.get_row_limit(),
+                    sender: this[this.sender_type].trim(),
+                    sender_type: this.sender_type,
+                    start_date: this.date_range[0],
+                    end_date: this.date_range[1]
+                    
+                }).then(response => {
+                    this.data_sender = this[this.sender_type].trim();
+                    this.data_sender_type = this.sender_type;
+                    this.data_date_range = this.date_range;
+                    this.update_route();
+                    this.recent_senders = UserSettings.get()
+                        .add_to_recent_list(
+                            this.set_prefix + this.sender_type,
+                            this[this.sender_type]
+                        );
+                    this.show_only_flagged = false;
+                    this.show_only_flagged_filter = null;
+                    
+                    /* setup table data */
+                    this.activity =
+                        new MailBvTable(response.data.activity, {
+                            _showDetails: true
+                        });
+                    this.combine_fields();
+                    this.activity
+                        .flag_fields()
+                        .get_field('connect_time')
+                        .add_tdClass('text-nowrap');
+                    this.update_activity_rowVariant();
+                    
+                }).catch(error => {
+                    this.$root.handleError(error);
+                    
+                }).finally(() => {
+                    this.$emit('loading', -1);
+                });
+
+            },
+
+            row_clicked: function(item, index, event) {
+                item._showDetails = ! item._showDetails;
+            },
+            
+        }
+        
+    })}).catch((e) => {
+        reject(e);
+    });
+    
+});

management/reporting/ui/panel-user-activity.html

@@ -0,0 +1,75 @@
+<div>
+  
+  <datalist id="panel-ua-users">
+    <option v-for="user in all_users">{{ user }}</option>
+  </datalist>
+
+  <b-form @submit.prevent="getChartData()" class="d-flex">
+    <b-input-group class="mb-3" style="width:30em">
+      <b-form-input class="h-auto" :autofocus="data_user_id===null" list="panel-ua-users" v-model="user_id" placeholder="Enter a user id/email address"></b-form-input>
+      <b-input-group-append>
+        <b-button variant="primary" @click="change_user">Change user</b-button>
+      </b-input-group-append>
+    </b-input-group>
+    <b-alert variant="warning" class="ml-2" :show="sent_mail && sent_mail.items.length>=get_row_limit() || received_mail && received_mail.items.length>=get_row_limit()"><sup>*</sup> Tables limited to {{ get_row_limit() }} rows <router-link to="/settings"><b-icon icon="gear-fill"></b-icon></router-link></b-alert>
+    <b-form-checkbox class="ml-auto" v-model="show_only_flagged" @change="show_only_flagged_change()">Flagged only</b-form-checkbox>
+  </b-form>
+
+  <b-tabs content-class="mt2" v-model="tab_index" v-if="sent_mail && received_mail">
+    <b-tab>
+      <template #title>
+        Sent mail<sup v-if="sent_mail.items.length >= get_row_limit()">*</sup> ({{sent_mail.unique_sends}} &rarr; {{sent_mail.items.length}})
+      </template>
+      <b-table
+        class="sticky-table-header-0 bg-light"
+        small
+        :filter="show_only_flagged_filter"
+        :filter-function="table_filter_cb"
+        tbody-tr-class="cursor-pointer"
+        details-td-class="cursor-default"
+        @row-clicked="row_clicked"
+        :items="sent_mail.items"
+        :fields="sent_mail.fields">
+        <template #row-details="row">
+          <b-card>
+            <div><strong>Relay</strong>: {{row.item.relay}}</div>
+            <div v-if="row.item.service != 'lmtp'"><strong>Connection</strong>:{{ row.item.delivery_connection_info }}</div>
+            <div><strong>Delivery</strong>: {{row.item.delivery_info}}</div>
+            <div v-if="row.item.spam_result"><strong>Spam score</strong>: {{sent_mail.x_fields.spam_score.formatter(row.item.spam_score)}}</div>
+          </b-card>
+        </template>
+        
+      </b-table>
+    </b-tab>
+    
+    <b-tab :title="`Received mail (${received_mail.items.length})`">
+      <template #title>
+        Received mail<sup v-if="received_mail.items.length >= get_row_limit()">*</sup> ({{received_mail.items.length}})
+      </template>
+      <b-table
+        class="sticky-table-header-0 bg-light"
+        small
+        :filter="show_only_flagged_filter"
+        :filter-function="table_filter_cb"
+        tbody-tr-class="cursor-pointer"
+        details-td-class="cursor-default"
+        @row-clicked="row_clicked"
+        :items="received_mail.items"
+        :fields="received_mail.fields">
+        <template #cell(envelope_from)='data'>
+          <wbr-text :text="data.value" :text_break_threshold="15"></wbr-text>
+        </template>
+        <template #row-details="row">
+          <b-card>
+            <div><strong>Connection disposition</strong>: {{ disposition_formatter(row.item.disposition) }}</div>
+            <div v-if="row.item.dkim_reason"><strong>Dkim reason</strong>: {{row.item.dkim_reason}}</div>
+            <div v-if="row.item.dmarc_reason"><strong>Dmarc reason</strong>: {{row.item.dmarc_reason}}</div>
+            <div v-if="row.item.postgrey_reason"><strong>Postgrey reason</strong>: {{row.item.postgrey_reason}}</div>
+            <div v-if="row.item.postgrey_delay"><strong>Postgrey delay</strong>: {{received_mail.x_fields.postgrey_delay.formatter(row.item.postgrey_delay)}}</div>
+            <div v-if="row.item.spam_result"><strong>Spam score</strong>: {{received_mail.x_fields.spam_score.formatter(row.item.spam_score)}}</div>
+          </b-card>
+        </template>
+      </b-table>
+    </b-tab>
+  </b-tabs>
+</div>

management/reporting/ui/panel-user-activity.js

@@ -0,0 +1,260 @@
+/*
+  details on the activity of a user
+*/
+
+Vue.component('panel-user-activity', function(resolve, reject) {
+    axios.get('reports/ui/panel-user-activity.html').then((response) => { resolve({
+
+        template: response.data,
+        
+        props: {
+            date_range: Array,  // YYYY-MM-DD strings (UTC)
+        },
+
+        components: {
+            'wbr-text': Vue.component('wbr-text'),
+        },
+        
+        data: function() {
+            var start_tab = this.$route.query.tab ?
+                Number(this.$route.query.tab) :
+                0;
+            return {
+                user_id: this.$route.query.user || '', /* v-model */
+                tab_index: start_tab, /* v-model */
+                show_only_flagged: false,
+                show_only_flagged_filter: null,
+                data_user_id: null, /* user_id for active table data */
+                data_date_range: null, /* date range for active table data */
+                sent_mail: null,
+                received_mail: null,
+                all_users: [],
+                disposition_formatter: ConnectionDisposition.formatter,
+            };
+        },
+
+        activated: function() {
+            const new_tab = Number(this.$route.query.tab);
+            const new_user = this.$route.query.user;
+            
+            if (new_user && new_user != this.user_id) {
+                this.user_id = new_user;
+                this.getChartData(isNaN(new_tab) ? 0 : new_tab);
+                return;
+            }
+            
+            // first time activated...
+            if (this.all_users.length == 0)
+                this.getChartData(new_tab);
+            
+            // see if props changed when deactive
+            else if (this.date_range && this.date_range !== this.data_date_range)
+                this.getChartData(new_tab);
+            else {
+                // ensure the route query contains "user=<data_user_id>"
+                if (!isNaN(new_tab)) this.tab_index = new_tab;
+                this.update_route();
+            }
+
+        },
+
+        watch: {
+            // watch props for changes
+            'date_range': function() {
+                this.getChartData();
+            }
+        },
+        
+        methods: {
+            update_route: function() {
+                // ensure the route contains query element
+                // "user=<data_user_id>" for the loaded data
+                if (this.data_user_id && this.data_user_id !== this.$route.query.user) {
+                    var route = Object.assign({}, this.$route);
+                    route.query = Object.assign({}, this.$route.query);
+                    route.query.user=this.data_user_id;
+                    this.$router.replace(route);
+                }
+            },
+            
+            change_user: function() {
+                this.getChartData(0);
+            },
+
+            combine_sent_mail_fields: function() {
+                // remove these fields...
+                this.sent_mail.combine_fields([
+                    'sent_id',
+                    'spam_score',
+                    'delivery_info',
+                    'delivery_connection_info',
+                ]);
+                
+                // combine fields 'envelope_from' and 'rcpt_to'
+                this.sent_mail.combine_fields(
+                    'envelope_from',
+                    'rcpt_to',
+                    (v, key, item) => {
+                        if (item.envelope_from == this.data_user_id)
+                            return v;
+                        return `${v} (FROM: ${item.envelope_from})`;
+                    });
+                
+                // combine fields 'relay', 'delivery_connection'
+                this.sent_mail.combine_fields(
+                    'delivery_connection',
+                    'relay',
+                    (v, key, item) => {
+                        if (item.service == 'lmtp')
+                            return '';
+                        var s = v.split('[', 1);
+                        // remove the ip address
+                        v = s[0];
+                        if (!item.delivery_connection ||
+                            item.delivery_connection == 'trusted' ||
+                            item.delivery_connection == 'verified')
+                        {
+                            return v;
+                        }
+                        return `${v}: ${item.delivery_connection}`;
+                    });
+
+            },
+
+            combine_received_mail_fields: function() {
+                // remove these fields
+                this.received_mail.combine_fields([
+                    'dkim_reason',
+                    'dmarc_reason',
+                    'postgrey_reason',
+                    'postgrey_delay',
+                    'spam_score'
+                    
+                ]);
+                // combine fields 'envelope_from' and 'sasl_username'
+                var f = this.received_mail.combine_fields(
+                    'sasl_username',
+                    'envelope_from',
+                    (v, key, item) => {
+                        if (!item.sasl_username || item.envelope_from == item.sasl_username)
+                            return v;
+                        return `${v} (${item.sasl_username})`;
+                    });
+                f.label = 'Evelope From (user)';
+            },
+
+            get_row_limit: function() {
+                return UserSettings.get().row_limit;
+            },
+
+            update_sent_mail_rowVariant: function() {
+                // there is 1 row for each recipient of a message
+                // - give all rows of the same message the same
+                // color
+                this.sent_mail.apply_rowVariant_grouping('info', item => {
+                    if (this.show_only_flagged && !item._flagged)
+                        return null;
+                    return item.sent_id;
+                });
+            },
+
+            show_only_flagged_change: function() {
+                // 'change' event callback for checkbox
+                this.update_sent_mail_rowVariant();
+                // trigger BV to filter or not filter via
+                // reactive `show_only_flagged_filter`
+                this.show_only_flagged_filter=
+                    (this.show_only_flagged ? 'yes' : null );                
+            },
+                
+
+            table_filter_cb: function(item, filter) {
+                // when filter is non-null, called by BV for each row
+                // to determine whether it will be filtered (false) or
+                // included in the output (true)
+                return item._flagged;
+            },
+            
+            
+            getChartData: function(switch_to_tab) {
+                if (this.all_users.length == 0) {
+                    this.$emit('loading', 1);
+                    axios.get('reports/uidata/user-list').then(response => {
+                        this.all_users = response.data;
+                        
+                    }).catch(error => {
+                        this.$root.handleError(error);
+                        
+                    }).finally(() => {
+                        this.$emit('loading', -1);
+                    });
+                }
+
+                
+                if (!this.date_range || !this.user_id) {
+                    return;
+                }
+
+                this.$emit('loading', 1);
+                const promise = axios.post('reports/uidata/user-activity', {
+                    row_limit: this.get_row_limit(),
+                    user_id: this.user_id.trim(),
+                    start_date: this.date_range[0],
+                    end_date: this.date_range[1]
+                    
+                }).then(response => {
+
+                    this.data_user_id = this.user_id.trim();
+                    this.data_date_range = this.date_range;
+                    if (!isNaN(switch_to_tab))
+                        this.tab_index = switch_to_tab;
+                    this.update_route();
+                    this.$emit('change', this.data_user_id);
+                    this.show_only_flagged = false;
+                    this.show_only_flagged_filter = null;
+
+                    /* setup sent_mail */
+                    this.sent_mail = new MailBvTable(
+                        response.data.sent_mail, {
+                            _showDetails: true
+                        });
+                    this.combine_sent_mail_fields();
+                    this.sent_mail
+                        .flag_fields()
+                        .get_field('connect_time')
+                        .add_tdClass('text-nowrap');
+                    this.update_sent_mail_rowVariant();
+                    
+
+                    /* setup received_mail */
+                    this.received_mail = new MailBvTable(
+                        response.data.received_mail, {
+                            _showDetails: true
+                        });
+                    this.combine_received_mail_fields();
+                    this.received_mail
+                        .flag_fields()
+                        .get_field('connect_time')
+                        .add_tdClass('text-nowrap');                    
+
+                }).catch(error => {
+                    this.$root.handleError(error);
+                    
+                }).finally(() => {
+                    this.$emit('loading', -1);
+                });
+
+                return promise;
+            },
+
+            row_clicked: function(item, index, event) {
+                item._showDetails = ! item._showDetails;
+            },
+            
+        }
+        
+    })}).catch((e) => {
+        reject(e);
+    });
+    
+});

management/reporting/ui/reports-page-header.js

@@ -0,0 +1,24 @@
+Vue.component('reports-page-header', {
+    props: {
+        loading_counter: { type:Number, required:true },
+    },
+    
+    components: {
+        'page-header': Vue.component('page-header'),
+    },
+    
+    template:
+    '<page-header '+
+        'header_text="Server Activity" :loading_counter="loading_counter">'+
+        '<template v-slot:links>'+
+        '  <b-navbar type="dark" variant="transparent" class="p-0">'+
+        '    <b-navbar-nav>'+
+        '      <b-nav-item href="/admin">Admin Panel</b-nav-item>'+
+        '      <b-nav-item to="/settings"><b-icon icon="gear-fill" aria-hidden="true"></b-icon></b-nav-item>'+
+        '    </b-navbar-nav>'+
+        '  </b-navbar>'+
+        '</template>'+
+        '</page-header>'
+    ,
+    
+});

management/reporting/ui/settings.js

@@ -0,0 +1,94 @@
+window.miabldap = window.miabldap || {};
+
+class CaptureConfig {
+    static get() {
+        return axios.get('/reports/capture/config').then(response => {
+            var cc = new CaptureConfig();
+            Object.assign(cc, response.data);
+            return cc;
+        });
+    }
+};
+
+
+class UserSettings {
+    static load() {
+        if (window.miabldap.user_settings) {
+            return Promise.resolve(window.miabldap.user_settings);
+        }
+        
+        var s = new UserSettings();
+        var json = localStorage.getItem('user_settings');
+        if (json) {
+            s.data = JSON.parse(json);
+        }
+        else {
+            s.data = {
+                row_limit: 1000
+            };
+        }
+        window.miabldap.user_settings = s;
+        return Promise.resolve(s);
+    }
+
+    static get() {
+        return window.miabldap.user_settings;
+    }
+
+    save() {
+        var json = JSON.stringify(this.data);
+        localStorage.setItem('user_settings', json);
+    }
+
+    _add_recent(list, val) {
+        var found = -1;
+        list.forEach((str, idx) => {
+            if (str.toLowerCase() == val.toLowerCase()) {
+                found = idx;
+            }
+        });
+        if (found >= 0) {
+            // move it to the top
+            list.splice(found, 1);
+        }
+        list.unshift(val);
+        while (list.length > 10) list.pop();
+    }
+    
+
+    /* row limit */
+    get row_limit() {
+        return this.data.row_limit;
+    }
+    
+    set row_limit(v) {
+        v = Number(v);
+        if (isNaN(v)) {
+            throw new ValueError("invalid")
+        }
+        else if (v < 5) {
+            throw new ValueError("minimum 5")
+        }
+        this.data.row_limit = v;
+        this.save();
+        return v;
+    }
+
+    get_recent_list(name) {
+        return this.data['recent_' + name];
+    }
+    
+    add_to_recent_list(name, value) {
+        const dataname = 'recent_' + name;
+        var v = this.data[dataname];
+        if (! v) {
+            this.data[dataname] = [ value ];
+            this.save();
+            return this.data[dataname];
+        }
+        this._add_recent(v, value);
+        this.data[dataname] = v;
+        this.save();
+        return v;
+    }
+};

management/reporting/ui/wbr-text.js

@@ -0,0 +1,48 @@
+/*
+ * This component adds <wbr> elements after all characters given by
+ * `break_chars` in the given text.
+ *
+ * <wbr> enables the browser to wrap long text at those points
+ * (without it the browser will only wrap at space and hyphen).
+ *
+ * Additionally, if `text_break_threshold` is greater than 0 and there
+ * is a segment of text that exceeds that length, the bootstrap css
+ * class "text-break" will be added to the <span>, which causes the
+ * browser to wrap at any character of the text.
+ */
+
+Vue.component('wbr-text', {
+    props: {
+        text: { type:String, required: true },
+        break_chars: { type:String, default:'@_.,:+=' },
+        text_break_threshold: { type:Number, default:0 },
+    },
+    
+    render: function(ce) {
+        var children = [];
+        var start=-1;
+        var idx=0;
+        var longest=-1;
+        while (idx < this.text.length) {
+            if (this.break_chars.indexOf(this.text[idx]) != -1) {
+                var sliver = this.text.substring(start+1, idx+1);
+                longest = Math.max(longest, sliver.length);
+                children.push(sliver);
+                children.push(ce('wbr'));
+                start=idx;
+            }
+            idx++;
+        }
+
+        if (start < this.text.length-1) {
+            var sliver = this.text.substring(start+1);
+            longest = Math.max(longest, sliver.length);
+            children.push(sliver);
+        }
+        
+        var data = { };
+        if (this.text_break_threshold>0 && longest>this.text_break_threshold)
+            data['class'] = { 'text-break': true };
+        return ce('span', data, children);
+    }
+});

management/reporting/uidata/DictCache.py

@@ -0,0 +1,33 @@
+import datetime
+import threading
+
+#
+# thread-safe dict cache
+#
+
+class DictCache(object):
+    def __init__(self, valid_for):
+        '''`valid_for` must be a datetime.timedelta object indicating how long
+        a cache item is valid
+
+        '''
+        self.obj = None
+        self.time = None
+        self.valid_for = valid_for
+        self.guard = threading.Lock()
+
+    def get(self):
+        now = datetime.datetime.now()
+        with self.guard:
+            if self.obj and (now - self.time) <= self.valid_for:
+                return self.obj.copy()
+
+    def set(self, obj):
+        with self.guard:
+            self.obj = obj.copy()
+            self.time = datetime.datetime.now()
+
+    def reset(self):
+        with self.guard:
+            self.obj = None
+            self.time = None

management/reporting/uidata/Timeseries.py

@@ -0,0 +1,119 @@
+import datetime
+import bisect
+
+class Timeseries(object):
+    def __init__(self, desc, start_date, end_date, binsize):
+        # start_date: 'YYYY-MM-DD' or 'YYYY-MM-DD HH:MM:SS'
+        #      start: 'YYYY-MM-DD HH:MM:SS'
+        self.start = self.full_datetime_str(start_date, False)
+        
+        #   end_date: 'YYYY-MM-DD' or 'YYYY-MM-DD HH:MM:SS'
+        #        end: 'YYYY-MM-DD HH:MM:SS'
+        self.end = self.full_datetime_str(end_date, True)
+
+        # binsize: integer in minutes
+        self.binsize = binsize
+
+        # timefmt is a format string for sqlite strftime() that puts a
+        # sqlite datetime into a "bin" date
+        self.timefmt='%Y-%m-%d'
+
+        # parsefmt is a date parser string to be used to re-interpret
+        # "bin" grouping dates (data.dates) to native dates
+        parsefmt='%Y-%m-%d'
+
+        b = self.binsizeWithUnit()
+
+        if b['unit'] == 'hour':
+            self.timefmt+=' %H:00:00'
+            parsefmt+=' %H:%M:%S'
+        elif b['unit'] == 'minute':
+            self.timefmt+=' %H:%M:00'
+            parsefmt+=' %H:%M:%S'
+
+        self.dates = []   # dates must be "bin" date strings
+        self.series = []
+        
+        self.data = {
+            'range': [ self.start, self.end ],
+            'range_parse_format': '%Y-%m-%d %H:%M:%S',
+            'binsize': self.binsize,
+            'date_parse_format': parsefmt,
+            'y': desc,
+            'dates': self.dates,
+            'series': self.series
+        }
+
+    def full_datetime_str(self, date_str, next_day):
+        if ':' in date_str:
+            return date_str
+        elif not next_day:
+            return date_str + " 00:00:00"
+        else:
+            d = datetime.datetime.strptime(date_str, '%Y-%m-%d')
+            d = d + datetime.timedelta(days=1)
+            return d.strftime('%Y-%m-%d 00:00:00')
+
+    def binsizeWithUnit(self):
+        # normalize binsize (which is a time span in minutes)
+        days = int(self.binsize / (24 * 60))
+        hours = int((self.binsize - days*24*60) / 60 )
+        mins = self.binsize - days*24*60 - hours*60
+        if days == 0 and hours == 0:
+            return {
+                'unit': 'minute',
+                'value': mins
+            }
+        
+        if days == 0:
+            return {
+                'unit': 'hour',
+                'value': hours
+            }
+        
+        return {
+            'unit': 'day',
+            'value': days
+        }
+
+    
+    def append_date(self, date_str):
+        '''date_str should be a "bin" date - that is a date formatted with
+        self.timefmt. 
+
+        1. it should be greater than the previous bin so that the date
+        list remains sorted
+
+        2. d3js does not require that all dates be added for a
+        timespan if there is no data for the bin
+
+        '''
+        self.dates.append(date_str)
+
+    def insert_date(self, date_str):
+        '''adds bin date if it does not exist and returns the new index.  if
+        the date already exists, returns the existing index.
+
+        '''
+        i = bisect.bisect_right(self.dates, date_str)
+        if i == len(self.dates):
+            self.dates.append(date_str)
+            return i
+        if self.dates[i] == date_str:
+            return i
+        self.dates.insert(i, date_str)
+        return i
+        
+    def add_series(self, id, name):
+        s = {
+            'id': id,
+            'name': name,
+            'values': []
+        }
+        self.series.append(s)
+        return s
+
+    
+    def asDict(self):
+        return self.data
+    

management/reporting/uidata/__init__.py

@@ -0,0 +1,9 @@
+from .exceptions import (InvalidArgsError)
+from .select_list_suggestions import select_list_suggestions
+from .messages_sent import messages_sent
+from .messages_received import messages_received
+from .user_activity import user_activity
+from .remote_sender_activity import remote_sender_activity
+from .flagged_connections import flagged_connections
+from .capture_db_stats import capture_db_stats
+from .capture_db_stats import clear_cache

management/reporting/uidata/capture_db_stats.py

@@ -0,0 +1,50 @@
+import datetime
+from .DictCache import DictCache
+
+#
+# because of the table scan (select_2 below), cache stats for 5
+# minutes
+#
+last_stats = DictCache(datetime.timedelta(minutes=5))
+
+def clear_cache():
+    last_stats.reset()
+    
+
+def capture_db_stats(conn):
+
+    stats = last_stats.get()
+    if stats:
+        return stats
+    
+    select_1 = 'SELECT min(connect_time) AS `min`, max(connect_time) AS `max`, count(*) AS `count` FROM mta_connection'
+
+    # table scan
+    select_2 = 'SELECT disposition, count(*) AS `count` FROM mta_connection GROUP BY disposition'
+    
+    c = conn.cursor()
+    stats = {
+        # all times are in this format: "YYYY-MM-DD HH:MM:SS" (utc)
+        'date_parse_format': '%Y-%m-%d %H:%M:%S'
+    }
+    try:
+        row = c.execute(select_1).fetchone()
+        stats['mta_connect'] = {
+            'connect_time': {
+                'min': row['min'],
+                'max': row['max'], # YYYY-MM-DD HH:MM:SS (utc)
+            },
+            'count': row['count'],
+            'disposition': {}
+        }
+
+        for row in c.execute(select_2):
+            stats['mta_connect']['disposition'][row['disposition']] =  {
+                'count': row['count']
+            }
+        
+    finally:
+        c.close()
+
+    last_stats.set(stats)
+    return stats

management/reporting/uidata/exceptions.py

@@ -0,0 +1,5 @@
+class MiabLdapError(Exception):
+    pass
+
+class InvalidArgsError(MiabLdapError):
+    pass

management/reporting/uidata/flagged_connections.1.sql

@@ -0,0 +1,14 @@
+--
+-- returns count of failed_login_attempt in each 'bin', which is the
+-- connection time rounded (as defined by {timefmt})
+--
+SELECT
+  strftime('{timefmt}',connect_time) AS `bin`,
+  count(*) AS `count`
+FROM mta_connection
+WHERE
+  disposition='failed_login_attempt' AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+GROUP BY strftime('{timefmt}',connect_time)
+ORDER BY connect_time

management/reporting/uidata/flagged_connections.2.sql

@@ -0,0 +1,14 @@
+--
+-- returns count of suspected_scanner in each 'bin', which is the
+-- connection time rounded (as defined by {timefmt})
+--
+SELECT
+  strftime('{timefmt}',connect_time) AS `bin`,
+  count(*) AS `count`
+FROM mta_connection
+WHERE
+  disposition='suspected_scanner' AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+GROUP BY strftime('{timefmt}',connect_time)
+ORDER BY connect_time

management/reporting/uidata/flagged_connections.3.sql

@@ -0,0 +1,8 @@
+SELECT failure_category, count(*) AS `count`
+FROM mta_connection
+JOIN mta_accept ON mta_accept.mta_conn_id = mta_connection.mta_conn_id
+WHERE
+   disposition='reject' AND
+   connect_time >=:start_date AND
+   connect_time <:end_date
+GROUP BY failure_category

management/reporting/uidata/flagged_connections.4.sql

@@ -0,0 +1,14 @@
+--
+-- top 10 servers getting rejected by category
+--
+SELECT CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END AS `remote_host`, mta_accept.failure_category AS `category`, count(*) AS `count` 
+FROM mta_connection
+JOIN mta_accept ON mta_accept.mta_conn_id = mta_connection.mta_conn_id
+WHERE
+  mta_connection.service='smtpd' AND
+  accept_status = 'reject' AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+GROUP BY CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END, mta_accept.failure_category
+ORDER BY count(*) DESC
+LIMIT 10

management/reporting/uidata/flagged_connections.5.sql

@@ -0,0 +1,12 @@
+--
+-- inbound mail using an insecure connection (no use of STARTTLS)
+--
+SELECT mta_connection.service AS `service`, sasl_username, envelope_from, rcpt_to, count(*) AS `count`
+FROM mta_connection
+LEFT JOIN mta_accept ON mta_accept.mta_conn_id = mta_connection.mta_conn_id
+LEFT JOIN mta_delivery ON mta_delivery.mta_accept_id = mta_accept.mta_accept_id
+WHERE 
+  disposition = 'insecure' AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+GROUP BY mta_connection.service, sasl_username, envelope_from, rcpt_to

management/reporting/uidata/flagged_connections.6.sql

@@ -0,0 +1,12 @@
+--
+-- outbound mail using an insecure connection (low grade encryption)
+--
+SELECT mta_delivery.service AS `service`, sasl_username, envelope_from, rcpt_to, count(*) AS `count`
+FROM mta_connection
+LEFT JOIN mta_accept ON mta_accept.mta_conn_id = mta_connection.mta_conn_id
+LEFT JOIN mta_delivery ON mta_delivery.mta_accept_id = mta_accept.mta_accept_id
+WHERE 
+  delivery_connection = 'untrusted' AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+GROUP BY mta_connection.service, sasl_username, envelope_from, rcpt_to

management/reporting/uidata/flagged_connections.py

@@ -0,0 +1,142 @@
+from .Timeseries import Timeseries
+from .exceptions import InvalidArgsError
+from .top import select_top
+
+with open(__file__.replace('.py','.1.sql')) as fp:
+    select_1 = fp.read()
+
+with open(__file__.replace('.py','.2.sql')) as fp:
+    select_2 = fp.read()
+
+with open(__file__.replace('.py','.3.sql')) as fp:
+    select_3 = fp.read()
+
+with open(__file__.replace('.py','.4.sql')) as fp:
+    select_4 = fp.read()
+
+with open(__file__.replace('.py','.5.sql')) as fp:
+    select_5 = fp.read()
+
+with open(__file__.replace('.py','.6.sql')) as fp:
+    select_6 = fp.read()
+
+
+def flagged_connections(conn, args):
+    try:
+        ts = Timeseries(
+            "Failed login attempts and suspected scanners over time",
+            args['start'],
+            args['end'],
+            args['binsize']
+        )
+    except KeyError:
+        raise InvalidArgsError()
+
+    c = conn.cursor()
+    
+    # pie chart for "connections by disposition"
+    select = 'SELECT disposition, count(*) AS `count` FROM mta_connection WHERE connect_time>=:start_date AND connect_time<:end_date GROUP BY disposition'
+    connections_by_disposition = []
+    for row in c.execute(select, {'start_date':ts.start, 'end_date':ts.end}):
+        connections_by_disposition.append({
+            'name': row[0],
+            'value': row[1]
+        })
+
+    # timeseries = failed logins count
+    s_failed_login = ts.add_series('failed_login_attempt', 'failed login attempts')
+    for row in c.execute(select_1.format(timefmt=ts.timefmt), {
+            'start_date': ts.start,
+            'end_date': ts.end
+    }):
+        ts.append_date(row['bin'])
+        s_failed_login['values'].append(row['count'])
+
+    # timeseries = suspected scanners count
+    s_scanner = ts.add_series('suspected_scanner', 'connections by suspected scanners')
+    for row in c.execute(select_2.format(timefmt=ts.timefmt), {
+            'start_date': ts.start,
+            'end_date': ts.end
+    }):
+        ts.insert_date(row['bin'])
+        s_scanner['values'].append(row['count'])
+
+
+    # pie chart for "disposition=='reject' grouped by failure_category"
+    reject_by_failure_category = []
+    for row in c.execute(select_3, {
+            'start_date': ts.start,
+            'end_date': ts.end
+    }):
+        reject_by_failure_category.append({
+            'name': row[0],
+            'value': row[1]
+        })
+        
+    # top 10 servers rejected by category
+    top_hosts_rejected = select_top(
+        c,
+        select_4,
+        ts.start,
+        ts.end,
+        "Top servers rejected by category",
+        [ 'remote_host', 'category', 'count' ],
+        [ 'text/hostname', 'text/plain', 'number/plain' ]
+    )
+
+    # insecure inbound connections - no limit
+    insecure_inbound = select_top(
+        c,
+        select_5,
+        ts.start,
+        ts.end,
+        "Insecure inbound connections (no use of STARTTLS)",
+        [
+            'service',
+            'sasl_username',
+            'envelope_from',
+            'rcpt_to',
+            'count'
+        ],
+        [
+            'text/plain', # service
+            'text/plain', # sasl_username
+            'text/email', # envelope_from
+            { 'type':'text/email', 'label':'Recipient' }, # rcpt_to
+            'number/plain', # count
+        ]
+    )
+
+    # insecure outbound connections - no limit
+    insecure_outbound = select_top(
+        c,
+        select_6,
+        ts.start,
+        ts.end,
+        "Insecure outbound connections (low grade encryption)",
+        [
+            'service',
+            'sasl_username',
+            'envelope_from',
+            'rcpt_to',
+            'count'
+        ],
+        [
+            'text/plain', # service
+            'text/plain', # sasl_username
+            'text/email', # envelope_from
+            { 'type':'text/email', 'label':'Recipient' }, # rcpt_to
+            'number/plain', # count
+        ]
+    )
+
+
+    
+    return {
+        'connections_by_disposition': connections_by_disposition,
+        'flagged': ts.asDict(),
+        'reject_by_failure_category': reject_by_failure_category,
+        'top_hosts_rejected': top_hosts_rejected,
+        'insecure_inbound': insecure_inbound,
+        'insecure_outbound': insecure_outbound,
+    }

management/reporting/uidata/messages_received.1.sql

@@ -0,0 +1,15 @@
+--
+-- returns count of messages received by smtpd in each 'bin', which is
+-- the connection time rounded (as defined by {timefmt})
+--
+SELECT
+  strftime('{timefmt}',connect_time) AS `bin`,
+  count(*) AS `count`
+FROM mta_accept
+JOIN mta_connection ON mta_connection.mta_conn_id = mta_accept.mta_conn_id
+WHERE
+  mta_connection.service = 'smtpd' AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+GROUP BY strftime('{timefmt}',connect_time)
+ORDER BY connect_time

management/reporting/uidata/messages_received.2.sql

@@ -0,0 +1,14 @@
+--
+-- top 10 senders (envelope_from) by message count
+--
+SELECT count(mta_accept_id) AS `count`, envelope_from AS `email`
+FROM mta_connection
+JOIN mta_accept on mta_accept.mta_conn_id = mta_connection.mta_conn_id
+WHERE
+  mta_connection.service = "smtpd" AND
+  accept_status != 'reject' AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+GROUP BY envelope_from
+ORDER BY count(mta_accept_id) DESC
+LIMIT 10

management/reporting/uidata/messages_received.3.sql

@@ -0,0 +1,12 @@
+--
+-- top 10 senders (envelope_from) by message size
+--
+SELECT sum(message_size) AS `size`, envelope_from AS `email`
+FROM mta_connection
+JOIN mta_accept on mta_accept.mta_conn_id = mta_connection.mta_conn_id
+WHERE mta_connection.service = "smtpd" AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+GROUP BY envelope_from
+ORDER BY sum(message_size) DESC
+LIMIT 10

management/reporting/uidata/messages_received.4.sql

@@ -0,0 +1,13 @@
+--
+-- top 10 remote servers/domains (remote hosts) by average spam score
+--
+SELECT CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END AS `remote_host`, avg(spam_score) AS avg_spam_score FROM mta_connection
+JOIN mta_accept ON mta_accept.mta_conn_id = mta_connection.mta_conn_id
+JOIN mta_delivery ON mta_accept.mta_accept_id = mta_delivery.mta_accept_id
+WHERE mta_connection.service='smtpd' AND
+  spam_score IS NOT NULL AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+GROUP BY CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END
+ORDER BY avg(spam_score) DESC
+LIMIT 10

management/reporting/uidata/messages_received.5.sql

@@ -0,0 +1,12 @@
+--
+-- top 10 users receiving the most spam
+--
+SELECT rcpt_to, count(*) AS count FROM mta_delivery
+JOIN mta_accept ON mta_accept.mta_accept_id = mta_delivery.mta_accept_id
+JOIN mta_connection ON mta_accept.mta_conn_id = mta_connection.mta_conn_id
+WHERE spam_result='spam' AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+GROUP BY rcpt_to
+ORDER BY count(*) DESC
+LIMIT 10

management/reporting/uidata/messages_received.py

@@ -0,0 +1,108 @@
+from .Timeseries import Timeseries
+from .exceptions import InvalidArgsError
+from .top import select_top
+
+with open(__file__.replace('.py','.1.sql')) as fp:
+    select_1 = fp.read()
+    
+with open(__file__.replace('.py','.2.sql')) as fp:
+    select_2 = fp.read()
+
+with open(__file__.replace('.py','.3.sql')) as fp:
+    select_3 = fp.read()
+    
+with open(__file__.replace('.py','.4.sql')) as fp:
+    select_4 = fp.read()
+
+with open(__file__.replace('.py','.5.sql')) as fp:
+    select_5 = fp.read()
+
+
+    
+def messages_received(conn, args):
+    '''
+    messages recived from the internet
+
+    '''
+    try:
+        ts = Timeseries(
+            "Messages received from the internet",
+            args['start'],
+            args['end'],
+            args['binsize']
+        )
+    except KeyError:
+        raise InvalidArgsError()
+
+    s_received = ts.add_series('received', 'messages received')
+    
+    c = conn.cursor()
+    try:
+        for row in c.execute(select_1.format(timefmt=ts.timefmt), {
+                'start_date':ts.start,
+                'end_date':ts.end
+        }):
+            ts.append_date(row['bin'])
+            s_received['values'].append(row['count'])
+
+
+        # top 10 senders (envelope_from) by message count
+        top_senders_by_count = select_top(
+            c,
+            select_2,
+            ts.start,
+            ts.end,
+            "Top 10 senders by count",
+            [ 'email', 'count' ],
+            [ 'text/email', 'number/plain' ]
+        )
+        
+        # top 10 senders (envelope_from) by message size
+        top_senders_by_size = select_top(
+            c,
+            select_3,
+            ts.start,
+            ts.end,
+            "Top 10 senders by size",
+            [ 'email', 'size' ],
+            [ 'text/email', 'number/size' ]
+        )
+
+        # top 10 remote servers/domains (remote hosts) by average spam score
+        top_hosts_by_spam_score = select_top(
+            c,
+            select_4,
+            ts.start,
+            ts.end,
+            "Top servers by average spam score",
+            [ 'remote_host', 'avg_spam_score' ],
+            [ 'text/hostname', { 'type':'decimal', 'places':2} ]
+        )
+        
+        # top 10 users receiving the most spam
+        top_user_receiving_spam = select_top(
+            c,
+            select_5,
+            ts.start,
+            ts.end,
+            "Top 10 users receiving spam",
+            [
+                'rcpt_to',
+                'count'
+            ],
+            [
+                { 'type': 'text', 'subtype':'email', 'label':'User' },
+                'number/plain'
+            ]
+        )
+
+    finally:
+        c.close()
+
+    return {
+        'top_senders_by_count': top_senders_by_count,
+        'top_senders_by_size': top_senders_by_size,
+        'top_hosts_by_spam_score': top_hosts_by_spam_score,
+        'top_user_receiving_spam': top_user_receiving_spam,
+        'ts_received': ts.asDict(),
+    }

management/reporting/uidata/messages_sent.1.sql

@@ -0,0 +1,16 @@
+--
+-- returns count of sent messages in each 'bin', which is the connection
+-- time rounded (as defined by {timefmt})
+--
+SELECT
+  strftime('{timefmt}',connect_time) AS `bin`,
+  count(*) AS `sent_count`
+FROM mta_accept
+JOIN mta_connection ON mta_connection.mta_conn_id = mta_accept.mta_conn_id
+JOIN mta_delivery ON mta_delivery.mta_accept_id = mta_accept.mta_accept_id
+WHERE
+  (mta_connection.service = 'submission' OR mta_connection.service = 'pickup') AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+GROUP BY strftime('{timefmt}',connect_time)
+ORDER BY connect_time

management/reporting/uidata/messages_sent.2.sql

@@ -0,0 +1,18 @@
+--
+-- returns count of sent messages delivered in each 'bin'/delivery
+-- service combination. the bin is the connection time rounded (as
+-- defined by {timefmt})
+--
+SELECT
+  strftime('{timefmt}',connect_time) AS `bin`,
+  mta_delivery.service AS `delivery_service`,
+  count(*) AS `delivery_count`
+FROM mta_accept
+JOIN mta_connection ON mta_connection.mta_conn_id = mta_accept.mta_conn_id
+JOIN mta_delivery ON mta_delivery.mta_accept_id = mta_accept.mta_accept_id
+WHERE
+  (mta_connection.service = 'submission' OR mta_connection.service = 'pickup') AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+GROUP BY strftime('{timefmt}',connect_time), mta_delivery.service
+ORDER BY connect_time

management/reporting/uidata/messages_sent.3.sql

@@ -0,0 +1,12 @@
+--
+-- top 10 senders by message count
+--
+select count(mta_accept_id) as count, sasl_username as username
+from mta_connection
+join mta_accept on mta_accept.mta_conn_id = mta_connection.mta_conn_id
+where mta_connection.service = "submission" AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+group by sasl_username
+order by count(mta_accept_id) DESC
+limit 10

management/reporting/uidata/messages_sent.4.sql

@@ -0,0 +1,12 @@
+--
+-- top 10 senders by message size
+--
+SELECT sum(message_size) AS message_size_total, sasl_username AS username
+FROM mta_connection
+JOIN mta_accept ON mta_accept.mta_conn_id = mta_connection.mta_conn_id
+WHERE mta_connection.service = "submission" AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+GROUP BY sasl_username
+ORDER BY sum(message_size) DESC
+LIMIT 10

management/reporting/uidata/messages_sent.py

@@ -0,0 +1,113 @@
+from .Timeseries import Timeseries
+from .exceptions import InvalidArgsError
+
+with open(__file__.replace('.py','.1.sql')) as fp:
+    select_1 = fp.read()
+    
+with open(__file__.replace('.py','.2.sql')) as fp:
+    select_2 = fp.read()
+
+with open(__file__.replace('.py','.3.sql')) as fp:
+    select_3 = fp.read()
+    
+with open(__file__.replace('.py','.4.sql')) as fp:
+    select_4 = fp.read()
+    
+
+def messages_sent(conn, args):
+    '''
+    messages sent by local users
+      - delivered locally & remotely
+
+    '''
+    try:
+        ts = Timeseries(
+            "Messages sent by users",
+            args['start'],
+            args['end'],
+            args['binsize']
+        )
+    except KeyError:
+        raise InvalidArgsError()
+
+    s_sent = ts.add_series('sent', 'messages sent')
+    s_local = ts.add_series('local', 'local recipients')
+    s_remote = ts.add_series('remote', 'remote recipients')
+    
+    c = conn.cursor()
+    try:
+        for row in c.execute(select_1.format(timefmt=ts.timefmt), {
+                'start_date':ts.start,
+                'end_date':ts.end
+        }):
+            ts.dates.append(row['bin'])
+            s_sent['values'].append(row['sent_count'])
+
+        date_idx = -1
+
+        # the returned bins are the same as select_1 because the
+        # querie's WHERE and JOINs are the same
+        for row in c.execute(select_2.format(timefmt=ts.timefmt), {
+                'start_date':ts.start,
+                'end_date':ts.end
+        }):
+            if date_idx>=0 and ts.dates[date_idx] == row['bin']:
+                if row['delivery_service']=='smtp':
+                    s_remote['values'][-1] = row['delivery_count']
+                elif row['delivery_service']=='lmtp':
+                    s_local['values'][-1] = row['delivery_count']
+                    
+            else:
+                date_idx += 1
+                if date_idx >= len(ts.dates):
+                    break
+                if row['delivery_service']=='smtp':
+                    s_remote['values'].append(row['delivery_count'])
+                    s_local['values'].append(0)
+                elif row['delivery_service']=='lmtp':
+                    s_remote['values'].append(0)
+                    s_local['values'].append(row['delivery_count'])
+
+
+        top_senders1 = {
+            'start': ts.start,
+            'end': ts.end,
+            'y': 'Top 10 users by count',
+            'fields': ['user','count'],
+            'field_types': ['text/email','number/plain'],
+            'items': []
+        }
+        for row in c.execute(select_3, {
+                'start_date':ts.start,
+                'end_date':ts.end
+        }):
+            top_senders1['items'].append({
+                'user': row['username'],
+                'count': row['count']
+            })
+            
+        top_senders2 = {
+            'start': ts.start,
+            'end': ts.end,
+            'y': 'Top 10 users by size',
+            'fields': ['user','size'],
+            'field_types': ['text/email','number/size'],
+            'items': []
+        }
+        for row in c.execute(select_4, {
+                'start_date':ts.start,
+                'end_date':ts.end
+        }):
+            top_senders2['items'].append({
+                'user': row['username'],
+                'size': row['message_size_total']
+            })
+
+    finally:
+        c.close()
+
+    return {
+        'top_senders_by_count': top_senders1,
+        'top_senders_by_size': top_senders2,
+        'ts_sent': ts.asDict(),
+    }

management/reporting/uidata/remote_sender_activity.1.sql

@@ -0,0 +1,19 @@
+--
+-- details on remote senders
+-- query: envelope_from
+--
+SELECT
+-- mta_connection
+connect_time, mta_connection.service AS `service`, sasl_username, disposition,
+-- mta_accept
+mta_accept.mta_accept_id AS mta_accept_id, spf_result, dkim_result, dkim_reason, dmarc_result, dmarc_reason, accept_status, failure_info, mta_accept.failure_category AS `category`,
+-- mta_delivery
+rcpt_to, postgrey_result, postgrey_reason, postgrey_delay, spam_score, spam_result, message_size
+FROM mta_connection
+JOIN mta_accept ON mta_accept.mta_conn_id = mta_connection.mta_conn_id
+LEFT JOIN mta_delivery ON mta_accept.mta_accept_id = mta_delivery.mta_accept_id
+WHERE
+  envelope_from = :envelope_from AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+ORDER BY connect_time

management/reporting/uidata/remote_sender_activity.2.sql

@@ -0,0 +1,20 @@
+--
+-- details on remote sender host
+-- query: remote_host or remote_ip
+--
+SELECT
+-- mta_connection
+connect_time, disposition,
+-- mta_accept
+mta_accept.mta_accept_id AS mta_accept_id, spf_result, dkim_result, dkim_reason, dmarc_result, dmarc_reason, accept_status, failure_info, mta_accept.failure_category AS `category`, envelope_from,
+-- mta_delivery
+rcpt_to, postgrey_result, postgrey_reason, postgrey_delay, spam_score, spam_result, message_size
+FROM mta_connection
+LEFT JOIN mta_accept ON mta_accept.mta_conn_id = mta_connection.mta_conn_id
+LEFT JOIN mta_delivery ON mta_accept.mta_accept_id = mta_delivery.mta_accept_id
+WHERE
+  (remote_host = :remote_host OR remote_ip = :remote_host) AND
+  mta_connection.service = 'smtpd' AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+ORDER BY connect_time

management/reporting/uidata/remote_sender_activity.py

@@ -0,0 +1,180 @@
+from .Timeseries import Timeseries
+from .exceptions import InvalidArgsError
+
+
+with open(__file__.replace('.py','.1.sql')) as fp:
+    select_1 = fp.read()
+
+with open(__file__.replace('.py','.2.sql')) as fp:
+    select_2 = fp.read()
+
+
+def remote_sender_activity(conn, args):
+    '''
+    details on remote senders (envelope from)
+    '''    
+
+    try:
+        sender = args['sender']
+        sender_type = args['sender_type']
+
+        if sender_type not in ['email', 'server']:
+            raise InvalidArgsError()
+
+        # use Timeseries to get a normalized start/end range
+        ts = Timeseries(
+            'Remote sender activity',
+            args['start_date'],
+            args['end_date'],
+            0
+        )
+    except KeyError:
+        raise InvalidArgsError()
+
+    # limit results
+    try:
+        limit = 'LIMIT ' + str(int(args.get('row_limit', 1000)));
+    except ValueError:
+        limit = 'LIMIT 1000'    
+
+    c = conn.cursor()
+
+    if sender_type == 'email':
+        select = select_1
+        fields = [
+            # mta_connection
+            'connect_time',
+            'service',
+            'sasl_username',
+            # mta_delivery
+            'rcpt_to',
+            # mta_accept
+            'disposition',
+            'accept_status',
+            'spf_result',
+            'dkim_result',
+            'dkim_reason',
+            'dmarc_result',
+            'dmarc_reason',
+            'failure_info',
+            'category',  # failure_category
+            # mta_delivery
+            'postgrey_result',
+            'postgrey_reason',
+            'postgrey_delay',
+            'spam_score',
+            'spam_result',
+            'message_size',
+            'sent_id',  # must be last
+        ]
+        field_types = [
+            { 'type':'datetime', 'format': '%Y-%m-%d %H:%M:%S' },# connect_time
+            'text/plain',    # service
+            'text/plain',    # sasl_username
+            { 'type':'text/email', 'label':'Recipient' },    # rcpt_to
+            'text/plain',    # disposition
+            'text/plain',    # accept_status
+            'text/plain',    # spf_result
+            'text/plain',    # dkim_result
+            'text/plain',    # dkim_reason
+            'text/plain',    # dmarc_result
+            'text/plain',    # dmarc_reason
+            'text/plain',    # failure_info
+            'text/plain',    # category (mta_accept.failure_category)
+            'text/plain',    # postgrey_result
+            'text/plain',    # postgrey_reason
+            { 'type':'time/span', 'unit':'s' }, # postgrey_delay
+            { 'type':'decimal', 'places':2 },     # spam_score
+            'text/plain',    # spam_result
+            'number/size',   # message_size
+            'number/plain',  # sent_id - must be last
+        ]
+        select_args = {
+            'envelope_from': sender,
+            'start_date': ts.start,
+            'end_date': ts.end
+        }
+
+    elif sender_type == 'server':
+        select = select_2
+        fields = [
+            # mta_connection
+            'connect_time',
+            # mta_accept
+            'envelope_from',
+            # mta_delivery
+            'rcpt_to',
+            'disposition',
+            # mta_accept
+            'accept_status',
+            'spf_result',
+            'dkim_result',
+            'dkim_reason',
+            'dmarc_result',
+            'dmarc_reason',
+            'failure_info',
+            'category',  # failure_category
+            # mta_delivery
+            'postgrey_result',
+            'postgrey_reason',
+            'postgrey_delay',
+            'spam_score',
+            'spam_result',
+            'message_size',
+            'sent_id',  # must be last
+        ]
+        field_types = [
+            { 'type':'datetime', 'format': '%Y-%m-%d %H:%M:%S' },# connect_time
+            { 'type':'text/email', 'label':'From' },    # envelope_from
+            { 'type':'text/email', 'label':'Recipient' },    # rcpt_to
+            'text/plain',    # disposition
+            'text/plain',    # accept_status
+            'text/plain',    # spf_result
+            'text/plain',    # dkim_result
+            'text/plain',    # dkim_reason
+            'text/plain',    # dmarc_result
+            'text/plain',    # dmarc_reason
+            'text/plain',    # failure_info
+            'text/plain',    # category (mta_accept.failure_category)
+            'text/plain',    # postgrey_result
+            'text/plain',    # postgrey_reason
+            { 'type':'time/span', 'unit':'s' }, # postgrey_delay
+            { 'type':'decimal', 'places':2 },     # spam_score
+            'text/plain',    # spam_result
+            'number/size',   # message_size
+            'number/plain',  # sent_id - must be last
+        ]
+        select_args = {
+            'remote_host': sender,
+            'start_date': ts.start,
+            'end_date': ts.end
+        }
+
+        
+    activity = {
+        'start': ts.start,
+        'end': ts.end,
+        'y': 'Remote sender activity',
+        'fields': fields,
+        'field_types': field_types,
+        'items': [],
+        'unique_sends': 0
+    }
+    last_mta_accept_id = -1
+    sent_id = 0
+    for row in c.execute(select + limit, select_args):
+        v = []
+        for key in activity['fields']:
+            if key != 'sent_id':
+                v.append(row[key])
+        if row['mta_accept_id'] is None or last_mta_accept_id != row['mta_accept_id']:
+            activity['unique_sends'] += 1
+            last_mta_accept_id = row['mta_accept_id']
+            sent_id += 1
+        v.append(sent_id)
+        activity['items'].append(v)
+
+        
+    return {
+        'activity': activity,
+    }

management/reporting/uidata/select_list_suggestions.py

@@ -0,0 +1,115 @@
+from .Timeseries import Timeseries
+from .exceptions import InvalidArgsError
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def select_list_suggestions(conn, args):
+
+    try:
+        query_type = args['type']
+        query = args['query'].strip()
+        ts = None
+        if 'start_date' in args:
+            # use Timeseries to get a normalized start/end range
+            ts = Timeseries(
+                'select list suggestions',
+                args['start_date'],
+                args['end_date'],
+                0
+            )
+    except KeyError:
+        raise InvalidArgsError()
+
+    # escape query with backslash for fuzzy match (LIKE)
+    query_escaped = query.replace("\\", "\\\\").replace("%","\\%").replace("_","\\_")
+    limit = 100
+
+    queries = {
+        'remote_host':  {
+            'select': "DISTINCT CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END",
+            'from': "mta_connection",
+            'join': {},
+            'order_by': "remote_host",
+            'where_exact': [ "(remote_host = ? OR remote_ip = ?)" ],
+            'args_exact': [ query, query ],
+            'where_fuzzy': [ "(remote_host LIKE ? ESCAPE '\\' OR remote_ip LIKE ? ESCAPE '\\')" ],
+            'args_fuzzy': [ '%'+query_escaped+'%', query_escaped+'%' ]
+        },
+        'rcpt_to': {
+            'select': "DISTINCT rcpt_to",
+            'from': 'mta_delivery',
+            'join': {},
+            'order_by': "rcpt_to",
+            'where_exact': [ "rcpt_to = ?" ],
+            'args_exact': [ query, ],
+            'where_fuzzy': [ "rcpt_to LIKE ? ESCAPE '\\'" ],
+            'args_fuzzy': [ '%'+query_escaped+'%' ]
+        },
+        'envelope_from': {
+            'select': "DISTINCT envelope_from",
+            'from': "mta_accept",
+            'join': {},
+            'order_by': 'envelope_from',
+            'where_exact': [ "envelope_from = ?" ],
+            'args_exact': [ query, ],
+            'where_fuzzy': [ "envelope_from LIKE ? ESCAPE '\\'" ],
+            'args_fuzzy': [ '%'+query_escaped+'%' ]
+        },
+    }
+
+    q = queries.get(query_type)
+    if not q:
+        raise InvalidArgError()
+
+    if ts:
+        q['where_exact'] += [ 'connect_time>=?', 'connect_time<?' ]
+        q['where_fuzzy'] += [ 'connect_time>=?', 'connect_time<?' ]
+        q['args_exact'] += [ ts.start, ts.end ];
+        q['args_fuzzy'] += [ ts.start, ts.end ];
+        cur_join = q['from']
+        if cur_join == 'mta_delivery':
+            q['join']['mta_accept'] = "mta_accept.mta_accept_id = mta_delivery.mta_accept_id"
+            cur_join = 'mta_accept'
+            
+        if cur_join == 'mta_accept':
+            q['join']['mta_connection'] = "mta_connection.mta_conn_id = mta_accept.mta_conn_id"
+
+
+    joins = []
+    for table in q['join']:
+        joins.append('JOIN ' + table + ' ON ' + q['join'][table])
+    joins =" ".join(joins)
+
+    c = conn.cursor()
+    try:
+        # 1. attempt to find an exact match first
+        where = ' AND '.join(q['where_exact'])
+        select = f"SELECT {q['select']} FROM {q['from']} {joins} WHERE {where} LIMIT {limit}"
+        log.debug(select)
+        c.execute(select, q['args_exact'])
+        row = c.fetchone()
+        if row:
+            return {
+                'exact': True,
+                'suggestions': [ row[0] ],
+                'limited': False
+            }
+
+        # 2. otherwise, do a fuzzy search and return all matches
+        where = ' AND '.join(q['where_fuzzy'])
+        select = f"SELECT {q['select']} FROM {q['from']} {joins} WHERE {where} ORDER BY {q['order_by']} LIMIT {limit}"
+        log.debug(select)
+        suggestions = []
+        for row in c.execute(select, q['args_fuzzy']):
+            suggestions.append(row[0])
+        return {
+            'exact': False,
+            'suggestions': suggestions,
+            'limited': len(suggestions)>=limit
+        }
+    
+    finally:
+        c.close()
+

management/reporting/uidata/top.py

@@ -0,0 +1,32 @@
+def select_top(c, select, start, end, y, fields, field_types):
+    '''`c` is a cursor
+
+    `select` is the select query `start` and `end` are the range in
+the format YYYY-MM-DD HH:MM:SS and the select query must have
+substitutes 'start_date' and 'end_date'.
+
+    `y` is a description of the dataset
+
+    `fields` are all fields to select by name
+
+    `field_types` are the corresponding field types the caller will
+    need to render the data visuals
+    '''
+    
+    top = {
+        'start': start,
+        'end': end,
+        'y': y,
+        'fields': fields,
+        'field_types': field_types,
+        'items': []
+    }
+    for row in c.execute(select, {
+            'start_date':start,
+            'end_date':end
+    }):
+        v = {}
+        for key in fields:
+            v[key] = row[key]            
+        top['items'].append(v)
+    return top

management/reporting/uidata/user_activity.1.sql

@@ -0,0 +1,17 @@
+--
+-- details on user sent mail
+--
+SELECT
+-- mta_connection
+connect_time, sasl_method,
+-- mta_accept
+mta_accept.mta_accept_id AS mta_accept_id, envelope_from,
+-- mta_delivery
+mta_delivery.service AS service, rcpt_to, spam_score, spam_result, message_size, status, relay, delivery_info, delivery_connection, delivery_connection_info
+FROM mta_accept
+JOIN mta_connection ON mta_accept.mta_conn_id = mta_connection.mta_conn_id
+JOIN mta_delivery ON mta_accept.mta_accept_id = mta_delivery.mta_accept_id
+WHERE sasl_username = :user_id AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+ORDER BY connect_time, mta_accept.mta_accept_id

management/reporting/uidata/user_activity.2.sql

@@ -0,0 +1,17 @@
+--
+-- details on user received mail
+--
+SELECT
+-- mta_connection
+connect_time, mta_connection.service AS service, sasl_username, disposition,
+-- mta_accept
+envelope_from, spf_result, dkim_result, dkim_reason, dmarc_result, dmarc_reason,
+-- mta_delivery
+postgrey_result, postgrey_reason, postgrey_delay, spam_score, spam_result, message_size
+FROM mta_accept
+JOIN mta_connection ON mta_accept.mta_conn_id = mta_connection.mta_conn_id
+JOIN mta_delivery ON mta_accept.mta_accept_id = mta_delivery.mta_accept_id
+WHERE rcpt_to = :user_id AND
+  connect_time >= :start_date AND
+  connect_time < :end_date
+ORDER BY connect_time

management/reporting/uidata/user_activity.py

@@ -0,0 +1,167 @@
+from .Timeseries import Timeseries
+from .exceptions import InvalidArgsError
+
+with open(__file__.replace('.py','.1.sql')) as fp:
+    select_1 = fp.read()
+
+with open(__file__.replace('.py','.2.sql')) as fp:
+    select_2 = fp.read()
+
+
+def user_activity(conn, args):
+    '''
+    details on user activity
+    '''    
+    try:
+        user_id = args['user_id']
+
+        # use Timeseries to get a normalized start/end range
+        ts = Timeseries(
+            'User activity',
+            args['start_date'],
+            args['end_date'],
+            0
+        )
+    except KeyError:
+        raise InvalidArgsError()
+
+    # limit results
+    try:
+        limit = 'LIMIT ' + str(int(args.get('row_limit', 1000)));
+    except ValueError:
+        limit = 'LIMIT 1000'    
+
+    #
+    # sent mail by user
+    #
+    c = conn.cursor()
+    
+    sent_mail = {
+        'start': ts.start,
+        'end': ts.end,
+        'y': 'Sent mail',
+        'fields': [
+            # mta_connection
+            'connect_time',
+            'sasl_method',
+            # mta_accept
+            'envelope_from',
+            # mta_delivery
+            'rcpt_to',
+            'service',
+            'spam_score',
+            'spam_result',
+            'message_size',
+            'status',
+            'relay',
+            'delivery_info',
+            'delivery_connection',
+            'delivery_connection_info',
+            'sent_id',  # must be last
+        ],
+        'field_types': [
+            { 'type':'datetime', 'format': '%Y-%m-%d %H:%M:%S' },# connect_time
+            'text/plain',    # sasl_method
+            'text/email',    # envelope_from
+            { 'type':'text/email', 'label':'Recipient' },    # rcpt_to
+            'text/plain',    # mta_delivery.service
+            { 'type':'decimal', 'places':2 },     # spam_score
+            'text/plain',    # spam_result
+            'number/size',   # message_size
+            'text/plain',    # status
+            'text/hostname', # relay
+            'text/plain',    # delivery_info
+            'text/plain',    # delivery_connection
+            'text/plain',    # delivery_connection_info
+            'number/plain',  # sent_id - must be last
+        ],
+        'items': [],
+        'unique_sends': 0
+    }
+    last_mta_accept_id = -1
+    sent_id = 0
+    for row in c.execute(select_1 + limit, {
+            'user_id': user_id,
+            'start_date': ts.start,
+            'end_date': ts.end
+    }):
+        v = []
+        for key in sent_mail['fields']:
+            if key != 'sent_id':
+                v.append(row[key])
+        if last_mta_accept_id != row['mta_accept_id']:
+            sent_mail['unique_sends'] += 1
+            last_mta_accept_id = row['mta_accept_id']
+            sent_id += 1
+        v.append(sent_id)
+        sent_mail['items'].append(v)
+
+
+    #
+    # received mail by user
+    #
+
+    received_mail = {
+        'start': ts.start,
+        'end': ts.end,
+        'y': 'Sent mail',
+        'fields': [
+            # mta_connection
+            'connect_time',
+            'service',
+            'sasl_username',
+            
+            # mta_accept
+            'envelope_from',
+            'disposition',
+            'spf_result',
+            'dkim_result',
+            'dkim_reason',
+            'dmarc_result',
+            'dmarc_reason',
+            
+            # mta_delivery
+            'postgrey_result',
+            'postgrey_reason',
+            'postgrey_delay',
+            'spam_score',
+            'spam_result',
+            'message_size',
+        ],
+        'field_types': [
+            { 'type':'datetime', 'format': '%Y-%m-%d %H:%M:%S' },# connect_time
+            'text/plain',    # mta_connection.service
+            'text/email',    # sasl_username
+            'text/email',    # envelope_from
+            'text/plain',    # disposition
+            'text/plain',    # spf_result
+            'text/plain',    # dkim_result
+            'text/plain',    # dkim_result
+            'text/plain',    # dmarc_result
+            'text/plain',    # dmarc_result
+            'text/plain',    # postgrey_result
+            'text/plain',    # postgrey_reason
+            { 'type':'time/span', 'unit':'s' },   # postgrey_delay
+            { 'type':'decimal', 'places':2 },     # spam_score
+            'text/plain',    # spam_result
+            'number/size',   # message_size
+        ],
+        'items': []
+    }
+
+    for row in c.execute(select_2 + limit, {
+            'user_id': user_id,
+            'start_date': ts.start,
+            'end_date': ts.end
+    }):
+        v = []
+        for key in received_mail['fields']:
+            v.append(row[key])
+        received_mail['items'].append(v)
+
+
+        
+    return {
+        'sent_mail': sent_mail,
+        'received_mail': received_mail
+    }

management/templates/index.html

@@ -109,6 +109,7 @@
             </li>
             <li><a href="#sync_guide" onclick="return show_panel(this);">Contacts/Calendar</a></li>
             <li><a href="#web" onclick="return show_panel(this);">Web</a></li>
+            <li><a href="/admin/reports/" onclick="return api_credentials[0]!=''">Activity</a></li>
           </ul>
           <ul class="nav navbar-nav navbar-right">
             <li><a href="#" onclick="do_logout(); return false;" style="color: white">Log out</a></li>

management/ui-common/authentication.js

@@ -0,0 +1,92 @@
+class Me {
+    /* construct with return value from GET /me */
+    constructor(me) {
+        Object.assign(this, me);
+    }
+
+    is_authenticated() {
+        return this.api_key || this.user_id;
+    }
+
+    get_email() {
+        return this.user_email || this.user_id;
+    }
+};
+
+
+/*
+ * axios interceptors for authentication
+ */
+
+function init_axios_interceptors() {
+
+    // requests: attach non-session based auth (admin panel)
+    axios.interceptors.request.use(request => {
+        var api_credentials = null;
+        // code from templates/index.html for "recall saved user
+        // credentials" (but, without the split(':'))
+        if (typeof sessionStorage != 'undefined' && sessionStorage.getItem("miab-cp-credentials"))
+            api_credentials = sessionStorage.getItem("miab-cp-credentials");
+        else if (typeof localStorage != 'undefined' && localStorage.getItem("miab-cp-credentials"))
+            api_credentials = localStorage.getItem("miab-cp-credentials");
+        // end
+
+        if (api_credentials) {
+            request.headers.authorization = 'Basic ' + window.btoa(api_credentials);
+        }
+        return request;
+    });
+
+
+    // reponses: redirect on authorization failure
+    axios.interceptors.response.use(
+        response => {
+            if (response.data &&
+                response.data.status === 'invalid' &&
+                response.config.headers.authorization)
+            {
+                var url = response.config.url;
+                if (response.config.baseURL) {
+                    var sep = ( response.config.baseURL.substr(-1) != '/' ?
+                                '/' : '' );
+                    url = response.config.baseURL + sep + url;
+                }
+            
+                if (url == '/admin/me')
+                {
+                    // non-session/admin login
+                    throw new AuthenticationError(
+                        null,
+                        'not authenticated',
+                        response
+                    );
+                }
+            }
+            return response;
+        },
+        
+        error => {
+            const auth_required_msg = 'Authentication required - you have been logged out of the server';
+            if (! error.response) {
+                throw error;
+            }
+            
+            if (error.response.status == 403 &&
+                error.response.data == 'login_required')
+            {
+                // session login
+                throw new AuthenticationError(error, auth_required_msg);
+            }
+            else if ((error.response.status == 403 ||
+                      error.response.status == 401) &&
+                     error.response.data &&
+                     error.response.data.status == 'error')
+            {
+                // admin
+                throw new AuthenticationError(error, auth_required_msg);
+            }
+            throw error;
+        }
+    );
+}
+

management/ui-common/exceptions.js

@@ -0,0 +1,18 @@
+class ValueError extends Error {
+    constructor(msg) {
+        super(msg);
+    }
+};
+
+class AssertionError extends Error {
+}
+
+class AuthenticationError extends Error {
+    constructor(caused_by_error, msg, response) {
+        super(msg);
+        this.caused_by = caused_by_error;
+        this.response = response;
+        if (!response && caused_by_error)
+            this.response = caused_by_error.response;
+    }
+};

management/ui-common/page-header.html

@@ -0,0 +1,11 @@
+<div class="text-center">
+  <h2 class="mb-0">{{ header_text }}</h2>
+  <small class="text-white-50">from the <a href="https://github.com/downtownallday/mailinabox-ldap/" target="_blank" class="text-white-50">MiaB-LDAP project</a></small>
+  <div class="d-flex align-items-center justify-content-between" style="margin-top:-1.5rem">
+    <div class="ml-1" style="min-width:1em">
+      <spinner v-show="loading_counter > 0"></spinner>
+    </div>
+    <slot name="links"><div>&nbsp;</div></slot>
+  </div>
+  
+</div>

management/ui-common/page-header.js

@@ -0,0 +1,19 @@
+Vue.component('spinner', {
+    template: '<span class="spinner-border spinner-border-sm"></span>'
+});
+
+Vue.component('page-header', function(resolve, reject) {
+    axios.get('ui-common/page-header.html').then((response) => { resolve({
+
+        props: {
+            header_text: { type: String, required: true },
+            loading_counter: { type: Number, required: true }
+        },
+        
+        template: response.data
+                        
+    })}).catch((e) => {
+        reject(e);
+    });
+
+});

management/ui-common/page-layout.html

@@ -0,0 +1,8 @@
+<div>
+  <div class="bg-dark text-white p-1">
+    <slot name="header"></slot>
+  </div>    
+  <div class="bg-light text-dark p-1">
+    <slot></slot>
+  </div>
+</div>

management/ui-common/page-layout.js

@@ -0,0 +1,10 @@
+Vue.component('page-layout', function(resolve, reject) {
+    axios.get('ui-common/page-layout.html').then((response) => { resolve({
+
+        template: response.data,
+        
+    })}).catch((e) => {
+        reject(e);
+    });
+
+});

management/ui-common/theme/.gitignore

@@ -0,0 +1 @@
+node_modules/

management/ui-common/theme/build.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+#
+# install bootstrap sources
+#
+if [ ! -e "node_modules/bootstrap" ]; then
+    npm install bootstrap
+    if [ $? -ne 0 ]; then
+        echo "Installing bootstrap using npm failed. Is npm install on your system?"
+        exit 1
+    fi
+fi
+
+
+#
+# install sass compiler
+#
+compiler="/usr/bin/sassc"
+if [ ! -x "$compiler" ]; then
+    sudo apt-get install sassc || exit 1
+fi
+
+
+#
+# compile our theme
+#
+b_dir="node_modules/bootstrap/scss"
+
+$compiler -I "$b_dir" --sourcemap --style compressed theme.scss ../ui-bootstrap.css
+if [ $? -eq 0 ]; then
+    echo "SUCCESS - files:"
+    ls -sh ../ui-bootstrap.*
+fi
+
+

management/ui-common/theme/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "theme",
+  "version": "1.0.0",
+  "description": "",
+  "dependencies": {
+    "bootstrap": "^4.5.3"
+  },
+  "devDependencies": {},
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "GPL-3.0-or-later"
+}

management/ui-common/theme/theme.scss

@@ -0,0 +1,95 @@
+/* variable overrides */
+$white:    #fff;
+$gray-100: #f8f9fa;
+$gray-200: #e9ecef;
+$gray-300: #dee2e6;
+$gray-400: #ced4da;
+$gray-500: #adb5bd;
+$gray-600: #6c757d;
+$gray-700: #495057;
+$gray-800: #343a40;
+$gray-900: #212529;
+$black:    #000;
+
+$blue:    cadetblue;
+
+$primary:       #446599; //#96a5b2;
+$secondary:     $gray-600; //#d8dfe5; //#f5f5f5;
+$success:       #b7dfb8;
+$info:          #ffd6b0;
+$warning:       #f0e68c;
+$danger:        #a91409;
+$light:         $gray-100;
+$dark:          #303e45;
+
+
+
+$body-bg:                   $gray-100; //$blue;
+$body-color:                $dark;
+
+$card-color:                $gray-900;
+$card-spacer-x:             0.75rem;
+$card-spacer-y:             0.25rem;
+
+$table-color:               $dark;
+$table-cell-padding:        0.375rem; //  .75rem;
+$table-cell-padding-sm:     0.15rem; //  .3rem;
+
+
+$font-size-base: 0.85rem; // Assumes the browser default, typically `16px`
+
+$input-btn-padding-y:     .25rem; //    .375rem !default;
+$input-btn-padding-x:     .5rem;  //    .75rem !default;
+
+$input-btn-padding-y-sm:  .12rem; //    .25rem !default;
+$input-btn-padding-x-sm:  .25rem; //    .5rem !default;
+
+//$input-btn-padding-y-lg:      .5rem !default;
+//$input-btn-padding-x-lg:      1rem !default;
+
+$alert-padding-y:                   .25rem;
+$alert-padding-x:                   .75rem;
+
+// $list-group-item-padding-y:         .75rem !default;
+// $list-group-item-padding-x:         1.25rem !default;
+$nav-link-padding-y: 0.20rem;
+
+
+/* bootstrap styles that we want */
+@import "functions";
+@import "variables";
+@import "mixins";
+@import "root";
+@import "reboot";
+@import "type";
+@import "images";
+@import "code";
+@import "grid";
+@import "tables";
+@import "forms";
+@import "buttons";
+@import "transitions";
+@import "dropdown";
+@import "button-group";
+@import "input-group";
+/* @import "custom-forms"; */
+@import "nav";
+@import "navbar";
+@import "card";
+@import "breadcrumb";
+@import "pagination";
+@import "badge"; 
+/* @import "jumbotron"; */
+@import "alert";
+@import "progress";
+/* @import "media"; */
+@import "list-group";
+@import "close";
+@import "toasts";
+@import "modal";
+@import "tooltip";
+@import "popover";
+/* @import "carousel"; */
+@import "spinners";
+@import "utilities";
+/* @import "print"; */