Split the User Activity/IMAP connections tab into two tables to better deal with the quantity of data

management/reporting/uidata/Timeseries.py

@@ -22,7 +22,7 @@ class Timeseries(object):
         # parsefmt is a date parser string to be used to re-interpret
         # "bin" grouping dates (data.dates) to native dates. server
         # always returns utc dates
-        parsefmt = '%Y-%m-%d %H:%M:%S'
+        self.parsefmt = '%Y-%m-%d %H:%M:%S'
 
         self.dates = []   # dates must be "bin" date strings
         self.series = []
@@ -31,7 +31,7 @@ class Timeseries(object):
             'range': [ self.start, self.end ],
             'range_parse_format': '%Y-%m-%d %H:%M:%S',
             'binsize': self.binsize,
-            'date_parse_format': parsefmt,
+            'date_parse_format': self.parsefmt,
             'y': desc,
             'dates': self.dates,
             'series': self.series

management/reporting/uidata/__init__.py

@@ -3,6 +3,7 @@ from .select_list_suggestions import select_list_suggestions
 from .messages_sent import messages_sent
 from .messages_received import messages_received
 from .user_activity import user_activity
+from .imap_details import imap_details
 from .remote_sender_activity import remote_sender_activity
 from .flagged_connections import flagged_connections
 from .capture_db_stats import capture_db_stats

management/reporting/uidata/imap_details.1.sql

@@ -0,0 +1,25 @@
+--
+-- details on user imap connections
+--
+SELECT
+  connect_time,
+  disconnect_time,
+  CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END AS `remote_host`,
+  sasl_method,
+  disconnect_reason,
+  connection_security,
+  disposition,
+  in_bytes,
+  out_bytes
+FROM
+  imap_connection
+WHERE
+  sasl_username = :user_id AND
+  connect_time >= :start_date AND
+  connect_time < :end_date AND
+  (:remote_host IS NULL OR
+     remote_host = :remote_host OR remote_ip = :remote_host) AND
+  (:disposition IS NULL OR
+     disposition = :disposition)
+ORDER BY
+  connect_time

management/reporting/uidata/imap_details.py

@@ -0,0 +1,83 @@
+from .Timeseries import Timeseries
+from .exceptions import InvalidArgsError
+
+with open(__file__.replace('.py','.1.sql')) as fp:
+    select_1 = fp.read()
+
+
+def imap_details(conn, args):
+    '''
+    details on imap connections
+    '''    
+    try:
+        user_id = args['user_id']
+
+        # use Timeseries to get a normalized start/end range
+        ts = Timeseries(
+            'IMAP details',
+            args['start_date'],
+            args['end_date'],
+            0
+        )
+
+        # optional
+        remote_host = args.get('remote_host')
+        disposition = args.get('disposition')
+        
+    except KeyError:
+        raise InvalidArgsError()
+
+    # limit results
+    try:
+        limit = 'LIMIT ' + str(int(args.get('row_limit', 1000)));
+    except ValueError:
+        limit = 'LIMIT 1000'
+
+
+    c = conn.cursor()
+
+    imap_details = {
+        'start': ts.start,
+        'end': ts.end,
+        'y': 'IMAP Details',
+        'fields': [
+            'connect_time',
+            'disconnect_time',
+            'remote_host',
+            'sasl_method',
+            'disconnect_reason',
+            'connection_security',
+            'disposition',
+            'in_bytes',
+            'out_bytes'
+        ],
+        'field_types': [
+            { 'type':'datetime', 'format': ts.parsefmt }, # connect_time
+            { 'type':'datetime', 'format': ts.parsefmt }, # disconnect_time
+            'text/plain',    # remote_host
+            'text/plain',    # sasl_method
+            'text/plain',    # disconnect_reason
+            'text/plain',    # connection_security
+            'text/plain',    # disposition
+            'number/size',   # in_bytes,
+            'number/size',   # out_bytes,
+        ],
+        'items': []
+    }
+
+    for row in c.execute(select_1 + limit, {
+            'user_id': user_id,
+            'start_date': ts.start,
+            'end_date': ts.end,
+            'remote_host': remote_host,
+            'disposition': disposition
+    }):
+        v = []
+        for key in imap_details['fields']:
+            v.append(row[key])
+        imap_details['items'].append(v)
+
+    
+    return {
+        'imap_details': imap_details
+    }

management/reporting/uidata/user_activity.3.sql

@@ -1,20 +1,22 @@
 --
--- details on user imap connections
+-- imap connection summary
 --
 SELECT
-  connect_time,
-  CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END AS `remote_host`,
-  sasl_method,
-  disconnect_reason,
-  connection_security,
+  count(*) as `count`,
   disposition,
-  in_bytes,
-  out_bytes
+  CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END AS `remote_host`,
+  sum(in_bytes) as `in_bytes`,
+  sum(out_bytes) as `out_bytes`,
+  min(connect_time) as `first_connection_time`,
+  max(connect_time) as `last_connection_time`
 FROM
   imap_connection
 WHERE
   sasl_username = :user_id AND
   connect_time >= :start_date AND
   connect_time < :end_date
+GROUP BY
+  disposition,
+  CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END
 ORDER BY
-  connect_time
+  `count` DESC, disposition

management/reporting/uidata/user_activity.py

@@ -200,50 +200,64 @@ def user_activity(conn, args):
 
 
     #
-    # imap connections by user
+    # IMAP connections by disposition, by remote host
+    #   Disposition
+    #   Remote host
+    #   Count
+    #   In bytes (sum)
+    #   Out bytes (sum)
+    #   % of total
     #
 
-    imap_details = {
+    imap_conn_summary = {
         'start': ts.start,
         'end': ts.end,
-        'y': 'IMAP Details',
+        'y': 'IMAP connection summary by host and disposition',
         'fields': [
-            'connect_time',
+            'count',
+            'total',
             'remote_host',
-            'sasl_method',
-            'disconnect_reason',
-            'connection_security',
             'disposition',
+            'first_connection_time',
+            'last_connection_time',
             'in_bytes',
-            'out_bytes'
+            'out_bytes',
         ],
         'field_types': [
-            { 'type':'datetime', 'format': '%Y-%m-%d %H:%M:%S' },# connect_time
+            'number',        # count
+            { 'type': 'number/percent', 'places': 1 }, # total
             'text/plain',    # remote_host
-            'text/plain',    # sasl_method
-            'text/plain',    # disconnect_reason
-            'text/plain',    # connection_security
             'text/plain',    # disposition
+            { 'type':'datetime', 'format': ts.parsefmt }, # first_conn_time
+            { 'type':'datetime', 'format': ts.parsefmt }, # last_conn_time
             'number/size',   # in_bytes,
             'number/size',   # out_bytes,
         ],
         'items': []
     }
 
+    count_field_idx = 0
+    total_field_idx = 1
+    total = 0
     for row in c.execute(select_3 + limit, {
             'user_id': user_id,
             'start_date': ts.start,
             'end_date': ts.end
     }):
         v = []
-        for key in imap_details['fields']:
-            v.append(row[key])
-        imap_details['items'].append(v)
+        for key in imap_conn_summary['fields']:
+            if key=='count':
+                total += row[key]
+            if key!='total':
+                v.append(row[key])
 
+        imap_conn_summary['items'].append(v)
 
+    for v in imap_conn_summary['items']:
+        v.insert(total_field_idx, v[count_field_idx] / total)
         
     return {
         'sent_mail': sent_mail,
         'received_mail': received_mail,
-        'imap_details': imap_details
+        'imap_conn_summary': imap_conn_summary
     }