From 36d9cbb4e8586e909b2a4d0b482322a44b3b1958 Mon Sep 17 00:00:00 2001
From: downtownallday <downtownallday@gmail.com>
Date: Mon, 12 Apr 2021 15:07:56 -0400
Subject: [PATCH] Split the User Activity/IMAP connections tab into two tables
 to better deal with the quantity of data

---
 management/daemon_reports.py                  | 12 +++
 management/reporting/ui/chart-table.js        |  5 +-
 management/reporting/ui/charting.js           |  2 +
 .../reporting/ui/panel-user-activity.html     | 36 ++++----
 .../reporting/ui/panel-user-activity.js       | 72 +++++++++++++---
 management/reporting/uidata/Timeseries.py     |  4 +-
 management/reporting/uidata/__init__.py       |  1 +
 .../reporting/uidata/imap_details.1.sql       | 25 ++++++
 management/reporting/uidata/imap_details.py   | 83 +++++++++++++++++++
 .../reporting/uidata/user_activity.3.sql      | 20 +++--
 management/reporting/uidata/user_activity.py  | 46 ++++++----
 11 files changed, 248 insertions(+), 58 deletions(-)
 create mode 100644 management/reporting/uidata/imap_details.1.sql
 create mode 100644 management/reporting/uidata/imap_details.py

diff --git a/management/daemon_reports.py b/management/daemon_reports.py
index fcebffcb..6dac25ea 100644
--- a/management/daemon_reports.py
+++ b/management/daemon_reports.py
@@ -102,6 +102,18 @@ def add_reports(app, env, authorized_personnel_only):
 		finally:
 			db_conn_factory.close(conn)
 
+	@app.route('/reports/uidata/imap-details', methods=['POST'])
+	@authorized_personnel_only
+	@json_payload
+	def get_imap_details(payload):
+		conn = db_conn_factory.connect()
+		try:
+			return jsonify(uidata.imap_details(conn, payload))
+		except uidata.InvalidArgsError as e:
+			return ('invalid request', 400)
+		finally:
+			db_conn_factory.close(conn)
+
 	@app.route('/reports/uidata/flagged-connections', methods=['POST'])
 	@authorized_personnel_only
 	@json_payload
diff --git a/management/reporting/ui/chart-table.js b/management/reporting/ui/chart-table.js
index 176427c5..96b42a51 100644
--- a/management/reporting/ui/chart-table.js
+++ b/management/reporting/ui/chart-table.js
@@ -2,7 +2,8 @@ export default Vue.component('chart-table', {
     props: {
         items: Array,
         fields: Array,
-        caption: String
+        caption: String,
+        small: { type:Boolean, default:true }
     },
 
     /* <b-table-lite striped small :fields="fields_x" :items="items" caption-top><template #table-caption><span class="text-nowrap">{{caption}}</span></template></b-table>*/
@@ -19,7 +20,7 @@ export default Vue.component('chart-table', {
         var table = ce('b-table-lite', {
             props: {
                 'striped': true,
-                'small': true,
+                'small': this.small,
                 'fields': this.fields_x,
                 'items': this.items,
                 'caption-top': true
diff --git a/management/reporting/ui/charting.js b/management/reporting/ui/charting.js
index bab22eab..fc8bfda6 100644
--- a/management/reporting/ui/charting.js
+++ b/management/reporting/ui/charting.js
@@ -493,6 +493,8 @@ export class BvTableField {
         }
         else if (ft.type == 'number') {
             if (ft.subtype == 'plain' ||
+                ft.subtype === null ||
+                ft.subtype === undefined ||
                 ft.subtype == 'decimal' && isNaN(ft.places)
                )
             {
diff --git a/management/reporting/ui/panel-user-activity.html b/management/reporting/ui/panel-user-activity.html
index 637a296e..9ceaf889 100644
--- a/management/reporting/ui/panel-user-activity.html
+++ b/management/reporting/ui/panel-user-activity.html
@@ -84,26 +84,30 @@
 
     <b-tab>
       <template #title>
-        IMAP Connections<sup v-if="imap_details.items.length >= get_row_limit()">*</sup> ({{imap_details.items.length}})
+        IMAP Connections
       </template>
+
       <b-table
-        class="sticky-table-header-0 bg-light"
-        small
+        tbody-tr-class="cursor-pointer"
+        selectable
+        select-mode="single"
         :filter="show_only_flagged_filter"
         :filter-function="table_filter_cb"
-        tbody-tr-class="cursor-pointer"
-        details-td-class="cursor-default"
-        @row-clicked="row_clicked"
-        :items="imap_details.items"
-        :fields="imap_details.fields">
-        <template #row-details="row">
-          <b-card>
-            <div><strong>Connection disposition</strong>: {{ disposition_formatter(row.item.disposition) }}</div>
-            <div><strong>Connection security</strong> {{ row.item.connection_security }}</div>
-            <div><strong>Disconnect reason</strong> {{ row.item.disconnect_reason }}</div>
-          </b-card>
-        </template>
-      </b-table>
+        :items="imap_conn_summary.items"
+        :fields="imap_conn_summary.fields"
+        @row-clicked="load_imap_details">
+      </b-table>       
+
+      <div v-if="imap_details" class="bg-white">
+        <div class="mt-3 text-center bg-info p-1">{{imap_details._desc}} ({{imap_details.items.length}} rows<sup v-if="imap_details.items.length >= get_row_limit()">*</sup>)</div>
+        <b-table
+          class="sticky-table-header-0"
+          small
+          :items="imap_details.items"
+          :fields="imap_details.fields">
+        </b-table>
+      </div>
+      
     </b-tab>
 
   </b-tabs>
diff --git a/management/reporting/ui/panel-user-activity.js b/management/reporting/ui/panel-user-activity.js
index 53ef6e54..b5d6c43c 100644
--- a/management/reporting/ui/panel-user-activity.js
+++ b/management/reporting/ui/panel-user-activity.js
@@ -3,9 +3,10 @@
 */
 
 import wbr_text from "./wbr-text.js";
+import chart_table from "./chart-table.js";
 import message_headers_view from "./message_headers_view.js";
 import UserSettings from "./settings.js";
-import { MailBvTable, ConnectionDisposition } from "./charting.js";
+import { BvTable, MailBvTable, ConnectionDisposition } from "./charting.js";
 
 
 export default Vue.component('panel-user-activity', function(resolve, reject) {
@@ -19,7 +20,8 @@ export default Vue.component('panel-user-activity', function(resolve, reject) {
 
         components: {
             'wbr-text': wbr_text,
-            'message-headers-view': message_headers_view
+            'message-headers-view': message_headers_view,
+            'chart-table': chart_table,
         },
         
         data: function() {
@@ -36,6 +38,7 @@ export default Vue.component('panel-user-activity', function(resolve, reject) {
                 data_date_range: null, /* date range for active table data */
                 sent_mail: null,
                 received_mail: null,
+                imap_conn_summary: null,
                 imap_details: null,
                 lmtp_id: null, /* for message headers modal */
                 all_users: [],
@@ -169,11 +172,19 @@ export default Vue.component('panel-user-activity', function(resolve, reject) {
                 f.label = 'Envelope From (user)';
             },
 
+            combine_imap_conn_summary_fields: function() {
+                // remove 'first_conn_time'
+                this.imap_conn_summary.combine_fields('first_connection_time');
+                // clear the label for the 'total' column (pct)
+                const f_total = this.imap_conn_summary.get_field('total');
+                f_total.label = '';
+            },
+
             combine_imap_details_fields: function() {
                 // remove these fields
                 this.imap_details.combine_fields([
-                    'disconnect_reason',
-                    'connection_security',
+                    'remote_host',
+                    'disposition',
                 ]);
             },
 
@@ -272,16 +283,21 @@ export default Vue.component('panel-user-activity', function(resolve, reject) {
                         .get_field('connect_time')
                         .add_tdClass('text-nowrap');
 
-                    /* setup imap_details */
-                    this.imap_details = new MailBvTable(
-                        response.data.imap_details, {
-                            _showDetails: true
+                    
+                    /* setup imap_conn_summary */
+                    this.imap_conn_summary = new MailBvTable(
+                        response.data.imap_conn_summary
+                    );
+                    this.combine_imap_conn_summary_fields();
+                    this.imap_conn_summary.flag_fields();
+                    ['last_connection_time','count']
+                        .forEach(name => {
+                            const f = this.imap_conn_summary.get_field(name);
+                            f.add_cls('text-nowrap', 'tdClass');
                         });
-                    this.combine_imap_details_fields();
-                    this.imap_details
-                        .flag_fields()
-                        .get_field('connect_time')
-                        .add_tdClass('text-nowrap');
+
+                    /* clear imap_details */
+                    this.imap_details = null;
 
                 }).catch(error => {
                     this.$root.handleError(error);
@@ -304,6 +320,36 @@ export default Vue.component('panel-user-activity', function(resolve, reject) {
                 
                 // show the modal dialog
                 this.$refs.message_headers_modal.show();
+            },
+
+            load_imap_details: function(item, index, event) {
+                this.$emit('loading', 1);
+                this.imap_details = null;
+                const promise = axios.post('reports/uidata/imap-details', {
+                    row_limit: this.get_row_limit(),
+                    user_id: this.user_id.trim(),
+                    start_date: this.date_range[0],
+                    end_date: this.date_range[1],
+                    disposition: item.disposition,
+                    remote_host: item.remote_host
+                }).then(response => {
+                    this.imap_details = new MailBvTable(
+                        response.data.imap_details
+                    );
+                    this.combine_imap_details_fields();
+                    this.imap_details.get_field('connect_time')
+                        .add_tdClass('text-nowrap');
+                    this.imap_details.get_field('disconnect_time')
+                        .add_tdClass('text-nowrap');
+                    this.imap_details._desc =
+                        `${item.remote_host}/${item.disposition}`;
+                    
+                }).catch(error => {
+                    this.$root.handleError(error);
+                    
+                }).finally( () => {
+                    this.$emit('loading', -1);
+                });
             }
             
         }
diff --git a/management/reporting/uidata/Timeseries.py b/management/reporting/uidata/Timeseries.py
index 73575187..b2e91259 100644
--- a/management/reporting/uidata/Timeseries.py
+++ b/management/reporting/uidata/Timeseries.py
@@ -22,7 +22,7 @@ class Timeseries(object):
         # parsefmt is a date parser string to be used to re-interpret
         # "bin" grouping dates (data.dates) to native dates. server
         # always returns utc dates
-        parsefmt = '%Y-%m-%d %H:%M:%S'
+        self.parsefmt = '%Y-%m-%d %H:%M:%S'
 
         self.dates = []   # dates must be "bin" date strings
         self.series = []
@@ -31,7 +31,7 @@ class Timeseries(object):
             'range': [ self.start, self.end ],
             'range_parse_format': '%Y-%m-%d %H:%M:%S',
             'binsize': self.binsize,
-            'date_parse_format': parsefmt,
+            'date_parse_format': self.parsefmt,
             'y': desc,
             'dates': self.dates,
             'series': self.series
diff --git a/management/reporting/uidata/__init__.py b/management/reporting/uidata/__init__.py
index 0afa0c46..ac3f9829 100644
--- a/management/reporting/uidata/__init__.py
+++ b/management/reporting/uidata/__init__.py
@@ -3,6 +3,7 @@ from .select_list_suggestions import select_list_suggestions
 from .messages_sent import messages_sent
 from .messages_received import messages_received
 from .user_activity import user_activity
+from .imap_details import imap_details
 from .remote_sender_activity import remote_sender_activity
 from .flagged_connections import flagged_connections
 from .capture_db_stats import capture_db_stats
diff --git a/management/reporting/uidata/imap_details.1.sql b/management/reporting/uidata/imap_details.1.sql
new file mode 100644
index 00000000..40e76085
--- /dev/null
+++ b/management/reporting/uidata/imap_details.1.sql
@@ -0,0 +1,25 @@
+--
+-- details on user imap connections
+--
+SELECT
+  connect_time,
+  disconnect_time,
+  CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END AS `remote_host`,
+  sasl_method,
+  disconnect_reason,
+  connection_security,
+  disposition,
+  in_bytes,
+  out_bytes
+FROM
+  imap_connection
+WHERE
+  sasl_username = :user_id AND
+  connect_time >= :start_date AND
+  connect_time < :end_date AND
+  (:remote_host IS NULL OR
+     remote_host = :remote_host OR remote_ip = :remote_host) AND
+  (:disposition IS NULL OR
+     disposition = :disposition)
+ORDER BY
+  connect_time
diff --git a/management/reporting/uidata/imap_details.py b/management/reporting/uidata/imap_details.py
new file mode 100644
index 00000000..36d18d9c
--- /dev/null
+++ b/management/reporting/uidata/imap_details.py
@@ -0,0 +1,83 @@
+from .Timeseries import Timeseries
+from .exceptions import InvalidArgsError
+
+with open(__file__.replace('.py','.1.sql')) as fp:
+    select_1 = fp.read()
+
+
+def imap_details(conn, args):
+    '''
+    details on imap connections
+    '''    
+    try:
+        user_id = args['user_id']
+
+        # use Timeseries to get a normalized start/end range
+        ts = Timeseries(
+            'IMAP details',
+            args['start_date'],
+            args['end_date'],
+            0
+        )
+
+        # optional
+        remote_host = args.get('remote_host')
+        disposition = args.get('disposition')
+        
+    except KeyError:
+        raise InvalidArgsError()
+
+    # limit results
+    try:
+        limit = 'LIMIT ' + str(int(args.get('row_limit', 1000)));
+    except ValueError:
+        limit = 'LIMIT 1000'
+
+
+    c = conn.cursor()
+
+    imap_details = {
+        'start': ts.start,
+        'end': ts.end,
+        'y': 'IMAP Details',
+        'fields': [
+            'connect_time',
+            'disconnect_time',
+            'remote_host',
+            'sasl_method',
+            'disconnect_reason',
+            'connection_security',
+            'disposition',
+            'in_bytes',
+            'out_bytes'
+        ],
+        'field_types': [
+            { 'type':'datetime', 'format': ts.parsefmt }, # connect_time
+            { 'type':'datetime', 'format': ts.parsefmt }, # disconnect_time
+            'text/plain',    # remote_host
+            'text/plain',    # sasl_method
+            'text/plain',    # disconnect_reason
+            'text/plain',    # connection_security
+            'text/plain',    # disposition
+            'number/size',   # in_bytes,
+            'number/size',   # out_bytes,
+        ],
+        'items': []
+    }
+
+    for row in c.execute(select_1 + limit, {
+            'user_id': user_id,
+            'start_date': ts.start,
+            'end_date': ts.end,
+            'remote_host': remote_host,
+            'disposition': disposition
+    }):
+        v = []
+        for key in imap_details['fields']:
+            v.append(row[key])
+        imap_details['items'].append(v)
+
+    
+    return {
+        'imap_details': imap_details
+    }
diff --git a/management/reporting/uidata/user_activity.3.sql b/management/reporting/uidata/user_activity.3.sql
index fbd98e25..ec39fba0 100644
--- a/management/reporting/uidata/user_activity.3.sql
+++ b/management/reporting/uidata/user_activity.3.sql
@@ -1,20 +1,22 @@
 --
--- details on user imap connections
+-- imap connection summary
 --
 SELECT
-  connect_time,
-  CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END AS `remote_host`,
-  sasl_method,
-  disconnect_reason,
-  connection_security,
+  count(*) as `count`,
   disposition,
-  in_bytes,
-  out_bytes
+  CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END AS `remote_host`,
+  sum(in_bytes) as `in_bytes`,
+  sum(out_bytes) as `out_bytes`,
+  min(connect_time) as `first_connection_time`,
+  max(connect_time) as `last_connection_time`
 FROM
   imap_connection
 WHERE
   sasl_username = :user_id AND
   connect_time >= :start_date AND
   connect_time < :end_date
+GROUP BY
+  disposition,
+  CASE WHEN remote_host='unknown' THEN remote_ip ELSE remote_host END
 ORDER BY
-  connect_time
+  `count` DESC, disposition
diff --git a/management/reporting/uidata/user_activity.py b/management/reporting/uidata/user_activity.py
index 31221466..1a7ca62f 100644
--- a/management/reporting/uidata/user_activity.py
+++ b/management/reporting/uidata/user_activity.py
@@ -200,50 +200,64 @@ def user_activity(conn, args):
 
 
     #
-    # imap connections by user
+    # IMAP connections by disposition, by remote host
+    #   Disposition
+    #   Remote host
+    #   Count
+    #   In bytes (sum)
+    #   Out bytes (sum)
+    #   % of total
     #
 
-    imap_details = {
+    imap_conn_summary = {
         'start': ts.start,
         'end': ts.end,
-        'y': 'IMAP Details',
+        'y': 'IMAP connection summary by host and disposition',
         'fields': [
-            'connect_time',
+            'count',
+            'total',
             'remote_host',
-            'sasl_method',
-            'disconnect_reason',
-            'connection_security',
             'disposition',
+            'first_connection_time',
+            'last_connection_time',
             'in_bytes',
-            'out_bytes'
+            'out_bytes',
         ],
         'field_types': [
-            { 'type':'datetime', 'format': '%Y-%m-%d %H:%M:%S' },# connect_time
+            'number',        # count
+            { 'type': 'number/percent', 'places': 1 }, # total
             'text/plain',    # remote_host
-            'text/plain',    # sasl_method
-            'text/plain',    # disconnect_reason
-            'text/plain',    # connection_security
             'text/plain',    # disposition
+            { 'type':'datetime', 'format': ts.parsefmt }, # first_conn_time
+            { 'type':'datetime', 'format': ts.parsefmt }, # last_conn_time
             'number/size',   # in_bytes,
             'number/size',   # out_bytes,
         ],
         'items': []
     }
 
+    count_field_idx = 0
+    total_field_idx = 1
+    total = 0
     for row in c.execute(select_3 + limit, {
             'user_id': user_id,
             'start_date': ts.start,
             'end_date': ts.end
     }):
         v = []
-        for key in imap_details['fields']:
-            v.append(row[key])
-        imap_details['items'].append(v)
+        for key in imap_conn_summary['fields']:
+            if key=='count':
+                total += row[key]
+            if key!='total':
+                v.append(row[key])
 
+        imap_conn_summary['items'].append(v)
 
+    for v in imap_conn_summary['items']:
+        v.insert(total_field_idx, v[count_field_idx] / total)
         
     return {
         'sent_mail': sent_mail,
         'received_mail': received_mail,
-        'imap_details': imap_details
+        'imap_conn_summary': imap_conn_summary
     }