Add ability to view message headers in the user activity panel

... and add message-id to output detail

management/daemon_reports.py

@@ -14,7 +14,7 @@ from functools import wraps
 from reporting.capture.db.SqliteConnFactory import SqliteConnFactory
 import reporting.uidata as uidata
 
-from mailconfig import get_mail_users
+from mailconfig import ( get_mail_users, validate_email )
 
 		
 def add_reports(app, env, authorized_personnel_only):
@@ -178,7 +178,7 @@ def add_reports(app, env, authorized_personnel_only):
 
 		r = subprocess.run(["systemctl", "reload", "miabldap-capture"])
 		if r.returncode != 0:
-			log.warning('systemctl reload faild for miabldap-capture: code=%s', r.returncode)
+			log.warning('systemctl reload failed for miabldap-capture: code=%s', r.returncode)
 		else:
 			# wait a sec for daemon to pick up new config
 			# TODO: monitor runtime config for mtime change
@@ -214,3 +214,41 @@ def add_reports(app, env, authorized_personnel_only):
 			return jsonify(uidata.capture_db_stats(conn))
 		finally:
 			db_conn_factory.close(conn)
+
+	@app.route('/reports/uidata/message-headers', methods=['POST'])
+	@authorized_personnel_only
+	@json_payload
+	def get_message_headers(payload):
+		try:
+			user_id = payload['user_id']
+			lmtp_id = payload['lmtp_id']
+		except KeyError:
+			return ('invalid request', 400)
+
+		if not validate_email(user_id, mode="user"):
+			return ('invalid email address', 400)
+
+		r = subprocess.run(
+			[
+				"/usr/bin/doveadm",
+				"fetch",
+				"-u",user_id,
+				"hdr",
+				"HEADER","received","LMTP id " + lmtp_id
+			],
+			encoding="utf8",
+			stdout=subprocess.PIPE,
+			stderr=subprocess.PIPE
+		)
+		
+		if r.returncode != 0:
+			log.error('retrieving message headers failed, code=%s, lmtp_id=%s, user_id=%s, stderr=%s', r.returncode, lmtp_id, user_id, r.stderr)
+			return Response(r.stderr, status=400, mimetype='text/plain')
+
+		else:
+			out = r.stdout.strip()
+			if out.startswith('hdr:\n'):
+				out = out[5:]
+			return Response(out, status=200, mimetype='text/plain')
+		
+		

management/reporting/ui/panel-user-activity.html

@@ -1,5 +1,9 @@
 <div>
   
+  <b-modal ref="message_headers_modal" hide-header no-fade ok-only no-close-on-backdrop size="lg" scrollable>
+    <message-headers-view :user_id="data_user_id" :lmtp_id="lmtp_id"></message-headers-view>
+  </b-modal>
+    
   <datalist id="panel-ua-users">
     <option v-for="user in all_users">{{ user }}</option>
   </datalist>
@@ -61,7 +65,7 @@
         </template>
         <template #row-details="row">
           <b-card>
-            <div><strong>Remote host</strong>: {{ row.item.remote_host }}[{{ row.item.remote_ip }}]</div>
+            <div><strong>Remote sender</strong>: {{ row.item.remote_host }}[{{ row.item.remote_ip }}]</div>
             <div><strong>Connection disposition</strong>: {{ disposition_formatter(row.item.disposition) }}</div>
             <div v-if="row.item.orig_to"><strong>Sent to alias</strong>: {{ row.item.orig_to }}</div>
             <div v-if="row.item.dkim_reason"><strong>Dkim reason</strong>: {{row.item.dkim_reason}}</div>
@@ -69,7 +73,9 @@
             <div v-if="row.item.postgrey_reason"><strong>Postgrey reason</strong>: {{row.item.postgrey_reason}}</div>
             <div v-if="row.item.postgrey_delay"><strong>Postgrey delay</strong>: {{received_mail.x_fields.postgrey_delay.formatter(row.item.postgrey_delay)}}</div>
             <div v-if="row.item.spam_result"><strong>Spam score</strong>: {{received_mail.x_fields.spam_score.formatter(row.item.spam_score)}}</div>
+            <div v-if="row.item.message_id"><strong>Message-ID</strong>: {{ row.item.message_id }}</div>
             <div v-if="row.item.failure_info"><strong>Failure info</strong>: {{row.item.failure_info}}</div>
+            <div v-if="row.item.lmtp_id"><a href="#" @click.prevent.stop="show_message_headers(row.item.lmtp_id)">Message headers</a></div>
           </b-card>
         </template>
       </b-table>

management/reporting/ui/panel-user-activity.js

@@ -3,6 +3,7 @@
 */
 
 import wbr_text from "./wbr-text.js";
+import message_headers_view from "./message_headers_view.js";
 import UserSettings from "./settings.js";
 import { MailBvTable, ConnectionDisposition } from "./charting.js";
 
@@ -18,6 +19,7 @@ export default Vue.component('panel-user-activity', function(resolve, reject) {
 
         components: {
             'wbr-text': wbr_text,
+            'message-headers-view': message_headers_view
         },
         
         data: function() {
@@ -35,6 +37,7 @@ export default Vue.component('panel-user-activity', function(resolve, reject) {
                 sent_mail: null,
                 received_mail: null,
                 imap_details: null,
+                lmtp_id: null, /* for message headers modal */
                 all_users: [],
                 disposition_formatter: ConnectionDisposition.formatter,
             };
@@ -150,7 +153,9 @@ export default Vue.component('panel-user-activity', function(resolve, reject) {
                     'postgrey_reason',
                     'postgrey_delay',
                     'spam_score',
-                    'orig_to'
+                    'orig_to',
+                    'message_id',
+                    'lmtp_id',
                 ]);
                 // combine fields 'envelope_from' and 'sasl_username'
                 var f = this.received_mail.combine_fields(
@@ -291,6 +296,15 @@ export default Vue.component('panel-user-activity', function(resolve, reject) {
             row_clicked: function(item, index, event) {
                 item._showDetails = ! item._showDetails;
             },
+
+            show_message_headers: function(lmtp_id) {
+                // set the lmtp_id that component message-headers-view
+                // searches for
+                this.lmtp_id = lmtp_id;
+                
+                // show the modal dialog
+                this.$refs.message_headers_modal.show();
+            }
             
         }
         

management/reporting/uidata/user_activity.2.sql

@@ -7,9 +7,9 @@ connect_time, mta_connection.service AS service, sasl_username, disposition,
 remote_host, remote_ip,
 -- mta_accept
 envelope_from, spf_result, dkim_result, dkim_reason, dmarc_result, dmarc_reason,
-failure_info,
+message_id, failure_info,
 -- mta_delivery
-postgrey_result, postgrey_reason, postgrey_delay, spam_score, spam_result, message_size, orig_to
+postgrey_result, postgrey_reason, postgrey_delay, spam_score, spam_result, message_size, orig_to, delivery_info
 FROM mta_accept
 JOIN mta_connection ON mta_accept.mta_conn_id = mta_connection.mta_conn_id
 JOIN mta_delivery ON mta_accept.mta_accept_id = mta_delivery.mta_accept_id

management/reporting/uidata/user_activity.py

@@ -124,6 +124,7 @@ def user_activity(conn, args):
             'dkim_reason',
             'dmarc_result',
             'dmarc_reason',
+            'message_id',
             'failure_info',
             
             # mta_delivery
@@ -134,6 +135,7 @@ def user_activity(conn, args):
             'spam_score',
             'spam_result',
             'message_size',
+            'lmtp_id',
         ],
         'field_types': [
             { 'type':'datetime', 'format': '%Y-%m-%d %H:%M:%S' },# connect_time
@@ -148,6 +150,7 @@ def user_activity(conn, args):
             'text/plain',    # dkim_result
             'text/plain',    # dmarc_result
             'text/plain',    # dmarc_reason
+            'text/plain',    # message_id
             'text/plain',    # failure_info
             'text/email',    # orig_to
             'text/plain',    # postgrey_result
@@ -156,6 +159,7 @@ def user_activity(conn, args):
             { 'type':'decimal', 'places':2 },     # spam_score
             'text/plain',    # spam_result
             'number/size',   # message_size
+            'text/plain',    # lmtp_id
         ],
         'items': []
     }
@@ -167,7 +171,31 @@ def user_activity(conn, args):
     }):
         v = []
         for key in received_mail['fields']:
-            v.append(row[key])
+            if key == 'lmtp_id':
+                # Extract the LMTP ID from delivery info, which looks
+                # like:
+                #
+                # "250 2.0.0 <user@domain.tld> oPHmBDvTaWA7UwAAlWWVsw
+                # Saved"
+                #
+                # When we know the LMTP ID, we can get the message
+                # headers using doveadm, like this:
+                #
+                # "/usr/bin/doveadm fetch -u "user@domain.tld" hdr
+                # HEADER received "LMTP id oPHmBDvTaWA7UwAAlWWVsw"
+                #
+                delivery_info = row['delivery_info']
+                valid = False
+                if delivery_info:
+                    parts = delivery_info.split(' ')
+                    if parts[0]=='250' and parts[1]=='2.0.0':
+                        v.append(parts[-2])
+                        valid = True
+                if not valid:
+                    v.append(None)
+                    
+            else:
+                v.append(row[key])
         received_mail['items'].append(v)