Add tests for dmarc reject and spf softfail

tests/suites/_mail-functions.sh

@@ -36,8 +36,11 @@ ensure_root_user() {
 
 dovecot_mailbox_home() {
 	local email="$1"
-	echo -n "${STORAGE_ROOT}/mail/mailboxes/"
+	local mailbox="${2:-INBOX}"
-	awk -F@ '{print $2"/"$1}' <<< "$email"
+	local path
+	/usr/bin/doveadm mailbox path -u "$email" "$mailbox"
+	#echo -n "${STORAGE_ROOT}/mail/mailboxes/"
+	#awk -F@ '{print $2"/"$1}' <<< "$email"
 }
 
 
@@ -60,16 +63,24 @@ start_log_capture() {
 
 start_mail_capture() {
 	local email="$1"
-	local newdir="$(dovecot_mailbox_home "$email")/new"
 	record "[Start mail capture $email]"
 	DOVECOT_CAPTURE_USER="$email"
-	DOVECOT_CAPTURE_FILECOUNT=0
+	DOVECOT_CAPTURE_FILECOUNT=()
-	if [ -e "$newdir" ]; then
+	DOVECOT_CAPTURE_MAILBOXES=(INBOX Spam)
-		DOVECOT_CAPTURE_FILECOUNT=$(ls "$newdir" 2>>$TEST_OF | wc -l)
+	for mailbox in ${DOVECOT_CAPTURE_MAILBOXES[@]}; do
-		[ $? -ne 0 ] && die "Error accessing mailbox of $email"
+		local mbhome
-	fi
+		mbhome="$(dovecot_mailbox_home "$email" "$mailbox" 2>>$TEST_OF)"
-	record "mailbox: $(dirname $newdir)"
+		[ $? -ne 0 ] && die "Error accessing $mailbox of $email"
-	record "mailbox has $DOVECOT_CAPTURE_FILECOUNT files"
+		local newdir="$mbhome/new"
+		local count=0
+		if [ -e "$newdir" ]; then
+			count=$(ls "$newdir" 2>>$TEST_OF | wc -l)
+			[ $? -ne 0 ] && die "Error accessing mailbox of $email"			
+		fi
+		DOVECOT_CAPTURE_FILECOUNT+=($count)
+		record "$mailbox location: $mbhome"
+		record "$mailbox has $count files"
+	done
 }
 
 dump_capture_logs() {
@@ -299,14 +310,23 @@ wait_mail() {
 }
 
 get_captured_mail_files() {
-	local newdir="$(dovecot_mailbox_home "$DOVECOT_CAPTURE_USER")/new"
+	local idx=0
-	local count
+	while [ $idx -lt ${#DOVECOT_CAPTURE_MAILBOXES[*]} ]; do
-	let count="$DOVECOT_CAPTURE_FILECOUNT + 1"
+		local mailbox=${DOVECOT_CAPTURE_MAILBOXES[$idx]}
-	[ ! -e "$newdir" ] && return 0
+		local filecount=${DOVECOT_CAPTURE_FILECOUNT[$idx]}
-	# output absolute path names
+		local mbhome
-	local file
+		mbhome="$(dovecot_mailbox_home "$DOVECOT_CAPTURE_USER" "$mailbox" 2>>$TEST_OF)"
-	for file in $(ls "$newdir" 2>>$TEST_OF | tail --lines=+${count}); do
+		[ $? -ne 0 ] && die "Error accessing mailbox of $email"
-		echo "$newdir/$file"
+		local newdir="$mbhome/new"
+		[ ! -e "$newdir" ] && return 0
+		local count
+		let count="$filecount + 1"
+		# output absolute path names
+		local file
+		for file in $(ls "$newdir" 2>>$TEST_OF | tail --lines=+${count}); do
+			echo "$newdir/$file"
+		done
+		let idx+=1
 	done
 }
 

tests/suites/mail-access.sh

@@ -31,7 +31,26 @@ _test_greylisting_x() {
 }
 
 
-postgrey_reset() {
+postgrey_whitelist_recipents() {
+	local wl="/etc/postgrey/whitelist_recipients.local"
+	rm -f "$wl"
+	local recipient
+	for recipient; do
+		echo "$recipient" >> "$wl" || \
+			die "Could not add postgrey whitelist recipient to $wl"
+	done
+	systemctl reload postgrey
+}
+
+
+postgrey_reset_whitelists() {
+	local wl="/etc/postgrey/whitelist_recipients.local"
+	rm -f "$wl"
+	systemctl reload postgrey
+}
+
+
+postgrey_reset_state() {
 	# when postgrey receives a message for processing that is suspect,
 	# it will:
 	#	1. initally reject it
@@ -52,6 +71,7 @@ postgrey_reset() {
 		systemctl start postgrey >>$TEST_OF 2>&1
 		die "unable to remove the postgrey database files"
 	fi
+
 	systemctl start postgrey >>$TEST_OF 2>&1 || die "unble to start postgrey"
 }
 
@@ -61,15 +81,15 @@ test_greylisting() {
 	test_start "greylisting"
 
 	# reset postgrey's database to start the cycle over
-	postgrey_reset
+	postgrey_reset_state
 
 	# create standard user alice
 	local alice="alice@somedomain.com"
 	create_user "$alice" "alice"
 
 	# IMPORTANT: bob's domain must be from one that has no SPF record
-	# in DNS. At the time of creation of this script, yahoo.com did
+	# in DNS. At the time of creation of this script, yahoo.com's
-	# not...
+	# is set to "?all" ("neutral" or "no policy statement")
 	local bob="bob@yahoo.com"
 	
 	# send to alice anonymously from bob
@@ -95,15 +115,18 @@ test_relay_prohibited() {
 }
 
 
-test_spf() {
+test_spf_fail() {
-	# test mail rejection due to SPF policy of FROM address
+	# test mail rejection due to SPF policy of envelope FROM address
-	test_start "spf"
+	test_start "spf-fail"
 	
 	# create standard user alice
 	local alice="alice@somedomain.com"
 	create_user "$alice" "alice"
 
 	# who we will impersonate
+	#
+	# note: we've configured policyd-spf to fail instead of softfail
+	# on google.com
 	local from="test@google.com"
 	local domain=$(awk -F@ '{print $2}' <<<"$from")
 
@@ -125,6 +148,135 @@ test_spf() {
 	test_end
 }
 
+test_spf_softfail() {
+	# When policyd-spf-postfix determines a message's SPF disposition
+	# is "Softfail", it attaches a "Received-SPF: Softfail" header
+	# (see RFC 7208) and allows the message to move along the postfix
+	# delivery chain.
+	#
+	# PR #1836 sets custom spam assassin rules that cause spam
+	# assassin to add a score of 5.0 for Softfail messages (see
+	# setup/dkim.sh and setup/spamassassin.sh)
+    #
+	# This test checks that a softfail message is properly scored.
+
+	local SPF_SOFTFAIL_SCORE="5.0"
+	
+	test_start "spf-softfail"
+
+	# create user alice
+	local alice="alice@somedomain.com"
+	local alice_pw="123alice"
+	create_user "$alice" "$alice_pw"
+	
+	# who we will impersonate
+	#
+	# important: the MAIL FROM address we choose here must currently
+	# have dns spf configuration set to softfail (eg: ~all)
+	#
+	local from="test@guest.com"
+	local domain=$(awk -F@ '{print $2}' <<<"$from")
+
+	# alice must be whitelisted to avoid greylisting
+	postgrey_whitelist_recipents "$alice"
+
+	# send to alice anonymously from imposter
+	start_log_capture
+	start_mail_capture "$alice"
+	record "[Test SPF for $domain FROM $from TO $alice]"
+	local output
+	local subject="$(generate_uuid)"
+	output="$($PYMAIL -no-delete -subj "$subject" -f $from -to $alice '' $PRIVATE_IP '' '' 2>&1)"
+	if assert_python_success $? "$output"; then	
+		if ! wait_mail; then
+			test_failure "Timeout waiting for mail"
+		else
+			record_captured_mail
+			local files=( $(get_captured_mail_files) )
+			local score
+			score=$(grep -F "SPF_SOFTFAIL" "$files" | grep -F '*' | awk '{print $2}')
+			floor_score=$(awk -F. '{print $1}' <<<"$score")
+			floor_expected=$(awk -F. '{print $1}' <<<"$SPF_SOFTFAIL_SCORE")
+			if [ -z "$score" ]; then
+				test_failure "No spam score for SPF_SOFTFAIL"
+			elif [ $floor_score -ne $floor_expected ]; then
+				test_failure "Got score $score, but exptected $SPF_SOFTFAIL_SCORE"
+			fi
+		fi
+		# clean up: delete the delivered mail
+		record "[delete delivered message]"
+		output="$($PYMAIL -no-send -subj $subject -timeout 1 $PRIMARY_HOSTNAME $alice "$alice_pw" 2>&1)"
+		record "$output"
+	fi
+	check_logs
+
+	# clean up
+	postgrey_reset_whitelists
+	delete_user "$alice"
+	test_end
+}
+
+
+test_dmarc_reject() {
+	# PR #1836 sets custom spam assassin rules that cause spam
+	# assassin to add a score of 10.0 for dmarc p=reject messages (see
+	# setup/dkim.sh and setup/spamassassin.sh)
+    #
+	# This test checks that a p=reject message is properly scored.
+
+	local DMARC_FAIL_REJECT_SCORE="10.0"
+	
+	test_start "dmarc-reject"
+
+	# create user alice - must be a domain that does not result in SPF Fail
+	local alice="alice@guest.com"
+	local alice_pw="alice123"
+	create_user "$alice" "$alice_pw"
+	
+	# who we will impersonate: their domain (_dmarc.domain.com dns TXT
+	# record), must have a dmarc policy with p=reject
+	local header_from="test@google.com"
+
+	# alice must be whitelisted to avoid greylisting
+	postgrey_whitelist_recipents "$alice"
+
+	# send to alice from alice with header From: imposter
+	start_log_capture
+	start_mail_capture "$alice"
+	record "[Test dmarc reject TO $alice, From: $header_from]"
+	local subject="$(generate_uuid)"
+	local output
+	output="$($PYMAIL -smptd -subj $subject -no-delete -f $alice -hfrom $header_from -to $alice '' $PRIVATE_IP 2>&1)"
+	
+	if assert_python_success $? "$output"; then	
+		if ! wait_mail; then
+			test_failure "Timeout waiting for mail"
+		else
+			record_captured_mail
+			local files=( $(get_captured_mail_files) )
+			local score
+			score=$(grep -F "DMARC_FAIL_REJECT" "$files" | grep -F '*' | awk '{print $2}')
+			floor_score=$(awk -F. '{print $1}' <<<"$score")
+			floor_expected=$(awk -F. '{print $1}' <<<"$DMARC_FAIL_REJECT_SCORE")
+			if [ -z "$score" ]; then
+				test_failure "No spam score for DMARC_FAIL_REJECT"
+			elif [ $floor_score -ne $floor_expected ]; then
+				test_failure "Got score $score, but exptected $DMARC_FAIL_REJECT_SCORE"
+			fi
+		fi
+		# clean up: delete the delivered mail
+		record "[delete delivered message]"
+		output="$($PYMAIL -no-send -subj $subject -timeout 1 $PRIMARY_HOSTNAME $alice "$alice_pw" 2>&1)"
+		record "$output"
+	fi
+	check_logs
+
+	# clean up
+	postgrey_reset_whitelists
+	delete_user "$alice"
+	test_end
+}
+
 
 test_mailbox_pipe() {
 	# postfix allows piped commands in aliases for local processing,
@@ -193,7 +345,9 @@ suite_start "mail-access" ensure_root_user
 
 test_greylisting
 test_relay_prohibited
-test_spf
+test_spf_fail
+test_spf_softfail
+test_dmarc_reject
 test_mailbox_pipe
 
 suite_end

tests/test_mail.py

@@ -11,8 +11,10 @@ def usage():
 	print("Usage: test_mail.py [options] hostname login password")
 	print("Send, then delete message")
 	print("  options")
+	print("    -smtpd: connect to port 25 and ignore login and password")
 	print("    -f <email>: use <email> as the MAIL FROM address")
 	print("    -to <email> <pass>: recipient of email and password")
+	print("    -hfrom <email>: header From: email")
 	print("    -subj <subject>: subject of the message (required with --no-send)")
 	print("    -no-send: don't send, just delete")
 	print("    -no-delete: don't delete, just send")
@@ -24,10 +26,12 @@ def if_unset(a,b):
 	return b if a is None else a
 
 # option defaults
+smtpd=False      # deliver mail to port 25, not submission (ignore login/pw)
 host=None        # smtp server address
 login=None       # smtp server login
 pw=None          # smtp server password
 emailfrom=None   # MAIL FROM address
+headerfrom=None  # Header From: address
 emailto=None     # RCPT TO address
 emailto_pw=None  # recipient password for imap login
 send_msg=True    # deliver message
@@ -43,9 +47,15 @@ while argi<len(sys.argv):
 	arg_remaining = len(sys.argv) - argi - 1
 	if not arg.startswith('-'):
 		break
-	if (arg=="-f" or arg=="-from") and arg_remaining>0:
+	if arg=="-smptd":
+		smtpd=True
+		argi+=1
+	elif (arg=="-f" or arg=="-from") and arg_remaining>0:
 		emailfrom=sys.argv[argi+1]
 		argi+=2
+	elif arg=="-hfrom" and arg_remaining>0:
+		headerfrom=sys.argv[argi+1]
+		argi+=2
 	elif arg=="-to" and arg_remaining>1:
 		emailto=sys.argv[argi+1]
 		emailto_pw=sys.argv[argi+2]
@@ -65,21 +75,28 @@ while argi<len(sys.argv):
 	else:
 		usage()
 		
-        
+if not smtpd:
-if len(sys.argv) - argi != 3: usage()
+	if len(sys.argv) - argi != 3: usage()
-host, login, pw = sys.argv[argi:argi+3]
+	host, login, pw = sys.argv[argi:argi+3]
-argi+=3
+	argi+=3
+	port=587
+else:
+	if len(sys.argv) - argi != 1: usage()
+	host = sys.argv[argi]
+	argi+=1
+	port=25
 
 emailfrom = if_unset(emailfrom, login)
+headerfrom = if_unset(headerfrom, emailfrom)
 emailto = if_unset(emailto, login)
 emailto_pw = if_unset(emailto_pw, pw)
 
-msg = """From: {emailfrom}
+msg = """From: {headerfrom}
 To: {emailto}
 Subject: {subject}
 
 This is a test message. It should be automatically deleted by the test script.""".format(
-	emailfrom=emailfrom,
+	headerfrom=headerfrom,
 	emailto=emailto,
 	subject=subject,
 	)
@@ -131,9 +148,9 @@ def imap_test_dkim(M, num):
 	pass
 
 
-def smtp_login(host, login, pw):
+def smtp_login(host, login, pw, port):
 	# Connect to the server on the SMTP submission TLS port.
-	server = smtplib.SMTP(host, 587)
+	server = smtplib.SMTP(host, port)
 	#server.set_debuglevel(1)
 	server.starttls()
 
@@ -163,7 +180,7 @@ def smtp_login(host, login, pw):
 
 if send_msg:
 	# Attempt to send a mail.
-	server = smtp_login(host, login, pw)
+	server = smtp_login(host, login, pw, port)
 	server.sendmail(emailfrom, [emailto], msg)
 	server.quit()
 	print("SMTP submission is OK.")
@@ -179,19 +196,22 @@ if delete_msg:
 		# Wait so the message can propagate to the inbox.
 		time.sleep(wait_cycle_sleep / 2)
 
-	while time.time() - start_time < wait_timeout:
+	while not found and time.time() - start_time < wait_timeout:
-		num = imap_search_for(M, subject)
+		for mailbox in ['INBOX', 'Spam']:
-		if num is not None:
+			M.select(mailbox)
-			# Delete the test message.
+			num = imap_search_for(M, subject)
-			found = True
+			if num is not None:
-			imap_test_dkim(M, num)
+				# Delete the test message.
-			M.store(num, '+FLAGS', '\\Deleted')
+				found = True
-			M.expunge()
+				imap_test_dkim(M, num)
-			print("Message %s deleted successfully." % num)
+				M.store(num, '+FLAGS', '\\Deleted')
-			break
+				M.expunge()
-		
+				print("Message %s deleted successfully from %s." % (num, mailbox))
-		print("Test message not present in the inbox yet...")
+				break
-		time.sleep(wait_cycle_sleep)
+
+		if not found:
+			print("Test message not present in the inbox yet...")
+			time.sleep(wait_cycle_sleep)
 		
 	M.close()
 	M.logout()