Add option to skip tests requiring remote smtp

.travis.yml

@@ -41,4 +41,4 @@ install:
   - sudo ./setup/start.sh -v
 
 script:
-  - sudo ./tests/runner.sh -dumpoutput
+  - sudo ./tests/runner.sh -dumpoutput -no-smtp-remote

tests/runner.sh

@@ -31,8 +31,9 @@ usage() {
 	echo "If no suite-name(s) given, all suites are run"
 	echo ""
 	echo "Options:"
-	echo "  -failfatal	   The runner will stop if any test fails"
-	echo "  -dumpoutput   After all tests have run, dump all failed test output"
+	echo "  -failfatal	    The runner will stop if any test fails"
+	echo "  -dumpoutput     After all tests have run, dump all failed test output"
+	echo "  -no-smtp-remote Skip tests requiring a remote SMTP server"
 	echo ""
 	echo "Output directory: $(dirname $0)/${base_outputdir}"
 	echo ""
@@ -49,6 +50,9 @@ while [ $# -gt 0 ]; do
 		-dumpoutput )
 			DUMP_FAILED_TESTS_OUTPUT="yes"
 			;;
+		-no-smtp-remote )
+			SKIP_REMOTE_SMTP_TESTS="yes"
+			;;
 		-* )
 			echo "Invalid argument $1" 1>&2
 			usage
@@ -76,7 +80,7 @@ fi
 
 echo ""
 echo "Done"
-echo "$OVERALL_COUNT tests ($OVERALL_SUCCESSES success/$OVERALL_FAILURES failures) in $OVERALL_COUNT_SUITES test suites"
+echo "$OVERALL_COUNT tests ($OVERALL_SUCCESSES success/$OVERALL_FAILURES failures/$OVERALL_SKIPPED skipped) in $OVERALL_COUNT_SUITES test suites"
 
 
 if [ $OVERALL_FAILURES -gt 0 ]; then

tests/suites/_init.sh

@@ -15,6 +15,7 @@ BASE_OUTPUTDIR="out"
 PYMAIL="./test_mail.py"
 declare -i OVERALL_SUCCESSES=0
 declare -i OVERALL_FAILURES=0
+declare -i OVERALL_SKIPPED=0
 declare -i OVERALL_COUNT=0
 declare -i OVERALL_COUNT_SUITES=0
 
@@ -26,6 +27,7 @@ F_RESET=$(echo -e "\033[39m")
 # options
 FAILURE_IS_FATAL=no
 DUMP_FAILED_TESTS_OUTPUT=no
+SKIP_REMOTE_SMTP_TESTS=no
 
 # record a list of output files for failed tests
 FAILED_TESTS_MANIFEST="$BASE_OUTPUTDIR/failed_tests_manifest.txt"
@@ -36,6 +38,7 @@ suite_start() {
 	let TEST_NUM=1
 	let SUITE_COUNT_SUCCESS=0
 	let SUITE_COUNT_FAILURE=0
+	let SUITE_COUNT_SKIPPED=0
 	let SUITE_COUNT_TOTAL=0
 	SUITE_NAME="$1"
 	OUTDIR="$BASE_OUTPUTDIR/$SUITE_NAME"
@@ -50,6 +53,7 @@ suite_end() {
 	echo "Suite $SUITE_NAME finished"
 	let OVERALL_SUCCESSES+=$SUITE_COUNT_SUCCESS
 	let OVERALL_FAILURES+=$SUITE_COUNT_FAILURE
+	let OVERALL_SKIPPED+=$SUITE_COUNT_SKIPPED
 	let OVERALL_COUNT+=$SUITE_COUNT_TOTAL
 	let OVERALL_COUNT_SUITES+=1
 }
@@ -110,6 +114,17 @@ test_end() {
 				exit 1
 			fi
 			;;
+		SKIPPED )
+			record "[SKIPPED]"
+			echo "SKIPPED"
+			local idx=0
+			while [ $idx -lt ${#TEST_STATE_MSG[*]} ]; do
+				record "${TEST_STATE_MSG[$idx]}"
+				echo "	   why: ${TEST_STATE_MSG[$idx]}"
+				let idx+=1
+			done
+			let SUITE_COUNT_SKIPPED+=1
+			;;
 		* )
 			record "[INVALID TEST STATE '$TEST_STATE']"
 			echo "Invalid TEST_STATE=$TEST_STATE"
@@ -131,6 +146,25 @@ test_failure() {
 	TEST_STATE_MSG+=( "$why" )
 }
 
+test_skip() {
+	local why="$1"
+	TEST_STATE="SKIPPED"
+	TEST_STATE_MSG+=( "$why" )
+}
+
+skip_test() {
+	# return 0 if we should skip the current test
+	if [ "$SKIP_REMOTE_SMTP_TESTS" == "yes" ] &&
+		   array_contains "remote-smtp" "$@";
+	then
+		test_skip "-no-smtp-remote option given"
+		return 0
+	fi
+	
+	return 1
+}
+
+
 have_test_failures() {
 	[ "$TEST_STATE" == "FAILURE" ] && return 0
 	return 1

tests/suites/mail-aliases.sh

@@ -136,6 +136,10 @@ test_trial_nonlocal_alias_delivery() {
 	# verify that mail sent to an alias with a non-local address
 	# (rfc822MailMember) can be delivered
 	test_start "trial-nonlocal-alias-delivery"
+	if skip_test remote-smtp; then
+		test_end
+		return 0
+	fi
 
 	# add alias
 	local alias="external@somedomain.com"

tests/suites/mail-basic.sh

@@ -30,6 +30,10 @@ test_trial_send_remote() {
 	# use sendmail -bv to test mail delivery without actually mailing
 	# anything
 	test_start "trial_send_remote"
+	if skip_test remote-smtp; then
+		test_end
+		return 0
+	fi
 	start_log_capture
 	sendmail_bv_send "test@google.com" 120
 	assert_check_logs