diff --git a/.travis.yml b/.travis.yml index fe04210d..21c47c28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,4 +41,4 @@ install: - sudo ./setup/start.sh -v script: - - sudo ./tests/runner.sh -dumpoutput + - sudo ./tests/runner.sh -dumpoutput -no-smtp-remote diff --git a/tests/runner.sh b/tests/runner.sh index 1b5e0f35..db0511a5 100755 --- a/tests/runner.sh +++ b/tests/runner.sh @@ -31,8 +31,9 @@ usage() { echo "If no suite-name(s) given, all suites are run" echo "" echo "Options:" - echo " -failfatal The runner will stop if any test fails" - echo " -dumpoutput After all tests have run, dump all failed test output" + echo " -failfatal The runner will stop if any test fails" + echo " -dumpoutput After all tests have run, dump all failed test output" + echo " -no-smtp-remote Skip tests requiring a remote SMTP server" echo "" echo "Output directory: $(dirname $0)/${base_outputdir}" echo "" @@ -49,6 +50,9 @@ while [ $# -gt 0 ]; do -dumpoutput ) DUMP_FAILED_TESTS_OUTPUT="yes" ;; + -no-smtp-remote ) + SKIP_REMOTE_SMTP_TESTS="yes" + ;; -* ) echo "Invalid argument $1" 1>&2 usage @@ -76,7 +80,7 @@ fi echo "" echo "Done" -echo "$OVERALL_COUNT tests ($OVERALL_SUCCESSES success/$OVERALL_FAILURES failures) in $OVERALL_COUNT_SUITES test suites" +echo "$OVERALL_COUNT tests ($OVERALL_SUCCESSES success/$OVERALL_FAILURES failures/$OVERALL_SKIPPED skipped) in $OVERALL_COUNT_SUITES test suites" if [ $OVERALL_FAILURES -gt 0 ]; then diff --git a/tests/suites/_init.sh b/tests/suites/_init.sh index 4189e282..2b29e576 100644 --- a/tests/suites/_init.sh +++ b/tests/suites/_init.sh @@ -15,6 +15,7 @@ BASE_OUTPUTDIR="out" PYMAIL="./test_mail.py" declare -i OVERALL_SUCCESSES=0 declare -i OVERALL_FAILURES=0 +declare -i OVERALL_SKIPPED=0 declare -i OVERALL_COUNT=0 declare -i OVERALL_COUNT_SUITES=0 @@ -26,6 +27,7 @@ F_RESET=$(echo -e "\033[39m") # options FAILURE_IS_FATAL=no DUMP_FAILED_TESTS_OUTPUT=no +SKIP_REMOTE_SMTP_TESTS=no # record a list of output files for failed tests FAILED_TESTS_MANIFEST="$BASE_OUTPUTDIR/failed_tests_manifest.txt" @@ -36,6 +38,7 @@ suite_start() { let TEST_NUM=1 let SUITE_COUNT_SUCCESS=0 let SUITE_COUNT_FAILURE=0 + let SUITE_COUNT_SKIPPED=0 let SUITE_COUNT_TOTAL=0 SUITE_NAME="$1" OUTDIR="$BASE_OUTPUTDIR/$SUITE_NAME" @@ -50,6 +53,7 @@ suite_end() { echo "Suite $SUITE_NAME finished" let OVERALL_SUCCESSES+=$SUITE_COUNT_SUCCESS let OVERALL_FAILURES+=$SUITE_COUNT_FAILURE + let OVERALL_SKIPPED+=$SUITE_COUNT_SKIPPED let OVERALL_COUNT+=$SUITE_COUNT_TOTAL let OVERALL_COUNT_SUITES+=1 } @@ -110,6 +114,17 @@ test_end() { exit 1 fi ;; + SKIPPED ) + record "[SKIPPED]" + echo "SKIPPED" + local idx=0 + while [ $idx -lt ${#TEST_STATE_MSG[*]} ]; do + record "${TEST_STATE_MSG[$idx]}" + echo " why: ${TEST_STATE_MSG[$idx]}" + let idx+=1 + done + let SUITE_COUNT_SKIPPED+=1 + ;; * ) record "[INVALID TEST STATE '$TEST_STATE']" echo "Invalid TEST_STATE=$TEST_STATE" @@ -131,6 +146,25 @@ test_failure() { TEST_STATE_MSG+=( "$why" ) } +test_skip() { + local why="$1" + TEST_STATE="SKIPPED" + TEST_STATE_MSG+=( "$why" ) +} + +skip_test() { + # return 0 if we should skip the current test + if [ "$SKIP_REMOTE_SMTP_TESTS" == "yes" ] && + array_contains "remote-smtp" "$@"; + then + test_skip "-no-smtp-remote option given" + return 0 + fi + + return 1 +} + + have_test_failures() { [ "$TEST_STATE" == "FAILURE" ] && return 0 return 1 diff --git a/tests/suites/mail-aliases.sh b/tests/suites/mail-aliases.sh index 8834b54a..d1814dd2 100644 --- a/tests/suites/mail-aliases.sh +++ b/tests/suites/mail-aliases.sh @@ -136,6 +136,10 @@ test_trial_nonlocal_alias_delivery() { # verify that mail sent to an alias with a non-local address # (rfc822MailMember) can be delivered test_start "trial-nonlocal-alias-delivery" + if skip_test remote-smtp; then + test_end + return 0 + fi # add alias local alias="external@somedomain.com" diff --git a/tests/suites/mail-basic.sh b/tests/suites/mail-basic.sh index fb73cae9..8994e268 100644 --- a/tests/suites/mail-basic.sh +++ b/tests/suites/mail-basic.sh @@ -30,6 +30,10 @@ test_trial_send_remote() { # use sendmail -bv to test mail delivery without actually mailing # anything test_start "trial_send_remote" + if skip_test remote-smtp; then + test_end + return 0 + fi start_log_capture sendmail_bv_send "test@google.com" 120 assert_check_logs