Dump the output from failed tests

.travis.yml

@@ -10,12 +10,22 @@ os: linux
 dist: bionic
   
 before_install:
-  - echo "Check the status of AppArmor"
+  - echo "==== DUMP: ENVIRONMENT ===="
+  - env
+  - echo "==== DUMP: AppArmor Status ===="
   - (sudo aa-status; true)
+  - echo "==== DUMP: Other ===="
+  - echo "UMASK: $(umask)"
+  - echo "==== System update ===="
   - sudo apt-get update
+  - echo "==== Install test/qa packages ===="
+  - sudo apt-get -y install python3-dnspython
+  - echo "==== Copy pre-built files ===="
+  - sudo mkdir -p /home/user-data/ssl
+  - sudo cp ./tests/assets/ssl/dh2048.pem /home/user-data/ssl
   
 install:
   - sudo ./setup/start.sh -v
 
 script:
-  - sudo ./tests/runner.sh
+  - sudo ./tests/runner.sh -dumpoutput

tests/assets/ssl/dh2048.pem

@@ -0,0 +1,8 @@
+-----BEGIN DH PARAMETERS-----
+MIIBCAKCAQEAp3b+6oqb6IFYiBOjEAA3/56OrevWokel23wfmhuu4U07vEntpkDV
+Rrp5AeYBsiZIibouj5ZeKj0g5OmlUljjv5a1SisHdHJnm2YbXmSTSfqAsKBV9E78
+XY5Fv/bPg/qIBdWmS+i/sVTyU9ah88AljiQnHNnBXv9m2ybEAsu6GHJN/TLykKjJ
+blhnrj284pPLRRIrN8A+gAipYa8Hlw4i2iaYWctadeLC47xP+FMZ1JUPt3mF80wk
+xEH3mKTGSY1HJ13mXfTcbkxlUSd/kT/3gxYpWnUwa2ItI05Conzf+lCMvyyXH7Ow
+RGTdjPKxYieEph8XglXV1cOeh6p4fEAN6wIBAg==
+-----END DH PARAMETERS-----

tests/runner.sh

@@ -32,6 +32,7 @@ usage() {
 	echo ""
 	echo "Options:"
 	echo "  -failfatal	   The runner will stop if any test fails"
+	echo "  -dumpoutput   After all tests have run, dump all failed test output"
 	echo ""
 	echo "Output directory: $(dirname $0)/${base_outputdir}"
 	echo ""
@@ -45,6 +46,9 @@ while [ $# -gt 0 ]; do
 			# failure is fatal (via global option, see _init.sh)
 			FAILURE_IS_FATAL=yes
 			;;
+		-dumpoutput )
+			DUMP_FAILED_TESTS_OUTPUT="yes"
+			;;
 		-* )
 			echo "Invalid argument $1" 1>&2
 			usage
@@ -74,8 +78,11 @@ echo ""
 echo "Done"
 echo "$OVERALL_COUNT tests ($OVERALL_SUCCESSES success/$OVERALL_FAILURES failures) in $OVERALL_COUNT_SUITES test suites"
 
+
 if [ $OVERALL_FAILURES -gt 0 ]; then
+	dump_failed_tests_output
 	exit 1
+	
 else
 	exit 0
 fi

tests/suites/_init.sh

@@ -25,6 +25,11 @@ F_RESET=$(echo -e "\033[39m")
 
 # options
 FAILURE_IS_FATAL=no
+DUMP_FAILED_TESTS_OUTPUT=no
+
+# record a list of output files for failed tests
+FAILED_TESTS_MANIFEST="$BASE_OUTPUTDIR/failed_tests_manifest.txt"
+rm -f "$FAILED_TESTS_MANIFEST"
 
 
 suite_start() {
@@ -95,11 +100,13 @@ test_end() {
 				echo "	   why: ${TEST_STATE_MSG[$idx]}"
 				let idx+=1
 			done
+			echo "$TEST_OF" >>$FAILED_TESTS_MANIFEST
 			echo "	   see: $(dirname $0)/$TEST_OF"
 			let SUITE_COUNT_FAILURE+=1
 			if [ "$FAILURE_IS_FATAL" == "yes" ]; then
 				record "FATAL: failures are fatal option enabled"
 				echo "FATAL: failures are fatal option enabled"
+				dump_failed_tests_output
 				exit 1
 			fi
 			;;
@@ -142,6 +149,7 @@ die() {
 	test_failure "a fatal error occurred"
 	test_end
 	echo "FATAL: $@"
+	dump_failed_tests_output
 	exit 1
 }
 
@@ -163,6 +171,22 @@ python_error() {
 	[ $? -eq 1 ] && echo "$output"
 }
 
+dump_failed_tests_output() {
+	if [ "$DUMP_FAILED_TESTS_OUTPUT" == "yes" ]; then
+		echo ""
+		echo "============================================================"
+		echo "OUTPUT OF FAILED TESTS"
+		echo "============================================================"
+		for file in $(cat $FAILED_TESTS_MANIFEST); do
+			echo ""
+			echo ""
+			echo "--------"
+			echo "-------- $file"
+			echo "--------"
+			cat "$file"
+		done
+	fi
+}
 
 
 ##