From b352545621369c162c9dc3a3baa24b8a8c3dbee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20M=C3=BCller?= Date: Tue, 28 Nov 2023 21:10:32 +0100 Subject: [PATCH] Improve integration test script Fix valgrind errors not being detected as failures when the tests otherwise finish, as the valgrind results are printed to stderr and not to the logfiles. Additionally, valgrind failures were not being detected because the error summary is not printed to stderr immediately but only after some time when valgrind has processed the summary. Find random unused port to bind to by opening a socket on port 0 and closing it immediately. This gives a port that will very likely be unused because the system cycles through all ports before reusing old ones obtained this way. Add `wait_for_launch` function to reduce duplicate code. Wait for server to launch before starting clients. Improve log messages. --- scripts/integration_test.sh | 123 +++++++++++++++++++----------------- 1 file changed, 64 insertions(+), 59 deletions(-) diff --git a/scripts/integration_test.sh b/scripts/integration_test.sh index 759f55f0e..09895662d 100755 --- a/scripts/integration_test.sh +++ b/scripts/integration_test.sh @@ -7,14 +7,14 @@ for arg in "$@" do if [ "$arg" == "-h" ] || [ "$arg" == "--help" ] then - echo "usage: $(basename "$0") [OPTION..] [build dir]" + echo "usage: $(basename "$0") [OPTION..]" echo "description:" echo " Runs a simple integration test of the client and server" - echo " binaries from the given build dir" + echo " binaries from the current build directory." echo "options:" - echo " --help|-h show this help" - echo " --verbose|-v verbose output" - echo " --valgrind-memcheck use valgrind's memcheck to run server and client" + echo " --help|-h show this help" + echo " --verbose|-v verbose output" + echo " --valgrind-memcheck use valgrind's memcheck to run server and client" exit 0 elif [ "$arg" == "-v" ] || [ "$arg" == "--verbose" ] then @@ -23,22 +23,23 @@ do then arg_valgrind_memcheck=1 else - echo "Error: unknown arg '$arg'" + echo "Error: unknown argument '$arg'" exit 1 fi done if [ ! -f DDNet ] then - echo "Error: client binary not found DDNet' not found" + echo "[-] Error: client binary 'DDNet' not found" exit 1 fi if [ ! -f DDNet-Server ] then - echo "Error: server binary not found DDNet-Server' not found" + echo "[-] Error: server binary 'DDNet-Server' not found" exit 1 fi +echo "[*] Setup" got_killed=0 function kill_all() { @@ -51,24 +52,27 @@ function kill_all() { if [ "$arg_verbose" == "1" ] then - echo "[*] shutting down test clients and server ..." + echo "[*] Shutting down test clients and server" fi - sleep 1 + if [[ ! -f fail_server.txt ]] then - echo "[*] shutting down server" + echo "[*] Shutting down server" echo "shutdown" > server.fifo fi + sleep 1 + local i for ((i=1;i<3;i++)) do if [[ ! -f fail_client$i.txt ]] then - echo "[*] shutting down client$i" + echo "[*] Shutting down client$i" echo "quit" > "client$i.fifo" fi done + sleep 1 } function cleanup() { @@ -85,8 +89,8 @@ function fail() echo "[-] $1 exited with code $2" } -# TODO: check for open ports instead -port=17822 +# Get unused port from the system by binding to port 0 and immediately closing the socket again +port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'); if [[ $OSTYPE == 'darwin'* ]]; then DETECT_LEAKS=0 @@ -100,13 +104,18 @@ export LSAN_OPTIONS=suppressions=../lsan.supp:print_suppressions=0 function print_results() { if [ "$arg_valgrind_memcheck" == "1" ]; then - if grep "ERROR SUMMARY" server.log client1.log client2.log | grep -q -v "ERROR SUMMARY: 0"; then - grep "^==" server.log client1.log client2.log + # Wait to ensure that the error summary was written to the stderr files because valgrind takes some time + # TODO: Instead wait for all started processes to finish + sleep 20 + if grep "== ERROR SUMMARY: " stderr_server.txt stderr_client1.txt stderr_client2.txt | grep -q -v "ERROR SUMMARY: 0"; then + echo "[-] Error: Valgrind has detected the following errors:" + grep "^==" stderr_server.txt stderr_client1.txt stderr_client2.txt return 1 fi else if test -n "$(find . -maxdepth 1 -name 'SAN.*' -print -quit)" then + echo "[-] Error: ASAN has detected the following errors:" cat SAN.* return 1 fi @@ -127,39 +136,54 @@ cd integration_test || exit 1 echo $'add_path ../data' } > storage.cfg +tool="" +client_args="cl_download_skins 0; + gfx_fullscreen 0; + snd_enable 0;" + if [ "$arg_valgrind_memcheck" == "1" ]; then tool="valgrind --tool=memcheck --gen-suppressions=all --suppressions=../memcheck.supp --track-origins=yes" - client_args="cl_menu_map \"\";" -else - tool="" - client_args="" + client_args="$client_args cl_menu_map \"\";" fi function wait_for_fifo() { local fifo="$1" local tries="$2" local fails=0 - # give the client time to launch and create the fifo file - # but assume after X secs that the client crashed before + # give the server/client time to launch and create the fifo file + # but assume after X secs that the server/client crashed before # being able to create the file while [[ ! -p "$fifo" ]] do fails="$((fails+1))" if [ "$arg_verbose" == "1" ] then - echo "[!] client fifos not found (attempts $fails/$tries)" + echo "[!] Note: $fifo not found (attempts $fails/$tries)" fi if [ "$fails" -gt "$tries" ] then + echo "[-] Error: $(basename "$fifo" .fifo) possibly crashed on launch" + kill_all print_results - echo "[-] Error: client possibly crashed on launch" exit 1 fi sleep 1 done } -echo "[*] launch server" +function wait_for_launch() { + local fifo="$1" + local baseDuration="$2" + if [ "$arg_valgrind_memcheck" == "1" ]; then + wait_for_fifo "$fifo" $((40 * baseDuration)) + sleep $((8 * baseDuration)) + else + wait_for_fifo "$fifo" $((10 * baseDuration)) + sleep "$baseDuration" + fi +} + +echo "[*] Launch server" $tool ../DDNet-Server \ "sv_input_fifo server.fifo; sv_rcon_password rcon; @@ -169,50 +193,34 @@ $tool ../DDNet-Server \ sv_register 0; sv_port $port" > stdout_server.txt 2> stderr_server.txt || fail server "$?" & -echo "[*] launch client 1" +wait_for_launch server.fifo 1 + +echo "[*] Launch client 1" $tool ../DDNet \ "cl_input_fifo client1.fifo; player_name client1; - cl_download_skins 0; - gfx_fullscreen 0; - snd_enable 0; logfile client1.log; $client_args connect localhost:$port" > stdout_client1.txt 2> stderr_client1.txt || fail client1 "$?" & -if [ "$arg_valgrind_memcheck" == "1" ]; then - wait_for_fifo client1.fifo 180 - sleep 40 -else - wait_for_fifo client1.fifo 50 - sleep 1 -fi +wait_for_launch client1.fifo 5 -echo "[*] start demo recording" +echo "[*] Start demo recording" echo "record server" > server.fifo echo "record client1" > client1.fifo sleep 1 -echo "[*] launch client 2" +echo "[*] Launch client 2" $tool ../DDNet \ "cl_input_fifo client2.fifo; player_name client2; - cl_download_skins 0; - gfx_fullscreen 0; - snd_enable 0; logfile client2.log; $client_args connect localhost:$port" > stdout_client2.txt 2> stderr_client2.txt || fail client2 "$?" & -if [ "$arg_valgrind_memcheck" == "1" ]; then - wait_for_fifo client2.fifo 180 - sleep 40 -else - wait_for_fifo client2.fifo 50 - sleep 2 -fi +wait_for_launch client2.fifo 5 -echo "[*] test chat and chat commands" +echo "[*] Test chat and chat commands" echo "say hello world" > client1.fifo echo "rcon_auth rcon" > client1.fifo sleep 1 @@ -236,7 +244,7 @@ say "/mc EOF sleep 1 -echo "[*] test rcon commands" +echo "[*] Test rcon commands" tr -d '\n' > client1.fifo << EOF rcon say hello from admin; rcon broadcast test; @@ -247,12 +255,12 @@ unban_all; EOF sleep 1 -echo "[*] stop demo recording" +echo "[*] Stop demo recording" echo "stoprecord" > server.fifo echo "stoprecord" > client1.fifo sleep 1 -echo "[*] test map change" +echo "[*] Test map change" echo "rcon sv_map Tutorial" > client1.fifo if [ "$arg_valgrind_memcheck" == "1" ]; then sleep 60 @@ -260,7 +268,7 @@ else sleep 15 fi -echo "[*] play demos" +echo "[*] Play demos" echo "play demos/server.demo" > client1.fifo echo "play demos/client1.demo" > client2.fifo if [ "$arg_valgrind_memcheck" == "1" ]; then @@ -271,8 +279,6 @@ fi # Kill all processes first so all outputs are fully written kill_all -wait -sleep 1 if ! grep -qE '^[0-9]{4}-[0-9]{2}-[0-9]{2} ([0-9]{2}:){2}[0-9]{2} I chat: 0:-2:client1: hello world$' server.log then @@ -331,7 +337,7 @@ do fi if [ ! -f "$logfile" ] then - echo "[-] Error: logfile '$logfile' not found." + echo "[-] Error: logfile '$logfile' not found" touch fail_logs.txt continue fi @@ -366,10 +372,9 @@ then cat "$fail" done print_results - echo "[-] Test failed. See errors above." + echo "[-] Test failed. See errors above" exit 1 -else - echo "[*] all tests passed" fi +echo "[*] All tests passed" print_results || exit 1