edg-l/ddnet
1
0
Fork 0
mirror of https://github.com/ddnet/ddnet.git synced 2024-11-17 21:48:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve integration test script

Browse source 
Fix valgrind errors not being detected as failures when the tests otherwise finish, as the valgrind results are printed to stderr and not to the logfiles. Additionally, valgrind failures were not being detected because the error summary is not printed to stderr immediately but only after some time when valgrind has processed the summary.

Find random unused port to bind to by opening a socket on port 0 and closing it immediately. This gives a port that will very likely be unused because the system cycles through all ports before reusing old ones obtained this way.

Add `wait_for_launch` function to reduce duplicate code. Wait for server to launch before starting clients.

Improve log messages.
This commit is contained in:
Robert Müller 2023-11-28 21:10:32 +01:00
parent f9fc85920f
commit b352545621
1 changed files with 64 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

117
scripts/integration_test.sh
View file

@ -7,10 +7,10 @@ for arg in "$@"
do do
if [ "$arg" == "-h" ] || [ "$arg" == "--help" ] if [ "$arg" == "-h" ] || [ "$arg" == "--help" ]
then then
echo "usage: $(basename "$0") [OPTION..] [build dir]" echo "usage: $(basename "$0") [OPTION..]"
echo "description:" echo "description:"
echo " Runs a simple integration test of the client and server" echo " Runs a simple integration test of the client and server"
echo " binaries from the given build dir" echo " binaries from the current build directory."
echo "options:" echo "options:"
echo " --help|-h show this help" echo " --help|-h show this help"
echo " --verbose|-v verbose output" echo " --verbose|-v verbose output"
@ -23,22 +23,23 @@ do
then then
arg_valgrind_memcheck=1 arg_valgrind_memcheck=1
else else
echo "Error: unknown arg '$arg'" echo "Error: unknown argument '$arg'"
exit 1 exit 1
fi fi
done done
if [ ! -f DDNet ] if [ ! -f DDNet ]
then then
echo "Error: client binary not found DDNet' not found" echo "[-] Error: client binary 'DDNet' not found"
exit 1 exit 1
fi fi
if [ ! -f DDNet-Server ] if [ ! -f DDNet-Server ]
then then
echo "Error: server binary not found DDNet-Server' not found" echo "[-] Error: server binary 'DDNet-Server' not found"
exit 1 exit 1
fi fi
echo "[*] Setup"
got_killed=0 got_killed=0
function kill_all() { function kill_all() {
@ -51,24 +52,27 @@ function kill_all() {
if [ "$arg_verbose" == "1" ] if [ "$arg_verbose" == "1" ]
then then
echo "[*] shutting down test clients and server ..." echo "[*] Shutting down test clients and server"
fi fi
sleep 1 sleep 1
if [[ ! -f fail_server.txt ]] if [[ ! -f fail_server.txt ]]
then then
echo "[*] shutting down server" echo "[*] Shutting down server"
echo "shutdown" > server.fifo echo "shutdown" > server.fifo
fi fi
sleep 1
local i local i
for ((i=1;i<3;i++)) for ((i=1;i<3;i++))
do do
if [[ ! -f fail_client$i.txt ]] if [[ ! -f fail_client$i.txt ]]
then then
echo "[*] shutting down client$i" echo "[*] Shutting down client$i"
echo "quit" > "client$i.fifo" echo "quit" > "client$i.fifo"
fi fi
done done
sleep 1
} }
function cleanup() { function cleanup() {
@ -85,8 +89,8 @@ function fail()
echo "[-] $1 exited with code $2" echo "[-] $1 exited with code $2"
} }
# TODO: check for open ports instead # Get unused port from the system by binding to port 0 and immediately closing the socket again
port=17822 port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()');
if [[ $OSTYPE == 'darwin'* ]]; then if [[ $OSTYPE == 'darwin'* ]]; then
DETECT_LEAKS=0 DETECT_LEAKS=0
@ -100,13 +104,18 @@ export LSAN_OPTIONS=suppressions=../lsan.supp:print_suppressions=0
function print_results() { function print_results() {
if [ "$arg_valgrind_memcheck" == "1" ]; then if [ "$arg_valgrind_memcheck" == "1" ]; then
if grep "ERROR SUMMARY" server.log client1.log client2.log | grep -q -v "ERROR SUMMARY: 0"; then # Wait to ensure that the error summary was written to the stderr files because valgrind takes some time
grep "^==" server.log client1.log client2.log # TODO: Instead wait for all started processes to finish
sleep 20
if grep "== ERROR SUMMARY: " stderr_server.txt stderr_client1.txt stderr_client2.txt | grep -q -v "ERROR SUMMARY: 0"; then
echo "[-] Error: Valgrind has detected the following errors:"
grep "^==" stderr_server.txt stderr_client1.txt stderr_client2.txt
return 1 return 1
fi fi
else else
if test -n "$(find . -maxdepth 1 -name 'SAN.*' -print -quit)" if test -n "$(find . -maxdepth 1 -name 'SAN.*' -print -quit)"
then then
echo "[-] Error: ASAN has detected the following errors:"
cat SAN.* cat SAN.*
return 1 return 1
fi fi
@ -127,39 +136,54 @@ cd integration_test || exit 1
echo $'add_path ../data' echo $'add_path ../data'
} > storage.cfg } > storage.cfg
tool=""
client_args="cl_download_skins 0;
gfx_fullscreen 0;
snd_enable 0;"
if [ "$arg_valgrind_memcheck" == "1" ]; then if [ "$arg_valgrind_memcheck" == "1" ]; then
tool="valgrind --tool=memcheck --gen-suppressions=all --suppressions=../memcheck.supp --track-origins=yes" tool="valgrind --tool=memcheck --gen-suppressions=all --suppressions=../memcheck.supp --track-origins=yes"
client_args="cl_menu_map \"\";" client_args="$client_args cl_menu_map \"\";"
else
tool=""
client_args=""
fi fi
function wait_for_fifo() { function wait_for_fifo() {
local fifo="$1" local fifo="$1"
local tries="$2" local tries="$2"
local fails=0 local fails=0
# give the client time to launch and create the fifo file # give the server/client time to launch and create the fifo file
# but assume after X secs that the client crashed before # but assume after X secs that the server/client crashed before
# being able to create the file # being able to create the file
while [[ ! -p "$fifo" ]] while [[ ! -p "$fifo" ]]
do do
fails="$((fails+1))" fails="$((fails+1))"
if [ "$arg_verbose" == "1" ] if [ "$arg_verbose" == "1" ]
then then
echo "[!] client fifos not found (attempts $fails/$tries)" echo "[!] Note: $fifo not found (attempts $fails/$tries)"
fi fi
if [ "$fails" -gt "$tries" ] if [ "$fails" -gt "$tries" ]
then then
echo "[-] Error: $(basename "$fifo" .fifo) possibly crashed on launch"
kill_all
print_results print_results
echo "[-] Error: client possibly crashed on launch"
exit 1 exit 1
fi fi
sleep 1 sleep 1
done done
} }
echo "[*] launch server" function wait_for_launch() {
local fifo="$1"
local baseDuration="$2"
if [ "$arg_valgrind_memcheck" == "1" ]; then
wait_for_fifo "$fifo" $((40 * baseDuration))
sleep $((8 * baseDuration))
else
wait_for_fifo "$fifo" $((10 * baseDuration))
sleep "$baseDuration"
fi
}
echo "[*] Launch server"
$tool ../DDNet-Server \ $tool ../DDNet-Server \
"sv_input_fifo server.fifo; "sv_input_fifo server.fifo;
sv_rcon_password rcon; sv_rcon_password rcon;
@ -169,50 +193,34 @@ $tool ../DDNet-Server \
sv_register 0; sv_register 0;
sv_port $port" > stdout_server.txt 2> stderr_server.txt || fail server "$?" & sv_port $port" > stdout_server.txt 2> stderr_server.txt || fail server "$?" &
echo "[*] launch client 1" wait_for_launch server.fifo 1
echo "[*] Launch client 1"
$tool ../DDNet \ $tool ../DDNet \
"cl_input_fifo client1.fifo; "cl_input_fifo client1.fifo;
player_name client1; player_name client1;
cl_download_skins 0;
gfx_fullscreen 0;
snd_enable 0;
logfile client1.log; logfile client1.log;
$client_args $client_args
connect localhost:$port" > stdout_client1.txt 2> stderr_client1.txt || fail client1 "$?" & connect localhost:$port" > stdout_client1.txt 2> stderr_client1.txt || fail client1 "$?" &
if [ "$arg_valgrind_memcheck" == "1" ]; then wait_for_launch client1.fifo 5
wait_for_fifo client1.fifo 180
sleep 40
else
wait_for_fifo client1.fifo 50
sleep 1
fi
echo "[*] start demo recording" echo "[*] Start demo recording"
echo "record server" > server.fifo echo "record server" > server.fifo
echo "record client1" > client1.fifo echo "record client1" > client1.fifo
sleep 1 sleep 1
echo "[*] launch client 2" echo "[*] Launch client 2"
$tool ../DDNet \ $tool ../DDNet \
"cl_input_fifo client2.fifo; "cl_input_fifo client2.fifo;
player_name client2; player_name client2;
cl_download_skins 0;
gfx_fullscreen 0;
snd_enable 0;
logfile client2.log; logfile client2.log;
$client_args $client_args
connect localhost:$port" > stdout_client2.txt 2> stderr_client2.txt || fail client2 "$?" & connect localhost:$port" > stdout_client2.txt 2> stderr_client2.txt || fail client2 "$?" &
if [ "$arg_valgrind_memcheck" == "1" ]; then wait_for_launch client2.fifo 5
wait_for_fifo client2.fifo 180
sleep 40
else
wait_for_fifo client2.fifo 50
sleep 2
fi
echo "[*] test chat and chat commands" echo "[*] Test chat and chat commands"
echo "say hello world" > client1.fifo echo "say hello world" > client1.fifo
echo "rcon_auth rcon" > client1.fifo echo "rcon_auth rcon" > client1.fifo
sleep 1 sleep 1
@ -236,7 +244,7 @@ say "/mc
EOF EOF
sleep 1 sleep 1
echo "[*] test rcon commands" echo "[*] Test rcon commands"
tr -d '\n' > client1.fifo << EOF tr -d '\n' > client1.fifo << EOF
rcon say hello from admin; rcon say hello from admin;
rcon broadcast test; rcon broadcast test;
@ -247,12 +255,12 @@ unban_all;
EOF EOF
sleep 1 sleep 1
echo "[*] stop demo recording" echo "[*] Stop demo recording"
echo "stoprecord" > server.fifo echo "stoprecord" > server.fifo
echo "stoprecord" > client1.fifo echo "stoprecord" > client1.fifo
sleep 1 sleep 1
echo "[*] test map change" echo "[*] Test map change"
echo "rcon sv_map Tutorial" > client1.fifo echo "rcon sv_map Tutorial" > client1.fifo
if [ "$arg_valgrind_memcheck" == "1" ]; then if [ "$arg_valgrind_memcheck" == "1" ]; then
sleep 60 sleep 60
@ -260,7 +268,7 @@ else
sleep 15 sleep 15
fi fi
echo "[*] play demos" echo "[*] Play demos"
echo "play demos/server.demo" > client1.fifo echo "play demos/server.demo" > client1.fifo
echo "play demos/client1.demo" > client2.fifo echo "play demos/client1.demo" > client2.fifo
if [ "$arg_valgrind_memcheck" == "1" ]; then if [ "$arg_valgrind_memcheck" == "1" ]; then
@ -271,8 +279,6 @@ fi
# Kill all processes first so all outputs are fully written # Kill all processes first so all outputs are fully written
kill_all kill_all
wait
sleep 1
if ! grep -qE '^[0-9]{4}-[0-9]{2}-[0-9]{2} ([0-9]{2}:){2}[0-9]{2} I chat: 0:-2:client1: hello world$' server.log if ! grep -qE '^[0-9]{4}-[0-9]{2}-[0-9]{2} ([0-9]{2}:){2}[0-9]{2} I chat: 0:-2:client1: hello world$' server.log
then then
@ -331,7 +337,7 @@ do
fi fi
if [ ! -f "$logfile" ] if [ ! -f "$logfile" ]
then then
echo "[-] Error: logfile '$logfile' not found." echo "[-] Error: logfile '$logfile' not found"
touch fail_logs.txt touch fail_logs.txt
continue continue
fi fi
@ -366,10 +372,9 @@ then
cat "$fail" cat "$fail"
done done
print_results print_results
echo "[-] Test failed. See errors above." echo "[-] Test failed. See errors above"
exit 1 exit 1
else
echo "[*] all tests passed"
fi fi
echo "[*] All tests passed"
print_results || exit 1 print_results || exit 1