Re-implement the boost tests execution.
All tests are now executed one at a time in order to avoid cross-test pollution. Fixes #6937
This commit is contained in:
Pentarctagon
Pentarctagon
parent
8fd558f0f1
commit
43a1ea2132
4 changed files with 261 additions and 2 deletions
4
.github/workflows/ci-main.yml
vendored
4
.github/workflows/ci-main.yml
vendored
|
|
@ -96,7 +96,7 @@ jobs:
|
|||
run: ./utils/CI/mp_test_executor.sh
|
||||
- name: Run unit tests
|
||||
if: success() || steps.build.outcome == 'success'
|
||||
run: ./utils/CI/test_executor.sh
|
||||
run: ./run_boost_tests
|
||||
|
||||
steam-runtime:
|
||||
runs-on: ubuntu-20.04
|
||||
|
|
@ -241,7 +241,7 @@ jobs:
|
|||
if: matrix.cfg == 'Release'
|
||||
run: ./run_wml_tests -g -c -t 30 -p "projectfiles/Xcode/build/$CFG/The Battle for Wesnoth.app/Contents/MacOS/The Battle for Wesnoth"
|
||||
- name: Run unit tests
|
||||
run: projectfiles/Xcode/build/"$CFG"/unit_tests --color_output --log_level=test_suite
|
||||
run: ./run_boost_tests --path=projectfiles/Xcode/build/"$CFG" --executable=unit_tests
|
||||
|
||||
windows:
|
||||
strategy:
|
||||
|
|
|
|||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -146,6 +146,7 @@ src/test
|
|||
error*.log
|
||||
boost_test_result.xml
|
||||
*build*.sh
|
||||
boost_tests.log
|
||||
|
||||
# translations
|
||||
locales
|
||||
|
|
|
|||
192
boost_test_schedule
Normal file
192
boost_test_schedule
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
test_fire_event
|
||||
test_gui2_iterator
|
||||
test_gui2
|
||||
test_make_test_fake
|
||||
test_gui2_visitor
|
||||
addons/validation
|
||||
addons/encoding
|
||||
cmdline_opts/test_empty_options
|
||||
cmdline_opts/test_default_options
|
||||
cmdline_opts/test_full_options
|
||||
cmdline_opts/test_positional_options
|
||||
test_config/test_config_attribute_value
|
||||
test_config/test_variable_info
|
||||
test_config/add_child_EmptyThis_newKey_AppendAndReturnNewEmptyChild
|
||||
test_config/add_child_NonEmptyThis_newOrExistingKey_lOrRValue_AppendAndReturnNewChild
|
||||
test_preproc_defines
|
||||
test_config_cache_defaults
|
||||
config_cache/test_load_config
|
||||
config_cache/test_non_clean_config_loading
|
||||
config_cache/test_macrosubstitution
|
||||
config_cache/test_transaction
|
||||
config_cache/test_define_loading
|
||||
config_cache/test_lead_spaces_loading
|
||||
filesystem/test_fs_game_path_reverse_engineering
|
||||
filesystem/test_fs_base
|
||||
filesystem/test_fs_enum
|
||||
filesystem/test_fs_binary_path
|
||||
filesystem/test_fs_wml_path
|
||||
filesystem/test_fs_search
|
||||
filesystem/test_fs_fluff
|
||||
formula_core/test_formula_basic_arithmetic
|
||||
formula_core/test_formula_basic_logic
|
||||
formula_core/test_formula_callable
|
||||
formula_core/test_formula_where_clause
|
||||
formula_core/test_formula_strings
|
||||
formula_core/test_formula_dice
|
||||
formula_core/test_formula_containers
|
||||
formula_core/test_formula_tokenizer
|
||||
formula_function/test_formula_function_substring
|
||||
formula_function/test_formula_function_length
|
||||
formula_function/test_formula_function_concatenate
|
||||
formula_function/test_formula_function_math
|
||||
formula_function/test_formula_function_trig
|
||||
formula_timespan/test_formula_timespan
|
||||
image_modification_parsing/test_modificaiton_queue_order
|
||||
image_modification_parsing/test_tc_modification_decoding
|
||||
image_modification_parsing/test_tc_modification_decoding_invalid_args
|
||||
image_modification_parsing/test_rc_modification_decoding
|
||||
image_modification_parsing/test_rc_modification_decoding_invalid_args
|
||||
image_modification_parsing/test_pal_modification_decoding
|
||||
image_modification_parsing/test_pal_modification_decoding_invalid_args
|
||||
image_modification_parsing/test_fl_modification_decoding_default
|
||||
image_modification_parsing/test_fl_modification_decoding_horiz
|
||||
image_modification_parsing/test_fl_modification_decoding_vert
|
||||
image_modification_parsing/test_fl_modification_decoding_horiz_and_vert
|
||||
image_modification_parsing/test_gs_modification_decoding
|
||||
image_modification_parsing/test_crop_modification_decoding_no_args
|
||||
image_modification_parsing/test_crop_modification_decoding_1_arg
|
||||
image_modification_parsing/test_crop_modification_decoding_2_args
|
||||
image_modification_parsing/test_crop_modification_decoding_3_args
|
||||
image_modification_parsing/test_crop_modification_decoding_4_args
|
||||
image_modification_parsing/test_blit_modification_decoding_1_arg
|
||||
image_modification_parsing/test_blit_modification_decoding_3_args
|
||||
image_modification_parsing/test_blit_modification_decoding_invalid_args
|
||||
image_modification_parsing/test_mask_modification_decoding_1_arg
|
||||
image_modification_parsing/test_mask_modification_decoding_3_args
|
||||
image_modification_parsing/test_mask_modification_decoding_invalid_args
|
||||
image_modification_parsing/test_l_modification_decoding_no_args
|
||||
image_modification_parsing/test_l_modification_decoding_1_arg
|
||||
image_modification_parsing/test_scale_modification_decoding_no_args
|
||||
image_modification_parsing/test_scale_modification_decoding_1_arg
|
||||
image_modification_parsing/test_scale_modification_decoding_2_args
|
||||
image_modification_parsing/test_o_modification_decoding_percent_args
|
||||
image_modification_parsing/test_o_modification_decoding_fraction_args
|
||||
image_modification_parsing/test_bl_modification_decoding_no_args
|
||||
image_modification_parsing/test_bl_modification_decoding
|
||||
image_modification_parsing/test_rgb_modification_decoding_no_args
|
||||
image_modification_parsing/test_r_modification_decoding
|
||||
image_modification_parsing/test_g_modification_decoding
|
||||
image_modification_parsing/test_b_modification_decoding
|
||||
image_modification_parsing/test_bg_modification_decoding_no_args
|
||||
image_modification_parsing/test_bg_modification_decoding_1_arg
|
||||
image_modification_parsing/test_bg_modification_decoding_2_args
|
||||
image_modification_parsing/test_bg_modification_decoding_3_args
|
||||
image_modification_parsing/test_bg_modification_decoding_4_args
|
||||
test_irdya_datetime/test_irdya_date_parse
|
||||
test_irdya_datetime/test_irdya_date_equal
|
||||
test_irdya_datetime/test_irdya_date_ordering
|
||||
test_lexical_cast_throw<bool>
|
||||
test_lexical_cast_throw<char>
|
||||
test_lexical_cast_throw<signed char>
|
||||
test_lexical_cast_throw<unsigned char>
|
||||
test_lexical_cast_throw<short>
|
||||
test_lexical_cast_throw<int>
|
||||
test_lexical_cast_throw<long>
|
||||
test_lexical_cast_throw<long long>
|
||||
test_lexical_cast_throw<unsigned short>
|
||||
test_lexical_cast_throw<unsigned int>
|
||||
test_lexical_cast_throw<unsigned long>
|
||||
test_lexical_cast_throw<unsigned long long>
|
||||
test_lexical_cast_throw<float>
|
||||
test_lexical_cast_throw<double>
|
||||
test_lexical_cast_throw<long double>
|
||||
test_lexical_cast_signed<signed char>
|
||||
test_lexical_cast_signed<short>
|
||||
test_lexical_cast_signed<int>
|
||||
test_lexical_cast_signed<long>
|
||||
test_lexical_cast_long_long
|
||||
test_lexical_cast_unsigned<unsigned char>
|
||||
test_lexical_cast_unsigned<unsigned short>
|
||||
test_lexical_cast_unsigned<unsigned int>
|
||||
test_lexical_cast_unsigned<unsigned long>
|
||||
test_lexical_cast_unsigned_long_long
|
||||
test_lexical_cast_bool
|
||||
test_lexical_cast_floating_point<float>
|
||||
test_lexical_cast_floating_point<double>
|
||||
test_lexical_cast_floating_point<long double>
|
||||
test_lexical_cast_result
|
||||
test_map_location/map_location_characterization_test_radial_mode
|
||||
test_map_location/reality_check_vector_negation
|
||||
test_map_location/reality_check_get_direction
|
||||
test_map_location/check_get_opposite_dir_refactor
|
||||
test_map_location/check_rotate
|
||||
test_map_location/check_rotate_around_center
|
||||
mp_connect/flg_map_settings2
|
||||
mp_connect/flg_map_settings3
|
||||
mp_connect/flg_map_settings4
|
||||
mp_connect/flg_map_settings5
|
||||
mp_connect/flg_map_settings6
|
||||
mp_connect/flg_map_settings7
|
||||
mp_connect/flg_map_settings8
|
||||
mp_connect/flg_map_settings9
|
||||
mp_connect/flg_map_settings10
|
||||
mp_connect/flg_map_settings11
|
||||
mp_connect/flg_map_settings12
|
||||
mp_connect/flg_map_settings13
|
||||
mp_connect/flg_map_settings14
|
||||
mp_connect/flg_map_settings15
|
||||
mp_connect/flg_map_settings16
|
||||
mp_connect/flg_map_settings17
|
||||
mp_connect/flg_map_settings18
|
||||
mp_connect/flg_map_settings19
|
||||
mp_connect/flg_map_settings20
|
||||
mp_connect/flg_map_settings21
|
||||
mp_connect/flg_map_settings22
|
||||
mp_connect/flg_map_settings23
|
||||
mp_connect/flg_map_settings24
|
||||
mp_connect/flg_map_settings25
|
||||
mp_connect/flg_map_settings26
|
||||
mp_connect/flg_map_settings27
|
||||
mp_connect/flg_map_settings28
|
||||
mp_connect/flg_map_settings29
|
||||
mp_connect/flg_no_map_settings1
|
||||
mp_connect/flg_no_map_settings2
|
||||
mp_connect/flg_no_map_settings3
|
||||
mp_connect/flg_no_map_settings4
|
||||
mp_connect/flg_no_map_settings5
|
||||
mp_connect/flg_no_map_settings6
|
||||
recall_list_suite/test_1
|
||||
rng/validate_mt19937
|
||||
rng/test_mt_rng_seed_manip
|
||||
rng/test_mt_rng_config_seed_manip
|
||||
rng/test_mt_rng_reproducibility
|
||||
rng/test_mt_rng_reproducibility2
|
||||
rng/test_mt_rng_reproducibility3
|
||||
rng/test_mt_rng_reproducibility4
|
||||
rng/test_mt_rng_reproducibility5
|
||||
rng/test_mt_rng_reproducibility_coverage
|
||||
rng/validate_get_random_int
|
||||
rng/validate_get_random_int2
|
||||
test_serialization_utils_and_unicode/utils_join_test
|
||||
test_serialization_utils_and_unicode/utils_split_test
|
||||
test_serialization_utils_and_unicode/utils_quoted_split_test
|
||||
test_serialization_utils_and_unicode/utils_map_split_test
|
||||
test_serialization_utils_and_unicode/utils_parenthetical_split_test
|
||||
test_serialization_utils_and_unicode/utils_square_parenthetical_split
|
||||
test_serialization_utils_and_unicode/utils_unicode_test
|
||||
test_serialization_utils_and_unicode/test_lowercase
|
||||
test_serialization_utils_and_unicode/test_wildcard_string_match
|
||||
test_serialization_utils_and_unicode/test_base64_encodings
|
||||
simple_wml/simple_wml_first_test
|
||||
teams/test_user_team_name
|
||||
unit_map_suite/test_1
|
||||
unit_map_suite/track_real_unit_by_underlying_id
|
||||
unit_map_suite/track_fake_unit_by_underlying_id
|
||||
unit_map_suite/track_real_unit_by_iterator
|
||||
unit_map_suite/track_fake_unit_by_iterator
|
||||
version/test_version_info
|
||||
whiteboard_side_actions_container/test_insertion
|
||||
whiteboard_side_actions_container/test_removal
|
||||
feature_test_WML_macro_define/macro_define_noArgument_ParseAsExpected
|
||||
feature_test_WML_macro_define/macro_define_1Argument_ParseAsExpected
|
||||
66
run_boost_tests
Executable file
66
run_boost_tests
Executable file
|
|
@ -0,0 +1,66 @@
|
|||
#!/usr/bin/env python3
|
||||
# encoding: utf-8
|
||||
"""
|
||||
This script runs a sequence of C++ unit test scenarios.
|
||||
"""
|
||||
|
||||
import argparse, enum, os, re, subprocess, sys
|
||||
|
||||
def get_tests(filename):
|
||||
test_list = []
|
||||
for line in open(filename, mode="rt"):
|
||||
line = line.strip()
|
||||
if line == "" or line.startswith("#"):
|
||||
continue
|
||||
|
||||
test_list.append(line)
|
||||
return test_list
|
||||
|
||||
def run_with_rerun_for_sdl_video(executable, test_name):
|
||||
"""A wrapper for subprocess.run with a workaround for the issue on CI of intermittently failing to initialise SDL.
|
||||
"""
|
||||
# Sanity check on the number of retries.
|
||||
# It's a rare failure, a single retry would probably be enough.
|
||||
sdl_retries = 0
|
||||
while sdl_retries < 10:
|
||||
res = subprocess.run([executable, "--run_test="+test_name, "--color_output"], timeout=20, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
retry = False
|
||||
if b"Could not initialize SDL_video" in res.stdout:
|
||||
retry = True
|
||||
if not retry:
|
||||
return res.returncode
|
||||
sdl_retries += 1
|
||||
print("Could not initialise SDL_video error, attempt", sdl_retries)
|
||||
|
||||
if __name__ == '__main__':
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("-e", "--executable", help="The name of the boost unit test executable. Default is bost_unit_tests.", default="boost_unit_tests")
|
||||
ap.add_argument("-p", "--path", metavar="dir", help="Path to wesnoth binary. By default assume it is with this script.", default=".")
|
||||
ap.add_argument("-l", "--list", metavar="filename", help="Loads list of tests from the given file.", default="boost_test_schedule")
|
||||
|
||||
options = ap.parse_args()
|
||||
|
||||
tests = get_tests(options.list)
|
||||
executable = options.executable
|
||||
success=0
|
||||
failure=0
|
||||
|
||||
for test in tests:
|
||||
returncode = 0
|
||||
try:
|
||||
returncode = run_with_rerun_for_sdl_video(os.path.join(options.path, executable), test)
|
||||
except subprocess.TimeoutExpired:
|
||||
failure+=1
|
||||
print("Test "+test+" timed out")
|
||||
else:
|
||||
if returncode == 0:
|
||||
success+=1
|
||||
print("Test "+test+" passed")
|
||||
else:
|
||||
failure+=1
|
||||
print("Test "+test+" failed with return code "+str(returncode))
|
||||
|
||||
print("Tests passed: "+str(success))
|
||||
print("Tests failed: "+str(failure))
|
||||
if failure != 0:
|
||||
exit(1)
|
||||
Loading…
Add table
Reference in a new issue