Separate unit test statuses for WML exceptions and strict warnings (#4599)
This is part of working out whether a subset of the "fail" tests could be run
in one Wesnoth instance. To run a test that returns TEST_FAIL_BROKE_STRICT with
any other test would require a mechanism to reset lg::broke_strict()'s flag.
All tests that fail with an {ASSERT} will also set the lg::broke_strict() flag,
the tests with the new status are only those that would pass without the strict
flag.
In the SUF tests, change a test from fail-on-success to breaks-strict, rename
the formula tests and add some comments. The rename is because "fai" is
"Formula AI", an obsolete name for WFL.
This commit is contained in:
Steve Cotton
GitHub
parent
6f5ed34a50
commit
7359fddecb
6 changed files with 41 additions and 20 deletions
|
|
@ -41,7 +41,7 @@
|
|||
[/event]
|
||||
)}
|
||||
|
||||
{GENERIC_UNIT_TEST filter_this_unit_fai (
|
||||
{GENERIC_UNIT_TEST filter_this_unit_formula (
|
||||
[event]
|
||||
name=prestart
|
||||
[modify_unit]
|
||||
|
|
@ -59,7 +59,7 @@
|
|||
[/event]
|
||||
)}
|
||||
|
||||
{GENERIC_UNIT_TEST filter_fai_unit (
|
||||
{GENERIC_UNIT_TEST filter_formula_unit (
|
||||
[event]
|
||||
name=prestart
|
||||
[modify_unit]
|
||||
|
|
@ -77,14 +77,19 @@
|
|||
[/event]
|
||||
)}
|
||||
|
||||
{GENERIC_UNIT_TEST filter_fai_unit_error (
|
||||
# Tests what happens when an invalid formula is given to an SUF.
|
||||
# This test requires strict mode, the expected behavior is to log
|
||||
# a warning but not interrupt the WML.
|
||||
{GENERIC_UNIT_TEST filter_formula_unit_error (
|
||||
[event]
|
||||
name=prestart
|
||||
{RETURN (
|
||||
[have_unit]
|
||||
id=bob
|
||||
formula="+ max_moves"
|
||||
[/have_unit]
|
||||
[not]
|
||||
[have_unit]
|
||||
id=bob
|
||||
formula="+ max_moves"
|
||||
[/have_unit]
|
||||
[/not]
|
||||
)}
|
||||
[/event]
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ class UnitTestResult(enum.Enum):
|
|||
TIMEOUT = 2
|
||||
FAIL_LOADING_REPLAY = 3
|
||||
FAIL_PLAYING_REPLAY = 4
|
||||
FAIL_BROKE_STRICT = 5
|
||||
FAIL_WML_EXCEPTION = 6
|
||||
|
||||
class TestCase:
|
||||
"""Represents a single line of the wml_test_schedule."""
|
||||
|
|
|
|||
|
|
@ -276,7 +276,7 @@ commandline_options::commandline_options (const std::vector<std::string>& args)
|
|||
po::options_description testing_opts("Testing options");
|
||||
testing_opts.add_options()
|
||||
("test,t", po::value<std::string>()->implicit_value(std::string()), "runs the game in a small test scenario. If specified, scenario <arg> will be used instead.")
|
||||
("unit,u", po::value<std::vector<std::string>>(), "runs a unit test scenario. The GUI is not shown and the exit code of the program reflects the victory / defeat conditions of the scenario.\n\t0 - PASS\n\t1 - FAIL\n\t3 - FAIL (INVALID REPLAY)\n\t4 - FAIL (ERRORED REPLAY)\n\tMultiple tests can be run by giving this option multiple times, in this case the test run will stop immediately after any test which doesn't PASS and the return code will be the status of the test that caused the stop.")
|
||||
("unit,u", po::value<std::vector<std::string>>(), "runs a unit test scenario. The GUI is not shown and the exit code of the program reflects the victory / defeat conditions of the scenario.\n\t0 - PASS\n\t1 - FAIL\n\t3 - FAIL (INVALID REPLAY)\n\t4 - FAIL (ERRORED REPLAY)\n\t5 - FAIL (BROKE STRICT)\n\t6 - FAIL (WML EXCEPTION)\n\tMultiple tests can be run by giving this option multiple times, in this case the test run will stop immediately after any test which doesn't PASS and the return code will be the status of the test that caused the stop.")
|
||||
("showgui", "don't run headlessly (for debugging a failing test)")
|
||||
("log-strict", po::value<std::string>(), "sets the strict level of the logger. any messages sent to log domains of this level or more severe will cause the unit test to fail regardless of the victory result.")
|
||||
("noreplaycheck", "don't try to validate replay of unit test.")
|
||||
|
|
|
|||
|
|
@ -519,6 +519,12 @@ game_launcher::unit_test_result game_launcher::unit_test()
|
|||
case unit_test_result::TEST_FAIL_PLAYING_REPLAY:
|
||||
describe_result = "FAIL TEST (ERRORED REPLAY)";
|
||||
break;
|
||||
case unit_test_result::TEST_FAIL_BROKE_STRICT:
|
||||
describe_result = "FAIL TEST (BROKE STRICT)";
|
||||
break;
|
||||
case unit_test_result::TEST_FAIL_WML_EXCEPTION:
|
||||
describe_result = "FAIL TEST (WML EXCEPTION)";
|
||||
break;
|
||||
default:
|
||||
describe_result = "FAIL TEST";
|
||||
break;
|
||||
|
|
@ -539,12 +545,17 @@ game_launcher::unit_test_result game_launcher::single_unit_test()
|
|||
try {
|
||||
campaign_controller ccontroller(state_, game_config_manager::get()->terrain_types(), true);
|
||||
LEVEL_RESULT res = ccontroller.play_game();
|
||||
if (!(res == LEVEL_RESULT::VICTORY) || lg::broke_strict()) {
|
||||
if (res != LEVEL_RESULT::VICTORY) {
|
||||
return unit_test_result::TEST_FAIL;
|
||||
}
|
||||
if (lg::broke_strict()) {
|
||||
// Test for LEVEL_RESULT::VICTORY before this, as the warning printed by
|
||||
// a failing ASSERT will also set broke_strict()'s flag.
|
||||
return unit_test_result::TEST_FAIL_BROKE_STRICT;
|
||||
}
|
||||
} catch(const wml_exception& e) {
|
||||
std::cerr << "Caught WML Exception:" << e.dev_message << std::endl;
|
||||
return unit_test_result::TEST_FAIL;
|
||||
return unit_test_result::TEST_FAIL_WML_EXCEPTION;
|
||||
}
|
||||
|
||||
savegame::clean_saves(state_.classification().label);
|
||||
|
|
|
|||
|
|
@ -64,8 +64,11 @@ public:
|
|||
enum class unit_test_result : int {
|
||||
TEST_PASS = 0,
|
||||
TEST_FAIL = 1,
|
||||
// 2 is reserved for timeouts
|
||||
TEST_FAIL_LOADING_REPLAY = 3,
|
||||
TEST_FAIL_PLAYING_REPLAY = 4,
|
||||
TEST_FAIL_BROKE_STRICT = 5,
|
||||
TEST_FAIL_WML_EXCEPTION = 6,
|
||||
};
|
||||
|
||||
bool init_video();
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
# Security test
|
||||
#
|
||||
0 cve_2018_1999023
|
||||
1 cve_2018_1999023_2
|
||||
5 cve_2018_1999023_2
|
||||
#
|
||||
# Test Check Victory (If this isn't working other tests may have dubious value)
|
||||
#
|
||||
|
|
@ -54,12 +54,12 @@
|
|||
0 unit_spawns_at_nearest_vacant_hex
|
||||
0 units_offmap_goto_recall
|
||||
0 test_move
|
||||
1 test_move_fail_1
|
||||
1 test_move_fail_2
|
||||
1 test_move_fail_3
|
||||
1 test_move_fail_4
|
||||
1 test_move_fail_5
|
||||
1 test_move_fail_6
|
||||
5 test_move_fail_1
|
||||
5 test_move_fail_2
|
||||
5 test_move_fail_3
|
||||
5 test_move_fail_4
|
||||
5 test_move_fail_5
|
||||
5 test_move_fail_6
|
||||
0 test_move_unit
|
||||
0 test_move_unit_in_circle
|
||||
0 sighted_events
|
||||
|
|
@ -175,9 +175,9 @@
|
|||
# Standard Unit Filter tests
|
||||
0 filter_this_unit_wml
|
||||
0 filter_this_unit_tl
|
||||
0 filter_this_unit_fai
|
||||
0 filter_fai_unit
|
||||
1 filter_fai_unit_error
|
||||
0 filter_this_unit_formula
|
||||
0 filter_formula_unit
|
||||
5 filter_formula_unit_error
|
||||
# Interrupt tag tests
|
||||
0 check_interrupts_break
|
||||
0 check_interrupts_return
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue