beenull/ladybird
1
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2024-11-22 07:30:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

LibTest: Add ability to turn test failure reporting on/off

Browse source 
This will be very useful as we add the randomized test cases and their
two loops ("generate+test many times" and "shrink once failure is
found"), because without this failure reporting we'd get many FAIL error
messages while still searching for the minimal one.

So, inside randomized test cases we want to only turn the error
reporting on for one last time after all the generating and shrinking.
This commit is contained in:
Martin Janiczek 2023-10-24 01:26:28 +02:00 committed by Andrew Kaster
parent 99e2d42a53
commit 1bcfead020
Notes: sideshowbarker 2024-07-17 09:56:35 +09:00 
Author: https://github.com/Janiczek
Commit: https://github.com/SerenityOS/serenity/commit/1bcfead020
Pull-request: https://github.com/SerenityOS/serenity/pull/21191
Reviewed-by: https://github.com/ADKaster
Reviewed-by: https://github.com/alimpfard
Reviewed-by: https://github.com/timschumi
3 changed files with 90 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

110
Userland/Libraries/LibTest/Macros.h
View file

@ -26,59 +26,71 @@ TestResult current_test_result();
Randomized::RandomnessSource& randomness_source(); Randomized::RandomnessSource& randomness_source();
void set_randomness_source(Randomized::RandomnessSource); void set_randomness_source(Randomized::RandomnessSource);
bool is_reporting_enabled();
void enable_reporting();
void disable_reporting();
} }
#define EXPECT_EQ(a, b) \ #define EXPECT_EQ(a, b) \
do { \ do { \
auto lhs = (a); \ auto lhs = (a); \
auto rhs = (b); \ auto rhs = (b); \
if (lhs != rhs) { \ if (lhs != rhs) { \
::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ if (::Test::is_reporting_enabled()) \
::Test::set_current_test_result(::Test::TestResult::Failed); \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", \
} \ __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \
::Test::set_current_test_result(::Test::TestResult::Failed); \
} \
} while (false) } while (false)
#define EXPECT_EQ_TRUTH(a, b) \ #define EXPECT_EQ_TRUTH(a, b) \
do { \ do { \
auto lhs = (a); \ auto lhs = (a); \
auto rhs = (b); \ auto rhs = (b); \
bool ltruth = static_cast<bool>(lhs); \ bool ltruth = static_cast<bool>(lhs); \
bool rtruth = static_cast<bool>(rhs); \ bool rtruth = static_cast<bool>(rhs); \
if (ltruth != rtruth) { \ if (ltruth != rtruth) { \
::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ_TRUTH({}, {}) failed with lhs={} ({}) and rhs={} ({})", \ if (::Test::is_reporting_enabled()) \
__FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, ltruth, FormatIfSupported { rhs }, rtruth); \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ_TRUTH({}, {}) failed with lhs={} ({}) and rhs={} ({})", \
::Test::set_current_test_result(::Test::TestResult::Failed); \ __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, ltruth, FormatIfSupported { rhs }, rtruth); \
} \ ::Test::set_current_test_result(::Test::TestResult::Failed); \
} \
} while (false) } while (false)
// If you're stuck and `EXPECT_EQ` seems to refuse to print anything useful, // If you're stuck and `EXPECT_EQ` seems to refuse to print anything useful,
// try this: It'll spit out a nice compiler error telling you why it doesn't print. // try this: It'll spit out a nice compiler error telling you why it doesn't print.
#define EXPECT_EQ_FORCE(a, b) \ #define EXPECT_EQ_FORCE(a, b) \
do { \ do { \
auto lhs = (a); \ auto lhs = (a); \
auto rhs = (b); \ auto rhs = (b); \
if (lhs != rhs) { \ if (lhs != rhs) { \
::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, lhs, rhs); \ if (::Test::is_reporting_enabled()) \
::Test::set_current_test_result(::Test::TestResult::Failed); \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", \
} \ __FILE__, __LINE__, #a, #b, lhs, rhs); \
::Test::set_current_test_result(::Test::TestResult::Failed); \
} \
} while (false) } while (false)
#define EXPECT_NE(a, b) \ #define EXPECT_NE(a, b) \
do { \ do { \
auto lhs = (a); \ auto lhs = (a); \
auto rhs = (b); \ auto rhs = (b); \
if (lhs == rhs) { \ if (lhs == rhs) { \
::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_NE({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ if (::Test::is_reporting_enabled()) \
::Test::set_current_test_result(::Test::TestResult::Failed); \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_NE({}, {}) failed with lhs={} and rhs={}", \
} \ __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \
::Test::set_current_test_result(::Test::TestResult::Failed); \
} \
} while (false) } while (false)
#define EXPECT(x) \ #define EXPECT(x) \
do { \ do { \
if (!(x)) { \ if (!(x)) { \
::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT({}) failed", __FILE__, __LINE__, #x); \ if (::Test::is_reporting_enabled()) \
::Test::set_current_test_result(::Test::TestResult::Failed); \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT({}) failed", __FILE__, __LINE__, #x); \
} \ ::Test::set_current_test_result(::Test::TestResult::Failed); \
} \
} while (false) } while (false)
#define EXPECT_APPROXIMATE_WITH_ERROR(a, b, err) \ #define EXPECT_APPROXIMATE_WITH_ERROR(a, b, err) \
@ -87,19 +99,21 @@ void set_randomness_source(Randomized::RandomnessSource);
auto expect_close_rhs = b; \ auto expect_close_rhs = b; \
auto expect_close_diff = static_cast<double>(expect_close_lhs) - static_cast<double>(expect_close_rhs); \ auto expect_close_diff = static_cast<double>(expect_close_lhs) - static_cast<double>(expect_close_rhs); \
if (AK::fabs(expect_close_diff) > (err)) { \ if (AK::fabs(expect_close_diff) > (err)) { \
::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_APPROXIMATE({}, {})" \ if (::Test::is_reporting_enabled()) \
" failed with lhs={}, rhs={}, (lhs-rhs)={}", \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_APPROXIMATE({}, {})" \
__FILE__, __LINE__, #a, #b, expect_close_lhs, expect_close_rhs, expect_close_diff); \ " failed with lhs={}, rhs={}, (lhs-rhs)={}", \
__FILE__, __LINE__, #a, #b, expect_close_lhs, expect_close_rhs, expect_close_diff); \
::Test::set_current_test_result(::Test::TestResult::Failed); \ ::Test::set_current_test_result(::Test::TestResult::Failed); \
} \ } \
} while (false) } while (false)
#define EXPECT_APPROXIMATE(a, b) EXPECT_APPROXIMATE_WITH_ERROR(a, b, 0.0000005) #define EXPECT_APPROXIMATE(a, b) EXPECT_APPROXIMATE_WITH_ERROR(a, b, 0.0000005)
#define FAIL(message) \ #define FAIL(message) \
do { \ do { \
::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: {}", __FILE__, __LINE__, message); \ if (::Test::is_reporting_enabled()) \
::Test::set_current_test_result(::Test::TestResult::Failed); \ ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: {}", __FILE__, __LINE__, message); \
::Test::set_current_test_result(::Test::TestResult::Failed); \
} while (false) } while (false)
// To use, specify the lambda to execute in a sub process and verify it exits: // To use, specify the lambda to execute in a sub process and verify it exits:

19
Userland/Libraries/LibTest/TestSuite.cpp
View file

@ -76,6 +76,24 @@ void set_suite_setup_function(Function<void()> setup)
TestSuite::the().set_suite_setup(move(setup)); TestSuite::the().set_suite_setup(move(setup));
} }
// Declared in Macros.h
bool is_reporting_enabled()
{
return TestSuite::the().is_reporting_enabled();
}
// Declared in Macros.h
void enable_reporting()
{
TestSuite::the().enable_reporting();
}
// Declared in Macros.h
void disable_reporting()
{
TestSuite::the().disable_reporting();
}
static DeprecatedString test_result_to_string(TestResult result) static DeprecatedString test_result_to_string(TestResult result)
{ {
switch (result) { switch (result) {
@ -164,6 +182,7 @@ int TestSuite::run(Vector<NonnullRefPtr<TestCase>> const& tests)
warnln("Running {} '{}'.", test_type, t->name()); warnln("Running {} '{}'.", test_type, t->name());
m_current_test_result = TestResult::NotRun; m_current_test_result = TestResult::NotRun;
enable_reporting();
u64 total_time = 0; u64 total_time = 0;
u64 sum_of_squared_times = 0; u64 sum_of_squared_times = 0;

9
Userland/Libraries/LibTest/TestSuite.h
View file

@ -53,6 +53,14 @@ public:
void set_randomness_source(Randomized::RandomnessSource source) { m_randomness_source = move(source); } void set_randomness_source(Randomized::RandomnessSource source) { m_randomness_source = move(source); }
Randomized::RandomnessSource& randomness_source() { return m_randomness_source; } Randomized::RandomnessSource& randomness_source() { return m_randomness_source; }
// Dictates whether FAIL(), EXPECT() and similar macros in LibTest/Macros.h
// print messages or not. This is important for randomized tests because
// they run the test function many times in a row, and we only want to
// report the _minimal_ (shrunk) failure to the user, not all of them.
bool is_reporting_enabled() { return m_reporting_enabled; }
void enable_reporting() { m_reporting_enabled = true; }
void disable_reporting() { m_reporting_enabled = false; }
private: private:
static TestSuite* s_global; static TestSuite* s_global;
Vector<NonnullRefPtr<TestCase>> m_cases; Vector<NonnullRefPtr<TestCase>> m_cases;
@ -63,6 +71,7 @@ private:
Function<void()> m_setup; Function<void()> m_setup;
TestResult m_current_test_result = TestResult::NotRun; TestResult m_current_test_result = TestResult::NotRun;
Randomized::RandomnessSource m_randomness_source = Randomized::RandomnessSource::live(); Randomized::RandomnessSource m_randomness_source = Randomized::RandomnessSource::live();
bool m_reporting_enabled = true;
}; };
} }