From 1bcfead0204262860de4b7e4d6e35b4d0397711d Mon Sep 17 00:00:00 2001 From: Martin Janiczek Date: Tue, 24 Oct 2023 01:26:28 +0200 Subject: [PATCH] LibTest: Add ability to turn test failure reporting on/off This will be very useful as we add the randomized test cases and their two loops ("generate+test many times" and "shrink once failure is found"), because without this failure reporting we'd get many FAIL error messages while still searching for the minimal one. So, inside randomized test cases we want to only turn the error reporting on for one last time after all the generating and shrinking. --- Userland/Libraries/LibTest/Macros.h | 110 +++++++++++++---------- Userland/Libraries/LibTest/TestSuite.cpp | 19 ++++ Userland/Libraries/LibTest/TestSuite.h | 9 ++ 3 files changed, 90 insertions(+), 48 deletions(-) diff --git a/Userland/Libraries/LibTest/Macros.h b/Userland/Libraries/LibTest/Macros.h index ed01cedfdac..004154ee3e7 100644 --- a/Userland/Libraries/LibTest/Macros.h +++ b/Userland/Libraries/LibTest/Macros.h @@ -26,59 +26,71 @@ TestResult current_test_result(); Randomized::RandomnessSource& randomness_source(); void set_randomness_source(Randomized::RandomnessSource); + +bool is_reporting_enabled(); +void enable_reporting(); +void disable_reporting(); } -#define EXPECT_EQ(a, b) \ - do { \ - auto lhs = (a); \ - auto rhs = (b); \ - if (lhs != rhs) { \ - ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ - ::Test::set_current_test_result(::Test::TestResult::Failed); \ - } \ +#define EXPECT_EQ(a, b) \ + do { \ + auto lhs = (a); \ + auto rhs = (b); \ + if (lhs != rhs) { \ + if (::Test::is_reporting_enabled()) \ + ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", \ + __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ + } \ } while (false) -#define EXPECT_EQ_TRUTH(a, b) \ - do { \ - auto lhs = (a); \ - auto rhs = (b); \ - bool ltruth = static_cast(lhs); \ - bool rtruth = static_cast(rhs); \ - if (ltruth != rtruth) { \ - ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ_TRUTH({}, {}) failed with lhs={} ({}) and rhs={} ({})", \ - __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, ltruth, FormatIfSupported { rhs }, rtruth); \ - ::Test::set_current_test_result(::Test::TestResult::Failed); \ - } \ +#define EXPECT_EQ_TRUTH(a, b) \ + do { \ + auto lhs = (a); \ + auto rhs = (b); \ + bool ltruth = static_cast(lhs); \ + bool rtruth = static_cast(rhs); \ + if (ltruth != rtruth) { \ + if (::Test::is_reporting_enabled()) \ + ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ_TRUTH({}, {}) failed with lhs={} ({}) and rhs={} ({})", \ + __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, ltruth, FormatIfSupported { rhs }, rtruth); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ + } \ } while (false) // If you're stuck and `EXPECT_EQ` seems to refuse to print anything useful, // try this: It'll spit out a nice compiler error telling you why it doesn't print. -#define EXPECT_EQ_FORCE(a, b) \ - do { \ - auto lhs = (a); \ - auto rhs = (b); \ - if (lhs != rhs) { \ - ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, lhs, rhs); \ - ::Test::set_current_test_result(::Test::TestResult::Failed); \ - } \ +#define EXPECT_EQ_FORCE(a, b) \ + do { \ + auto lhs = (a); \ + auto rhs = (b); \ + if (lhs != rhs) { \ + if (::Test::is_reporting_enabled()) \ + ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", \ + __FILE__, __LINE__, #a, #b, lhs, rhs); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ + } \ } while (false) -#define EXPECT_NE(a, b) \ - do { \ - auto lhs = (a); \ - auto rhs = (b); \ - if (lhs == rhs) { \ - ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_NE({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ - ::Test::set_current_test_result(::Test::TestResult::Failed); \ - } \ +#define EXPECT_NE(a, b) \ + do { \ + auto lhs = (a); \ + auto rhs = (b); \ + if (lhs == rhs) { \ + if (::Test::is_reporting_enabled()) \ + ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_NE({}, {}) failed with lhs={} and rhs={}", \ + __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ + } \ } while (false) -#define EXPECT(x) \ - do { \ - if (!(x)) { \ - ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT({}) failed", __FILE__, __LINE__, #x); \ - ::Test::set_current_test_result(::Test::TestResult::Failed); \ - } \ +#define EXPECT(x) \ + do { \ + if (!(x)) { \ + if (::Test::is_reporting_enabled()) \ + ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT({}) failed", __FILE__, __LINE__, #x); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ + } \ } while (false) #define EXPECT_APPROXIMATE_WITH_ERROR(a, b, err) \ @@ -87,19 +99,21 @@ void set_randomness_source(Randomized::RandomnessSource); auto expect_close_rhs = b; \ auto expect_close_diff = static_cast(expect_close_lhs) - static_cast(expect_close_rhs); \ if (AK::fabs(expect_close_diff) > (err)) { \ - ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_APPROXIMATE({}, {})" \ - " failed with lhs={}, rhs={}, (lhs-rhs)={}", \ - __FILE__, __LINE__, #a, #b, expect_close_lhs, expect_close_rhs, expect_close_diff); \ + if (::Test::is_reporting_enabled()) \ + ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_APPROXIMATE({}, {})" \ + " failed with lhs={}, rhs={}, (lhs-rhs)={}", \ + __FILE__, __LINE__, #a, #b, expect_close_lhs, expect_close_rhs, expect_close_diff); \ ::Test::set_current_test_result(::Test::TestResult::Failed); \ } \ } while (false) #define EXPECT_APPROXIMATE(a, b) EXPECT_APPROXIMATE_WITH_ERROR(a, b, 0.0000005) -#define FAIL(message) \ - do { \ - ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: {}", __FILE__, __LINE__, message); \ - ::Test::set_current_test_result(::Test::TestResult::Failed); \ +#define FAIL(message) \ + do { \ + if (::Test::is_reporting_enabled()) \ + ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: {}", __FILE__, __LINE__, message); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } while (false) // To use, specify the lambda to execute in a sub process and verify it exits: diff --git a/Userland/Libraries/LibTest/TestSuite.cpp b/Userland/Libraries/LibTest/TestSuite.cpp index f02321d01a7..76076de110b 100644 --- a/Userland/Libraries/LibTest/TestSuite.cpp +++ b/Userland/Libraries/LibTest/TestSuite.cpp @@ -76,6 +76,24 @@ void set_suite_setup_function(Function setup) TestSuite::the().set_suite_setup(move(setup)); } +// Declared in Macros.h +bool is_reporting_enabled() +{ + return TestSuite::the().is_reporting_enabled(); +} + +// Declared in Macros.h +void enable_reporting() +{ + TestSuite::the().enable_reporting(); +} + +// Declared in Macros.h +void disable_reporting() +{ + TestSuite::the().disable_reporting(); +} + static DeprecatedString test_result_to_string(TestResult result) { switch (result) { @@ -164,6 +182,7 @@ int TestSuite::run(Vector> const& tests) warnln("Running {} '{}'.", test_type, t->name()); m_current_test_result = TestResult::NotRun; + enable_reporting(); u64 total_time = 0; u64 sum_of_squared_times = 0; diff --git a/Userland/Libraries/LibTest/TestSuite.h b/Userland/Libraries/LibTest/TestSuite.h index 9ee1a6466fa..f4816bb1c22 100644 --- a/Userland/Libraries/LibTest/TestSuite.h +++ b/Userland/Libraries/LibTest/TestSuite.h @@ -53,6 +53,14 @@ public: void set_randomness_source(Randomized::RandomnessSource source) { m_randomness_source = move(source); } Randomized::RandomnessSource& randomness_source() { return m_randomness_source; } + // Dictates whether FAIL(), EXPECT() and similar macros in LibTest/Macros.h + // print messages or not. This is important for randomized tests because + // they run the test function many times in a row, and we only want to + // report the _minimal_ (shrunk) failure to the user, not all of them. + bool is_reporting_enabled() { return m_reporting_enabled; } + void enable_reporting() { m_reporting_enabled = true; } + void disable_reporting() { m_reporting_enabled = false; } + private: static TestSuite* s_global; Vector> m_cases; @@ -63,6 +71,7 @@ private: Function m_setup; TestResult m_current_test_result = TestResult::NotRun; Randomized::RandomnessSource m_randomness_source = Randomized::RandomnessSource::live(); + bool m_reporting_enabled = true; }; }