diff --git a/Userland/Libraries/LibTest/Macros.h b/Userland/Libraries/LibTest/Macros.h index ed01cedfdac..004154ee3e7 100644 --- a/Userland/Libraries/LibTest/Macros.h +++ b/Userland/Libraries/LibTest/Macros.h @@ -26,59 +26,71 @@ TestResult current_test_result(); Randomized::RandomnessSource& randomness_source(); void set_randomness_source(Randomized::RandomnessSource); + +bool is_reporting_enabled(); +void enable_reporting(); +void disable_reporting(); } -#define EXPECT_EQ(a, b) \ - do { \ - auto lhs = (a); \ - auto rhs = (b); \ - if (lhs != rhs) { \ - ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ - ::Test::set_current_test_result(::Test::TestResult::Failed); \ - } \ +#define EXPECT_EQ(a, b) \ + do { \ + auto lhs = (a); \ + auto rhs = (b); \ + if (lhs != rhs) { \ + if (::Test::is_reporting_enabled()) \ + ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", \ + __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ + } \ } while (false) -#define EXPECT_EQ_TRUTH(a, b) \ - do { \ - auto lhs = (a); \ - auto rhs = (b); \ - bool ltruth = static_cast(lhs); \ - bool rtruth = static_cast(rhs); \ - if (ltruth != rtruth) { \ - ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ_TRUTH({}, {}) failed with lhs={} ({}) and rhs={} ({})", \ - __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, ltruth, FormatIfSupported { rhs }, rtruth); \ - ::Test::set_current_test_result(::Test::TestResult::Failed); \ - } \ +#define EXPECT_EQ_TRUTH(a, b) \ + do { \ + auto lhs = (a); \ + auto rhs = (b); \ + bool ltruth = static_cast(lhs); \ + bool rtruth = static_cast(rhs); \ + if (ltruth != rtruth) { \ + if (::Test::is_reporting_enabled()) \ + ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ_TRUTH({}, {}) failed with lhs={} ({}) and rhs={} ({})", \ + __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, ltruth, FormatIfSupported { rhs }, rtruth); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ + } \ } while (false) // If you're stuck and `EXPECT_EQ` seems to refuse to print anything useful, // try this: It'll spit out a nice compiler error telling you why it doesn't print. -#define EXPECT_EQ_FORCE(a, b) \ - do { \ - auto lhs = (a); \ - auto rhs = (b); \ - if (lhs != rhs) { \ - ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, lhs, rhs); \ - ::Test::set_current_test_result(::Test::TestResult::Failed); \ - } \ +#define EXPECT_EQ_FORCE(a, b) \ + do { \ + auto lhs = (a); \ + auto rhs = (b); \ + if (lhs != rhs) { \ + if (::Test::is_reporting_enabled()) \ + ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", \ + __FILE__, __LINE__, #a, #b, lhs, rhs); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ + } \ } while (false) -#define EXPECT_NE(a, b) \ - do { \ - auto lhs = (a); \ - auto rhs = (b); \ - if (lhs == rhs) { \ - ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_NE({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ - ::Test::set_current_test_result(::Test::TestResult::Failed); \ - } \ +#define EXPECT_NE(a, b) \ + do { \ + auto lhs = (a); \ + auto rhs = (b); \ + if (lhs == rhs) { \ + if (::Test::is_reporting_enabled()) \ + ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_NE({}, {}) failed with lhs={} and rhs={}", \ + __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ + } \ } while (false) -#define EXPECT(x) \ - do { \ - if (!(x)) { \ - ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT({}) failed", __FILE__, __LINE__, #x); \ - ::Test::set_current_test_result(::Test::TestResult::Failed); \ - } \ +#define EXPECT(x) \ + do { \ + if (!(x)) { \ + if (::Test::is_reporting_enabled()) \ + ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT({}) failed", __FILE__, __LINE__, #x); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ + } \ } while (false) #define EXPECT_APPROXIMATE_WITH_ERROR(a, b, err) \ @@ -87,19 +99,21 @@ void set_randomness_source(Randomized::RandomnessSource); auto expect_close_rhs = b; \ auto expect_close_diff = static_cast(expect_close_lhs) - static_cast(expect_close_rhs); \ if (AK::fabs(expect_close_diff) > (err)) { \ - ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_APPROXIMATE({}, {})" \ - " failed with lhs={}, rhs={}, (lhs-rhs)={}", \ - __FILE__, __LINE__, #a, #b, expect_close_lhs, expect_close_rhs, expect_close_diff); \ + if (::Test::is_reporting_enabled()) \ + ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_APPROXIMATE({}, {})" \ + " failed with lhs={}, rhs={}, (lhs-rhs)={}", \ + __FILE__, __LINE__, #a, #b, expect_close_lhs, expect_close_rhs, expect_close_diff); \ ::Test::set_current_test_result(::Test::TestResult::Failed); \ } \ } while (false) #define EXPECT_APPROXIMATE(a, b) EXPECT_APPROXIMATE_WITH_ERROR(a, b, 0.0000005) -#define FAIL(message) \ - do { \ - ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: {}", __FILE__, __LINE__, message); \ - ::Test::set_current_test_result(::Test::TestResult::Failed); \ +#define FAIL(message) \ + do { \ + if (::Test::is_reporting_enabled()) \ + ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: {}", __FILE__, __LINE__, message); \ + ::Test::set_current_test_result(::Test::TestResult::Failed); \ } while (false) // To use, specify the lambda to execute in a sub process and verify it exits: diff --git a/Userland/Libraries/LibTest/TestSuite.cpp b/Userland/Libraries/LibTest/TestSuite.cpp index f02321d01a7..76076de110b 100644 --- a/Userland/Libraries/LibTest/TestSuite.cpp +++ b/Userland/Libraries/LibTest/TestSuite.cpp @@ -76,6 +76,24 @@ void set_suite_setup_function(Function setup) TestSuite::the().set_suite_setup(move(setup)); } +// Declared in Macros.h +bool is_reporting_enabled() +{ + return TestSuite::the().is_reporting_enabled(); +} + +// Declared in Macros.h +void enable_reporting() +{ + TestSuite::the().enable_reporting(); +} + +// Declared in Macros.h +void disable_reporting() +{ + TestSuite::the().disable_reporting(); +} + static DeprecatedString test_result_to_string(TestResult result) { switch (result) { @@ -164,6 +182,7 @@ int TestSuite::run(Vector> const& tests) warnln("Running {} '{}'.", test_type, t->name()); m_current_test_result = TestResult::NotRun; + enable_reporting(); u64 total_time = 0; u64 sum_of_squared_times = 0; diff --git a/Userland/Libraries/LibTest/TestSuite.h b/Userland/Libraries/LibTest/TestSuite.h index 9ee1a6466fa..f4816bb1c22 100644 --- a/Userland/Libraries/LibTest/TestSuite.h +++ b/Userland/Libraries/LibTest/TestSuite.h @@ -53,6 +53,14 @@ public: void set_randomness_source(Randomized::RandomnessSource source) { m_randomness_source = move(source); } Randomized::RandomnessSource& randomness_source() { return m_randomness_source; } + // Dictates whether FAIL(), EXPECT() and similar macros in LibTest/Macros.h + // print messages or not. This is important for randomized tests because + // they run the test function many times in a row, and we only want to + // report the _minimal_ (shrunk) failure to the user, not all of them. + bool is_reporting_enabled() { return m_reporting_enabled; } + void enable_reporting() { m_reporting_enabled = true; } + void disable_reporting() { m_reporting_enabled = false; } + private: static TestSuite* s_global; Vector> m_cases; @@ -63,6 +71,7 @@ private: Function m_setup; TestResult m_current_test_result = TestResult::NotRun; Randomized::RandomnessSource m_randomness_source = Randomized::RandomnessSource::live(); + bool m_reporting_enabled = true; }; }