LibTest: Expand test result bool to a TestResult

This will be used in the randomized tests a lot more than it is in the
unit tests / benchmarks; randomized tests will run the test function
multiple times, check the result and optionally start shrinking the
failing input. Generators will also be able to fail, resulting in some
of the new TestResult variants.

Userland/Libraries/LibTest/Macros.h

@@ -11,6 +11,7 @@
 #include <AK/CheckedFormatString.h>
 #include <AK/Math.h>
 #include <LibTest/CrashTest.h>
+#include <LibTest/TestResult.h>
 
 namespace AK {
 template<typename... Parameters>
@@ -18,8 +19,9 @@ void warnln(CheckedFormatString<Parameters...>&& fmtstr, Parameters const&...);
 }
 
 namespace Test {
-// Declare a helper so that we can call it from VERIFY in included headers
-void current_test_case_did_fail();
+// Declare helpers so that we can call them from VERIFY in included headers
+// the setter for TestResult is already declared in TestResult.h
+TestResult current_test_result();
 }
 
 #define EXPECT_EQ(a, b)                                                                                                                                                                      \
@@ -28,7 +30,7 @@ void current_test_case_did_fail();
         auto rhs = (b);                                                                                                                                                                      \
         if (lhs != rhs) {                                                                                                                                                                    \
             ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \
-            ::Test::current_test_case_did_fail();                                                                                                                                            \
+            ::Test::set_current_test_result(::Test::TestResult::Failed);                                                                                                                     \
         }                                                                                                                                                                                    \
     } while (false)
 
@@ -41,7 +43,7 @@ void current_test_case_did_fail();
         if (ltruth != rtruth) {                                                                                           \
             ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ_TRUTH({}, {}) failed with lhs={} ({}) and rhs={} ({})", \
                 __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, ltruth, FormatIfSupported { rhs }, rtruth);        \
-            ::Test::current_test_case_did_fail();                                                                         \
+            ::Test::set_current_test_result(::Test::TestResult::Failed);                                                  \
         }                                                                                                                 \
     } while (false)
 
@@ -53,7 +55,7 @@ void current_test_case_did_fail();
         auto rhs = (b);                                                                                                                          \
         if (lhs != rhs) {                                                                                                                        \
             ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_EQ({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, lhs, rhs); \
-            ::Test::current_test_case_did_fail();                                                                                                \
+            ::Test::set_current_test_result(::Test::TestResult::Failed);                                                                         \
         }                                                                                                                                        \
     } while (false)
 
@@ -63,7 +65,7 @@ void current_test_case_did_fail();
         auto rhs = (b);                                                                                                                                                                      \
         if (lhs == rhs) {                                                                                                                                                                    \
             ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_NE({}, {}) failed with lhs={} and rhs={}", __FILE__, __LINE__, #a, #b, FormatIfSupported { lhs }, FormatIfSupported { rhs }); \
-            ::Test::current_test_case_did_fail();                                                                                                                                            \
+            ::Test::set_current_test_result(::Test::TestResult::Failed);                                                                                                                     \
         }                                                                                                                                                                                    \
     } while (false)
 
@@ -71,7 +73,7 @@ void current_test_case_did_fail();
     do {                                                                                             \
         if (!(x)) {                                                                                  \
             ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT({}) failed", __FILE__, __LINE__, #x); \
-            ::Test::current_test_case_did_fail();                                                    \
+            ::Test::set_current_test_result(::Test::TestResult::Failed);                             \
         }                                                                                            \
     } while (false)
 
@@ -84,7 +86,7 @@ void current_test_case_did_fail();
             ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: EXPECT_APPROXIMATE({}, {})"                             \
                          " failed with lhs={}, rhs={}, (lhs-rhs)={}",                                           \
                 __FILE__, __LINE__, #a, #b, expect_close_lhs, expect_close_rhs, expect_close_diff);             \
-            ::Test::current_test_case_did_fail();                                                               \
+            ::Test::set_current_test_result(::Test::TestResult::Failed);                                        \
         }                                                                                                       \
     } while (false)
 
@@ -93,32 +95,32 @@ void current_test_case_did_fail();
 #define FAIL(message)                                                                  \
     do {                                                                               \
         ::AK::warnln("\033[31;1mFAIL\033[0m: {}:{}: {}", __FILE__, __LINE__, message); \
-        ::Test::current_test_case_did_fail();                                          \
+        ::Test::set_current_test_result(::Test::TestResult::Failed);                   \
     } while (false)
 
 // To use, specify the lambda to execute in a sub process and verify it exits:
 //  EXPECT_CRASH("This should fail", []{
 //      return Test::Crash::Failure::DidNotCrash;
 //  });
-#define EXPECT_CRASH(test_message, test_func)       \
-    do {                                            \
-        Test::Crash crash(test_message, test_func); \
-        if (!crash.run())                           \
-            ::Test::current_test_case_did_fail();   \
+#define EXPECT_CRASH(test_message, test_func)                            \
+    do {                                                                 \
+        Test::Crash crash(test_message, test_func);                      \
+        if (!crash.run())                                                \
+            ::Test::set_current_test_result(::Test::TestResult::Failed); \
     } while (false)
 
-#define EXPECT_CRASH_WITH_SIGNAL(test_message, signal, test_func) \
-    do {                                                          \
-        Test::Crash crash(test_message, test_func, (signal));     \
-        if (!crash.run())                                         \
-            ::Test::current_test_case_did_fail();                 \
+#define EXPECT_CRASH_WITH_SIGNAL(test_message, signal, test_func)        \
+    do {                                                                 \
+        Test::Crash crash(test_message, test_func, (signal));            \
+        if (!crash.run())                                                \
+            ::Test::set_current_test_result(::Test::TestResult::Failed); \
     } while (false)
 
-#define EXPECT_NO_CRASH(test_message, test_func)       \
-    do {                                               \
-        Test::Crash crash(test_message, test_func, 0); \
-        if (!crash.run())                              \
-            ::Test::current_test_case_did_fail();      \
+#define EXPECT_NO_CRASH(test_message, test_func)                         \
+    do {                                                                 \
+        Test::Crash crash(test_message, test_func, 0);                   \
+        if (!crash.run())                                                \
+            ::Test::set_current_test_result(::Test::TestResult::Failed); \
     } while (false)
 
 #define TRY_OR_FAIL(expression)                                                                      \

Userland/Libraries/LibTest/TestResult.h

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2023, Martin Janiczek <martin@janiczek.cz>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+namespace Test {
+
+// TestResult signals to the TestSuite how the TestCase execution went.
+enum class TestResult {
+    NotRun,
+
+    // Test fn ran to completion without setting any of the below flags
+    Passed,
+
+    // Didn't get through EXPECT(...).
+    Failed,
+};
+
+// Used eg. to signal we've ran out of prerecorded random bits.
+// Defined in TestSuite.cpp
+void set_current_test_result(TestResult);
+
+} // namespace Test

Userland/Libraries/LibTest/TestSuite.cpp

@@ -9,6 +9,7 @@
 
 #include <AK/Function.h>
 #include <LibCore/ArgsParser.h>
+#include <LibTest/TestResult.h>
 #include <LibTest/TestSuite.h>
 #include <math.h>
 #include <stdlib.h>
@@ -40,9 +41,15 @@ private:
 };
 
 // Declared in Macros.h
-void current_test_case_did_fail()
+TestResult current_test_result()
 {
-    TestSuite::the().current_test_case_did_fail();
+    return TestSuite::the().current_test_result();
+}
+
+// Declared in Macros.h
+void set_current_test_result(TestResult result)
+{
+    TestSuite::the().set_current_test_result(result);
 }
 
 // Declared in TestCase.h
@@ -57,6 +64,20 @@ void set_suite_setup_function(Function<void()> setup)
     TestSuite::the().set_suite_setup(move(setup));
 }
 
+static DeprecatedString test_result_to_string(TestResult result)
+{
+    switch (result) {
+    case TestResult::NotRun:
+        return "Not run";
+    case TestResult::Passed:
+        return "Completed";
+    case TestResult::Failed:
+        return "Failed";
+    default:
+        return "Unknown TestResult";
+    }
+}
+
 int TestSuite::main(DeprecatedString const& suite_name, Span<StringView> arguments)
 {
     m_suite_name = suite_name;
@@ -116,8 +137,11 @@ Vector<NonnullRefPtr<TestCase>> TestSuite::find_cases(DeprecatedString const& se
 int TestSuite::run(Vector<NonnullRefPtr<TestCase>> const& tests)
 {
     size_t test_count = 0;
+    size_t test_passed_count = 0;
     size_t test_failed_count = 0;
     size_t benchmark_count = 0;
+    size_t benchmark_passed_count = 0;
+    size_t benchmark_failed_count = 0;
     TestElapsedTimer global_timer;
 
     for (auto const& t : tests) {
@@ -125,7 +149,7 @@ int TestSuite::run(Vector<NonnullRefPtr<TestCase>> const& tests)
         auto const repetitions = t->is_benchmark() ? m_benchmark_repetitions : 1;
 
         warnln("Running {} '{}'.", test_type, t->name());
-        m_current_test_case_passed = true;
+        m_current_test_result = TestResult::NotRun;
 
         u64 total_time = 0;
         u64 sum_of_squared_times = 0;
@@ -140,6 +164,10 @@ int TestSuite::run(Vector<NonnullRefPtr<TestCase>> const& tests)
             sum_of_squared_times += iteration_time * iteration_time;
             min_time = min(min_time, iteration_time);
             max_time = max(max_time, iteration_time);
+
+            // Non-randomized tests don't touch the test result when passing.
+            if (m_current_test_result == TestResult::NotRun)
+                m_current_test_result = TestResult::Passed;
         }
 
         if (repetitions != 1) {
@@ -148,22 +176,40 @@ int TestSuite::run(Vector<NonnullRefPtr<TestCase>> const& tests)
             double standard_deviation = sqrt((sum_of_squared_times + repetitions * average_squared - 2 * total_time * average) / (repetitions - 1));
 
             dbgln("{} {} '{}' on average in {:.1f}±{:.1f}ms (min={}ms, max={}ms, total={}ms)",
-                m_current_test_case_passed ? "Completed" : "Failed", test_type, t->name(),
+                test_result_to_string(m_current_test_result), test_type, t->name(),
                 average, standard_deviation, min_time, max_time, total_time);
         } else {
-            dbgln("{} {} '{}' in {}ms", m_current_test_case_passed ? "Completed" : "Failed", test_type, t->name(), total_time);
+            dbgln("{} {} '{}' in {}ms", test_result_to_string(m_current_test_result), test_type, t->name(), total_time);
         }
 
         if (t->is_benchmark()) {
             m_benchtime += total_time;
             benchmark_count++;
+
+            switch (m_current_test_result) {
+            case TestResult::Passed:
+                benchmark_passed_count++;
+                break;
+            case TestResult::Failed:
+                benchmark_failed_count++;
+                break;
+            default:
+                break;
+            }
         } else {
             m_testtime += total_time;
             test_count++;
-        }
 
-        if (!m_current_test_case_passed) {
-            test_failed_count++;
+            switch (m_current_test_result) {
+            case TestResult::Passed:
+                test_passed_count++;
+                break;
+            case TestResult::Failed:
+                test_failed_count++;
+                break;
+            default:
+                break;
+            }
         }
     }
 
@@ -175,10 +221,29 @@ int TestSuite::run(Vector<NonnullRefPtr<TestCase>> const& tests)
         m_benchtime,
         global_timer.elapsed_milliseconds() - (m_testtime + m_benchtime));
 
-    if (test_count != 0)
-        dbgln("Out of {} tests, {} passed and {} failed.", test_count, test_count - test_failed_count, test_failed_count);
+    if (test_count != 0) {
+        if (test_passed_count == test_count) {
+            dbgln("All {} tests passed.", test_count);
+        } else if (test_passed_count + test_failed_count == test_count) {
+            dbgln("Out of {} tests, {} passed and {} failed.", test_count, test_passed_count, test_failed_count);
+        } else {
+            dbgln("Out of {} tests, {} passed, {} failed and {} didn't finish for other reasons.", test_count, test_passed_count, test_failed_count, test_count - test_passed_count - test_failed_count);
+        }
+    }
 
-    return (int)test_failed_count;
-}
+    if (benchmark_count != 0) {
+        if (benchmark_passed_count == benchmark_count) {
+            dbgln("All {} benchmarks passed.", benchmark_count);
+        } else if (benchmark_passed_count + benchmark_failed_count == benchmark_count) {
+            dbgln("Out of {} benchmarks, {} passed and {} failed.", benchmark_count, benchmark_passed_count, benchmark_failed_count);
+        } else {
+            dbgln("Out of {} benchmarks, {} passed, {} failed and {} didn't finish for other reasons.", benchmark_count, benchmark_passed_count, benchmark_failed_count, benchmark_count - benchmark_passed_count - benchmark_failed_count);
+        }
+    }
 
+    // We have multiple TestResults, all except for Passed being "bad".
+    // Let's get a count of them:
+    return (int)(test_count - test_passed_count + benchmark_count - benchmark_passed_count);
 }
+
+} // namespace Test

Userland/Libraries/LibTest/TestSuite.h

@@ -13,6 +13,7 @@
 #include <AK/Function.h>
 #include <AK/Vector.h>
 #include <LibTest/TestCase.h>
+#include <LibTest/TestResult.h>
 
 namespace Test {
 
@@ -40,7 +41,8 @@ public:
         m_cases.append(test_case);
     }
 
-    void current_test_case_did_fail() { m_current_test_case_passed = false; }
+    TestResult current_test_result() const { return m_current_test_result; }
+    void set_current_test_result(TestResult result) { m_current_test_result = result; }
 
     void set_suite_setup(Function<void()> setup) { m_setup = move(setup); }
 
@@ -51,8 +53,8 @@ private:
     u64 m_benchtime = 0;
     DeprecatedString m_suite_name;
     u64 m_benchmark_repetitions = 1;
-    bool m_current_test_case_passed = true;
     Function<void()> m_setup;
+    TestResult m_current_test_result = TestResult::NotRun;
 };
 
 }