From c91511b883a4a342daecf447a0ca0f1ba0995b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?kleines=20Filmr=C3=B6llchen?= Date: Mon, 25 Jul 2022 13:28:16 +0200 Subject: [PATCH] Meta+Tests: Allow running FLAC spec tests The FLAC "spec tests", or rather the test suite by xiph that exercises weird FLAC features and edge cases, can be found at https://github.com/ietf-wg-cellar/flac-test-files and is a good challenge for our FLAC decoder to become more spec compliant. Running these tests is similar to LibWasm spec tests, you need to pass INCLUDE_FLAC_SPEC_TESTS to CMake. As of integrating these tests, 23 out of 63 fail. :yakplus: --- CMakeLists.txt | 1 + Documentation/AdvancedBuildInstructions.md | 1 + Meta/CMake/common_options.cmake | 1 + Meta/CMake/flac_spec_tests.cmake | 29 +++++++++++ Meta/Lagom/CMakeLists.txt | 7 +++ Tests/CMakeLists.txt | 1 + Tests/LibAudio/CMakeLists.txt | 9 ++++ Tests/LibAudio/TestFLACSpec.cpp | 59 ++++++++++++++++++++++ 8 files changed, 108 insertions(+) create mode 100644 Meta/CMake/flac_spec_tests.cmake create mode 100644 Tests/LibAudio/CMakeLists.txt create mode 100644 Tests/LibAudio/TestFLACSpec.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 3ce94fa769f..977403de9e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -206,6 +206,7 @@ option(BUILD_EVERYTHING "Build all optional components" ON) include(utils) include(wasm_spec_tests) +include(flac_spec_tests) serenity_component( Tests diff --git a/Documentation/AdvancedBuildInstructions.md b/Documentation/AdvancedBuildInstructions.md index 94c7199429c..c35513dbd3d 100644 --- a/Documentation/AdvancedBuildInstructions.md +++ b/Documentation/AdvancedBuildInstructions.md @@ -62,6 +62,7 @@ There are some optional features that can be enabled during compilation that are - `ENABLE_JAKT`: builds the `jakt` compiler as a Lagom host tool and enables building applications and libraries that are written in the jakt language. - `JAKT_SOURCE_DIR`: `jakt` developer's local checkout of the jakt programming language for rapid testing. To use a local checkout, set to an absolute path when changing the CMake cache of Lagom. e.g. ``cmake -S Meta/Lagom -B Build/lagom -DENABLE_JAKT=ON -DJAKT_SOURCE_DIR=/home/me/jakt`` - `INCLUDE_WASM_SPEC_TESTS`: downloads and includes the WebAssembly spec testsuite tests. In order to use this option, you will need to install `prettier` and `wabt`. wabt version 1.0.23 or higher is required to pre-process the WebAssembly spec testsuite. +- `INCLUDE_FLAC_SPEC_TESTS`: downloads and includes the xiph.org FLAC test suite. - `SERENITY_TOOLCHAIN`: Specifies whether to use the established GNU toolchain, or the experimental Clang-based toolchain for building SerenityOS. See the [Clang-based toolchain](#clang-based-toolchain) section below. - `SERENITY_ARCH`: Specifies which architecture to build for. Currently supported options are `i686` and `x86_64`. `x86_64` requires a separate toolchain build from `i686`. - `BUILD_`: builds the specified component, e.g. `BUILD_HEARTS` (note: must be all caps). Check the components.ini file in your build directory for a list of available components. Make sure to run `ninja clean` and `rm -rf Build/i686/Root` after disabling components. These options can be easily configured by using the `ConfigureComponents` utility. See the [Component Configuration](#component-configuration) section below. diff --git a/Meta/CMake/common_options.cmake b/Meta/CMake/common_options.cmake index da879ffaed3..8ca08c1a764 100644 --- a/Meta/CMake/common_options.cmake +++ b/Meta/CMake/common_options.cmake @@ -12,6 +12,7 @@ serenity_option(ENABLE_COMPILETIME_HEADER_CHECK OFF CACHE BOOL "Enable compileti serenity_option(ENABLE_TIME_ZONE_DATABASE_DOWNLOAD ON CACHE BOOL "Enable download of the IANA Time Zone Database at build time") serenity_option(ENABLE_UNICODE_DATABASE_DOWNLOAD ON CACHE BOOL "Enable download of Unicode UCD and CLDR files at build time") serenity_option(INCLUDE_WASM_SPEC_TESTS OFF CACHE BOOL "Download and include the WebAssembly spec testsuite") +serenity_option(INCLUDE_FLAC_SPEC_TESTS OFF CACHE BOOL "Download and include the FLAC spec testsuite") serenity_option(HACKSTUDIO_BUILD OFF CACHE BOOL "Automatically enabled when building from HackStudio") diff --git a/Meta/CMake/flac_spec_tests.cmake b/Meta/CMake/flac_spec_tests.cmake new file mode 100644 index 00000000000..2f37b75595b --- /dev/null +++ b/Meta/CMake/flac_spec_tests.cmake @@ -0,0 +1,29 @@ +include(utils) + +if(INCLUDE_FLAC_SPEC_TESTS) + if (CMAKE_PROJECT_NAME STREQUAL "SerenityOS") + set(SOURCE_DIR "${SerenityOS_SOURCE_DIR}") + else() + set(SOURCE_DIR "${SERENITY_PROJECT_ROOT}") + endif() + set(FLAC_SPEC_TEST_GZ_URL https://github.com/ietf-wg-cellar/flac-test-files/archive/refs/heads/main.tar.gz) + + set(FLAC_TEST_PATH ${CMAKE_BINARY_DIR}/Tests/LibAudio/FLAC CACHE PATH "Location of FLAC tests") + set(FLAC_SPEC_TEST_GZ_PATH ${FLAC_TEST_PATH}/flac-spec-testsuite.tar.gz) + set(FLAC_SPEC_TEST_PATH ${FLAC_TEST_PATH}/SpecTests) + + if(NOT EXISTS ${FLAC_SPEC_TEST_GZ_PATH}) + message(STATUS "Downloading the IETF CELLAR FLAC testsuite from ${FLAC_SPEC_TEST_GZ_URL}...") + download_file(${FLAC_SPEC_TEST_GZ_URL} ${FLAC_SPEC_TEST_GZ_PATH}) + endif() + + if(EXISTS ${FLAC_SPEC_TEST_GZ_PATH} AND NOT EXISTS ${FLAC_SPEC_TEST_PATH}) + file(MAKE_DIRECTORY ${FLAC_SPEC_TEST_PATH}) + message(STATUS "Extracting the FLAC testsuite from ${FLAC_SPEC_TEST_GZ_PATH}...") + execute_process(COMMAND "${TAR_TOOL}" -xzf ${FLAC_SPEC_TEST_GZ_PATH} -C ${FLAC_TEST_PATH} RESULT_VARIABLE tar_result) + if (NOT tar_result EQUAL 0) + message(FATAL_ERROR "Failed to unzip ${FLAC_TEST_PATH} from ${FLAC_SPEC_TEST_GZ_PATH} with status ${tar_result}") + endif() + file(RENAME "${FLAC_TEST_PATH}/flac-test-files-main/subset" ${FLAC_SPEC_TEST_PATH}) + endif() +endif() diff --git a/Meta/Lagom/CMakeLists.txt b/Meta/Lagom/CMakeLists.txt index 9e23947e613..d759939a642 100644 --- a/Meta/Lagom/CMakeLists.txt +++ b/Meta/Lagom/CMakeLists.txt @@ -61,6 +61,7 @@ if (ENABLE_FUZZERS_LIBFUZZER OR ENABLE_FUZZERS_OSSFUZZ) endif() include(wasm_spec_tests) +include(flac_spec_tests) include(lagom_compile_options) include(GNUInstallDirs) # make sure to include before we mess w/RPATH @@ -643,6 +644,12 @@ if (BUILD_LAGOM) lagom_test(${source} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../Tests/AK) endforeach() + # LibAudio + file(GLOB LIBAUDIO_TEST_SOURCES CONFIGURE_DEPENDS "../../Tests/LibAudio/*.cpp") + foreach(source ${LIBAUDIO_TEST_SOURCES}) + lagom_test(${source} LIBS LibAudio WORKING_DIRECTORY ${FLAC_TEST_PATH}) + endforeach() + # LibCore lagom_test(../../Tests/LibCore/TestLibCoreIODevice.cpp WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../Tests/LibCore) diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index 1023b38265f..92d6ea160f4 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -1,5 +1,6 @@ add_subdirectory(AK) add_subdirectory(Kernel) +add_subdirectory(LibAudio) add_subdirectory(LibC) add_subdirectory(LibCompress) add_subdirectory(LibCore) diff --git a/Tests/LibAudio/CMakeLists.txt b/Tests/LibAudio/CMakeLists.txt new file mode 100644 index 00000000000..5e70e6c5249 --- /dev/null +++ b/Tests/LibAudio/CMakeLists.txt @@ -0,0 +1,9 @@ +set(TEST_SOURCES + TestFLACSpec.cpp +) + +foreach(source IN LISTS TEST_SOURCES) + serenity_test("${source}" LibAudio LIBS LibAudio) +endforeach() + +install(DIRECTORY ${FLAC_SPEC_TEST_PATH} DESTINATION usr/Tests/LibAudio) diff --git a/Tests/LibAudio/TestFLACSpec.cpp b/Tests/LibAudio/TestFLACSpec.cpp new file mode 100644 index 00000000000..23a040d1e16 --- /dev/null +++ b/Tests/LibAudio/TestFLACSpec.cpp @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2022, kleines Filmröllchen + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include + +struct FlacTest : Test::TestCase { + FlacTest(LexicalPath path) + : Test::TestCase( + String::formatted("flac_spec_test_{}", path.basename()), [this]() { run(); }, false) + , m_path(std::move(path)) + { + } + + void run() const + { + auto loader = Audio::FlacLoaderPlugin { m_path.string() }; + if (auto result = loader.initialize(); result.is_error()) { + FAIL(String::formatted("{} (at {})", result.error().description, result.error().index)); + return; + } + + while (true) { + auto maybe_samples = loader.get_more_samples(2 * MiB); + if (maybe_samples.is_error()) { + FAIL(String::formatted("{} (at {})", maybe_samples.error().description, maybe_samples.error().index)); + return; + } + if (maybe_samples.value().is_empty()) + return; + } + } + + LexicalPath m_path; +}; + +struct DiscoverFLACTestsHack { + DiscoverFLACTestsHack() + { + // FIXME: Also run (our own) tests in this directory. + auto test_iterator = Core::DirIterator { "./SpecTests", Core::DirIterator::Flags::SkipParentAndBaseDir }; + + while (test_iterator.has_next()) { + auto file = LexicalPath { test_iterator.next_full_path() }; + if (file.extension() == "flac"sv) { + Test::add_test_case_to_suite(make_ref_counted(move(file))); + } + } + } +}; +// Hack taken from TEST_CASE; the above constructor will run as part of global initialization before the tests are actually executed +static struct DiscoverFLACTestsHack hack;