Meta+Tests: Allow running FLAC spec tests

The FLAC "spec tests", or rather the test suite by xiph that exercises
weird FLAC features and edge cases, can be found at
https://github.com/ietf-wg-cellar/flac-test-files and is a good
challenge for our FLAC decoder to become more spec compliant. Running
these tests is similar to LibWasm spec tests, you need to pass
INCLUDE_FLAC_SPEC_TESTS to CMake.

As of integrating these tests, 23 out of 63 fail. :yakplus:
This commit is contained in:
kleines Filmröllchen 2022-07-25 13:28:16 +02:00 committed by Linus Groh
parent 6587638ffe
commit c91511b883
Notes: sideshowbarker 2024-07-17 07:32:40 +09:00
8 changed files with 108 additions and 0 deletions

View file

@ -206,6 +206,7 @@ option(BUILD_EVERYTHING "Build all optional components" ON)
include(utils)
include(wasm_spec_tests)
include(flac_spec_tests)
serenity_component(
Tests

View file

@ -62,6 +62,7 @@ There are some optional features that can be enabled during compilation that are
- `ENABLE_JAKT`: builds the `jakt` compiler as a Lagom host tool and enables building applications and libraries that are written in the jakt language.
- `JAKT_SOURCE_DIR`: `jakt` developer's local checkout of the jakt programming language for rapid testing. To use a local checkout, set to an absolute path when changing the CMake cache of Lagom. e.g. ``cmake -S Meta/Lagom -B Build/lagom -DENABLE_JAKT=ON -DJAKT_SOURCE_DIR=/home/me/jakt``
- `INCLUDE_WASM_SPEC_TESTS`: downloads and includes the WebAssembly spec testsuite tests. In order to use this option, you will need to install `prettier` and `wabt`. wabt version 1.0.23 or higher is required to pre-process the WebAssembly spec testsuite.
- `INCLUDE_FLAC_SPEC_TESTS`: downloads and includes the xiph.org FLAC test suite.
- `SERENITY_TOOLCHAIN`: Specifies whether to use the established GNU toolchain, or the experimental Clang-based toolchain for building SerenityOS. See the [Clang-based toolchain](#clang-based-toolchain) section below.
- `SERENITY_ARCH`: Specifies which architecture to build for. Currently supported options are `i686` and `x86_64`. `x86_64` requires a separate toolchain build from `i686`.
- `BUILD_<component>`: builds the specified component, e.g. `BUILD_HEARTS` (note: must be all caps). Check the components.ini file in your build directory for a list of available components. Make sure to run `ninja clean` and `rm -rf Build/i686/Root` after disabling components. These options can be easily configured by using the `ConfigureComponents` utility. See the [Component Configuration](#component-configuration) section below.

View file

@ -12,6 +12,7 @@ serenity_option(ENABLE_COMPILETIME_HEADER_CHECK OFF CACHE BOOL "Enable compileti
serenity_option(ENABLE_TIME_ZONE_DATABASE_DOWNLOAD ON CACHE BOOL "Enable download of the IANA Time Zone Database at build time")
serenity_option(ENABLE_UNICODE_DATABASE_DOWNLOAD ON CACHE BOOL "Enable download of Unicode UCD and CLDR files at build time")
serenity_option(INCLUDE_WASM_SPEC_TESTS OFF CACHE BOOL "Download and include the WebAssembly spec testsuite")
serenity_option(INCLUDE_FLAC_SPEC_TESTS OFF CACHE BOOL "Download and include the FLAC spec testsuite")
serenity_option(HACKSTUDIO_BUILD OFF CACHE BOOL "Automatically enabled when building from HackStudio")

View file

@ -0,0 +1,29 @@
include(utils)
if(INCLUDE_FLAC_SPEC_TESTS)
if (CMAKE_PROJECT_NAME STREQUAL "SerenityOS")
set(SOURCE_DIR "${SerenityOS_SOURCE_DIR}")
else()
set(SOURCE_DIR "${SERENITY_PROJECT_ROOT}")
endif()
set(FLAC_SPEC_TEST_GZ_URL https://github.com/ietf-wg-cellar/flac-test-files/archive/refs/heads/main.tar.gz)
set(FLAC_TEST_PATH ${CMAKE_BINARY_DIR}/Tests/LibAudio/FLAC CACHE PATH "Location of FLAC tests")
set(FLAC_SPEC_TEST_GZ_PATH ${FLAC_TEST_PATH}/flac-spec-testsuite.tar.gz)
set(FLAC_SPEC_TEST_PATH ${FLAC_TEST_PATH}/SpecTests)
if(NOT EXISTS ${FLAC_SPEC_TEST_GZ_PATH})
message(STATUS "Downloading the IETF CELLAR FLAC testsuite from ${FLAC_SPEC_TEST_GZ_URL}...")
download_file(${FLAC_SPEC_TEST_GZ_URL} ${FLAC_SPEC_TEST_GZ_PATH})
endif()
if(EXISTS ${FLAC_SPEC_TEST_GZ_PATH} AND NOT EXISTS ${FLAC_SPEC_TEST_PATH})
file(MAKE_DIRECTORY ${FLAC_SPEC_TEST_PATH})
message(STATUS "Extracting the FLAC testsuite from ${FLAC_SPEC_TEST_GZ_PATH}...")
execute_process(COMMAND "${TAR_TOOL}" -xzf ${FLAC_SPEC_TEST_GZ_PATH} -C ${FLAC_TEST_PATH} RESULT_VARIABLE tar_result)
if (NOT tar_result EQUAL 0)
message(FATAL_ERROR "Failed to unzip ${FLAC_TEST_PATH} from ${FLAC_SPEC_TEST_GZ_PATH} with status ${tar_result}")
endif()
file(RENAME "${FLAC_TEST_PATH}/flac-test-files-main/subset" ${FLAC_SPEC_TEST_PATH})
endif()
endif()

View file

@ -61,6 +61,7 @@ if (ENABLE_FUZZERS_LIBFUZZER OR ENABLE_FUZZERS_OSSFUZZ)
endif()
include(wasm_spec_tests)
include(flac_spec_tests)
include(lagom_compile_options)
include(GNUInstallDirs) # make sure to include before we mess w/RPATH
@ -643,6 +644,12 @@ if (BUILD_LAGOM)
lagom_test(${source} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../Tests/AK)
endforeach()
# LibAudio
file(GLOB LIBAUDIO_TEST_SOURCES CONFIGURE_DEPENDS "../../Tests/LibAudio/*.cpp")
foreach(source ${LIBAUDIO_TEST_SOURCES})
lagom_test(${source} LIBS LibAudio WORKING_DIRECTORY ${FLAC_TEST_PATH})
endforeach()
# LibCore
lagom_test(../../Tests/LibCore/TestLibCoreIODevice.cpp WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../Tests/LibCore)

View file

@ -1,5 +1,6 @@
add_subdirectory(AK)
add_subdirectory(Kernel)
add_subdirectory(LibAudio)
add_subdirectory(LibC)
add_subdirectory(LibCompress)
add_subdirectory(LibCore)

View file

@ -0,0 +1,9 @@
set(TEST_SOURCES
TestFLACSpec.cpp
)
foreach(source IN LISTS TEST_SOURCES)
serenity_test("${source}" LibAudio LIBS LibAudio)
endforeach()
install(DIRECTORY ${FLAC_SPEC_TEST_PATH} DESTINATION usr/Tests/LibAudio)

View file

@ -0,0 +1,59 @@
/*
* Copyright (c) 2022, kleines Filmröllchen <filmroellchen@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/LexicalPath.h>
#include <AK/Types.h>
#include <LibAudio/FlacLoader.h>
#include <LibCore/DirIterator.h>
#include <LibCore/Stream.h>
#include <LibTest/TestCase.h>
struct FlacTest : Test::TestCase {
FlacTest(LexicalPath path)
: Test::TestCase(
String::formatted("flac_spec_test_{}", path.basename()), [this]() { run(); }, false)
, m_path(std::move(path))
{
}
void run() const
{
auto loader = Audio::FlacLoaderPlugin { m_path.string() };
if (auto result = loader.initialize(); result.is_error()) {
FAIL(String::formatted("{} (at {})", result.error().description, result.error().index));
return;
}
while (true) {
auto maybe_samples = loader.get_more_samples(2 * MiB);
if (maybe_samples.is_error()) {
FAIL(String::formatted("{} (at {})", maybe_samples.error().description, maybe_samples.error().index));
return;
}
if (maybe_samples.value().is_empty())
return;
}
}
LexicalPath m_path;
};
struct DiscoverFLACTestsHack {
DiscoverFLACTestsHack()
{
// FIXME: Also run (our own) tests in this directory.
auto test_iterator = Core::DirIterator { "./SpecTests", Core::DirIterator::Flags::SkipParentAndBaseDir };
while (test_iterator.has_next()) {
auto file = LexicalPath { test_iterator.next_full_path() };
if (file.extension() == "flac"sv) {
Test::add_test_case_to_suite(make_ref_counted<FlacTest>(move(file)));
}
}
}
};
// Hack taken from TEST_CASE; the above constructor will run as part of global initialization before the tests are actually executed
static struct DiscoverFLACTestsHack hack;