LibVideo: Implement CICP color space conversion

This adds a struct called CodingIndependentCodePoints and related enums
that are used by video codecs to define its color space that frames
must be converted from when displaying a video.

Pre-multiplied matrices and lookup tables are stored to avoid most of
the floating point division and exponentiation in the conversion.

Userland/Applications/VideoPlayer/main.cpp

@@ -12,6 +12,7 @@
 #include <LibGUI/Window.h>
 #include <LibGfx/Bitmap.h>
 #include <LibMain/Main.h>
+#include <LibVideo/Color/ColorConverter.h>
 #include <LibVideo/MatroskaReader.h>
 #include <LibVideo/VP9/Decoder.h>
 
@@ -91,6 +92,15 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
         auto uv_subsampling_y = vp9_decoder.get_uv_subsampling_y();
         auto uv_subsampling_x = vp9_decoder.get_uv_subsampling_x();
         Gfx::IntSize uv_size { y_size.width() >> uv_subsampling_x, y_size.height() >> uv_subsampling_y };
+        auto cicp = vp9_decoder.get_cicp_color_space();
+        cicp.default_code_points_if_unspecified(Video::ColorPrimaries::BT709, Video::TransferCharacteristics::BT709, Video::MatrixCoefficients::BT709);
+
+        auto color_converter_result = Video::ColorConverter::create(vp9_decoder.get_bit_depth(), cicp);
+        if (color_converter_result.is_error()) {
+            outln("Cannot convert video colors: {}", color_converter_result.release_error().string_literal());
+            return;
+        }
+        auto color_converter = color_converter_result.release_value();
 
         for (auto y_row = 0u; y_row < video_track.pixel_height; y_row++) {
             auto uv_row = y_row >> uv_subsampling_y;
@@ -99,17 +109,10 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
                 auto uv_column = y_column >> uv_subsampling_x;
 
                 auto y = output_y[y_row * y_size.width() + y_column];
-                auto cb = output_u[uv_row * uv_size.width() + uv_column];
-                auto cr = output_v[uv_row * uv_size.width() + uv_column];
-                // Convert from Rec.709 YCbCr to RGB.
-                auto r_float = floorf(clamp(y + (cr - 128) * 219.0f / 224.0f * 1.5748f, 0, 255));
-                auto g_float = floorf(clamp(y + (cb - 128) * 219.0f / 224.0f * -0.0722f * 1.8556f / 0.7152f + (cr - 128) * 219.0f / 224.0f * -0.2126f * 1.5748f / 0.7152f, 0, 255));
-                auto b_float = floorf(clamp(y + (cb - 128) * 219.0f / 224.0f * 1.8556f, 0, 255));
-                auto r = static_cast<u8>(r_float);
-                auto g = static_cast<u8>(g_float);
-                auto b = static_cast<u8>(b_float);
-
-                image->set_pixel(y_column, y_row, Gfx::Color(r, g, b));
+                auto u = output_u[uv_row * uv_size.width() + uv_column];
+                auto v = output_v[uv_row * uv_size.width() + uv_column];
+
+                image->set_pixel(y_column, y_row, color_converter.convert_yuv_to_full_range_rgb(y, u, v));
             }
         }
 

Userland/Libraries/LibVideo/CMakeLists.txt

@@ -1,4 +1,7 @@
 set(SOURCES
+    Color/ColorConverter.cpp
+    Color/ColorPrimaries.cpp
+    Color/TransferCharacteristics.cpp
     MatroskaReader.cpp
     VP9/BitStream.cpp
     VP9/Decoder.cpp

Userland/Libraries/LibVideo/Color/CodingIndependentCodePoints.h

@@ -0,0 +1,228 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/StringView.h>
+
+namespace Video {
+
+// CICP is defined by H.273:
+// https://www.itu.int/rec/T-REC-H.273/en
+// See the Section 8.
+// Current edition is from 07/21.
+
+enum class ColorPrimaries : u8 {
+    Reserved = 0,
+    BT709 = 1,
+    Unspecified = 2, // Used by codecs to indicate that an alternative value may be used
+    BT470M = 4,
+    BT470BG = 5,
+    BT601 = 6,
+    SMPTE240 = 7,
+    GenericFilm = 8,
+    BT2020 = 9,
+    XYZ = 10,
+    SMPTE431 = 11,
+    SMPTE432 = 12,
+    EBU3213 = 22,
+    // All other values are also Reserved for later use.
+};
+
+enum class TransferCharacteristics : u8 {
+    Reserved = 0,
+    BT709 = 1,
+    Unspecified = 2, // Used by codecs to indicate that an alternative value may be used
+    BT470M = 4,
+    BT470BG = 5,
+    BT601 = 6, // BT.601 or Rec. 601
+    SMPTE240 = 7,
+    Linear = 8,
+    Log100 = 9,
+    Log100Sqrt10 = 10,
+    IEC61966 = 11,
+    BT1361 = 12,
+    SRGB = 13,
+    BT2020BitDepth10 = 14,
+    BT2020BitDepth12 = 15,
+    SMPTE2084 = 16, // Also known as PQ
+    SMPTE428 = 17,
+    HLG = 18,
+    // All other values are also Reserved for later use.
+};
+
+enum class MatrixCoefficients : u8 {
+    Identity = 0, // Applies no transformation to input values
+    BT709 = 1,
+    Unspecified = 2, // Used by codecs to indicate that an alternative value may be used
+    FCC = 4,
+    BT470BG = 5,
+    BT601 = 6,
+    SMPTE240 = 7,
+    YCgCo = 8,
+    BT2020NonConstantLuminance = 9,
+    BT2020ConstantLuminance = 10,
+    SMPTE2085 = 11,
+    ChromaticityDerivedNonConstantLuminance = 12,
+    ChromaticityDerivedConstantLuminance = 13,
+    ICtCp = 14,
+    // All other values are Reserved for later use.
+};
+
+enum class ColorRange : u8 {
+    Studio = 0, // Y range 16..235, UV range 16..240
+    Full = 1,   // 0..255
+};
+
+// https://en.wikipedia.org/wiki/Coding-independent_code_points
+struct CodingIndependentCodePoints {
+public:
+    constexpr CodingIndependentCodePoints(ColorPrimaries color_primaries, TransferCharacteristics transfer_characteristics, MatrixCoefficients matrix_coefficients, ColorRange color_range)
+        : m_color_primaries(color_primaries)
+        , m_transfer_characteristics(transfer_characteristics)
+        , m_matrix_coefficients(matrix_coefficients)
+        , m_color_range(color_range)
+    {
+    }
+
+    constexpr ColorPrimaries color_primaries() const { return m_color_primaries; }
+    constexpr void set_color_primaries(ColorPrimaries value) { m_color_primaries = value; }
+    constexpr TransferCharacteristics transfer_characteristics() const { return m_transfer_characteristics; }
+    constexpr void set_transfer_characteristics(TransferCharacteristics value) { m_transfer_characteristics = value; }
+    constexpr MatrixCoefficients matrix_coefficients() const { return m_matrix_coefficients; }
+    constexpr void set_matrix_coefficients(MatrixCoefficients value) { m_matrix_coefficients = value; }
+    constexpr ColorRange color_range() const { return m_color_range; }
+    constexpr void set_color_range(ColorRange value) { m_color_range = value; }
+
+    constexpr void default_code_points_if_unspecified(ColorPrimaries cp, TransferCharacteristics tc, MatrixCoefficients mc)
+    {
+        if (color_primaries() == ColorPrimaries::Unspecified)
+            set_color_primaries(cp);
+        if (transfer_characteristics() == TransferCharacteristics::Unspecified)
+            set_transfer_characteristics(tc);
+        if (matrix_coefficients() == MatrixCoefficients::Unspecified)
+            set_matrix_coefficients(mc);
+    }
+
+private:
+    ColorPrimaries m_color_primaries;
+    TransferCharacteristics m_transfer_characteristics;
+    MatrixCoefficients m_matrix_coefficients;
+    ColorRange m_color_range;
+};
+
+constexpr StringView color_primaries_to_string(ColorPrimaries color_primaries)
+{
+    switch (color_primaries) {
+    case ColorPrimaries::Reserved:
+        return "Reserved"sv;
+    case ColorPrimaries::BT709:
+        return "BT.709"sv;
+    case ColorPrimaries::Unspecified:
+        return "Unspecified"sv;
+    case ColorPrimaries::BT470M:
+        return "BT.470 System M"sv;
+    case ColorPrimaries::BT470BG:
+        return "BT.470 System B, G"sv;
+    case ColorPrimaries::BT601:
+        return "BT.601"sv;
+    case ColorPrimaries::SMPTE240:
+        return "SMPTE ST 240"sv;
+    case ColorPrimaries::GenericFilm:
+        return "Generic film"sv;
+    case ColorPrimaries::BT2020:
+        return "BT.2020"sv;
+    case ColorPrimaries::XYZ:
+        return "CIE 1931 XYZ"sv;
+    case ColorPrimaries::SMPTE431:
+        return "SMPTE RP 431"sv;
+    case ColorPrimaries::SMPTE432:
+        return "SMPTE EG 432"sv;
+    case ColorPrimaries::EBU3213:
+        return "EBU Tech 3213"sv;
+    }
+    return "Reserved"sv;
+};
+
+constexpr StringView transfer_characteristics_to_string(TransferCharacteristics transfer_characteristics)
+{
+    switch (transfer_characteristics) {
+    case TransferCharacteristics::Reserved:
+        return "Reserved"sv;
+    case TransferCharacteristics::BT709:
+        return "BT.709"sv;
+    case TransferCharacteristics::Unspecified:
+        return "Unspecified"sv;
+    case TransferCharacteristics::BT470M:
+        return "BT.470 System M"sv;
+    case TransferCharacteristics::BT470BG:
+        return "BT.470 System B, G"sv;
+    case TransferCharacteristics::BT601:
+        return "BT.601"sv;
+    case TransferCharacteristics::SMPTE240:
+        return "SMPTE ST 240"sv;
+    case TransferCharacteristics::Linear:
+        return "Linear"sv;
+    case TransferCharacteristics::Log100:
+        return "Logarithmic (100:1 range)"sv;
+    case TransferCharacteristics::Log100Sqrt10:
+        return "Logarithmic (100xSqrt(10):1 range)"sv;
+    case TransferCharacteristics::IEC61966:
+        return "IEC 61966"sv;
+    case TransferCharacteristics::BT1361:
+        return "BT.1361"sv;
+    case TransferCharacteristics::SRGB:
+        return "sRGB"sv;
+    case TransferCharacteristics::BT2020BitDepth10:
+        return "BT.2020 (10-bit)"sv;
+    case TransferCharacteristics::BT2020BitDepth12:
+        return "BT.2020 (12-bit)"sv;
+    case TransferCharacteristics::SMPTE2084:
+        return "SMPTE ST 2084 (PQ)"sv;
+    case TransferCharacteristics::SMPTE428:
+        return "SMPTE ST 428"sv;
+    case TransferCharacteristics::HLG:
+        return "ARIB STD-B67 (HLG, BT.2100)"sv;
+    }
+    return "Reserved"sv;
+};
+
+constexpr StringView matrix_coefficients_to_string(MatrixCoefficients matrix_coefficients)
+{
+    switch (matrix_coefficients) {
+    case MatrixCoefficients::Identity:
+        return "Identity"sv;
+    case MatrixCoefficients::BT709:
+        return "BT.709"sv;
+    case MatrixCoefficients::Unspecified:
+        return "Unspecified"sv;
+    case MatrixCoefficients::FCC:
+        return "FCC (CFR 73.682)"sv;
+    case MatrixCoefficients::BT470BG:
+        return "BT.470 System B, G"sv;
+    case MatrixCoefficients::BT601:
+        return "BT.601"sv;
+    case MatrixCoefficients::SMPTE240:
+        return "SMPTE ST 240"sv;
+    case MatrixCoefficients::YCgCo:
+        return "YCgCo"sv;
+    case MatrixCoefficients::BT2020NonConstantLuminance:
+        return "BT.2020, non-constant luminance"sv;
+    case MatrixCoefficients::BT2020ConstantLuminance:
+        return "BT.2020, constant luminance"sv;
+    case MatrixCoefficients::SMPTE2085:
+        return "SMPTE ST 2085"sv;
+    case MatrixCoefficients::ChromaticityDerivedNonConstantLuminance:
+        return "Chromaticity-derived, non-constant luminance"sv;
+    case MatrixCoefficients::ChromaticityDerivedConstantLuminance:
+        return "Chromaticity-derived, constant luminance"sv;
+    case MatrixCoefficients::ICtCp:
+        return "BT.2100 ICtCp"sv;
+    }
+    return "Reserved"sv;
+};
+
+}

Userland/Libraries/LibVideo/Color/ColorConverter.cpp

@@ -0,0 +1,263 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Format.h>
+#include <AK/Math.h>
+#include <AK/StdLibExtras.h>
+#include <LibGfx/Matrix4x4.h>
+#include <LibVideo/Color/ColorPrimaries.h>
+#include <LibVideo/Color/TransferCharacteristics.h>
+
+#include "ColorConverter.h"
+
+namespace Video {
+
+// Tonemapping methods are outlined here:
+// https://64.github.io/tonemapping/
+
+template<typename T>
+ALWAYS_INLINE constexpr T scalar_to_color_vector(float value)
+{
+    if constexpr (IsSame<T, Gfx::VectorN<4, float>>) {
+        return Gfx::VectorN<4, float>(value, value, value, 1.0f);
+    } else if constexpr (IsSame<T, Gfx::VectorN<3, float>>) {
+        return Gfx::VectorN<3, float>(value, value, value);
+    } else {
+        static_assert(IsFloatingPoint<T>);
+        return static_cast<T>(value);
+    }
+}
+
+template<typename T>
+ALWAYS_INLINE constexpr T hable_tonemapping_partial(T value)
+{
+    constexpr auto a = scalar_to_color_vector<T>(0.15f);
+    constexpr auto b = scalar_to_color_vector<T>(0.5f);
+    constexpr auto c = scalar_to_color_vector<T>(0.1f);
+    constexpr auto d = scalar_to_color_vector<T>(0.2f);
+    constexpr auto e = scalar_to_color_vector<T>(0.02f);
+    constexpr auto f = scalar_to_color_vector<T>(0.3f);
+    return ((value * (a * value + c * b) + d * e) / (value * (a * value + b) + d * f)) - e / f;
+}
+
+template<typename T>
+ALWAYS_INLINE constexpr T hable_tonemapping(T value)
+{
+    constexpr auto exposure_bias = scalar_to_color_vector<T>(2.0f);
+    value = hable_tonemapping_partial<T>(value * exposure_bias);
+    constexpr auto scale = scalar_to_color_vector<T>(1.0f) / scalar_to_color_vector<T>(hable_tonemapping_partial(11.2f));
+    return value * scale;
+}
+
+DecoderErrorOr<ColorConverter> ColorConverter::create(u8 bit_depth, CodingIndependentCodePoints cicp)
+{
+    // We'll need to apply tonemapping for linear HDR values.
+    bool should_tonemap = false;
+    switch (cicp.transfer_characteristics()) {
+    case TransferCharacteristics::SMPTE2084:
+        should_tonemap = true;
+        break;
+    case TransferCharacteristics::HLG:
+        should_tonemap = true;
+        break;
+    default:
+        break;
+    }
+
+    // Conversion process:
+    // 1. Scale integer YUV values with maximum values of (1 << bit_depth) - 1 into
+    //    float 0..1 range.
+    //    This can be done with a 3x3 scaling matrix.
+    size_t maximum_value = (1u << bit_depth) - 1;
+    float scale = 1.0 / maximum_value;
+    FloatMatrix4x4 integer_scaling_matrix = {
+        scale, 0.0f, 0.0f, 0.0f, // y
+        0.0f, scale, 0.0f, 0.0f, // u
+        0.0f, 0.0f, scale, 0.0f, // v
+        0.0f, 0.0f, 0.0f, 1.0f,  // w
+    };
+
+    // 2. Scale YUV values into usable ranges.
+    //    For studio range, Y range is 16..235, and UV is 16..240.
+    //    UV values should be scaled to a range of -1..1.
+    //    This can be done in a 4x4 matrix with translation and scaling.
+    float y_min;
+    float y_max;
+    float uv_min;
+    float uv_max;
+    if (cicp.color_range() == ColorRange::Studio) {
+        y_min = 16.0f / 255.0f;
+        y_max = 235.0f / 255.0f;
+        uv_min = y_min;
+        uv_max = 240.0f / 255.0f;
+    } else {
+        y_min = 0.0f;
+        y_max = 1.0f;
+        uv_min = 0.0f;
+        uv_max = 1.0f;
+    }
+    auto clip_y_scale = 1.0f / (y_max - y_min);
+    auto clip_uv_scale = 2.0f / (uv_max - uv_min);
+
+    FloatMatrix4x4 range_scaling_matrix = {
+        clip_y_scale, 0.0f, 0.0f, -y_min * clip_y_scale,             // y
+        0.0f, clip_uv_scale, 0.0f, -(uv_min * clip_uv_scale + 1.0f), // u
+        0.0f, 0.0f, clip_uv_scale, -(uv_min * clip_uv_scale + 1.0f), // v
+        0.0f, 0.0f, 0.0f, 1.0f,                                      // w
+    };
+
+    // 3. Convert YUV values to RGB.
+    //    This is done with coefficients that can be put into a 3x3 matrix
+    //    and combined with the above 4x4 matrix to combine steps 1 and 2.
+    FloatMatrix4x4 color_conversion_matrix;
+
+    // https://kdashg.github.io/misc/colors/from-coeffs.html
+    switch (cicp.matrix_coefficients()) {
+    case MatrixCoefficients::BT709:
+        color_conversion_matrix = {
+            1.0f, 0.0f, 0.78740f, 0.0f,       // y
+            1.0f, -0.09366f, -0.23406f, 0.0f, // u
+            1.0f, 0.92780f, 0.0f, 0.0f,       // v
+            0.0f, 0.0f, 0.0f, 1.0f,           // w
+        };
+        break;
+    case MatrixCoefficients::BT601:
+        color_conversion_matrix = {
+            1.0f, 0.0f, 0.70100f, 0.0f,       // y
+            1.0f, -0.17207f, -0.35707f, 0.0f, // u
+            1.0f, 0.88600f, 0.0f, 0.0f,       // v
+            0.0f, 0.0f, 0.0f, 1.0f,           // w
+        };
+        break;
+    case MatrixCoefficients::BT2020ConstantLuminance:
+    case MatrixCoefficients::BT2020NonConstantLuminance:
+        color_conversion_matrix = {
+            1.0f, 0.0f, 0.73730f, 0.0f,       // y
+            1.0f, -0.08228f, -0.28568f, 0.0f, // u
+            1.0f, 0.94070f, 0.0f, 0.0f,       // v
+            0.0f, 0.0f, 0.0f, 1.0f,           // w
+        };
+        break;
+    default:
+        return DecoderError::format(DecoderErrorCategory::Invalid, "Matrix coefficients {} not supported", matrix_coefficients_to_string(cicp.matrix_coefficients()));
+    }
+
+    // 4. Apply the inverse transfer function to convert RGB values to the
+    //    linear color space.
+    //    This will be turned into a lookup table and interpolated to speed
+    //    up the conversion.
+    auto to_linear_lookup_table = InterpolatedLookupTable<to_linear_size>::create(
+        [&](float value) {
+            return TransferCharacteristicsConversion::to_linear_luminance(value, cicp.transfer_characteristics());
+        });
+
+    // 5. Convert the RGB color to CIE XYZ coordinates using the input color
+    //    primaries and then to the output color primaries.
+    //    This is done with two 3x3 matrices that can be combined into one
+    //    matrix multiplication.
+    ColorPrimaries output_cp = ColorPrimaries::BT709;
+    FloatMatrix3x3 color_primaries_matrix = TRY(get_conversion_matrix(cicp.color_primaries(), output_cp));
+
+    // 6. Apply the output transfer function. For HDR color spaces, this
+    //    should apply tonemapping as well.
+    //    Use a lookup table as with step 3.
+    TransferCharacteristics output_tc = TransferCharacteristics::SRGB;
+    switch (cicp.transfer_characteristics()) {
+    case TransferCharacteristics::Unspecified:
+        break;
+    case TransferCharacteristics::BT709:
+    case TransferCharacteristics::BT601:
+    case TransferCharacteristics::BT2020BitDepth10:
+    case TransferCharacteristics::BT2020BitDepth12:
+        // BT.601, BT.709 and BT.2020 have a similar transfer function to sRGB, and other applications
+        // (Chromium, VLC) seem to keep video output in those transfer characteristics.
+        output_tc = TransferCharacteristics::BT709;
+        break;
+    default:
+        break;
+    }
+
+    auto to_non_linear_lookup_table = InterpolatedLookupTable<to_non_linear_size>::create(
+        [&](float value) {
+            return TransferCharacteristicsConversion::to_non_linear_luminance(value, output_tc);
+        });
+
+    // Expand color primaries matrix with identity elements.
+    FloatMatrix4x4 color_primaries_matrix_4x4 = {
+        color_primaries_matrix.elements()[0][0],
+        color_primaries_matrix.elements()[0][1],
+        color_primaries_matrix.elements()[0][2],
+        0.0f, // y
+        color_primaries_matrix.elements()[1][0],
+        color_primaries_matrix.elements()[1][1],
+        color_primaries_matrix.elements()[1][2],
+        0.0f, // u
+        color_primaries_matrix.elements()[2][0],
+        color_primaries_matrix.elements()[2][1],
+        color_primaries_matrix.elements()[2][2],
+        0.0f, // v
+        0.0f,
+        0.0f,
+        0.0f,
+        1.0f, // w
+    };
+
+    bool should_skip_color_remapping = output_cp == cicp.color_primaries() && output_tc == cicp.transfer_characteristics();
+    FloatMatrix4x4 input_conversion_matrix = color_conversion_matrix * range_scaling_matrix * integer_scaling_matrix;
+
+    return ColorConverter(bit_depth, cicp, should_skip_color_remapping, should_tonemap, input_conversion_matrix, to_linear_lookup_table, color_primaries_matrix_4x4, to_non_linear_lookup_table);
+}
+
+ALWAYS_INLINE FloatVector4 max_zero(FloatVector4 vector)
+{
+    return { max(0.0f, vector.x()), max(0.0f, vector.y()), max(0.0f, vector.z()), vector.w() };
+}
+
+// Referencing https://en.wikipedia.org/wiki/YCbCr
+Gfx::Color ColorConverter::convert_yuv_to_full_range_rgb(u16 y, u16 u, u16 v)
+{
+    FloatVector4 color_vector = { static_cast<float>(y), static_cast<float>(u), static_cast<float>(v), 1.0f };
+    color_vector = m_input_conversion_matrix * color_vector;
+
+    if (m_should_skip_color_remapping) {
+        color_vector.clamp(0.0f, 1.0f);
+    } else {
+        color_vector = max_zero(color_vector);
+        color_vector = m_to_linear_lookup.do_lookup(color_vector);
+
+        if (m_cicp.transfer_characteristics() == TransferCharacteristics::HLG) {
+            static auto hlg_ootf_lookup_table = InterpolatedLookupTable<32, 1000>::create(
+                [](float value) {
+                    return AK::pow(value, 1.2f - 1.0f);
+                });
+            // See: https://en.wikipedia.org/wiki/Hybrid_log-gamma under a bolded section "HLG reference OOTF"
+            float luminance = (0.2627f * color_vector.x() + 0.6780f * color_vector.y() + 0.0593f * color_vector.z()) * 1000.0f;
+            float coefficient = hlg_ootf_lookup_table.do_lookup(luminance);
+            color_vector = { color_vector.x() * coefficient, color_vector.y() * coefficient, color_vector.z() * coefficient, 1.0f };
+        }
+
+        // FIXME: We could implement gamut compression here:
+        //        https://github.com/jedypod/gamut-compress/blob/master/docs/gamut-compress-algorithm.md
+        //        This would allow the color values outside the output gamut to be
+        //        preserved relative to values within the gamut instead of clipping. The
+        //        downside is that this requires a pass over the image before conversion
+        //        back into gamut is done to find the maximum color values to compress.
+        //        The compression would have to be somewhat temporally consistent as well.
+        color_vector = m_color_space_conversion_matrix * color_vector;
+        color_vector = max_zero(color_vector);
+        if (m_should_tonemap)
+            color_vector = hable_tonemapping(color_vector);
+        color_vector = m_to_non_linear_lookup.do_lookup(color_vector);
+        color_vector = max_zero(color_vector);
+    }
+
+    u8 r = static_cast<u8>(color_vector.x() * 255.0f);
+    u8 g = static_cast<u8>(color_vector.y() * 255.0f);
+    u8 b = static_cast<u8>(color_vector.z() * 255.0f);
+    return Gfx::Color(r, g, b);
+}
+
+}

Userland/Libraries/LibVideo/Color/ColorConverter.h

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Array.h>
+#include <AK/Function.h>
+#include <LibGfx/Color.h>
+#include <LibGfx/Matrix4x4.h>
+#include <LibVideo/Color/CodingIndependentCodePoints.h>
+#include <LibVideo/DecoderError.h>
+
+namespace Video {
+
+template<size_t N, size_t Scale = 1>
+struct InterpolatedLookupTable {
+public:
+    static InterpolatedLookupTable<N, Scale> create(Function<float(float)> transfer_function)
+    {
+        // We'll allocate one extra index to allow the values to reach 1.0.
+        InterpolatedLookupTable<N, Scale> lookup_table;
+        float index_to_value_mult = static_cast<float>(Scale) / maximum_value;
+        for (size_t i = 0; i < N; i++) {
+            float value = i * index_to_value_mult;
+            value = transfer_function(value);
+            lookup_table.m_lookup_table[i] = value;
+        }
+        return lookup_table;
+    }
+
+    float do_lookup(float value) const
+    {
+        float float_index = value * (maximum_value / static_cast<float>(Scale));
+        if (float_index > maximum_value) [[unlikely]]
+            float_index = maximum_value;
+        size_t index = static_cast<size_t>(float_index);
+        float partial_index = float_index - index;
+        value = m_lookup_table[index] * (1.0f - partial_index) + m_lookup_table[index + 1] * partial_index;
+        return value;
+    }
+
+    FloatVector4 do_lookup(FloatVector4 vector) const
+    {
+        return {
+            do_lookup(vector.x()),
+            do_lookup(vector.y()),
+            do_lookup(vector.z()),
+            vector.w()
+        };
+    }
+
+private:
+    static constexpr size_t maximum_value = N - 2;
+
+    Array<float, N> m_lookup_table;
+};
+
+class ColorConverter final {
+
+public:
+    static DecoderErrorOr<ColorConverter> create(u8 bit_depth, CodingIndependentCodePoints cicp);
+
+    Gfx::Color convert_yuv_to_full_range_rgb(u16 y, u16 u, u16 v);
+
+private:
+    static constexpr size_t to_linear_size = 64;
+    static constexpr size_t to_non_linear_size = 64;
+
+    ColorConverter(u8 bit_depth, CodingIndependentCodePoints cicp, bool should_skip_color_remapping, bool should_tonemap, FloatMatrix4x4 input_conversion_matrix, InterpolatedLookupTable<to_linear_size> to_linear_lookup, FloatMatrix4x4 color_space_conversion_matrix, InterpolatedLookupTable<to_non_linear_size> to_non_linear_lookup)
+        : m_bit_depth(bit_depth)
+        , m_cicp(cicp)
+        , m_should_skip_color_remapping(should_skip_color_remapping)
+        , m_should_tonemap(should_tonemap)
+        , m_input_conversion_matrix(input_conversion_matrix)
+        , m_to_linear_lookup(move(to_linear_lookup))
+        , m_color_space_conversion_matrix(color_space_conversion_matrix)
+        , m_to_non_linear_lookup(move(to_non_linear_lookup))
+    {
+    }
+    u8 m_bit_depth;
+    CodingIndependentCodePoints m_cicp;
+    bool m_should_skip_color_remapping;
+    bool m_should_tonemap;
+    FloatMatrix4x4 m_input_conversion_matrix;
+    InterpolatedLookupTable<to_linear_size> m_to_linear_lookup;
+    FloatMatrix4x4 m_color_space_conversion_matrix;
+    InterpolatedLookupTable<to_non_linear_size> m_to_non_linear_lookup;
+};
+
+}

Userland/Libraries/LibVideo/Color/ColorPrimaries.cpp

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibGfx/Vector2.h>
+#include <LibGfx/Vector3.h>
+
+#include "ColorPrimaries.h"
+
+namespace Video {
+
+ALWAYS_INLINE constexpr FloatVector3 primaries_to_xyz(FloatVector2 primaries)
+{
+    // https://en.wikipedia.org/wiki/CIE_1931_color_space#CIE_xy_chromaticity_diagram_and_the_CIE_xyY_color_space
+    // Luminosity is set to 1.0, so the equations are simplified.
+    auto const x = primaries.x();
+    auto const y = primaries.y();
+    return {
+        x / y,
+        1.0f,
+        (1.0f - x - y) / y
+    };
+}
+
+ALWAYS_INLINE constexpr FloatMatrix3x3 vectors_to_matrix(FloatVector3 a, FloatVector3 b, FloatVector3 c)
+{
+    return FloatMatrix3x3(
+        a.x(), a.y(), a.z(),
+        b.x(), b.y(), b.z(),
+        c.x(), c.y(), c.z());
+}
+
+ALWAYS_INLINE constexpr FloatMatrix3x3 primaries_matrix(FloatVector2 red, FloatVector2 green, FloatVector2 blue)
+{
+    return vectors_to_matrix(primaries_to_xyz(red), primaries_to_xyz(green), primaries_to_xyz(blue)).transpose();
+}
+
+ALWAYS_INLINE constexpr FloatVector3 matrix_row(FloatMatrix3x3 matrix, size_t row)
+{
+    return { matrix.elements()[row][0], matrix.elements()[row][1], matrix.elements()[row][2] };
+}
+
+ALWAYS_INLINE constexpr FloatMatrix3x3 generate_rgb_to_xyz_matrix(FloatVector2 red_xy, FloatVector2 green_xy, FloatVector2 blue_xy, FloatVector2 white_xy)
+{
+    // http://www.brucelindbloom.com/index.html?Eqn_RGB_XYZ_Matrix.html
+    const FloatMatrix3x3 matrix = primaries_matrix(red_xy, green_xy, blue_xy);
+    const FloatVector3 scale_vector = matrix.inverse() * primaries_to_xyz(white_xy);
+    return vectors_to_matrix(matrix_row(matrix, 0) * scale_vector, matrix_row(matrix, 1) * scale_vector, matrix_row(matrix, 2) * scale_vector);
+}
+
+constexpr FloatVector2 ILLUMINANT_D65 = { 0.3127f, 0.3290f };
+
+constexpr FloatVector2 BT_709_RED = { 0.64f, 0.33f };
+constexpr FloatVector2 BT_709_GREEN = { 0.30f, 0.60f };
+constexpr FloatVector2 BT_709_BLUE = { 0.15f, 0.06f };
+
+constexpr FloatVector2 BT_2020_RED = { 0.708f, 0.292f };
+constexpr FloatVector2 BT_2020_GREEN = { 0.170f, 0.797f };
+constexpr FloatVector2 BT_2020_BLUE = { 0.131f, 0.046f };
+
+constexpr FloatMatrix3x3 bt_2020_rgb_to_xyz = generate_rgb_to_xyz_matrix(BT_2020_RED, BT_2020_GREEN, BT_2020_BLUE, ILLUMINANT_D65);
+constexpr FloatMatrix3x3 bt_709_rgb_to_xyz = generate_rgb_to_xyz_matrix(BT_709_RED, BT_709_GREEN, BT_709_BLUE, ILLUMINANT_D65);
+
+DecoderErrorOr<FloatMatrix3x3> get_conversion_matrix(ColorPrimaries input_primaries, ColorPrimaries output_primaries)
+{
+    FloatMatrix3x3 input_conversion_matrix;
+    switch (input_primaries) {
+    case ColorPrimaries::BT709:
+        input_conversion_matrix = bt_709_rgb_to_xyz;
+        break;
+    case ColorPrimaries::BT2020:
+        input_conversion_matrix = bt_2020_rgb_to_xyz;
+        break;
+    default:
+        return DecoderError::format(DecoderErrorCategory::NotImplemented, "Conversion of primaries {} is not implemented", color_primaries_to_string(input_primaries));
+    }
+
+    FloatMatrix3x3 output_conversion_matrix;
+    switch (output_primaries) {
+    case ColorPrimaries::BT709:
+        output_conversion_matrix = bt_709_rgb_to_xyz.inverse();
+        break;
+    case ColorPrimaries::BT2020:
+        output_conversion_matrix = bt_2020_rgb_to_xyz.inverse();
+        break;
+    default:
+        return DecoderError::format(DecoderErrorCategory::NotImplemented, "Conversion of primaries {} is not implemented", color_primaries_to_string(output_primaries));
+    }
+
+    return output_conversion_matrix * input_conversion_matrix;
+}
+
+}

Userland/Libraries/LibVideo/Color/ColorPrimaries.h

@@ -0,0 +1,17 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibGfx/Matrix3x3.h>
+#include <LibVideo/Color/CodingIndependentCodePoints.h>
+#include <LibVideo/DecoderError.h>
+
+namespace Video {
+
+DecoderErrorOr<FloatMatrix3x3> get_conversion_matrix(ColorPrimaries input_primaries, ColorPrimaries output_primaries);
+
+}

Userland/Libraries/LibVideo/Color/TransferCharacteristics.cpp

@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Format.h>
+#include <AK/Math.h>
+#include <AK/StdLibExtras.h>
+
+#include "TransferCharacteristics.h"
+
+namespace Video {
+
+// SDR maximum luminance in candelas per meter squared
+constexpr float sdr_max_luminance = 120.0f;
+
+// sRGB
+constexpr float srgb_inverse_beta = 0.0031308f;
+constexpr float srgb_inverse_linear_coef = 12.92f;
+constexpr float srgb_gamma = 2.4f;
+constexpr float srgb_alpha = 1.055f;
+
+// BT.601/BT.709/BT.2020 constants
+constexpr float bt_601_beta = 0.018053968510807f;
+constexpr float bt_601_linear_coef = 4.5f;
+constexpr float bt_601_alpha = 1.0f + 5.5f * bt_601_beta;
+constexpr float bt_601_gamma = 0.45f;
+
+// Perceptual quantizer (SMPTE ST 2084) constants
+constexpr float pq_m1 = 2610.0f / 16384.0f;
+constexpr float pq_m2 = 128.0f * 2523.0f / 4096.0f;
+constexpr float pq_c1 = 3424.0f / 4096.0f;
+constexpr float pq_c2 = 32.0f * 2413.0f / 4096.0f;
+constexpr float pq_c3 = 32.0f * 2392.0f / 4096.0f;
+constexpr float pq_max_luminance = 10000.0f;
+
+// Hybrid log-gamma constants
+constexpr float hlg_a = 0.17883277f;
+constexpr float hlg_b = 0.28466892f;
+constexpr float hlg_c = 0.55991073f;
+
+float TransferCharacteristicsConversion::to_linear_luminance(float value, TransferCharacteristics transfer_function)
+{
+    switch (transfer_function) {
+    case TransferCharacteristics::BT709:
+    case TransferCharacteristics::BT601:
+    case TransferCharacteristics::BT2020BitDepth10:
+    case TransferCharacteristics::BT2020BitDepth12:
+        // https://en.wikipedia.org/wiki/Rec._601#Transfer_characteristics
+        // https://en.wikipedia.org/wiki/Rec._709#Transfer_characteristics
+        // https://en.wikipedia.org/wiki/Rec._2020#Transfer_characteristics
+        // These three share identical OETFs.
+        if (value < bt_601_beta * bt_601_linear_coef)
+            return value / bt_601_linear_coef;
+        return AK::pow((value + (bt_601_alpha - 1.0f)) / bt_601_alpha, 1.0f / bt_601_gamma);
+    case TransferCharacteristics::SRGB:
+        // https://color.org/sRGB.pdf
+        if (value < srgb_inverse_linear_coef * srgb_inverse_beta)
+            return value / srgb_inverse_linear_coef;
+        return AK::pow((value + (srgb_alpha - 1.0f)) / srgb_alpha, srgb_gamma);
+    case TransferCharacteristics::SMPTE2084: {
+        // https://en.wikipedia.org/wiki/Perceptual_quantizer
+        auto gamma_adjusted = AK::pow(value, 1.0f / pq_m2);
+        auto numerator = max(gamma_adjusted - pq_c1, 0.0f);
+        auto denominator = pq_c2 - pq_c3 * gamma_adjusted;
+        return AK::pow(numerator / denominator, 1.0f / pq_m1) * (pq_max_luminance / sdr_max_luminance);
+    }
+    case TransferCharacteristics::HLG:
+        // https://en.wikipedia.org/wiki/Hybrid_log-gamma
+        if (value < 0.5f)
+            return (value * value) / 3.0f;
+        return (AK::exp((value - hlg_c) / hlg_a) + hlg_b) / 12.0f;
+    default:
+        dbgln("Unsupported transfer function {}", static_cast<u8>(transfer_function));
+        VERIFY_NOT_REACHED();
+    }
+}
+
+float TransferCharacteristicsConversion::to_non_linear_luminance(float value, TransferCharacteristics transfer_function)
+{
+    switch (transfer_function) {
+    case TransferCharacteristics::BT709:
+    case TransferCharacteristics::BT601:
+    case TransferCharacteristics::BT2020BitDepth10:
+    case TransferCharacteristics::BT2020BitDepth12:
+        // https://en.wikipedia.org/wiki/Rec._601#Transfer_characteristics
+        // https://en.wikipedia.org/wiki/Rec._709#Transfer_characteristics
+        // https://en.wikipedia.org/wiki/Rec._2020#Transfer_characteristics
+        // These three share identical OETFs.
+        if (value < bt_601_beta)
+            return bt_601_linear_coef * value;
+        return bt_601_alpha * AK::pow(value, bt_601_gamma) - (bt_601_alpha - 1.0f);
+    case TransferCharacteristics::SRGB:
+        // https://color.org/sRGB.pdf
+        if (value < srgb_inverse_beta)
+            return value * srgb_inverse_linear_coef;
+        return srgb_alpha * AK::pow(value, 1.0f / srgb_gamma) - (srgb_alpha - 1.0f);
+    case TransferCharacteristics::SMPTE2084: {
+        // https://en.wikipedia.org/wiki/Perceptual_quantizer
+        auto linear_value = AK::pow(value * (sdr_max_luminance / pq_max_luminance), pq_m1);
+        auto numerator = pq_c1 + pq_c2 * linear_value;
+        auto denominator = 1 + pq_c3 * linear_value;
+        return AK::pow(numerator / denominator, pq_m2);
+    }
+    case TransferCharacteristics::HLG:
+        // https://en.wikipedia.org/wiki/Hybrid_log-gamma
+        if (value < 1.0f / 12.0f)
+            return AK::sqrt(value * 3.0f);
+        return hlg_a * AK::log(12.0f * value - hlg_b) + hlg_c;
+    default:
+        dbgln("Unsupported transfer function {}", static_cast<u8>(transfer_function));
+        VERIFY_NOT_REACHED();
+    }
+}
+
+FloatVector4 TransferCharacteristicsConversion::hlg_opto_optical_transfer_function(FloatVector4 const& vector, float gamma, float gain)
+{
+    float luminance = (0.2627f * vector.x() + 0.6780f * vector.y() + 0.0593f * vector.z()) * 1000.0f;
+    float coefficient = gain * AK::pow(luminance, gamma - 1.0f);
+    return FloatVector4(vector.x() * coefficient, vector.y() * coefficient, vector.z() * coefficient, vector.w());
+}
+
+}

Userland/Libraries/LibVideo/Color/TransferCharacteristics.h

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibGfx/Vector4.h>
+#include <LibVideo/Color/CodingIndependentCodePoints.h>
+
+namespace Video {
+
+class TransferCharacteristicsConversion {
+public:
+    static float to_linear_luminance(float value, TransferCharacteristics transfer_function);
+
+    static float to_non_linear_luminance(float value, TransferCharacteristics transfer_function);
+
+    // https://en.wikipedia.org/wiki/Hybrid_log-gamma
+    // See "HLG reference OOTF"
+    static FloatVector4 hlg_opto_optical_transfer_function(FloatVector4 const& vector, float gamma, float gain);
+};
+
+}

Userland/Libraries/LibVideo/DecoderError.h

@@ -25,6 +25,8 @@ enum class DecoderErrorCategory : u32 {
     Memory,
     // The input is corrupted.
     Corrupted,
+    // Invalid call.
+    Invalid,
     // The input uses features that are not yet implemented.
     NotImplemented,
 };

Userland/Libraries/LibVideo/VP9/Decoder.cpp

@@ -7,6 +7,7 @@
 
 #include <AK/IntegralMath.h>
 #include <LibGfx/Size.h>
+#include <LibVideo/Color/CodingIndependentCodePoints.h>
 
 #include "Decoder.h"
 #include "Utilities.h"
@@ -149,6 +150,70 @@ bool Decoder::get_uv_subsampling_x()
     return m_parser->m_subsampling_x;
 }
 
+CodingIndependentCodePoints Decoder::get_cicp_color_space()
+{
+    ColorPrimaries color_primaries;
+    TransferCharacteristics transfer_characteristics;
+    MatrixCoefficients matrix_coefficients;
+
+    switch (m_parser->m_color_space) {
+    case ColorSpace::Unknown:
+        color_primaries = ColorPrimaries::Unspecified;
+        transfer_characteristics = TransferCharacteristics::Unspecified;
+        matrix_coefficients = MatrixCoefficients::Unspecified;
+        break;
+    case ColorSpace::Bt601:
+        color_primaries = ColorPrimaries::BT601;
+        transfer_characteristics = TransferCharacteristics::BT601;
+        matrix_coefficients = MatrixCoefficients::BT601;
+        break;
+    case ColorSpace::Bt709:
+        color_primaries = ColorPrimaries::BT709;
+        transfer_characteristics = TransferCharacteristics::BT709;
+        matrix_coefficients = MatrixCoefficients::BT709;
+        break;
+    case ColorSpace::Smpte170:
+        // https://www.kernel.org/doc/html/v4.9/media/uapi/v4l/pixfmt-007.html#colorspace-smpte-170m-v4l2-colorspace-smpte170m
+        color_primaries = ColorPrimaries::BT601;
+        transfer_characteristics = TransferCharacteristics::BT709;
+        matrix_coefficients = MatrixCoefficients::BT601;
+        break;
+    case ColorSpace::Smpte240:
+        color_primaries = ColorPrimaries::SMPTE240;
+        transfer_characteristics = TransferCharacteristics::SMPTE240;
+        matrix_coefficients = MatrixCoefficients::SMPTE240;
+        break;
+    case ColorSpace::Bt2020:
+        color_primaries = ColorPrimaries::BT2020;
+        // Bit depth doesn't actually matter to our transfer functions since we
+        // convert in floats of range 0-1 (for now?), but just for correctness set
+        // the TC to match the bit depth here.
+        if (m_parser->m_bit_depth == 12)
+            transfer_characteristics = TransferCharacteristics::BT2020BitDepth12;
+        else if (m_parser->m_bit_depth == 10)
+            transfer_characteristics = TransferCharacteristics::BT2020BitDepth10;
+        else
+            transfer_characteristics = TransferCharacteristics::BT709;
+        matrix_coefficients = MatrixCoefficients::BT2020NonConstantLuminance;
+        break;
+    case ColorSpace::RGB:
+        color_primaries = ColorPrimaries::BT709;
+        transfer_characteristics = TransferCharacteristics::Linear;
+        matrix_coefficients = MatrixCoefficients::Identity;
+        break;
+    case ColorSpace::Reserved:
+        VERIFY_NOT_REACHED();
+        break;
+    }
+
+    return { color_primaries, transfer_characteristics, matrix_coefficients, m_parser->m_color_range };
+}
+
+u8 Decoder::get_bit_depth()
+{
+    return m_parser->m_bit_depth;
+}
+
 u8 Decoder::merge_prob(u8 pre_prob, u8 count_0, u8 count_1, u8 count_sat, u8 max_update_factor)
 {
     auto total_decode_count = count_0 + count_1;

Userland/Libraries/LibVideo/VP9/Decoder.h

@@ -10,6 +10,7 @@
 #include <AK/ByteBuffer.h>
 #include <AK/Error.h>
 #include <AK/Span.h>
+#include <LibVideo/Color/CodingIndependentCodePoints.h>
 #include <LibVideo/DecoderError.h>
 
 #include "Parser.h"
@@ -32,6 +33,8 @@ public:
     Gfx::Size<size_t> get_y_plane_size();
     bool get_uv_subsampling_y();
     bool get_uv_subsampling_x();
+    CodingIndependentCodePoints get_cicp_color_space();
+    u8 get_bit_depth();
 
 private:
     typedef i32 Intermediate;

Userland/Libraries/LibVideo/VP9/Enums.h

@@ -27,11 +27,6 @@ enum ColorSpace : u8 {
     RGB = 7
 };
 
-enum ColorRange {
-    StudioSwing,
-    FullSwing
-};
-
 enum InterpolationFilter : u8 {
     EightTap = 0,
     EightTapSmooth = 1,

Userland/Libraries/LibVideo/VP9/Parser.cpp

@@ -136,8 +136,8 @@ DecoderErrorOr<FrameType> Parser::read_frame_type()
 DecoderErrorOr<ColorRange> Parser::read_color_range()
 {
     if (TRY_READ(m_bit_stream->read_bit()))
-        return FullSwing;
-    return StudioSwing;
+        return ColorRange::Full;
+    return ColorRange::Studio;
 }
 
 /* (6.2) */
@@ -273,7 +273,7 @@ DecoderErrorOr<void> Parser::color_config()
             m_subsampling_y = true;
         }
     } else {
-        m_color_range = FullSwing;
+        m_color_range = ColorRange::Full;
         if (m_profile == 1 || m_profile == 3) {
             m_subsampling_x = false;
             m_subsampling_y = false;

Userland/Libraries/LibVideo/VP9/Parser.h

@@ -12,6 +12,7 @@
 #include <AK/Span.h>
 #include <AK/Vector.h>
 #include <LibGfx/Forward.h>
+#include <LibVideo/Color/CodingIndependentCodePoints.h>
 #include <LibVideo/DecoderError.h>
 
 #include "BitStream.h"