beenull/ladybird
1
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2024-11-26 01:20:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

LibVideo: Implement CICP color space conversion

Browse source 
This adds a struct called CodingIndependentCodePoints and related enums
that are used by video codecs to define its color space that frames
must be converted from when displaying a video.

Pre-multiplied matrices and lookup tables are stored to avoid most of
the floating point division and exponentiation in the conversion.
This commit is contained in:
Zaggy1024 2022-10-10 05:03:57 -05:00 committed by Andreas Kling
parent ba79de0439
commit cd127b65c3
Notes: sideshowbarker 2024-07-17 05:07:39 +09:00 
Author: https://github.com/Zaggy1024
Commit: https://github.com/SerenityOS/serenity/commit/cd127b65c3
Pull-request: https://github.com/SerenityOS/serenity/pull/15542
Reviewed-by: https://github.com/ADKaster ✅
Reviewed-by: https://github.com/Hendiadyoin1
Reviewed-by: https://github.com/davidot
Reviewed-by: https://github.com/gmta
15 changed files with 935 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
Userland/Applications/VideoPlayer/main.cpp
View file

@ -12,6 +12,7 @@
#include <LibGUI/Window.h>
#include <LibGfx/Bitmap.h>
#include <LibMain/Main.h>
#include <LibVideo/Color/ColorConverter.h>
#include <LibVideo/MatroskaReader.h>
#include <LibVideo/VP9/Decoder.h>
@ -91,6 +92,15 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
auto uv_subsampling_y = vp9_decoder.get_uv_subsampling_y();
auto uv_subsampling_x = vp9_decoder.get_uv_subsampling_x();
Gfx::IntSize uv_size { y_size.width() >> uv_subsampling_x, y_size.height() >> uv_subsampling_y };
auto cicp = vp9_decoder.get_cicp_color_space();
cicp.default_code_points_if_unspecified(Video::ColorPrimaries::BT709, Video::TransferCharacteristics::BT709, Video::MatrixCoefficients::BT709);
auto color_converter_result = Video::ColorConverter::create(vp9_decoder.get_bit_depth(), cicp);
if (color_converter_result.is_error()) {
outln("Cannot convert video colors: {}", color_converter_result.release_error().string_literal());
return;
}
auto color_converter = color_converter_result.release_value();
for (auto y_row = 0u; y_row < video_track.pixel_height; y_row++) {
auto uv_row = y_row >> uv_subsampling_y;
@ -99,17 +109,10 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
auto uv_column = y_column >> uv_subsampling_x;
auto y = output_y[y_row * y_size.width() + y_column];
auto cb = output_u[uv_row * uv_size.width() + uv_column];
auto cr = output_v[uv_row * uv_size.width() + uv_column];
// Convert from Rec.709 YCbCr to RGB.
auto r_float = floorf(clamp(y + (cr - 128) * 219.0f / 224.0f * 1.5748f, 0, 255));
auto g_float = floorf(clamp(y + (cb - 128) * 219.0f / 224.0f * -0.0722f * 1.8556f / 0.7152f + (cr - 128) * 219.0f / 224.0f * -0.2126f * 1.5748f / 0.7152f, 0, 255));
auto b_float = floorf(clamp(y + (cb - 128) * 219.0f / 224.0f * 1.8556f, 0, 255));
auto r = static_cast<u8>(r_float);
auto g = static_cast<u8>(g_float);
auto b = static_cast<u8>(b_float);
auto u = output_u[uv_row * uv_size.width() + uv_column];
auto v = output_v[uv_row * uv_size.width() + uv_column];
image->set_pixel(y_column, y_row, Gfx::Color(r, g, b));
image->set_pixel(y_column, y_row, color_converter.convert_yuv_to_full_range_rgb(y, u, v));
}
}

3
Userland/Libraries/LibVideo/CMakeLists.txt
View file

@ -1,4 +1,7 @@
set(SOURCES
Color/ColorConverter.cpp
Color/ColorPrimaries.cpp
Color/TransferCharacteristics.cpp
MatroskaReader.cpp
VP9/BitStream.cpp
VP9/Decoder.cpp

228
Userland/Libraries/LibVideo/Color/CodingIndependentCodePoints.h Normal file
View file

@ -0,0 +1,228 @@
/*
* Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/StringView.h>
namespace Video {
// CICP is defined by H.273:
// https://www.itu.int/rec/T-REC-H.273/en
// See the Section 8.
// Current edition is from 07/21.
enum class ColorPrimaries : u8 {
Reserved = 0,
BT709 = 1,
Unspecified = 2, // Used by codecs to indicate that an alternative value may be used
BT470M = 4,
BT470BG = 5,
BT601 = 6,
SMPTE240 = 7,
GenericFilm = 8,
BT2020 = 9,
XYZ = 10,
SMPTE431 = 11,
SMPTE432 = 12,
EBU3213 = 22,
// All other values are also Reserved for later use.
};
enum class TransferCharacteristics : u8 {
Reserved = 0,
BT709 = 1,
Unspecified = 2, // Used by codecs to indicate that an alternative value may be used
BT470M = 4,
BT470BG = 5,
BT601 = 6, // BT.601 or Rec. 601
SMPTE240 = 7,
Linear = 8,
Log100 = 9,
Log100Sqrt10 = 10,
IEC61966 = 11,
BT1361 = 12,
SRGB = 13,
BT2020BitDepth10 = 14,
BT2020BitDepth12 = 15,
SMPTE2084 = 16, // Also known as PQ
SMPTE428 = 17,
HLG = 18,
// All other values are also Reserved for later use.
};
enum class MatrixCoefficients : u8 {
Identity = 0, // Applies no transformation to input values
BT709 = 1,
Unspecified = 2, // Used by codecs to indicate that an alternative value may be used
FCC = 4,
BT470BG = 5,
BT601 = 6,
SMPTE240 = 7,
YCgCo = 8,
BT2020NonConstantLuminance = 9,
BT2020ConstantLuminance = 10,
SMPTE2085 = 11,
ChromaticityDerivedNonConstantLuminance = 12,
ChromaticityDerivedConstantLuminance = 13,
ICtCp = 14,
// All other values are Reserved for later use.
};
enum class ColorRange : u8 {
Studio = 0, // Y range 16..235, UV range 16..240
Full = 1, // 0..255
};
// https://en.wikipedia.org/wiki/Coding-independent_code_points
struct CodingIndependentCodePoints {
public:
constexpr CodingIndependentCodePoints(ColorPrimaries color_primaries, TransferCharacteristics transfer_characteristics, MatrixCoefficients matrix_coefficients, ColorRange color_range)
: m_color_primaries(color_primaries)
, m_transfer_characteristics(transfer_characteristics)
, m_matrix_coefficients(matrix_coefficients)
, m_color_range(color_range)
{
}
constexpr ColorPrimaries color_primaries() const { return m_color_primaries; }
constexpr void set_color_primaries(ColorPrimaries value) { m_color_primaries = value; }
constexpr TransferCharacteristics transfer_characteristics() const { return m_transfer_characteristics; }
constexpr void set_transfer_characteristics(TransferCharacteristics value) { m_transfer_characteristics = value; }
constexpr MatrixCoefficients matrix_coefficients() const { return m_matrix_coefficients; }
constexpr void set_matrix_coefficients(MatrixCoefficients value) { m_matrix_coefficients = value; }
constexpr ColorRange color_range() const { return m_color_range; }
constexpr void set_color_range(ColorRange value) { m_color_range = value; }
constexpr void default_code_points_if_unspecified(ColorPrimaries cp, TransferCharacteristics tc, MatrixCoefficients mc)
{
if (color_primaries() == ColorPrimaries::Unspecified)
set_color_primaries(cp);
if (transfer_characteristics() == TransferCharacteristics::Unspecified)
set_transfer_characteristics(tc);
if (matrix_coefficients() == MatrixCoefficients::Unspecified)
set_matrix_coefficients(mc);
}
private:
ColorPrimaries m_color_primaries;
TransferCharacteristics m_transfer_characteristics;
MatrixCoefficients m_matrix_coefficients;
ColorRange m_color_range;
};
constexpr StringView color_primaries_to_string(ColorPrimaries color_primaries)
{
switch (color_primaries) {
case ColorPrimaries::Reserved:
return "Reserved"sv;
case ColorPrimaries::BT709:
return "BT.709"sv;
case ColorPrimaries::Unspecified:
return "Unspecified"sv;
case ColorPrimaries::BT470M:
return "BT.470 System M"sv;
case ColorPrimaries::BT470BG:
return "BT.470 System B, G"sv;
case ColorPrimaries::BT601:
return "BT.601"sv;
case ColorPrimaries::SMPTE240:
return "SMPTE ST 240"sv;
case ColorPrimaries::GenericFilm:
return "Generic film"sv;
case ColorPrimaries::BT2020:
return "BT.2020"sv;
case ColorPrimaries::XYZ:
return "CIE 1931 XYZ"sv;
case ColorPrimaries::SMPTE431:
return "SMPTE RP 431"sv;
case ColorPrimaries::SMPTE432:
return "SMPTE EG 432"sv;
case ColorPrimaries::EBU3213:
return "EBU Tech 3213"sv;
}
return "Reserved"sv;
};
constexpr StringView transfer_characteristics_to_string(TransferCharacteristics transfer_characteristics)
{
switch (transfer_characteristics) {
case TransferCharacteristics::Reserved:
return "Reserved"sv;
case TransferCharacteristics::BT709:
return "BT.709"sv;
case TransferCharacteristics::Unspecified:
return "Unspecified"sv;
case TransferCharacteristics::BT470M:
return "BT.470 System M"sv;
case TransferCharacteristics::BT470BG:
return "BT.470 System B, G"sv;
case TransferCharacteristics::BT601:
return "BT.601"sv;
case TransferCharacteristics::SMPTE240:
return "SMPTE ST 240"sv;
case TransferCharacteristics::Linear:
return "Linear"sv;
case TransferCharacteristics::Log100:
return "Logarithmic (100:1 range)"sv;
case TransferCharacteristics::Log100Sqrt10:
return "Logarithmic (100xSqrt(10):1 range)"sv;
case TransferCharacteristics::IEC61966:
return "IEC 61966"sv;
case TransferCharacteristics::BT1361:
return "BT.1361"sv;
case TransferCharacteristics::SRGB:
return "sRGB"sv;
case TransferCharacteristics::BT2020BitDepth10:
return "BT.2020 (10-bit)"sv;
case TransferCharacteristics::BT2020BitDepth12:
return "BT.2020 (12-bit)"sv;
case TransferCharacteristics::SMPTE2084:
return "SMPTE ST 2084 (PQ)"sv;
case TransferCharacteristics::SMPTE428:
return "SMPTE ST 428"sv;
case TransferCharacteristics::HLG:
return "ARIB STD-B67 (HLG, BT.2100)"sv;
}
return "Reserved"sv;
};
constexpr StringView matrix_coefficients_to_string(MatrixCoefficients matrix_coefficients)
{
switch (matrix_coefficients) {
case MatrixCoefficients::Identity:
return "Identity"sv;
case MatrixCoefficients::BT709:
return "BT.709"sv;
case MatrixCoefficients::Unspecified:
return "Unspecified"sv;
case MatrixCoefficients::FCC:
return "FCC (CFR 73.682)"sv;
case MatrixCoefficients::BT470BG:
return "BT.470 System B, G"sv;
case MatrixCoefficients::BT601:
return "BT.601"sv;
case MatrixCoefficients::SMPTE240:
return "SMPTE ST 240"sv;
case MatrixCoefficients::YCgCo:
return "YCgCo"sv;
case MatrixCoefficients::BT2020NonConstantLuminance:
return "BT.2020, non-constant luminance"sv;
case MatrixCoefficients::BT2020ConstantLuminance:
return "BT.2020, constant luminance"sv;
case MatrixCoefficients::SMPTE2085:
return "SMPTE ST 2085"sv;
case MatrixCoefficients::ChromaticityDerivedNonConstantLuminance:
return "Chromaticity-derived, non-constant luminance"sv;
case MatrixCoefficients::ChromaticityDerivedConstantLuminance:
return "Chromaticity-derived, constant luminance"sv;
case MatrixCoefficients::ICtCp:
return "BT.2100 ICtCp"sv;
}
return "Reserved"sv;
};
}

263
Userland/Libraries/LibVideo/Color/ColorConverter.cpp Normal file
View file

@ -0,0 +1,263 @@
/*
* Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/Format.h>
#include <AK/Math.h>
#include <AK/StdLibExtras.h>
#include <LibGfx/Matrix4x4.h>
#include <LibVideo/Color/ColorPrimaries.h>
#include <LibVideo/Color/TransferCharacteristics.h>
#include "ColorConverter.h"
namespace Video {
// Tonemapping methods are outlined here:
// https://64.github.io/tonemapping/
template<typename T>
ALWAYS_INLINE constexpr T scalar_to_color_vector(float value)
{
if constexpr (IsSame<T, Gfx::VectorN<4, float>>) {
return Gfx::VectorN<4, float>(value, value, value, 1.0f);
} else if constexpr (IsSame<T, Gfx::VectorN<3, float>>) {
return Gfx::VectorN<3, float>(value, value, value);
} else {
static_assert(IsFloatingPoint<T>);
return static_cast<T>(value);
}
}
template<typename T>
ALWAYS_INLINE constexpr T hable_tonemapping_partial(T value)
{
constexpr auto a = scalar_to_color_vector<T>(0.15f);
constexpr auto b = scalar_to_color_vector<T>(0.5f);
constexpr auto c = scalar_to_color_vector<T>(0.1f);
constexpr auto d = scalar_to_color_vector<T>(0.2f);
constexpr auto e = scalar_to_color_vector<T>(0.02f);
constexpr auto f = scalar_to_color_vector<T>(0.3f);
return ((value * (a * value + c * b) + d * e) / (value * (a * value + b) + d * f)) - e / f;
}
template<typename T>
ALWAYS_INLINE constexpr T hable_tonemapping(T value)
{
constexpr auto exposure_bias = scalar_to_color_vector<T>(2.0f);
value = hable_tonemapping_partial<T>(value * exposure_bias);
constexpr auto scale = scalar_to_color_vector<T>(1.0f) / scalar_to_color_vector<T>(hable_tonemapping_partial(11.2f));
return value * scale;
}
DecoderErrorOr<ColorConverter> ColorConverter::create(u8 bit_depth, CodingIndependentCodePoints cicp)
{
// We'll need to apply tonemapping for linear HDR values.
bool should_tonemap = false;
switch (cicp.transfer_characteristics()) {
case TransferCharacteristics::SMPTE2084:
should_tonemap = true;
break;
case TransferCharacteristics::HLG:
should_tonemap = true;
break;
default:
break;
}
// Conversion process:
// 1. Scale integer YUV values with maximum values of (1 << bit_depth) - 1 into
// float 0..1 range.
// This can be done with a 3x3 scaling matrix.
size_t maximum_value = (1u << bit_depth) - 1;
float scale = 1.0 / maximum_value;
FloatMatrix4x4 integer_scaling_matrix = {
scale, 0.0f, 0.0f, 0.0f, // y
0.0f, scale, 0.0f, 0.0f, // u
0.0f, 0.0f, scale, 0.0f, // v
0.0f, 0.0f, 0.0f, 1.0f, // w
};
// 2. Scale YUV values into usable ranges.
// For studio range, Y range is 16..235, and UV is 16..240.
// UV values should be scaled to a range of -1..1.
// This can be done in a 4x4 matrix with translation and scaling.
float y_min;
float y_max;
float uv_min;
float uv_max;
if (cicp.color_range() == ColorRange::Studio) {
y_min = 16.0f / 255.0f;
y_max = 235.0f / 255.0f;
uv_min = y_min;
uv_max = 240.0f / 255.0f;
} else {
y_min = 0.0f;
y_max = 1.0f;
uv_min = 0.0f;
uv_max = 1.0f;
}
auto clip_y_scale = 1.0f / (y_max - y_min);
auto clip_uv_scale = 2.0f / (uv_max - uv_min);
FloatMatrix4x4 range_scaling_matrix = {
clip_y_scale, 0.0f, 0.0f, -y_min * clip_y_scale, // y
0.0f, clip_uv_scale, 0.0f, -(uv_min * clip_uv_scale + 1.0f), // u
0.0f, 0.0f, clip_uv_scale, -(uv_min * clip_uv_scale + 1.0f), // v
0.0f, 0.0f, 0.0f, 1.0f, // w
};
// 3. Convert YUV values to RGB.
// This is done with coefficients that can be put into a 3x3 matrix
// and combined with the above 4x4 matrix to combine steps 1 and 2.
FloatMatrix4x4 color_conversion_matrix;
// https://kdashg.github.io/misc/colors/from-coeffs.html
switch (cicp.matrix_coefficients()) {
case MatrixCoefficients::BT709:
color_conversion_matrix = {
1.0f, 0.0f, 0.78740f, 0.0f, // y
1.0f, -0.09366f, -0.23406f, 0.0f, // u
1.0f, 0.92780f, 0.0f, 0.0f, // v
0.0f, 0.0f, 0.0f, 1.0f, // w
};
break;
case MatrixCoefficients::BT601:
color_conversion_matrix = {
1.0f, 0.0f, 0.70100f, 0.0f, // y
1.0f, -0.17207f, -0.35707f, 0.0f, // u
1.0f, 0.88600f, 0.0f, 0.0f, // v
0.0f, 0.0f, 0.0f, 1.0f, // w
};
break;
case MatrixCoefficients::BT2020ConstantLuminance:
case MatrixCoefficients::BT2020NonConstantLuminance:
color_conversion_matrix = {
1.0f, 0.0f, 0.73730f, 0.0f, // y
1.0f, -0.08228f, -0.28568f, 0.0f, // u
1.0f, 0.94070f, 0.0f, 0.0f, // v
0.0f, 0.0f, 0.0f, 1.0f, // w
};
break;
default:
return DecoderError::format(DecoderErrorCategory::Invalid, "Matrix coefficients {} not supported", matrix_coefficients_to_string(cicp.matrix_coefficients()));
}
// 4. Apply the inverse transfer function to convert RGB values to the
// linear color space.
// This will be turned into a lookup table and interpolated to speed
// up the conversion.
auto to_linear_lookup_table = InterpolatedLookupTable<to_linear_size>::create(
[&](float value) {
return TransferCharacteristicsConversion::to_linear_luminance(value, cicp.transfer_characteristics());
});
// 5. Convert the RGB color to CIE XYZ coordinates using the input color
// primaries and then to the output color primaries.
// This is done with two 3x3 matrices that can be combined into one
// matrix multiplication.
ColorPrimaries output_cp = ColorPrimaries::BT709;
FloatMatrix3x3 color_primaries_matrix = TRY(get_conversion_matrix(cicp.color_primaries(), output_cp));
// 6. Apply the output transfer function. For HDR color spaces, this
// should apply tonemapping as well.
// Use a lookup table as with step 3.
TransferCharacteristics output_tc = TransferCharacteristics::SRGB;
switch (cicp.transfer_characteristics()) {
case TransferCharacteristics::Unspecified:
break;
case TransferCharacteristics::BT709:
case TransferCharacteristics::BT601:
case TransferCharacteristics::BT2020BitDepth10:
case TransferCharacteristics::BT2020BitDepth12:
// BT.601, BT.709 and BT.2020 have a similar transfer function to sRGB, and other applications
// (Chromium, VLC) seem to keep video output in those transfer characteristics.
output_tc = TransferCharacteristics::BT709;
break;
default:
break;
}
auto to_non_linear_lookup_table = InterpolatedLookupTable<to_non_linear_size>::create(
[&](float value) {
return TransferCharacteristicsConversion::to_non_linear_luminance(value, output_tc);
});
// Expand color primaries matrix with identity elements.
FloatMatrix4x4 color_primaries_matrix_4x4 = {
color_primaries_matrix.elements()[0][0],
color_primaries_matrix.elements()[0][1],
color_primaries_matrix.elements()[0][2],
0.0f, // y
color_primaries_matrix.elements()[1][0],
color_primaries_matrix.elements()[1][1],
color_primaries_matrix.elements()[1][2],
0.0f, // u
color_primaries_matrix.elements()[2][0],
color_primaries_matrix.elements()[2][1],
color_primaries_matrix.elements()[2][2],
0.0f, // v
0.0f,
0.0f,
0.0f,
1.0f, // w
};
bool should_skip_color_remapping = output_cp == cicp.color_primaries() && output_tc == cicp.transfer_characteristics();
FloatMatrix4x4 input_conversion_matrix = color_conversion_matrix * range_scaling_matrix * integer_scaling_matrix;
return ColorConverter(bit_depth, cicp, should_skip_color_remapping, should_tonemap, input_conversion_matrix, to_linear_lookup_table, color_primaries_matrix_4x4, to_non_linear_lookup_table);
}
ALWAYS_INLINE FloatVector4 max_zero(FloatVector4 vector)
{
return { max(0.0f, vector.x()), max(0.0f, vector.y()), max(0.0f, vector.z()), vector.w() };
}
// Referencing https://en.wikipedia.org/wiki/YCbCr
Gfx::Color ColorConverter::convert_yuv_to_full_range_rgb(u16 y, u16 u, u16 v)
{
FloatVector4 color_vector = { static_cast<float>(y), static_cast<float>(u), static_cast<float>(v), 1.0f };
color_vector = m_input_conversion_matrix * color_vector;
if (m_should_skip_color_remapping) {
color_vector.clamp(0.0f, 1.0f);
} else {
color_vector = max_zero(color_vector);
color_vector = m_to_linear_lookup.do_lookup(color_vector);
if (m_cicp.transfer_characteristics() == TransferCharacteristics::HLG) {
static auto hlg_ootf_lookup_table = InterpolatedLookupTable<32, 1000>::create(
[](float value) {
return AK::pow(value, 1.2f - 1.0f);
});
// See: https://en.wikipedia.org/wiki/Hybrid_log-gamma under a bolded section "HLG reference OOTF"
float luminance = (0.2627f * color_vector.x() + 0.6780f * color_vector.y() + 0.0593f * color_vector.z()) * 1000.0f;
float coefficient = hlg_ootf_lookup_table.do_lookup(luminance);
color_vector = { color_vector.x() * coefficient, color_vector.y() * coefficient, color_vector.z() * coefficient, 1.0f };
}
// FIXME: We could implement gamut compression here:
// https://github.com/jedypod/gamut-compress/blob/master/docs/gamut-compress-algorithm.md
// This would allow the color values outside the output gamut to be
// preserved relative to values within the gamut instead of clipping. The
// downside is that this requires a pass over the image before conversion
// back into gamut is done to find the maximum color values to compress.
// The compression would have to be somewhat temporally consistent as well.
color_vector = m_color_space_conversion_matrix * color_vector;
color_vector = max_zero(color_vector);
if (m_should_tonemap)
color_vector = hable_tonemapping(color_vector);
color_vector = m_to_non_linear_lookup.do_lookup(color_vector);
color_vector = max_zero(color_vector);
}
u8 r = static_cast<u8>(color_vector.x() * 255.0f);
u8 g = static_cast<u8>(color_vector.y() * 255.0f);
u8 b = static_cast<u8>(color_vector.z() * 255.0f);
return Gfx::Color(r, g, b);
}
}

93
Userland/Libraries/LibVideo/Color/ColorConverter.h Normal file
View file

@ -0,0 +1,93 @@
/*
* Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Array.h>
#include <AK/Function.h>
#include <LibGfx/Color.h>
#include <LibGfx/Matrix4x4.h>
#include <LibVideo/Color/CodingIndependentCodePoints.h>
#include <LibVideo/DecoderError.h>
namespace Video {
template<size_t N, size_t Scale = 1>
struct InterpolatedLookupTable {
public:
static InterpolatedLookupTable<N, Scale> create(Function<float(float)> transfer_function)
{
// We'll allocate one extra index to allow the values to reach 1.0.
InterpolatedLookupTable<N, Scale> lookup_table;
float index_to_value_mult = static_cast<float>(Scale) / maximum_value;
for (size_t i = 0; i < N; i++) {
float value = i * index_to_value_mult;
value = transfer_function(value);
lookup_table.m_lookup_table[i] = value;
}
return lookup_table;
}
float do_lookup(float value) const
{
float float_index = value * (maximum_value / static_cast<float>(Scale));
if (float_index > maximum_value) [[unlikely]]
float_index = maximum_value;
size_t index = static_cast<size_t>(float_index);
float partial_index = float_index - index;
value = m_lookup_table[index] * (1.0f - partial_index) + m_lookup_table[index + 1] * partial_index;
return value;
}
FloatVector4 do_lookup(FloatVector4 vector) const
{
return {
do_lookup(vector.x()),
do_lookup(vector.y()),
do_lookup(vector.z()),
vector.w()
};
}
private:
static constexpr size_t maximum_value = N - 2;
Array<float, N> m_lookup_table;
};
class ColorConverter final {
public:
static DecoderErrorOr<ColorConverter> create(u8 bit_depth, CodingIndependentCodePoints cicp);
Gfx::Color convert_yuv_to_full_range_rgb(u16 y, u16 u, u16 v);
private:
static constexpr size_t to_linear_size = 64;
static constexpr size_t to_non_linear_size = 64;
ColorConverter(u8 bit_depth, CodingIndependentCodePoints cicp, bool should_skip_color_remapping, bool should_tonemap, FloatMatrix4x4 input_conversion_matrix, InterpolatedLookupTable<to_linear_size> to_linear_lookup, FloatMatrix4x4 color_space_conversion_matrix, InterpolatedLookupTable<to_non_linear_size> to_non_linear_lookup)
: m_bit_depth(bit_depth)
, m_cicp(cicp)
, m_should_skip_color_remapping(should_skip_color_remapping)
, m_should_tonemap(should_tonemap)
, m_input_conversion_matrix(input_conversion_matrix)
, m_to_linear_lookup(move(to_linear_lookup))
, m_color_space_conversion_matrix(color_space_conversion_matrix)
, m_to_non_linear_lookup(move(to_non_linear_lookup))
{
}
u8 m_bit_depth;
CodingIndependentCodePoints m_cicp;
bool m_should_skip_color_remapping;
bool m_should_tonemap;
FloatMatrix4x4 m_input_conversion_matrix;
InterpolatedLookupTable<to_linear_size> m_to_linear_lookup;
FloatMatrix4x4 m_color_space_conversion_matrix;
InterpolatedLookupTable<to_non_linear_size> m_to_non_linear_lookup;
};
}

95
Userland/Libraries/LibVideo/Color/ColorPrimaries.cpp Normal file
View file

@ -0,0 +1,95 @@
/*
* Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibGfx/Vector2.h>
#include <LibGfx/Vector3.h>
#include "ColorPrimaries.h"
namespace Video {
ALWAYS_INLINE constexpr FloatVector3 primaries_to_xyz(FloatVector2 primaries)
{
// https://en.wikipedia.org/wiki/CIE_1931_color_space#CIE_xy_chromaticity_diagram_and_the_CIE_xyY_color_space
// Luminosity is set to 1.0, so the equations are simplified.
auto const x = primaries.x();
auto const y = primaries.y();
return {
x / y,
1.0f,
(1.0f - x - y) / y
};
}
ALWAYS_INLINE constexpr FloatMatrix3x3 vectors_to_matrix(FloatVector3 a, FloatVector3 b, FloatVector3 c)
{
return FloatMatrix3x3(
a.x(), a.y(), a.z(),
b.x(), b.y(), b.z(),
c.x(), c.y(), c.z());
}
ALWAYS_INLINE constexpr FloatMatrix3x3 primaries_matrix(FloatVector2 red, FloatVector2 green, FloatVector2 blue)
{
return vectors_to_matrix(primaries_to_xyz(red), primaries_to_xyz(green), primaries_to_xyz(blue)).transpose();
}
ALWAYS_INLINE constexpr FloatVector3 matrix_row(FloatMatrix3x3 matrix, size_t row)
{
return { matrix.elements()[row][0], matrix.elements()[row][1], matrix.elements()[row][2] };
}
ALWAYS_INLINE constexpr FloatMatrix3x3 generate_rgb_to_xyz_matrix(FloatVector2 red_xy, FloatVector2 green_xy, FloatVector2 blue_xy, FloatVector2 white_xy)
{
// http://www.brucelindbloom.com/index.html?Eqn_RGB_XYZ_Matrix.html
const FloatMatrix3x3 matrix = primaries_matrix(red_xy, green_xy, blue_xy);
const FloatVector3 scale_vector = matrix.inverse() * primaries_to_xyz(white_xy);
return vectors_to_matrix(matrix_row(matrix, 0) * scale_vector, matrix_row(matrix, 1) * scale_vector, matrix_row(matrix, 2) * scale_vector);
}
constexpr FloatVector2 ILLUMINANT_D65 = { 0.3127f, 0.3290f };
constexpr FloatVector2 BT_709_RED = { 0.64f, 0.33f };
constexpr FloatVector2 BT_709_GREEN = { 0.30f, 0.60f };
constexpr FloatVector2 BT_709_BLUE = { 0.15f, 0.06f };
constexpr FloatVector2 BT_2020_RED = { 0.708f, 0.292f };
constexpr FloatVector2 BT_2020_GREEN = { 0.170f, 0.797f };
constexpr FloatVector2 BT_2020_BLUE = { 0.131f, 0.046f };
constexpr FloatMatrix3x3 bt_2020_rgb_to_xyz = generate_rgb_to_xyz_matrix(BT_2020_RED, BT_2020_GREEN, BT_2020_BLUE, ILLUMINANT_D65);
constexpr FloatMatrix3x3 bt_709_rgb_to_xyz = generate_rgb_to_xyz_matrix(BT_709_RED, BT_709_GREEN, BT_709_BLUE, ILLUMINANT_D65);
DecoderErrorOr<FloatMatrix3x3> get_conversion_matrix(ColorPrimaries input_primaries, ColorPrimaries output_primaries)
{
FloatMatrix3x3 input_conversion_matrix;
switch (input_primaries) {
case ColorPrimaries::BT709:
input_conversion_matrix = bt_709_rgb_to_xyz;
break;
case ColorPrimaries::BT2020:
input_conversion_matrix = bt_2020_rgb_to_xyz;
break;
default:
return DecoderError::format(DecoderErrorCategory::NotImplemented, "Conversion of primaries {} is not implemented", color_primaries_to_string(input_primaries));
}
FloatMatrix3x3 output_conversion_matrix;
switch (output_primaries) {
case ColorPrimaries::BT709:
output_conversion_matrix = bt_709_rgb_to_xyz.inverse();
break;
case ColorPrimaries::BT2020:
output_conversion_matrix = bt_2020_rgb_to_xyz.inverse();
break;
default:
return DecoderError::format(DecoderErrorCategory::NotImplemented, "Conversion of primaries {} is not implemented", color_primaries_to_string(output_primaries));
}
return output_conversion_matrix * input_conversion_matrix;
}
}

17
Userland/Libraries/LibVideo/Color/ColorPrimaries.h Normal file
View file

@ -0,0 +1,17 @@
/*
* Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <LibGfx/Matrix3x3.h>
#include <LibVideo/Color/CodingIndependentCodePoints.h>
#include <LibVideo/DecoderError.h>
namespace Video {
DecoderErrorOr<FloatMatrix3x3> get_conversion_matrix(ColorPrimaries input_primaries, ColorPrimaries output_primaries);
}

124
Userland/Libraries/LibVideo/Color/TransferCharacteristics.cpp Normal file
View file

@ -0,0 +1,124 @@
/*
* Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/Format.h>
#include <AK/Math.h>
#include <AK/StdLibExtras.h>
#include "TransferCharacteristics.h"
namespace Video {
// SDR maximum luminance in candelas per meter squared
constexpr float sdr_max_luminance = 120.0f;
// sRGB
constexpr float srgb_inverse_beta = 0.0031308f;
constexpr float srgb_inverse_linear_coef = 12.92f;
constexpr float srgb_gamma = 2.4f;
constexpr float srgb_alpha = 1.055f;
// BT.601/BT.709/BT.2020 constants
constexpr float bt_601_beta = 0.018053968510807f;
constexpr float bt_601_linear_coef = 4.5f;
constexpr float bt_601_alpha = 1.0f + 5.5f * bt_601_beta;
constexpr float bt_601_gamma = 0.45f;
// Perceptual quantizer (SMPTE ST 2084) constants
constexpr float pq_m1 = 2610.0f / 16384.0f;
constexpr float pq_m2 = 128.0f * 2523.0f / 4096.0f;
constexpr float pq_c1 = 3424.0f / 4096.0f;
constexpr float pq_c2 = 32.0f * 2413.0f / 4096.0f;
constexpr float pq_c3 = 32.0f * 2392.0f / 4096.0f;
constexpr float pq_max_luminance = 10000.0f;
// Hybrid log-gamma constants
constexpr float hlg_a = 0.17883277f;
constexpr float hlg_b = 0.28466892f;
constexpr float hlg_c = 0.55991073f;
float TransferCharacteristicsConversion::to_linear_luminance(float value, TransferCharacteristics transfer_function)
{
switch (transfer_function) {
case TransferCharacteristics::BT709:
case TransferCharacteristics::BT601:
case TransferCharacteristics::BT2020BitDepth10:
case TransferCharacteristics::BT2020BitDepth12:
// https://en.wikipedia.org/wiki/Rec._601#Transfer_characteristics
// https://en.wikipedia.org/wiki/Rec._709#Transfer_characteristics
// https://en.wikipedia.org/wiki/Rec._2020#Transfer_characteristics
// These three share identical OETFs.
if (value < bt_601_beta * bt_601_linear_coef)
return value / bt_601_linear_coef;
return AK::pow((value + (bt_601_alpha - 1.0f)) / bt_601_alpha, 1.0f / bt_601_gamma);
case TransferCharacteristics::SRGB:
// https://color.org/sRGB.pdf
if (value < srgb_inverse_linear_coef * srgb_inverse_beta)
return value / srgb_inverse_linear_coef;
return AK::pow((value + (srgb_alpha - 1.0f)) / srgb_alpha, srgb_gamma);
case TransferCharacteristics::SMPTE2084: {
// https://en.wikipedia.org/wiki/Perceptual_quantizer
auto gamma_adjusted = AK::pow(value, 1.0f / pq_m2);
auto numerator = max(gamma_adjusted - pq_c1, 0.0f);
auto denominator = pq_c2 - pq_c3 * gamma_adjusted;
return AK::pow(numerator / denominator, 1.0f / pq_m1) * (pq_max_luminance / sdr_max_luminance);
}
case TransferCharacteristics::HLG:
// https://en.wikipedia.org/wiki/Hybrid_log-gamma
if (value < 0.5f)
return (value * value) / 3.0f;
return (AK::exp((value - hlg_c) / hlg_a) + hlg_b) / 12.0f;
default:
dbgln("Unsupported transfer function {}", static_cast<u8>(transfer_function));
VERIFY_NOT_REACHED();
}
}
float TransferCharacteristicsConversion::to_non_linear_luminance(float value, TransferCharacteristics transfer_function)
{
switch (transfer_function) {
case TransferCharacteristics::BT709:
case TransferCharacteristics::BT601:
case TransferCharacteristics::BT2020BitDepth10:
case TransferCharacteristics::BT2020BitDepth12:
// https://en.wikipedia.org/wiki/Rec._601#Transfer_characteristics
// https://en.wikipedia.org/wiki/Rec._709#Transfer_characteristics
// https://en.wikipedia.org/wiki/Rec._2020#Transfer_characteristics
// These three share identical OETFs.
if (value < bt_601_beta)
return bt_601_linear_coef * value;
return bt_601_alpha * AK::pow(value, bt_601_gamma) - (bt_601_alpha - 1.0f);
case TransferCharacteristics::SRGB:
// https://color.org/sRGB.pdf
if (value < srgb_inverse_beta)
return value * srgb_inverse_linear_coef;
return srgb_alpha * AK::pow(value, 1.0f / srgb_gamma) - (srgb_alpha - 1.0f);
case TransferCharacteristics::SMPTE2084: {
// https://en.wikipedia.org/wiki/Perceptual_quantizer
auto linear_value = AK::pow(value * (sdr_max_luminance / pq_max_luminance), pq_m1);
auto numerator = pq_c1 + pq_c2 * linear_value;
auto denominator = 1 + pq_c3 * linear_value;
return AK::pow(numerator / denominator, pq_m2);
}
case TransferCharacteristics::HLG:
// https://en.wikipedia.org/wiki/Hybrid_log-gamma
if (value < 1.0f / 12.0f)
return AK::sqrt(value * 3.0f);
return hlg_a * AK::log(12.0f * value - hlg_b) + hlg_c;
default:
dbgln("Unsupported transfer function {}", static_cast<u8>(transfer_function));
VERIFY_NOT_REACHED();
}
}
FloatVector4 TransferCharacteristicsConversion::hlg_opto_optical_transfer_function(FloatVector4 const& vector, float gamma, float gain)
{
float luminance = (0.2627f * vector.x() + 0.6780f * vector.y() + 0.0593f * vector.z()) * 1000.0f;
float coefficient = gain * AK::pow(luminance, gamma - 1.0f);
return FloatVector4(vector.x() * coefficient, vector.y() * coefficient, vector.z() * coefficient, vector.w());
}
}

25
Userland/Libraries/LibVideo/Color/TransferCharacteristics.h Normal file
View file

@ -0,0 +1,25 @@
/*
* Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <LibGfx/Vector4.h>
#include <LibVideo/Color/CodingIndependentCodePoints.h>
namespace Video {
class TransferCharacteristicsConversion {
public:
static float to_linear_luminance(float value, TransferCharacteristics transfer_function);
static float to_non_linear_luminance(float value, TransferCharacteristics transfer_function);
// https://en.wikipedia.org/wiki/Hybrid_log-gamma
// See "HLG reference OOTF"
static FloatVector4 hlg_opto_optical_transfer_function(FloatVector4 const& vector, float gamma, float gain);
};
}

2
Userland/Libraries/LibVideo/DecoderError.h
View file

@ -25,6 +25,8 @@ enum class DecoderErrorCategory : u32 {
Memory,
// The input is corrupted.
Corrupted,
// Invalid call.
Invalid,
// The input uses features that are not yet implemented.
NotImplemented,
};

65
Userland/Libraries/LibVideo/VP9/Decoder.cpp
View file

@ -7,6 +7,7 @@
#include <AK/IntegralMath.h>
#include <LibGfx/Size.h>
#include <LibVideo/Color/CodingIndependentCodePoints.h>
#include "Decoder.h"
#include "Utilities.h"
@ -149,6 +150,70 @@ bool Decoder::get_uv_subsampling_x()
return m_parser->m_subsampling_x;
}
CodingIndependentCodePoints Decoder::get_cicp_color_space()
{
ColorPrimaries color_primaries;
TransferCharacteristics transfer_characteristics;
MatrixCoefficients matrix_coefficients;
switch (m_parser->m_color_space) {
case ColorSpace::Unknown:
color_primaries = ColorPrimaries::Unspecified;
transfer_characteristics = TransferCharacteristics::Unspecified;
matrix_coefficients = MatrixCoefficients::Unspecified;
break;
case ColorSpace::Bt601:
color_primaries = ColorPrimaries::BT601;
transfer_characteristics = TransferCharacteristics::BT601;
matrix_coefficients = MatrixCoefficients::BT601;
break;
case ColorSpace::Bt709:
color_primaries = ColorPrimaries::BT709;
transfer_characteristics = TransferCharacteristics::BT709;
matrix_coefficients = MatrixCoefficients::BT709;
break;
case ColorSpace::Smpte170:
// https://www.kernel.org/doc/html/v4.9/media/uapi/v4l/pixfmt-007.html#colorspace-smpte-170m-v4l2-colorspace-smpte170m
color_primaries = ColorPrimaries::BT601;
transfer_characteristics = TransferCharacteristics::BT709;
matrix_coefficients = MatrixCoefficients::BT601;
break;
case ColorSpace::Smpte240:
color_primaries = ColorPrimaries::SMPTE240;
transfer_characteristics = TransferCharacteristics::SMPTE240;
matrix_coefficients = MatrixCoefficients::SMPTE240;
break;
case ColorSpace::Bt2020:
color_primaries = ColorPrimaries::BT2020;
// Bit depth doesn't actually matter to our transfer functions since we
// convert in floats of range 0-1 (for now?), but just for correctness set
// the TC to match the bit depth here.
if (m_parser->m_bit_depth == 12)
transfer_characteristics = TransferCharacteristics::BT2020BitDepth12;
else if (m_parser->m_bit_depth == 10)
transfer_characteristics = TransferCharacteristics::BT2020BitDepth10;
else
transfer_characteristics = TransferCharacteristics::BT709;
matrix_coefficients = MatrixCoefficients::BT2020NonConstantLuminance;
break;
case ColorSpace::RGB:
color_primaries = ColorPrimaries::BT709;
transfer_characteristics = TransferCharacteristics::Linear;
matrix_coefficients = MatrixCoefficients::Identity;
break;
case ColorSpace::Reserved:
VERIFY_NOT_REACHED();
break;
}
return { color_primaries, transfer_characteristics, matrix_coefficients, m_parser->m_color_range };
}
u8 Decoder::get_bit_depth()
{
return m_parser->m_bit_depth;
}
u8 Decoder::merge_prob(u8 pre_prob, u8 count_0, u8 count_1, u8 count_sat, u8 max_update_factor)
{
auto total_decode_count = count_0 + count_1;

3
Userland/Libraries/LibVideo/VP9/Decoder.h
View file

@ -10,6 +10,7 @@
#include <AK/ByteBuffer.h>
#include <AK/Error.h>
#include <AK/Span.h>
#include <LibVideo/Color/CodingIndependentCodePoints.h>
#include <LibVideo/DecoderError.h>
#include "Parser.h"
@ -32,6 +33,8 @@ public:
Gfx::Size<size_t> get_y_plane_size();
bool get_uv_subsampling_y();
bool get_uv_subsampling_x();
CodingIndependentCodePoints get_cicp_color_space();
u8 get_bit_depth();
private:
typedef i32 Intermediate;

5
Userland/Libraries/LibVideo/VP9/Enums.h
View file

@ -27,11 +27,6 @@ enum ColorSpace : u8 {
RGB = 7
};
enum ColorRange {
StudioSwing,
FullSwing
};
enum InterpolationFilter : u8 {
EightTap = 0,
EightTapSmooth = 1,

6
Userland/Libraries/LibVideo/VP9/Parser.cpp
View file

@ -136,8 +136,8 @@ DecoderErrorOr<FrameType> Parser::read_frame_type()
DecoderErrorOr<ColorRange> Parser::read_color_range()
{
if (TRY_READ(m_bit_stream->read_bit()))
return FullSwing;
return StudioSwing;
return ColorRange::Full;
return ColorRange::Studio;
}
/* (6.2) */
@ -273,7 +273,7 @@ DecoderErrorOr<void> Parser::color_config()
m_subsampling_y = true;
}
} else {
m_color_range = FullSwing;
m_color_range = ColorRange::Full;
if (m_profile == 1 || m_profile == 3) {
m_subsampling_x = false;
m_subsampling_y = false;

1
Userland/Libraries/LibVideo/VP9/Parser.h
View file

@ -12,6 +12,7 @@
#include <AK/Span.h>
#include <AK/Vector.h>
#include <LibGfx/Forward.h>
#include <LibVideo/Color/CodingIndependentCodePoints.h>
#include <LibVideo/DecoderError.h>
#include "BitStream.h"