LibGfx: Extend Unicode support for BitmapFonts

Removes the concept of Type enumeration in favor of a bitmask which
represents 544 potential byte ranges of 256 characters per bit,
supporting the current unicode code point set (0x0000-0x10FFFF).
Range positions are indexed in a vector for code point lookup and
conversion.

Co-authored-by: Lynn <lynn@foldr.moe>

Userland/Libraries/LibGfx/BitmapFont.cpp

@@ -17,7 +17,7 @@ struct [[gnu::packed]] FontFileHeader {
     char magic[4];
     u8 glyph_width;
     u8 glyph_height;
-    u8 type;
+    u16 range_mask_size;
     u8 is_variable_width;
     u8 glyph_spacing;
     u8 baseline;
@@ -26,34 +26,95 @@ struct [[gnu::packed]] FontFileHeader {
     u16 weight;
     char name[32];
     char family[32];
-    u16 unused;
+    u8 unused;
 };
 
 static_assert(AssertSize<FontFileHeader, 80>());
 
+static constexpr size_t s_max_glyph_count = 0x110000;
+static constexpr size_t s_max_range_mask_size = s_max_glyph_count / (256 * 8);
+
 NonnullRefPtr<Font> BitmapFont::clone() const
 {
+    auto* new_range_mask = static_cast<u8*>(malloc(m_range_mask_size));
+    memcpy(new_range_mask, m_range_mask, m_range_mask_size);
     size_t bytes_per_glyph = sizeof(u32) * glyph_height();
-    auto* new_rows = static_cast<unsigned*>(kmalloc_array(m_glyph_count, bytes_per_glyph));
+    auto* new_rows = static_cast<u32*>(kmalloc_array(m_glyph_count, bytes_per_glyph));
     memcpy(new_rows, m_rows, bytes_per_glyph * m_glyph_count);
     auto* new_widths = static_cast<u8*>(malloc(m_glyph_count));
     memcpy(new_widths, m_glyph_widths, m_glyph_count);
-    return adopt_ref(*new BitmapFont(m_name, m_family, new_rows, new_widths, m_fixed_width, m_glyph_width, m_glyph_height, m_glyph_spacing, m_type, m_baseline, m_mean_line, m_presentation_size, m_weight, true));
+    return adopt_ref(*new BitmapFont(m_name, m_family, new_rows, new_widths, m_fixed_width, m_glyph_width, m_glyph_height, m_glyph_spacing, m_range_mask_size, new_range_mask, m_baseline, m_mean_line, m_presentation_size, m_weight, true));
 }
 
-NonnullRefPtr<BitmapFont> BitmapFont::create(u8 glyph_height, u8 glyph_width, bool fixed, FontTypes type)
+NonnullRefPtr<BitmapFont> BitmapFont::create(u8 glyph_height, u8 glyph_width, bool fixed, size_t glyph_count)
 {
+    glyph_count += 256 - (glyph_count % 256);
+    glyph_count = min(glyph_count, s_max_glyph_count);
+    size_t glyphs_per_range = 8 * 256;
+    u16 range_mask_size = ceil_div(glyph_count, glyphs_per_range);
+    auto* new_range_mask = static_cast<u8*>(calloc(range_mask_size, 1));
+    for (size_t i = 0; i < glyph_count; i += 256) {
+        new_range_mask[i / 256 / 8] |= 1 << (i / 256 % 8);
+    }
     size_t bytes_per_glyph = sizeof(u32) * glyph_height;
-    size_t count = glyph_count_by_type(type);
+    auto* new_rows = static_cast<u32*>(calloc(glyph_count, bytes_per_glyph));
-    auto* new_rows = static_cast<unsigned*>(calloc(count, bytes_per_glyph));
+    auto* new_widths = static_cast<u8*>(calloc(glyph_count, 1));
-    auto* new_widths = static_cast<u8*>(calloc(count, 1));
+    return adopt_ref(*new BitmapFont("Untitled", "Untitled", new_rows, new_widths, fixed, glyph_width, glyph_height, 1, range_mask_size, new_range_mask, 0, 0, 0, 400, true));
-    return adopt_ref(*new BitmapFont("Untitled", "Untitled", new_rows, new_widths, fixed, glyph_width, glyph_height, 1, type, 0, 0, 0, 400, true));
 }
 
-BitmapFont::BitmapFont(String name, String family, unsigned* rows, u8* widths, bool is_fixed_width, u8 glyph_width, u8 glyph_height, u8 glyph_spacing, FontTypes type, u8 baseline, u8 mean_line, u8 presentation_size, u16 weight, bool owns_arrays)
+NonnullRefPtr<BitmapFont> BitmapFont::unmasked_character_set() const
+{
+    auto* new_range_mask = static_cast<u8*>(malloc(s_max_range_mask_size));
+    constexpr u8 max_bits { 0b1111'1111 };
+    memset(new_range_mask, max_bits, s_max_range_mask_size);
+    size_t bytes_per_glyph = sizeof(u32) * glyph_height();
+    auto* new_rows = static_cast<u32*>(kmalloc_array(s_max_glyph_count, bytes_per_glyph));
+    auto* new_widths = static_cast<u8*>(calloc(s_max_glyph_count, 1));
+    for (size_t code_point = 0; code_point < s_max_glyph_count; ++code_point) {
+        auto index = glyph_index(code_point);
+        if (index.has_value()) {
+            memcpy(&new_widths[code_point], &m_glyph_widths[index.value()], 1);
+            memcpy(&new_rows[code_point * glyph_height()], &m_rows[index.value() * glyph_height()], bytes_per_glyph);
+        }
+    }
+    return adopt_ref(*new BitmapFont(m_name, m_family, new_rows, new_widths, m_fixed_width, m_glyph_width, m_glyph_height, m_glyph_spacing, s_max_range_mask_size, new_range_mask, m_baseline, m_mean_line, m_presentation_size, m_weight, true));
+}
+
+NonnullRefPtr<BitmapFont> BitmapFont::masked_character_set() const
+{
+    auto* new_range_mask = static_cast<u8*>(calloc(s_max_range_mask_size, 1));
+    u16 new_range_mask_size { 0 };
+    for (size_t i = 0; i < s_max_glyph_count; ++i) {
+        if (m_glyph_widths[i] > 0) {
+            new_range_mask[i / 256 / 8] |= 1 << (i / 256 % 8);
+            if (i / 256 / 8 + 1 > new_range_mask_size)
+                new_range_mask_size = i / 256 / 8 + 1;
+        }
+    }
+    size_t new_glyph_count { 0 };
+    for (size_t i = 0; i < new_range_mask_size; ++i) {
+        new_glyph_count += 256 * __builtin_popcount(new_range_mask[i]);
+    }
+    size_t bytes_per_glyph = sizeof(u32) * m_glyph_height;
+    auto* new_rows = static_cast<u32*>(calloc(new_glyph_count, bytes_per_glyph));
+    auto* new_widths = static_cast<u8*>(calloc(new_glyph_count, 1));
+    for (size_t i = 0, j = 0; i < s_max_glyph_count; ++i) {
+        if (!(new_range_mask[i / 256 / 8] & 1 << (i / 256 % 8))) {
+            j++;
+            i += 255;
+            continue;
+        }
+        memcpy(&new_widths[i - j * 256], &m_glyph_widths[i], 1);
+        memcpy(&new_rows[(i - j * 256) * glyph_height()], &m_rows[i * glyph_height()], bytes_per_glyph);
+    }
+    return adopt_ref(*new BitmapFont(m_name, m_family, new_rows, new_widths, m_fixed_width, m_glyph_width, m_glyph_height, m_glyph_spacing, new_range_mask_size, new_range_mask, m_baseline, m_mean_line, m_presentation_size, m_weight, true));
+}
+
+BitmapFont::BitmapFont(String name, String family, u32* rows, u8* widths, bool is_fixed_width, u8 glyph_width, u8 glyph_height, u8 glyph_spacing, u16 range_mask_size, u8* range_mask, u8 baseline, u8 mean_line, u8 presentation_size, u16 weight, bool owns_arrays)
     : m_name(name)
     , m_family(family)
-    , m_type(type)
+    , m_range_mask_size(range_mask_size)
+    , m_range_mask(range_mask)
     , m_rows(rows)
     , m_glyph_widths(widths)
     , m_glyph_width(glyph_width)
@@ -68,12 +129,22 @@ BitmapFont::BitmapFont(String name, String family, unsigned* rows, u8* widths, b
     , m_fixed_width(is_fixed_width)
     , m_owns_arrays(owns_arrays)
 {
+    VERIFY(m_range_mask);
     VERIFY(m_rows);
     VERIFY(m_glyph_widths);
 
     update_x_height();
 
-    m_glyph_count = glyph_count_by_type(m_type);
+    for (size_t i = 0, index = 0; i < m_range_mask_size; ++i) {
+        for (size_t j = 0; j < 8; ++j) {
+            if (m_range_mask[i] & (1 << j)) {
+                m_glyph_count += 256;
+                m_range_indices.append(index++);
+            } else {
+                m_range_indices.append({});
+            }
+        }
+    }
 
     if (!m_fixed_width) {
         u8 maximum = 0;
@@ -92,6 +163,7 @@ BitmapFont::~BitmapFont()
     if (m_owns_arrays) {
         free(m_glyph_widths);
         free(m_rows);
+        free(m_range_mask);
     }
 }
 
@@ -112,60 +184,14 @@ RefPtr<BitmapFont> BitmapFont::load_from_memory(const u8* data)
         return nullptr;
     }
 
-    FontTypes type;
+    size_t bytes_per_glyph = sizeof(u32) * header.glyph_height;
-    if (header.type == 0)
+    size_t glyph_count { 0 };
-        type = FontTypes::Default;
+    u8* range_mask = const_cast<u8*>(data + sizeof(FontFileHeader));
-    else if (header.type == 1)
+    for (size_t i = 0; i < header.range_mask_size; ++i)
-        type = FontTypes::LatinExtendedA;
+        glyph_count += 256 * __builtin_popcount(range_mask[i]);
-    else if (header.type == 2)
+    u32* rows = (u32*)(range_mask + header.range_mask_size);
-        type = FontTypes::Cyrillic;
+    u8* widths = (u8*)(rows) + glyph_count * bytes_per_glyph;
-    else if (header.type == 3)
+    return adopt_ref(*new BitmapFont(String(header.name), String(header.family), rows, widths, !header.is_variable_width, header.glyph_width, header.glyph_height, header.glyph_spacing, header.range_mask_size, range_mask, header.baseline, header.mean_line, header.presentation_size, header.weight));
-        type = FontTypes::Hebrew;
-    else
-        VERIFY_NOT_REACHED();
-
-    size_t count = glyph_count_by_type(type);
-    size_t bytes_per_glyph = sizeof(unsigned) * header.glyph_height;
-
-    auto* rows = const_cast<unsigned*>((const unsigned*)(data + sizeof(FontFileHeader)));
-    u8* widths = (u8*)(rows) + count * bytes_per_glyph;
-    return adopt_ref(*new BitmapFont(String(header.name), String(header.family), rows, widths, !header.is_variable_width, header.glyph_width, header.glyph_height, header.glyph_spacing, type, header.baseline, header.mean_line, header.presentation_size, header.weight));
-}
-
-size_t BitmapFont::glyph_count_by_type(FontTypes type)
-{
-    if (type == FontTypes::Default)
-        return 256;
-
-    if (type == FontTypes::LatinExtendedA)
-        return 384;
-
-    if (type == FontTypes::Cyrillic)
-        return 1280;
-
-    if (type == FontTypes::Hebrew)
-        return 1536;
-
-    dbgln("Unknown font type: {}", (int)type);
-    VERIFY_NOT_REACHED();
-}
-
-String BitmapFont::type_name_by_type(FontTypes type)
-{
-    if (type == FontTypes::Default)
-        return "Default";
-
-    if (type == FontTypes::LatinExtendedA)
-        return "LatinExtendedA";
-
-    if (type == FontTypes::Cyrillic)
-        return "Cyrillic";
-
-    if (type == FontTypes::Hebrew)
-        return "Hebrew";
-
-    dbgln("Unknown font type: {}", (int)type);
-    VERIFY_NOT_REACHED();
 }
 
 RefPtr<BitmapFont> BitmapFont::load_from_file(String const& path)
@@ -192,7 +218,7 @@ bool BitmapFont::write_to_file(String const& path)
     memcpy(header.magic, "!Fnt", 4);
     header.glyph_width = m_glyph_width;
     header.glyph_height = m_glyph_height;
-    header.type = m_type;
+    header.range_mask_size = m_range_mask_size;
     header.baseline = m_baseline;
     header.mean_line = m_mean_line;
     header.is_variable_width = !m_fixed_width;
@@ -202,17 +228,16 @@ bool BitmapFont::write_to_file(String const& path)
     memcpy(header.name, m_name.characters(), min(m_name.length(), sizeof(header.name) - 1));
     memcpy(header.family, m_family.characters(), min(m_family.length(), sizeof(header.family) - 1));
 
-    size_t bytes_per_glyph = sizeof(unsigned) * m_glyph_height;
-    size_t count = glyph_count_by_type(m_type);
-
     auto stream_result = Core::OutputFileStream::open_buffered(path);
     if (stream_result.is_error())
         return false;
     auto& stream = stream_result.value();
 
+    size_t bytes_per_glyph = sizeof(u32) * m_glyph_height;
     stream << ReadonlyBytes { &header, sizeof(header) };
-    stream << ReadonlyBytes { m_rows, count * bytes_per_glyph };
+    stream << ReadonlyBytes { m_range_mask, m_range_mask_size };
-    stream << ReadonlyBytes { m_glyph_widths, count };
+    stream << ReadonlyBytes { m_rows, m_glyph_count * bytes_per_glyph };
+    stream << ReadonlyBytes { m_glyph_widths, m_glyph_count };
 
     stream.flush();
     if (stream.handle_any_error())
@@ -223,7 +248,20 @@ bool BitmapFont::write_to_file(String const& path)
 
 Glyph BitmapFont::glyph(u32 code_point) const
 {
-    auto width = glyph_width(code_point);
+    // Note: Until all fonts support the 0xFFFD replacement
+    // character, fall back to painting '?' if necessary.
+    auto index = glyph_index(code_point).value_or('?');
+    auto width = m_glyph_widths[index];
+    return Glyph(
+        GlyphBitmap(&m_rows[index * m_glyph_height], { width, m_glyph_height }),
+        0,
+        width,
+        m_glyph_height);
+}
+
+Glyph BitmapFont::raw_glyph(u32 code_point) const
+{
+    auto width = m_glyph_widths[code_point];
     return Glyph(
         GlyphBitmap(&m_rows[code_point * m_glyph_height], { width, m_glyph_height }),
         0,
@@ -231,18 +269,46 @@ Glyph BitmapFont::glyph(u32 code_point) const
         m_glyph_height);
 }
 
+Optional<size_t> BitmapFont::glyph_index(u32 code_point) const
+{
+    auto index = code_point / 256;
+    if (index >= m_range_indices.size())
+        return {};
+    if (!m_range_indices[index].has_value())
+        return {};
+    return m_range_indices[index].value() * 256 + code_point % 256;
+}
+
+bool BitmapFont::contains_glyph(u32 code_point) const
+{
+    auto index = glyph_index(code_point);
+    return index.has_value() && m_glyph_widths[index.value()] > 0;
+}
+
+u8 BitmapFont::glyph_width(u32 code_point) const
+{
+    if (is_ascii(code_point) && !is_ascii_printable(code_point))
+        return 0;
+    auto index = glyph_index(code_point);
+    return m_fixed_width || !index.has_value() ? m_glyph_width : m_glyph_widths[index.value()];
+}
+
 int BitmapFont::glyph_or_emoji_width_for_variable_width_font(u32 code_point) const
 {
-    if (code_point < m_glyph_count) {
+    // FIXME: This is a hack in lieu of proper code point identification.
-        if (m_glyph_widths[code_point] > 0)
+    // 0xFFFF is arbitrary but also the end of the Basic Multilingual Plane.
+    if (code_point < 0xFFFF) {
+        auto index = glyph_index(code_point);
+        if (!index.has_value())
+            return glyph_width(0xFFFD);
+        if (m_glyph_widths[index.value()] > 0)
             return glyph_width(code_point);
-        else
+        return glyph_width(0xFFFD);
-            return glyph_width('?');
     }
 
     auto* emoji = Emoji::emoji_for_code_point(code_point);
     if (emoji == nullptr)
-        return glyph_width('?');
+        return glyph_width(0xFFFD);
     return emoji->size().width();
 }
 
@@ -275,39 +341,6 @@ ALWAYS_INLINE int BitmapFont::unicode_view_width(T const& view) const
     return longest_width;
 }
 
-void BitmapFont::set_type(FontTypes type)
-{
-    if (type == m_type)
-        return;
-
-    if (type == FontTypes::Default)
-        return;
-
-    size_t new_glyph_count = glyph_count_by_type(type);
-    if (new_glyph_count <= m_glyph_count) {
-        m_glyph_count = new_glyph_count;
-        return;
-    }
-
-    int item_count_to_copy = min(m_glyph_count, new_glyph_count);
-
-    size_t bytes_per_glyph = sizeof(u32) * glyph_height();
-
-    auto* new_rows = static_cast<unsigned*>(calloc(new_glyph_count, bytes_per_glyph));
-    memcpy(new_rows, m_rows, bytes_per_glyph * item_count_to_copy);
-
-    auto* new_widths = static_cast<u8*>(calloc(new_glyph_count, 1));
-    memcpy(new_widths, m_glyph_widths, item_count_to_copy);
-
-    kfree(m_rows);
-    kfree(m_glyph_widths);
-
-    m_type = type;
-    m_glyph_count = new_glyph_count;
-    m_rows = new_rows;
-    m_glyph_widths = new_widths;
-}
-
 String BitmapFont::qualified_name() const
 {
     return String::formatted("{} {} {}", family(), presentation_size(), weight());

Userland/Libraries/LibGfx/BitmapFont.h

@@ -17,20 +17,13 @@
 
 namespace Gfx {
 
-// Note: Perhaps put glyph count directly in header
-// and sidestep FontType conflation/sync maintenance
-enum FontTypes {
-    Default = 0,
-    LatinExtendedA,
-    Cyrillic,
-    Hebrew,
-    __Count
-};
-
 class BitmapFont final : public Font {
 public:
     NonnullRefPtr<Font> clone() const override;
-    static NonnullRefPtr<BitmapFont> create(u8 glyph_height, u8 glyph_width, bool fixed, FontTypes type);
+    static NonnullRefPtr<BitmapFont> create(u8 glyph_height, u8 glyph_width, bool fixed, size_t glyph_count);
+
+    NonnullRefPtr<BitmapFont> masked_character_set() const;
+    NonnullRefPtr<BitmapFont> unmasked_character_set() const;
 
     static RefPtr<BitmapFont> load_from_file(String const& path);
     bool write_to_file(String const& path);
@@ -44,15 +37,9 @@ public:
     void set_weight(u16 weight) { m_weight = weight; }
 
     Glyph glyph(u32 code_point) const override;
-    bool contains_glyph(u32 code_point) const override { return code_point < (u32)glyph_count() && m_glyph_widths[code_point] > 0; }
+    Glyph raw_glyph(u32 code_point) const;
+    bool contains_glyph(u32 code_point) const override;
 
-    u8 glyph_width(size_t ch) const override
-    {
-        if (is_ascii(ch) && !is_ascii_printable(ch))
-            return 0;
-
-        return m_fixed_width ? m_glyph_width : m_glyph_widths[ch];
-    }
     ALWAYS_INLINE int glyph_or_emoji_width(u32 code_point) const override
     {
         if (m_fixed_width)
@@ -62,7 +49,8 @@ public:
     u8 glyph_height() const override { return m_glyph_height; }
     int x_height() const override { return m_x_height; }
 
-    u8 raw_glyph_width(size_t ch) const { return m_glyph_widths[ch]; }
+    u8 glyph_width(u32 code_point) const override;
+    u8 raw_glyph_width(u32 code_point) const { return m_glyph_widths[code_point]; }
 
     u8 min_glyph_width() const override { return m_min_glyph_width; }
     u8 max_glyph_width() const override { return m_max_glyph_width; }
@@ -95,16 +83,17 @@ public:
     u8 glyph_spacing() const override { return m_glyph_spacing; }
     void set_glyph_spacing(u8 spacing) { m_glyph_spacing = spacing; }
 
-    void set_glyph_width(size_t ch, u8 width)
+    void set_glyph_width(u32 code_point, u8 width)
     {
         VERIFY(m_glyph_widths);
-        m_glyph_widths[ch] = width;
+        m_glyph_widths[code_point] = width;
     }
 
     size_t glyph_count() const override { return m_glyph_count; }
+    Optional<size_t> glyph_index(u32 code_point) const;
 
-    FontTypes type() { return m_type; }
+    u16 range_size() const { return m_range_mask_size; }
-    void set_type(FontTypes type);
+    bool is_range_empty(u32 code_point) const { return !(m_range_mask[code_point / 256 / 8] & 1 << (code_point / 256 % 8)); }
 
     String family() const override { return m_family; }
     void set_family(String family) { m_family = move(family); }
@@ -112,11 +101,10 @@ public:
 
     String qualified_name() const override;
 
-    static size_t glyph_count_by_type(FontTypes type);
-    static String type_name_by_type(FontTypes type);
-
 private:
-    BitmapFont(String name, String family, unsigned* rows, u8* widths, bool is_fixed_width, u8 glyph_width, u8 glyph_height, u8 glyph_spacing, FontTypes type, u8 baseline, u8 mean_line, u8 presentation_size, u16 weight, bool owns_arrays = false);
+    BitmapFont(String name, String family, u32* rows, u8* widths, bool is_fixed_width,
+        u8 glyph_width, u8 glyph_height, u8 glyph_spacing, u16 range_mask_size, u8* range_mask,
+        u8 baseline, u8 mean_line, u8 presentation_size, u16 weight, bool owns_arrays = false);
 
     static RefPtr<BitmapFont> load_from_memory(u8 const*);
 
@@ -128,10 +116,13 @@ private:
 
     String m_name;
     String m_family;
-    FontTypes m_type;
+    size_t m_glyph_count { 0 };
-    size_t m_glyph_count { 256 };
 
-    unsigned* m_rows { nullptr };
+    u16 m_range_mask_size { 0 };
+    u8* m_range_mask { nullptr };
+    Vector<Optional<size_t>> m_range_indices;
+
+    u32* m_rows { nullptr };
     u8* m_glyph_widths { nullptr };
     RefPtr<MappedFile> m_mapped_file;
 

Userland/Libraries/LibGfx/Font.h

@@ -92,7 +92,7 @@ public:
     virtual Glyph glyph(u32 code_point) const = 0;
     virtual bool contains_glyph(u32 code_point) const = 0;
 
-    virtual u8 glyph_width(size_t ch) const = 0;
+    virtual u8 glyph_width(u32 code_point) const = 0;
     virtual int glyph_or_emoji_width(u32 code_point) const = 0;
     virtual u8 glyph_height() const = 0;
     virtual int x_height() const = 0;

Userland/Libraries/LibGfx/TrueTypeFont/Font.cpp

@@ -540,7 +540,7 @@ Gfx::Glyph ScaledFont::glyph(u32 code_point) const
     return Gfx::Glyph(bitmap, metrics.left_side_bearing, metrics.advance_width, metrics.ascender);
 }
 
-u8 ScaledFont::glyph_width(size_t code_point) const
+u8 ScaledFont::glyph_width(u32 code_point) const
 {
     auto id = glyph_id_for_code_point(code_point);
     auto metrics = glyph_metrics(id);

Userland/Libraries/LibGfx/TrueTypeFont/Font.h

@@ -125,7 +125,7 @@ public:
     virtual u16 weight() const override { return m_font->weight(); }
     virtual Gfx::Glyph glyph(u32 code_point) const override;
     virtual bool contains_glyph(u32 code_point) const override { return m_font->glyph_id_for_code_point(code_point) > 0; }
-    virtual u8 glyph_width(size_t ch) const override;
+    virtual u8 glyph_width(u32 code_point) const override;
     virtual int glyph_or_emoji_width(u32 code_point) const override;
     virtual u8 glyph_height() const override { return m_point_height; }
     virtual int x_height() const override { return m_point_height; }      // FIXME: Read from font