From 71b4433b0d83714852e882a83808a7d4bbeb6117 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Pur Date: Mon, 7 Jun 2021 12:03:09 +0430 Subject: [PATCH] LibWeb+LibSyntax: Implement nested syntax highlighters And use them to highlight javascript in HTML source. This commit also changes how TextDocumentSpan::data is interpreted, as it used to be an opaque pointer, but everyone stuffed an enum value inside it, which made the values not unique to each highlighter; that field is now a u64 serial id. The syntax highlighters don't need to change their ways of stuffing token types into that field, but a highlighter that calls another nested highlighter needs to register the nested types for use with token pairs. --- .../Spreadsheet/CellSyntaxHighlighter.cpp | 4 +- .../Libraries/LibCpp/SyntaxHighlighter.cpp | 22 ++--- Userland/Libraries/LibCpp/SyntaxHighlighter.h | 8 +- .../Libraries/LibGUI/GMLSyntaxHighlighter.cpp | 14 +-- .../Libraries/LibGUI/GMLSyntaxHighlighter.h | 6 +- .../Libraries/LibGUI/INISyntaxHighlighter.cpp | 14 +-- .../Libraries/LibGUI/INISyntaxHighlighter.h | 6 +- Userland/Libraries/LibGUI/TextDocument.h | 2 +- .../Libraries/LibJS/SyntaxHighlighter.cpp | 20 ++-- Userland/Libraries/LibJS/SyntaxHighlighter.h | 8 +- .../Libraries/LibSQL/SyntaxHighlighter.cpp | 14 +-- Userland/Libraries/LibSQL/SyntaxHighlighter.h | 6 +- Userland/Libraries/LibSyntax/Highlighter.cpp | 17 +++- Userland/Libraries/LibSyntax/Highlighter.h | 91 +++++++++++++++++-- .../SyntaxHighlighter/SyntaxHighlighter.cpp | 45 +++++++-- .../SyntaxHighlighter/SyntaxHighlighter.h | 8 +- Userland/Shell/SyntaxHighlighter.cpp | 30 +++--- Userland/Shell/SyntaxHighlighter.h | 8 +- 18 files changed, 219 insertions(+), 104 deletions(-) diff --git a/Userland/Applications/Spreadsheet/CellSyntaxHighlighter.cpp b/Userland/Applications/Spreadsheet/CellSyntaxHighlighter.cpp index 4e74b7b6ae9..bfdbe73c9ce 100644 --- a/Userland/Applications/Spreadsheet/CellSyntaxHighlighter.cpp +++ b/Userland/Applications/Spreadsheet/CellSyntaxHighlighter.cpp @@ -31,7 +31,7 @@ void CellSyntaxHighlighter::rehighlight(const Palette& palette) false, false, }, - nullptr, + (u64)-1, false); if (m_cell && m_cell->exception()) { @@ -47,7 +47,7 @@ void CellSyntaxHighlighter::rehighlight(const Palette& palette) false, false, }, - nullptr, + (u64)-1, false }); } m_client->do_update(); diff --git a/Userland/Libraries/LibCpp/SyntaxHighlighter.cpp b/Userland/Libraries/LibCpp/SyntaxHighlighter.cpp index 563fe1db764..9446b5e7f48 100644 --- a/Userland/Libraries/LibCpp/SyntaxHighlighter.cpp +++ b/Userland/Libraries/LibCpp/SyntaxHighlighter.cpp @@ -43,15 +43,15 @@ static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, Cpp:: } } -bool SyntaxHighlighter::is_identifier(void* token) const +bool SyntaxHighlighter::is_identifier(u64 token) const { - auto cpp_token = static_cast(reinterpret_cast(token)); + auto cpp_token = static_cast(token); return cpp_token == Cpp::Token::Type::Identifier; } -bool SyntaxHighlighter::is_navigatable(void* token) const +bool SyntaxHighlighter::is_navigatable(u64 token) const { - auto cpp_token = static_cast(reinterpret_cast(token)); + auto cpp_token = static_cast(token); return cpp_token == Cpp::Token::Type::IncludePath; } @@ -72,7 +72,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette) span.attributes.color = style.color; span.attributes.bold = style.bold; span.is_skippable = token.type() == Cpp::Token::Type::Whitespace; - span.data = reinterpret_cast(token.type()); + span.data = static_cast(token.type()); spans.append(span); } m_client->do_set_spans(move(spans)); @@ -83,20 +83,20 @@ void SyntaxHighlighter::rehighlight(Palette const& palette) m_client->do_update(); } -Vector SyntaxHighlighter::matching_token_pairs() const +Vector SyntaxHighlighter::matching_token_pairs_impl() const { static Vector pairs; if (pairs.is_empty()) { - pairs.append({ reinterpret_cast(Cpp::Token::Type::LeftCurly), reinterpret_cast(Cpp::Token::Type::RightCurly) }); - pairs.append({ reinterpret_cast(Cpp::Token::Type::LeftParen), reinterpret_cast(Cpp::Token::Type::RightParen) }); - pairs.append({ reinterpret_cast(Cpp::Token::Type::LeftBracket), reinterpret_cast(Cpp::Token::Type::RightBracket) }); + pairs.append({ static_cast(Cpp::Token::Type::LeftCurly), static_cast(Cpp::Token::Type::RightCurly) }); + pairs.append({ static_cast(Cpp::Token::Type::LeftParen), static_cast(Cpp::Token::Type::RightParen) }); + pairs.append({ static_cast(Cpp::Token::Type::LeftBracket), static_cast(Cpp::Token::Type::RightBracket) }); } return pairs; } -bool SyntaxHighlighter::token_types_equal(void* token1, void* token2) const +bool SyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const { - return static_cast(reinterpret_cast(token1)) == static_cast(reinterpret_cast(token2)); + return static_cast(token1) == static_cast(token2); } SyntaxHighlighter::~SyntaxHighlighter() diff --git a/Userland/Libraries/LibCpp/SyntaxHighlighter.h b/Userland/Libraries/LibCpp/SyntaxHighlighter.h index 0df0bbc59d8..5526c02b105 100644 --- a/Userland/Libraries/LibCpp/SyntaxHighlighter.h +++ b/Userland/Libraries/LibCpp/SyntaxHighlighter.h @@ -15,15 +15,15 @@ public: SyntaxHighlighter() { } virtual ~SyntaxHighlighter() override; - virtual bool is_identifier(void*) const override; - virtual bool is_navigatable(void*) const override; + virtual bool is_identifier(u64) const override; + virtual bool is_navigatable(u64) const override; virtual Syntax::Language language() const override { return Syntax::Language::Cpp; } virtual void rehighlight(Palette const&) override; protected: - virtual Vector matching_token_pairs() const override; - virtual bool token_types_equal(void*, void*) const override; + virtual Vector matching_token_pairs_impl() const override; + virtual bool token_types_equal(u64, u64) const override; }; } diff --git a/Userland/Libraries/LibGUI/GMLSyntaxHighlighter.cpp b/Userland/Libraries/LibGUI/GMLSyntaxHighlighter.cpp index 2a0a40a21b0..20ef918cfa2 100644 --- a/Userland/Libraries/LibGUI/GMLSyntaxHighlighter.cpp +++ b/Userland/Libraries/LibGUI/GMLSyntaxHighlighter.cpp @@ -33,9 +33,9 @@ static Syntax::TextStyle style_for_token_type(const Gfx::Palette& palette, GMLTo } } -bool GMLSyntaxHighlighter::is_identifier(void* token) const +bool GMLSyntaxHighlighter::is_identifier(u64 token) const { - auto ini_token = static_cast(reinterpret_cast(token)); + auto ini_token = static_cast(token); return ini_token == GUI::GMLToken::Type::Identifier; } @@ -54,7 +54,7 @@ void GMLSyntaxHighlighter::rehighlight(const Palette& palette) span.attributes.color = style.color; span.attributes.bold = style.bold; span.is_skippable = false; - span.data = reinterpret_cast(token.m_type); + span.data = static_cast(token.m_type); spans.append(span); } m_client->do_set_spans(move(spans)); @@ -65,18 +65,18 @@ void GMLSyntaxHighlighter::rehighlight(const Palette& palette) m_client->do_update(); } -Vector GMLSyntaxHighlighter::matching_token_pairs() const +Vector GMLSyntaxHighlighter::matching_token_pairs_impl() const { static Vector pairs; if (pairs.is_empty()) { - pairs.append({ reinterpret_cast(GMLToken::Type::LeftCurly), reinterpret_cast(GMLToken::Type::RightCurly) }); + pairs.append({ static_cast(GMLToken::Type::LeftCurly), static_cast(GMLToken::Type::RightCurly) }); } return pairs; } -bool GMLSyntaxHighlighter::token_types_equal(void* token1, void* token2) const +bool GMLSyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const { - return static_cast(reinterpret_cast(token1)) == static_cast(reinterpret_cast(token2)); + return static_cast(token1) == static_cast(token2); } GMLSyntaxHighlighter::~GMLSyntaxHighlighter() diff --git a/Userland/Libraries/LibGUI/GMLSyntaxHighlighter.h b/Userland/Libraries/LibGUI/GMLSyntaxHighlighter.h index a8a944dbc01..399671ac6db 100644 --- a/Userland/Libraries/LibGUI/GMLSyntaxHighlighter.h +++ b/Userland/Libraries/LibGUI/GMLSyntaxHighlighter.h @@ -15,14 +15,14 @@ public: GMLSyntaxHighlighter() { } virtual ~GMLSyntaxHighlighter() override; - virtual bool is_identifier(void*) const override; + virtual bool is_identifier(u64) const override; virtual Syntax::Language language() const override { return Syntax::Language::GML; } virtual void rehighlight(const Palette&) override; protected: - virtual Vector matching_token_pairs() const override; - virtual bool token_types_equal(void*, void*) const override; + virtual Vector matching_token_pairs_impl() const override; + virtual bool token_types_equal(u64, u64) const override; }; } diff --git a/Userland/Libraries/LibGUI/INISyntaxHighlighter.cpp b/Userland/Libraries/LibGUI/INISyntaxHighlighter.cpp index dcfe00f8c31..5a72560803e 100644 --- a/Userland/Libraries/LibGUI/INISyntaxHighlighter.cpp +++ b/Userland/Libraries/LibGUI/INISyntaxHighlighter.cpp @@ -32,9 +32,9 @@ static Syntax::TextStyle style_for_token_type(const Gfx::Palette& palette, IniTo } } -bool IniSyntaxHighlighter::is_identifier(void* token) const +bool IniSyntaxHighlighter::is_identifier(u64 token) const { - auto ini_token = static_cast(reinterpret_cast(token)); + auto ini_token = static_cast(token); return ini_token == GUI::IniToken::Type::Name; } @@ -53,7 +53,7 @@ void IniSyntaxHighlighter::rehighlight(const Palette& palette) span.attributes.color = style.color; span.attributes.bold = style.bold; span.is_skippable = token.m_type == IniToken::Type::Whitespace; - span.data = reinterpret_cast(token.m_type); + span.data = static_cast(token.m_type); spans.append(span); } m_client->do_set_spans(move(spans)); @@ -64,18 +64,18 @@ void IniSyntaxHighlighter::rehighlight(const Palette& palette) m_client->do_update(); } -Vector IniSyntaxHighlighter::matching_token_pairs() const +Vector IniSyntaxHighlighter::matching_token_pairs_impl() const { static Vector pairs; if (pairs.is_empty()) { - pairs.append({ reinterpret_cast(IniToken::Type::LeftBracket), reinterpret_cast(IniToken::Type::RightBracket) }); + pairs.append({ static_cast(IniToken::Type::LeftBracket), static_cast(IniToken::Type::RightBracket) }); } return pairs; } -bool IniSyntaxHighlighter::token_types_equal(void* token1, void* token2) const +bool IniSyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const { - return static_cast(reinterpret_cast(token1)) == static_cast(reinterpret_cast(token2)); + return static_cast(token1) == static_cast(token2); } IniSyntaxHighlighter::~IniSyntaxHighlighter() diff --git a/Userland/Libraries/LibGUI/INISyntaxHighlighter.h b/Userland/Libraries/LibGUI/INISyntaxHighlighter.h index fd14127211a..2639c542470 100644 --- a/Userland/Libraries/LibGUI/INISyntaxHighlighter.h +++ b/Userland/Libraries/LibGUI/INISyntaxHighlighter.h @@ -15,14 +15,14 @@ public: IniSyntaxHighlighter() { } virtual ~IniSyntaxHighlighter() override; - virtual bool is_identifier(void*) const override; + virtual bool is_identifier(u64) const override; virtual Syntax::Language language() const override { return Syntax::Language::INI; } virtual void rehighlight(const Palette&) override; protected: - virtual Vector matching_token_pairs() const override; - virtual bool token_types_equal(void*, void*) const override; + virtual Vector matching_token_pairs_impl() const override; + virtual bool token_types_equal(u64, u64) const override; }; } diff --git a/Userland/Libraries/LibGUI/TextDocument.h b/Userland/Libraries/LibGUI/TextDocument.h index 503e80580ec..dfd3daf7196 100644 --- a/Userland/Libraries/LibGUI/TextDocument.h +++ b/Userland/Libraries/LibGUI/TextDocument.h @@ -26,7 +26,7 @@ namespace GUI { struct TextDocumentSpan { TextRange range; Gfx::TextAttributes attributes; - void* data { nullptr }; + u64 data { 0 }; bool is_skippable { false }; }; diff --git a/Userland/Libraries/LibJS/SyntaxHighlighter.cpp b/Userland/Libraries/LibJS/SyntaxHighlighter.cpp index d5b56708271..5fd14f644fa 100644 --- a/Userland/Libraries/LibJS/SyntaxHighlighter.cpp +++ b/Userland/Libraries/LibJS/SyntaxHighlighter.cpp @@ -38,13 +38,13 @@ static Syntax::TextStyle style_for_token_type(const Gfx::Palette& palette, JS::T } } -bool SyntaxHighlighter::is_identifier(void* token) const +bool SyntaxHighlighter::is_identifier(u64 token) const { - auto js_token = static_cast(reinterpret_cast(token)); + auto js_token = static_cast(static_cast(token)); return js_token == JS::TokenType::Identifier; } -bool SyntaxHighlighter::is_navigatable([[maybe_unused]] void* token) const +bool SyntaxHighlighter::is_navigatable([[maybe_unused]] u64 token) const { return false; } @@ -83,7 +83,7 @@ void SyntaxHighlighter::rehighlight(const Palette& palette) span.attributes.color = style.color; span.attributes.bold = style.bold; span.is_skippable = is_trivia; - span.data = reinterpret_cast(static_cast(type)); + span.data = static_cast(type); spans.append(span); dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "{}{} @ '{}' {}:{} - {}:{}", @@ -111,20 +111,20 @@ void SyntaxHighlighter::rehighlight(const Palette& palette) m_client->do_update(); } -Vector SyntaxHighlighter::matching_token_pairs() const +Vector SyntaxHighlighter::matching_token_pairs_impl() const { static Vector pairs; if (pairs.is_empty()) { - pairs.append({ reinterpret_cast(JS::TokenType::CurlyOpen), reinterpret_cast(JS::TokenType::CurlyClose) }); - pairs.append({ reinterpret_cast(JS::TokenType::ParenOpen), reinterpret_cast(JS::TokenType::ParenClose) }); - pairs.append({ reinterpret_cast(JS::TokenType::BracketOpen), reinterpret_cast(JS::TokenType::BracketClose) }); + pairs.append({ static_cast(JS::TokenType::CurlyOpen), static_cast(JS::TokenType::CurlyClose) }); + pairs.append({ static_cast(JS::TokenType::ParenOpen), static_cast(JS::TokenType::ParenClose) }); + pairs.append({ static_cast(JS::TokenType::BracketOpen), static_cast(JS::TokenType::BracketClose) }); } return pairs; } -bool SyntaxHighlighter::token_types_equal(void* token1, void* token2) const +bool SyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const { - return static_cast(reinterpret_cast(token1)) == static_cast(reinterpret_cast(token2)); + return static_cast(token1) == static_cast(token2); } SyntaxHighlighter::~SyntaxHighlighter() diff --git a/Userland/Libraries/LibJS/SyntaxHighlighter.h b/Userland/Libraries/LibJS/SyntaxHighlighter.h index 41f34a1e974..b65f88356bc 100644 --- a/Userland/Libraries/LibJS/SyntaxHighlighter.h +++ b/Userland/Libraries/LibJS/SyntaxHighlighter.h @@ -15,15 +15,15 @@ public: SyntaxHighlighter() { } virtual ~SyntaxHighlighter() override; - virtual bool is_identifier(void*) const override; - virtual bool is_navigatable(void*) const override; + virtual bool is_identifier(u64) const override; + virtual bool is_navigatable(u64) const override; virtual Syntax::Language language() const override { return Syntax::Language::JavaScript; } virtual void rehighlight(const Palette&) override; protected: - virtual Vector matching_token_pairs() const override; - virtual bool token_types_equal(void*, void*) const override; + virtual Vector matching_token_pairs_impl() const override; + virtual bool token_types_equal(u64, u64) const override; }; } diff --git a/Userland/Libraries/LibSQL/SyntaxHighlighter.cpp b/Userland/Libraries/LibSQL/SyntaxHighlighter.cpp index 5d53f4e0791..db0b308763c 100644 --- a/Userland/Libraries/LibSQL/SyntaxHighlighter.cpp +++ b/Userland/Libraries/LibSQL/SyntaxHighlighter.cpp @@ -33,9 +33,9 @@ static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, Token } } -bool SyntaxHighlighter::is_identifier(void* token) const +bool SyntaxHighlighter::is_identifier(u64 token) const { - auto sql_token = static_cast(reinterpret_cast(token)); + auto sql_token = static_cast(static_cast(token)); return sql_token == SQL::TokenType::Identifier; } @@ -66,7 +66,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette) auto style = style_for_token_type(palette, token.type()); span.attributes.color = style.color; span.attributes.bold = style.bold; - span.data = reinterpret_cast(static_cast(token.type())); + span.data = static_cast(token.type()); spans.append(span); dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "{} @ '{}' {}:{} - {}:{}", @@ -91,18 +91,18 @@ void SyntaxHighlighter::rehighlight(Palette const& palette) m_client->do_update(); } -Vector SyntaxHighlighter::matching_token_pairs() const +Vector SyntaxHighlighter::matching_token_pairs_impl() const { static Vector pairs; if (pairs.is_empty()) { - pairs.append({ reinterpret_cast(TokenType::ParenOpen), reinterpret_cast(TokenType::ParenClose) }); + pairs.append({ static_cast(TokenType::ParenOpen), static_cast(TokenType::ParenClose) }); } return pairs; } -bool SyntaxHighlighter::token_types_equal(void* token1, void* token2) const +bool SyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const { - return static_cast(reinterpret_cast(token1)) == static_cast(reinterpret_cast(token2)); + return static_cast(token1) == static_cast(token2); } SyntaxHighlighter::~SyntaxHighlighter() diff --git a/Userland/Libraries/LibSQL/SyntaxHighlighter.h b/Userland/Libraries/LibSQL/SyntaxHighlighter.h index a5fa012aa3e..b8f17b85274 100644 --- a/Userland/Libraries/LibSQL/SyntaxHighlighter.h +++ b/Userland/Libraries/LibSQL/SyntaxHighlighter.h @@ -15,14 +15,14 @@ public: SyntaxHighlighter() { } virtual ~SyntaxHighlighter() override; - virtual bool is_identifier(void*) const override; + virtual bool is_identifier(u64) const override; virtual Syntax::Language language() const override { return Syntax::Language::SQL; } virtual void rehighlight(Palette const&) override; protected: - virtual Vector matching_token_pairs() const override; - virtual bool token_types_equal(void*, void*) const override; + virtual Vector matching_token_pairs_impl() const override; + virtual bool token_types_equal(u64, u64) const override; }; } diff --git a/Userland/Libraries/LibSyntax/Highlighter.cpp b/Userland/Libraries/LibSyntax/Highlighter.cpp index c80ee331047..df5dbeed5fc 100644 --- a/Userland/Libraries/LibSyntax/Highlighter.cpp +++ b/Userland/Libraries/LibSyntax/Highlighter.cpp @@ -23,7 +23,7 @@ void Highlighter::highlight_matching_token_pair() Backward, }; - auto find_span_of_type = [&](auto i, void* type, void* not_type, Direction direction) -> Optional { + auto find_span_of_type = [&](auto i, u64 type, u64 not_type, Direction direction) -> Optional { size_t nesting_level = 0; bool forward = direction == Direction::Forward; @@ -130,4 +130,19 @@ void Highlighter::cursor_did_change() highlight_matching_token_pair(); } +Vector Highlighter::matching_token_pairs() const +{ + auto own_pairs = matching_token_pairs_impl(); + own_pairs.ensure_capacity(own_pairs.size() + m_nested_token_pairs.size()); + for (auto& nested_pair : m_nested_token_pairs) + own_pairs.append(nested_pair); + return own_pairs; +} + +void Highlighter::register_nested_token_pairs(Vector pairs) +{ + for (auto& pair : pairs) + m_nested_token_pairs.set(pair); +} + } diff --git a/Userland/Libraries/LibSyntax/Highlighter.h b/Userland/Libraries/LibSyntax/Highlighter.h index de0b6bdc2b2..7a2541f96e3 100644 --- a/Userland/Libraries/LibSyntax/Highlighter.h +++ b/Userland/Libraries/LibSyntax/Highlighter.h @@ -41,26 +41,30 @@ public: virtual void rehighlight(const Palette&) = 0; virtual void highlight_matching_token_pair(); - virtual bool is_identifier(void*) const { return false; }; - virtual bool is_navigatable(void*) const { return false; }; + virtual bool is_identifier(u64) const { return false; }; + virtual bool is_navigatable(u64) const { return false; }; void attach(HighlighterClient&); void detach(); void cursor_did_change(); + struct MatchingTokenPair { + u64 open; + u64 close; + }; + Vector matching_token_pairs() const; + protected: Highlighter() { } // FIXME: This should be WeakPtr somehow HighlighterClient* m_client { nullptr }; - struct MatchingTokenPair { - void* open; - void* close; - }; - - virtual Vector matching_token_pairs() const = 0; - virtual bool token_types_equal(void*, void*) const = 0; + virtual Vector matching_token_pairs_impl() const = 0; + virtual bool token_types_equal(u64, u64) const = 0; + void register_nested_token_pairs(Vector); + void clear_nested_token_pairs() { m_nested_token_pairs.clear(); } + size_t first_free_token_kind_serial_value() const { return m_nested_token_pairs.size(); } struct BuddySpan { int index { -1 }; @@ -69,6 +73,75 @@ protected: bool m_has_brace_buddies { false }; BuddySpan m_brace_buddies[2]; + HashTable m_nested_token_pairs; +}; + +class ProxyHighlighterClient final : public Syntax::HighlighterClient { +public: + ProxyHighlighterClient(Syntax::HighlighterClient& client, GUI::TextPosition start, u64 nested_kind_start_value, StringView source) + : m_document(client.get_document()) + , m_text(source) + , m_start(start) + , m_nested_kind_start_value(nested_kind_start_value) + { + } + + Vector corrected_spans() const + { + Vector spans { m_spans }; + for (auto& entry : spans) { + entry.range.start() = { + entry.range.start().line() + m_start.line(), + entry.range.start().line() == 0 ? entry.range.start().column() + m_start.column() : entry.range.start().column(), + }; + entry.range.end() = { + entry.range.end().line() + m_start.line(), + entry.range.end().line() == 0 ? entry.range.end().column() + m_start.column() : entry.range.end().column(), + }; + if (entry.data != (u64)-1) + entry.data += m_nested_kind_start_value; + } + + return spans; + } + + Vector corrected_token_pairs(Vector pairs) const + { + for (auto& pair : pairs) { + pair.close += m_nested_kind_start_value; + pair.open += m_nested_kind_start_value; + } + return pairs; + } + +private: + virtual Vector& spans() override { return m_spans; } + virtual const Vector& spans() const override { return m_spans; } + virtual void set_span_at_index(size_t index, GUI::TextDocumentSpan span) override { m_spans.at(index) = move(span); } + + virtual String highlighter_did_request_text() const override { return m_text; } + virtual void highlighter_did_request_update() override { } + virtual GUI::TextDocument& highlighter_did_request_document() override { return m_document; } + virtual GUI::TextPosition highlighter_did_request_cursor() const override { return {}; } + virtual void highlighter_did_set_spans(Vector spans) override { m_spans = move(spans); } + + Vector m_spans; + GUI::TextDocument& m_document; + StringView m_text; + GUI::TextPosition m_start; + u64 m_nested_kind_start_value { 0 }; }; } + +template<> +struct AK::Traits : public AK::GenericTraits { + static unsigned hash(Syntax::Highlighter::MatchingTokenPair const& pair) + { + return pair_int_hash(u64_hash(pair.open), u64_hash(pair.close)); + } + static bool equals(Syntax::Highlighter::MatchingTokenPair const& a, Syntax::Highlighter::MatchingTokenPair const& b) + { + return a.open == b.open && a.close == b.close; + } +}; diff --git a/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.cpp b/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.cpp index df66bede635..fe899e5940d 100644 --- a/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.cpp +++ b/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.cpp @@ -6,6 +6,7 @@ */ #include +#include #include #include @@ -18,16 +19,17 @@ enum class AugmentedTokenKind : u32 { CloseTag, Comment, Doctype, + __Count, }; -bool SyntaxHighlighter::is_identifier(void* token) const +bool SyntaxHighlighter::is_identifier(u64 token) const { if (!token) return false; return false; } -bool SyntaxHighlighter::is_navigatable(void*) const +bool SyntaxHighlighter::is_navigatable(u64) const { return false; } @@ -35,8 +37,8 @@ bool SyntaxHighlighter::is_navigatable(void*) const void SyntaxHighlighter::rehighlight(Palette const& palette) { dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "(HTML::SyntaxHighlighter) starting rehighlight"); - (void)palette; auto text = m_client->get_text(); + clear_nested_token_pairs(); Vector spans; auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, AugmentedTokenKind kind) { @@ -51,7 +53,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette) { end_line, end_column }, }, move(attributes), - (void*)kind, + static_cast(kind), false); }; @@ -61,6 +63,9 @@ void SyntaxHighlighter::rehighlight(Palette const& palette) Javascript, CSS, } state { State::HTML }; + StringBuilder substring_builder; + GUI::TextPosition substring_start_position; + for (;;) { auto token = tokenizer.next_token(); if (!token.has_value() || token.value().is_end_of_file()) @@ -71,19 +76,41 @@ void SyntaxHighlighter::rehighlight(Palette const& palette) if (token->tag_name() == "script"sv) { tokenizer.switch_to(HTMLTokenizer::State::ScriptData); state = State::Javascript; + substring_start_position = { token->end_position().line, token->end_position().column }; } else if (token->tag_name() == "style"sv) { tokenizer.switch_to(HTMLTokenizer::State::RAWTEXT); state = State::CSS; + substring_start_position = { token->end_position().line, token->end_position().column }; } } else if (token->is_end_tag()) { if (token->tag_name().is_one_of("script"sv, "style"sv)) { if (state == State::Javascript) { - // FIXME: Highlight javascript code here instead. + Syntax::ProxyHighlighterClient proxy_client { + *m_client, + substring_start_position, + static_cast(AugmentedTokenKind::__Count) + first_free_token_kind_serial_value(), + substring_builder.string_view() + }; + { + JS::SyntaxHighlighter highlighter; + highlighter.attach(proxy_client); + highlighter.rehighlight(palette); + highlighter.detach(); + register_nested_token_pairs(proxy_client.corrected_token_pairs(highlighter.matching_token_pairs())); + } + + spans.append(proxy_client.corrected_spans()); + substring_builder.clear(); } else if (state == State::CSS) { // FIXME: Highlight CSS code here instead. + substring_builder.clear(); } state = State::HTML; } + } else if (state != State::HTML) { + VERIFY(token->is_character()); + substring_builder.append_code_point(token->code_point()); + continue; } size_t token_start_offset = token->is_end_tag() ? 1 : 0; @@ -135,7 +162,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette) if constexpr (SYNTAX_HIGHLIGHTING_DEBUG) { dbgln("(HTML::SyntaxHighlighter) list of all spans:"); for (auto& span : spans) - dbgln("{}, {}", span.range, span.attributes.color); + dbgln("{}, {} - {}", span.range, span.attributes.color, span.data); dbgln("(HTML::SyntaxHighlighter) end of list"); } @@ -145,16 +172,16 @@ void SyntaxHighlighter::rehighlight(Palette const& palette) m_client->do_update(); } -Vector SyntaxHighlighter::matching_token_pairs() const +Vector SyntaxHighlighter::matching_token_pairs_impl() const { static Vector pairs; if (pairs.is_empty()) { - pairs.append({ (void*)AugmentedTokenKind::OpenTag, (void*)AugmentedTokenKind::CloseTag }); + pairs.append({ static_cast(AugmentedTokenKind::OpenTag), static_cast(AugmentedTokenKind::CloseTag) }); } return pairs; } -bool SyntaxHighlighter::token_types_equal(void* token0, void* token1) const +bool SyntaxHighlighter::token_types_equal(u64 token0, u64 token1) const { return token0 == token1; } diff --git a/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h b/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h index 6d412c940ae..79f38c3d98e 100644 --- a/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h +++ b/Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h @@ -15,15 +15,15 @@ public: SyntaxHighlighter() = default; virtual ~SyntaxHighlighter() override = default; - virtual bool is_identifier(void*) const override; - virtual bool is_navigatable(void*) const override; + virtual bool is_identifier(u64) const override; + virtual bool is_navigatable(u64) const override; virtual Syntax::Language language() const override { return Syntax::Language::HTML; } virtual void rehighlight(Palette const&) override; protected: - virtual Vector matching_token_pairs() const override; - virtual bool token_types_equal(void*, void*) const override; + virtual Vector matching_token_pairs_impl() const override; + virtual bool token_types_equal(u64, u64) const override; size_t m_line { 1 }; size_t m_column { 0 }; diff --git a/Userland/Shell/SyntaxHighlighter.cpp b/Userland/Shell/SyntaxHighlighter.cpp index a1434f66bcc..64a819cca02 100644 --- a/Userland/Shell/SyntaxHighlighter.cpp +++ b/Userland/Shell/SyntaxHighlighter.cpp @@ -68,7 +68,7 @@ private: GUI::TextDocumentSpan span; set_offset_range_start(span.range, node->position().start_line); set_offset_range_end(span.range, node->position().end_line); - span.data = (void*)static_cast(node->kind()); + span.data = static_cast(node->kind()); span.is_skippable = false; m_spans.append(move(span)); @@ -143,12 +143,12 @@ private: auto& start_span = span_for_node(node); start_span.attributes.color = m_palette.syntax_punctuation(); start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 }); - start_span.data = (void*)static_cast(AugmentedTokenKind::OpenParen); + start_span.data = static_cast(AugmentedTokenKind::OpenParen); auto& end_span = span_for_node(node); end_span.attributes.color = m_palette.syntax_punctuation(); set_offset_range_start(end_span.range, node->position().end_line, 1); - end_span.data = (void*)static_cast(AugmentedTokenKind::CloseParen); + end_span.data = static_cast(AugmentedTokenKind::CloseParen); } virtual void visit(const AST::CloseFdRedirection* node) override { @@ -289,12 +289,12 @@ private: auto& start_span = span_for_node(node); start_span.attributes.color = m_palette.syntax_punctuation(); start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 }); - start_span.data = (void*)static_cast(AugmentedTokenKind::OpenParen); + start_span.data = static_cast(AugmentedTokenKind::OpenParen); auto& end_span = span_for_node(node); end_span.attributes.color = m_palette.syntax_punctuation(); set_offset_range_start(end_span.range, node->position().end_line, 1); - end_span.data = (void*)static_cast(AugmentedTokenKind::CloseParen); + end_span.data = static_cast(AugmentedTokenKind::CloseParen); } } virtual void visit(const AST::IfCond* node) override @@ -328,7 +328,7 @@ private: auto& start_span = span_for_node(node); start_span.attributes.color = m_palette.syntax_punctuation(); start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 }); - start_span.data = (void*)static_cast(AugmentedTokenKind::OpenParen); + start_span.data = static_cast(AugmentedTokenKind::OpenParen); // Function name auto& name_span = span_for_node(node); @@ -340,7 +340,7 @@ private: auto& end_span = span_for_node(node); end_span.attributes.color = m_palette.syntax_punctuation(); set_offset_range_start(end_span.range, node->position().end_line, 1); - end_span.data = (void*)static_cast(AugmentedTokenKind::CloseParen); + end_span.data = static_cast(AugmentedTokenKind::CloseParen); } virtual void visit(const AST::Join* node) override @@ -500,7 +500,7 @@ private: start_span.range.set_start({ decl.name->position().end_line.line_number, decl.name->position().end_line.line_column }); start_span.range.set_end({ decl.value->position().start_line.line_number, decl.value->position().start_line.line_column + 1 }); start_span.attributes.color = m_palette.syntax_punctuation(); - start_span.data = (void*)static_cast(AugmentedTokenKind::OpenParen); + start_span.data = static_cast(AugmentedTokenKind::OpenParen); } } virtual void visit(const AST::WriteAppendRedirection* node) override @@ -518,7 +518,7 @@ private: bool m_is_first_in_command { false }; }; -bool SyntaxHighlighter::is_identifier(void* token) const +bool SyntaxHighlighter::is_identifier(u64 token) const { if (!token) return false; @@ -529,7 +529,7 @@ bool SyntaxHighlighter::is_identifier(void* token) const || kind == (size_t)AST::Node::Kind::Tilde; } -bool SyntaxHighlighter::is_navigatable(void*) const +bool SyntaxHighlighter::is_navigatable(u64) const { return false; } @@ -551,7 +551,7 @@ void SyntaxHighlighter::rehighlight(const Palette& palette) if constexpr (SYNTAX_HIGHLIGHTING_DEBUG) { for (auto& span : spans) { - dbgln("Kind {}, range {}.", reinterpret_cast(span.data), span.range); + dbgln("Kind {}, range {}.", span.data, span.range); } } @@ -561,19 +561,19 @@ void SyntaxHighlighter::rehighlight(const Palette& palette) m_client->do_update(); } -Vector SyntaxHighlighter::matching_token_pairs() const +Vector SyntaxHighlighter::matching_token_pairs_impl() const { static Vector pairs; if (pairs.is_empty()) { pairs.append({ - (void*)static_cast(AugmentedTokenKind::OpenParen), - (void*)static_cast(AugmentedTokenKind::CloseParen), + static_cast(AugmentedTokenKind::OpenParen), + static_cast(AugmentedTokenKind::CloseParen), }); } return pairs; } -bool SyntaxHighlighter::token_types_equal(void* token0, void* token1) const +bool SyntaxHighlighter::token_types_equal(u64 token0, u64 token1) const { return token0 == token1; } diff --git a/Userland/Shell/SyntaxHighlighter.h b/Userland/Shell/SyntaxHighlighter.h index 306d4257397..5cc860cf4d0 100644 --- a/Userland/Shell/SyntaxHighlighter.h +++ b/Userland/Shell/SyntaxHighlighter.h @@ -15,15 +15,15 @@ public: SyntaxHighlighter() { } virtual ~SyntaxHighlighter() override; - virtual bool is_identifier(void*) const override; - virtual bool is_navigatable(void*) const override; + virtual bool is_identifier(u64) const override; + virtual bool is_navigatable(u64) const override; virtual Syntax::Language language() const override { return Syntax::Language::Shell; } virtual void rehighlight(const Palette&) override; protected: - virtual Vector matching_token_pairs() const override; - virtual bool token_types_equal(void*, void*) const override; + virtual Vector matching_token_pairs_impl() const override; + virtual bool token_types_equal(u64, u64) const override; }; }