beenull/ladybird
1
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2024-11-22 15:40:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

LibWeb+LibSyntax: Implement nested syntax highlighters

Browse source 
And use them to highlight javascript in HTML source.
This commit also changes how TextDocumentSpan::data is interpreted,
as it used to be an opaque pointer, but everyone stuffed an enum value
inside it, which made the values not unique to each highlighter;
that field is now a u64 serial id.
The syntax highlighters don't need to change their ways of stuffing
token types into that field, but a highlighter that calls another
nested highlighter needs to register the nested types for use with
token pairs.
This commit is contained in:
Ali Mohammad Pur 2021-06-07 12:03:09 +04:30 committed by Ali Mohammad Pur
parent 3bac14e19e
commit 71b4433b0d
Notes: sideshowbarker 2024-07-18 12:43:01 +09:00 
Author: https://github.com/alimpfard
Commit: https://github.com/SerenityOS/serenity/commit/71b4433b0d8
Pull-request: https://github.com/SerenityOS/serenity/pull/7866
18 changed files with 219 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Userland/Applications/Spreadsheet/CellSyntaxHighlighter.cpp
View file

@ -31,7 +31,7 @@ void CellSyntaxHighlighter::rehighlight(const Palette& palette)
false,
false,
},
nullptr,
(u64)-1,
false);
if (m_cell && m_cell->exception()) {
@ -47,7 +47,7 @@ void CellSyntaxHighlighter::rehighlight(const Palette& palette)
false,
false,
},
nullptr,
(u64)-1,
false });
}
m_client->do_update();

22
Userland/Libraries/LibCpp/SyntaxHighlighter.cpp
View file

@ -43,15 +43,15 @@ static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, Cpp::
}
}
bool SyntaxHighlighter::is_identifier(void* token) const
bool SyntaxHighlighter::is_identifier(u64 token) const
{
auto cpp_token = static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token));
auto cpp_token = static_cast<Cpp::Token::Type>(token);
return cpp_token == Cpp::Token::Type::Identifier;
}
bool SyntaxHighlighter::is_navigatable(void* token) const
bool SyntaxHighlighter::is_navigatable(u64 token) const
{
auto cpp_token = static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token));
auto cpp_token = static_cast<Cpp::Token::Type>(token);
return cpp_token == Cpp::Token::Type::IncludePath;
}
@ -72,7 +72,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
span.attributes.color = style.color;
span.attributes.bold = style.bold;
span.is_skippable = token.type() == Cpp::Token::Type::Whitespace;
span.data = reinterpret_cast<void*>(token.type());
span.data = static_cast<u64>(token.type());
spans.append(span);
}
m_client->do_set_spans(move(spans));
@ -83,20 +83,20 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
m_client->do_update();
}
Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
{
static Vector<SyntaxHighlighter::MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({ reinterpret_cast<void*>(Cpp::Token::Type::LeftCurly), reinterpret_cast<void*>(Cpp::Token::Type::RightCurly) });
pairs.append({ reinterpret_cast<void*>(Cpp::Token::Type::LeftParen), reinterpret_cast<void*>(Cpp::Token::Type::RightParen) });
pairs.append({ reinterpret_cast<void*>(Cpp::Token::Type::LeftBracket), reinterpret_cast<void*>(Cpp::Token::Type::RightBracket) });
pairs.append({ static_cast<u64>(Cpp::Token::Type::LeftCurly), static_cast<u64>(Cpp::Token::Type::RightCurly) });
pairs.append({ static_cast<u64>(Cpp::Token::Type::LeftParen), static_cast<u64>(Cpp::Token::Type::RightParen) });
pairs.append({ static_cast<u64>(Cpp::Token::Type::LeftBracket), static_cast<u64>(Cpp::Token::Type::RightBracket) });
}
return pairs;
}
bool SyntaxHighlighter::token_types_equal(void* token1, void* token2) const
bool SyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
{
return static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token1)) == static_cast<Cpp::Token::Type>(reinterpret_cast<size_t>(token2));
return static_cast<Cpp::Token::Type>(token1) == static_cast<Cpp::Token::Type>(token2);
}
SyntaxHighlighter::~SyntaxHighlighter()

8
Userland/Libraries/LibCpp/SyntaxHighlighter.h
View file

@ -15,15 +15,15 @@ public:
SyntaxHighlighter() { }
virtual ~SyntaxHighlighter() override;
virtual bool is_identifier(void*) const override;
virtual bool is_navigatable(void*) const override;
virtual bool is_identifier(u64) const override;
virtual bool is_navigatable(u64) const override;
virtual Syntax::Language language() const override { return Syntax::Language::Cpp; }
virtual void rehighlight(Palette const&) override;
protected:
virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
virtual bool token_types_equal(void*, void*) const override;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
virtual bool token_types_equal(u64, u64) const override;
};
}

14
Userland/Libraries/LibGUI/GMLSyntaxHighlighter.cpp
View file

@ -33,9 +33,9 @@ static Syntax::TextStyle style_for_token_type(const Gfx::Palette& palette, GMLTo
}
}
bool GMLSyntaxHighlighter::is_identifier(void* token) const
bool GMLSyntaxHighlighter::is_identifier(u64 token) const
{
auto ini_token = static_cast<GUI::GMLToken::Type>(reinterpret_cast<size_t>(token));
auto ini_token = static_cast<GUI::GMLToken::Type>(token);
return ini_token == GUI::GMLToken::Type::Identifier;
}
@ -54,7 +54,7 @@ void GMLSyntaxHighlighter::rehighlight(const Palette& palette)
span.attributes.color = style.color;
span.attributes.bold = style.bold;
span.is_skippable = false;
span.data = reinterpret_cast<void*>(token.m_type);
span.data = static_cast<u64>(token.m_type);
spans.append(span);
}
m_client->do_set_spans(move(spans));
@ -65,18 +65,18 @@ void GMLSyntaxHighlighter::rehighlight(const Palette& palette)
m_client->do_update();
}
Vector<GMLSyntaxHighlighter::MatchingTokenPair> GMLSyntaxHighlighter::matching_token_pairs() const
Vector<GMLSyntaxHighlighter::MatchingTokenPair> GMLSyntaxHighlighter::matching_token_pairs_impl() const
{
static Vector<MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({ reinterpret_cast<void*>(GMLToken::Type::LeftCurly), reinterpret_cast<void*>(GMLToken::Type::RightCurly) });
pairs.append({ static_cast<u64>(GMLToken::Type::LeftCurly), static_cast<u64>(GMLToken::Type::RightCurly) });
}
return pairs;
}
bool GMLSyntaxHighlighter::token_types_equal(void* token1, void* token2) const
bool GMLSyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
{
return static_cast<GUI::GMLToken::Type>(reinterpret_cast<size_t>(token1)) == static_cast<GUI::GMLToken::Type>(reinterpret_cast<size_t>(token2));
return static_cast<GUI::GMLToken::Type>(token1) == static_cast<GUI::GMLToken::Type>(token2);
}
GMLSyntaxHighlighter::~GMLSyntaxHighlighter()

6
Userland/Libraries/LibGUI/GMLSyntaxHighlighter.h
View file

@ -15,14 +15,14 @@ public:
GMLSyntaxHighlighter() { }
virtual ~GMLSyntaxHighlighter() override;
virtual bool is_identifier(void*) const override;
virtual bool is_identifier(u64) const override;
virtual Syntax::Language language() const override { return Syntax::Language::GML; }
virtual void rehighlight(const Palette&) override;
protected:
virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
virtual bool token_types_equal(void*, void*) const override;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
virtual bool token_types_equal(u64, u64) const override;
};
}

14
Userland/Libraries/LibGUI/INISyntaxHighlighter.cpp
View file

@ -32,9 +32,9 @@ static Syntax::TextStyle style_for_token_type(const Gfx::Palette& palette, IniTo
}
}
bool IniSyntaxHighlighter::is_identifier(void* token) const
bool IniSyntaxHighlighter::is_identifier(u64 token) const
{
auto ini_token = static_cast<GUI::IniToken::Type>(reinterpret_cast<size_t>(token));
auto ini_token = static_cast<GUI::IniToken::Type>(token);
return ini_token == GUI::IniToken::Type::Name;
}
@ -53,7 +53,7 @@ void IniSyntaxHighlighter::rehighlight(const Palette& palette)
span.attributes.color = style.color;
span.attributes.bold = style.bold;
span.is_skippable = token.m_type == IniToken::Type::Whitespace;
span.data = reinterpret_cast<void*>(token.m_type);
span.data = static_cast<u64>(token.m_type);
spans.append(span);
}
m_client->do_set_spans(move(spans));
@ -64,18 +64,18 @@ void IniSyntaxHighlighter::rehighlight(const Palette& palette)
m_client->do_update();
}
Vector<IniSyntaxHighlighter::MatchingTokenPair> IniSyntaxHighlighter::matching_token_pairs() const
Vector<IniSyntaxHighlighter::MatchingTokenPair> IniSyntaxHighlighter::matching_token_pairs_impl() const
{
static Vector<MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({ reinterpret_cast<void*>(IniToken::Type::LeftBracket), reinterpret_cast<void*>(IniToken::Type::RightBracket) });
pairs.append({ static_cast<u64>(IniToken::Type::LeftBracket), static_cast<u64>(IniToken::Type::RightBracket) });
}
return pairs;
}
bool IniSyntaxHighlighter::token_types_equal(void* token1, void* token2) const
bool IniSyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
{
return static_cast<GUI::IniToken::Type>(reinterpret_cast<size_t>(token1)) == static_cast<GUI::IniToken::Type>(reinterpret_cast<size_t>(token2));
return static_cast<GUI::IniToken::Type>(token1) == static_cast<GUI::IniToken::Type>(token2);
}
IniSyntaxHighlighter::~IniSyntaxHighlighter()

6
Userland/Libraries/LibGUI/INISyntaxHighlighter.h
View file

@ -15,14 +15,14 @@ public:
IniSyntaxHighlighter() { }
virtual ~IniSyntaxHighlighter() override;
virtual bool is_identifier(void*) const override;
virtual bool is_identifier(u64) const override;
virtual Syntax::Language language() const override { return Syntax::Language::INI; }
virtual void rehighlight(const Palette&) override;
protected:
virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
virtual bool token_types_equal(void*, void*) const override;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
virtual bool token_types_equal(u64, u64) const override;
};
}

2
Userland/Libraries/LibGUI/TextDocument.h
View file

@ -26,7 +26,7 @@ namespace GUI {
struct TextDocumentSpan {
TextRange range;
Gfx::TextAttributes attributes;
void* data { nullptr };
u64 data { 0 };
bool is_skippable { false };
};

20
Userland/Libraries/LibJS/SyntaxHighlighter.cpp
View file

@ -38,13 +38,13 @@ static Syntax::TextStyle style_for_token_type(const Gfx::Palette& palette, JS::T
}
}
bool SyntaxHighlighter::is_identifier(void* token) const
bool SyntaxHighlighter::is_identifier(u64 token) const
{
auto js_token = static_cast<JS::TokenType>(reinterpret_cast<size_t>(token));
auto js_token = static_cast<JS::TokenType>(static_cast<size_t>(token));
return js_token == JS::TokenType::Identifier;
}
bool SyntaxHighlighter::is_navigatable([[maybe_unused]] void* token) const
bool SyntaxHighlighter::is_navigatable([[maybe_unused]] u64 token) const
{
return false;
}
@ -83,7 +83,7 @@ void SyntaxHighlighter::rehighlight(const Palette& palette)
span.attributes.color = style.color;
span.attributes.bold = style.bold;
span.is_skippable = is_trivia;
span.data = reinterpret_cast<void*>(static_cast<size_t>(type));
span.data = static_cast<u64>(type);
spans.append(span);
dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "{}{} @ '{}' {}:{} - {}:{}",
@ -111,20 +111,20 @@ void SyntaxHighlighter::rehighlight(const Palette& palette)
m_client->do_update();
}
Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
{
static Vector<Syntax::Highlighter::MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({ reinterpret_cast<void*>(JS::TokenType::CurlyOpen), reinterpret_cast<void*>(JS::TokenType::CurlyClose) });
pairs.append({ reinterpret_cast<void*>(JS::TokenType::ParenOpen), reinterpret_cast<void*>(JS::TokenType::ParenClose) });
pairs.append({ reinterpret_cast<void*>(JS::TokenType::BracketOpen), reinterpret_cast<void*>(JS::TokenType::BracketClose) });
pairs.append({ static_cast<u64>(JS::TokenType::CurlyOpen), static_cast<u64>(JS::TokenType::CurlyClose) });
pairs.append({ static_cast<u64>(JS::TokenType::ParenOpen), static_cast<u64>(JS::TokenType::ParenClose) });
pairs.append({ static_cast<u64>(JS::TokenType::BracketOpen), static_cast<u64>(JS::TokenType::BracketClose) });
}
return pairs;
}
bool SyntaxHighlighter::token_types_equal(void* token1, void* token2) const
bool SyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
{
return static_cast<JS::TokenType>(reinterpret_cast<size_t>(token1)) == static_cast<JS::TokenType>(reinterpret_cast<size_t>(token2));
return static_cast<JS::TokenType>(token1) == static_cast<JS::TokenType>(token2);
}
SyntaxHighlighter::~SyntaxHighlighter()

8
Userland/Libraries/LibJS/SyntaxHighlighter.h
View file

@ -15,15 +15,15 @@ public:
SyntaxHighlighter() { }
virtual ~SyntaxHighlighter() override;
virtual bool is_identifier(void*) const override;
virtual bool is_navigatable(void*) const override;
virtual bool is_identifier(u64) const override;
virtual bool is_navigatable(u64) const override;
virtual Syntax::Language language() const override { return Syntax::Language::JavaScript; }
virtual void rehighlight(const Palette&) override;
protected:
virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
virtual bool token_types_equal(void*, void*) const override;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
virtual bool token_types_equal(u64, u64) const override;
};
}

14
Userland/Libraries/LibSQL/SyntaxHighlighter.cpp
View file

@ -33,9 +33,9 @@ static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, Token
}
}
bool SyntaxHighlighter::is_identifier(void* token) const
bool SyntaxHighlighter::is_identifier(u64 token) const
{
auto sql_token = static_cast<SQL::TokenType>(reinterpret_cast<size_t>(token));
auto sql_token = static_cast<SQL::TokenType>(static_cast<size_t>(token));
return sql_token == SQL::TokenType::Identifier;
}
@ -66,7 +66,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
auto style = style_for_token_type(palette, token.type());
span.attributes.color = style.color;
span.attributes.bold = style.bold;
span.data = reinterpret_cast<void*>(static_cast<size_t>(token.type()));
span.data = static_cast<u64>(token.type());
spans.append(span);
dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "{} @ '{}' {}:{} - {}:{}",
@ -91,18 +91,18 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
m_client->do_update();
}
Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
Vector<SyntaxHighlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
{
static Vector<SyntaxHighlighter::MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({ reinterpret_cast<void*>(TokenType::ParenOpen), reinterpret_cast<void*>(TokenType::ParenClose) });
pairs.append({ static_cast<u64>(TokenType::ParenOpen), static_cast<u64>(TokenType::ParenClose) });
}
return pairs;
}
bool SyntaxHighlighter::token_types_equal(void* token1, void* token2) const
bool SyntaxHighlighter::token_types_equal(u64 token1, u64 token2) const
{
return static_cast<TokenType>(reinterpret_cast<size_t>(token1)) == static_cast<TokenType>(reinterpret_cast<size_t>(token2));
return static_cast<TokenType>(token1) == static_cast<TokenType>(token2);
}
SyntaxHighlighter::~SyntaxHighlighter()

6
Userland/Libraries/LibSQL/SyntaxHighlighter.h
View file

@ -15,14 +15,14 @@ public:
SyntaxHighlighter() { }
virtual ~SyntaxHighlighter() override;
virtual bool is_identifier(void*) const override;
virtual bool is_identifier(u64) const override;
virtual Syntax::Language language() const override { return Syntax::Language::SQL; }
virtual void rehighlight(Palette const&) override;
protected:
virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
virtual bool token_types_equal(void*, void*) const override;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
virtual bool token_types_equal(u64, u64) const override;
};
}

17
Userland/Libraries/LibSyntax/Highlighter.cpp
View file

@ -23,7 +23,7 @@ void Highlighter::highlight_matching_token_pair()
Backward,
};
auto find_span_of_type = [&](auto i, void* type, void* not_type, Direction direction) -> Optional<size_t> {
auto find_span_of_type = [&](auto i, u64 type, u64 not_type, Direction direction) -> Optional<size_t> {
size_t nesting_level = 0;
bool forward = direction == Direction::Forward;
@ -130,4 +130,19 @@ void Highlighter::cursor_did_change()
highlight_matching_token_pair();
}
Vector<Highlighter::MatchingTokenPair> Highlighter::matching_token_pairs() const
{
auto own_pairs = matching_token_pairs_impl();
own_pairs.ensure_capacity(own_pairs.size() + m_nested_token_pairs.size());
for (auto& nested_pair : m_nested_token_pairs)
own_pairs.append(nested_pair);
return own_pairs;
}
void Highlighter::register_nested_token_pairs(Vector<MatchingTokenPair> pairs)
{
for (auto& pair : pairs)
m_nested_token_pairs.set(pair);
}
}

91
Userland/Libraries/LibSyntax/Highlighter.h
View file

@ -41,26 +41,30 @@ public:
virtual void rehighlight(const Palette&) = 0;
virtual void highlight_matching_token_pair();
virtual bool is_identifier(void*) const { return false; };
virtual bool is_navigatable(void*) const { return false; };
virtual bool is_identifier(u64) const { return false; };
virtual bool is_navigatable(u64) const { return false; };
void attach(HighlighterClient&);
void detach();
void cursor_did_change();
struct MatchingTokenPair {
u64 open;
u64 close;
};
Vector<MatchingTokenPair> matching_token_pairs() const;
protected:
Highlighter() { }
// FIXME: This should be WeakPtr somehow
HighlighterClient* m_client { nullptr };
struct MatchingTokenPair {
void* open;
void* close;
};
virtual Vector<MatchingTokenPair> matching_token_pairs() const = 0;
virtual bool token_types_equal(void*, void*) const = 0;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const = 0;
virtual bool token_types_equal(u64, u64) const = 0;
void register_nested_token_pairs(Vector<MatchingTokenPair>);
void clear_nested_token_pairs() { m_nested_token_pairs.clear(); }
size_t first_free_token_kind_serial_value() const { return m_nested_token_pairs.size(); }
struct BuddySpan {
int index { -1 };
@ -69,6 +73,75 @@ protected:
bool m_has_brace_buddies { false };
BuddySpan m_brace_buddies[2];
HashTable<MatchingTokenPair> m_nested_token_pairs;
};
class ProxyHighlighterClient final : public Syntax::HighlighterClient {
public:
ProxyHighlighterClient(Syntax::HighlighterClient& client, GUI::TextPosition start, u64 nested_kind_start_value, StringView source)
: m_document(client.get_document())
, m_text(source)
, m_start(start)
, m_nested_kind_start_value(nested_kind_start_value)
{
}
Vector<GUI::TextDocumentSpan> corrected_spans() const
{
Vector<GUI::TextDocumentSpan> spans { m_spans };
for (auto& entry : spans) {
entry.range.start() = {
entry.range.start().line() + m_start.line(),
entry.range.start().line() == 0 ? entry.range.start().column() + m_start.column() : entry.range.start().column(),
};
entry.range.end() = {
entry.range.end().line() + m_start.line(),
entry.range.end().line() == 0 ? entry.range.end().column() + m_start.column() : entry.range.end().column(),
};
if (entry.data != (u64)-1)
entry.data += m_nested_kind_start_value;
}
return spans;
}
Vector<Syntax::Highlighter::MatchingTokenPair> corrected_token_pairs(Vector<Syntax::Highlighter::MatchingTokenPair> pairs) const
{
for (auto& pair : pairs) {
pair.close += m_nested_kind_start_value;
pair.open += m_nested_kind_start_value;
}
return pairs;
}
private:
virtual Vector<GUI::TextDocumentSpan>& spans() override { return m_spans; }
virtual const Vector<GUI::TextDocumentSpan>& spans() const override { return m_spans; }
virtual void set_span_at_index(size_t index, GUI::TextDocumentSpan span) override { m_spans.at(index) = move(span); }
virtual String highlighter_did_request_text() const override { return m_text; }
virtual void highlighter_did_request_update() override { }
virtual GUI::TextDocument& highlighter_did_request_document() override { return m_document; }
virtual GUI::TextPosition highlighter_did_request_cursor() const override { return {}; }
virtual void highlighter_did_set_spans(Vector<GUI::TextDocumentSpan> spans) override { m_spans = move(spans); }
Vector<GUI::TextDocumentSpan> m_spans;
GUI::TextDocument& m_document;
StringView m_text;
GUI::TextPosition m_start;
u64 m_nested_kind_start_value { 0 };
};
}
template<>
struct AK::Traits<Syntax::Highlighter::MatchingTokenPair> : public AK::GenericTraits<Syntax::Highlighter::MatchingTokenPair> {
static unsigned hash(Syntax::Highlighter::MatchingTokenPair const& pair)
{
return pair_int_hash(u64_hash(pair.open), u64_hash(pair.close));
}
static bool equals(Syntax::Highlighter::MatchingTokenPair const& a, Syntax::Highlighter::MatchingTokenPair const& b)
{
return a.open == b.open && a.close == b.close;
}
};

45
Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.cpp
View file

@ -6,6 +6,7 @@
*/
#include <AK/Debug.h>
#include <LibJS/SyntaxHighlighter.h>
#include <LibWeb/HTML/Parser/HTMLTokenizer.h>
#include <LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h>
@ -18,16 +19,17 @@ enum class AugmentedTokenKind : u32 {
CloseTag,
Comment,
Doctype,
__Count,
};
bool SyntaxHighlighter::is_identifier(void* token) const
bool SyntaxHighlighter::is_identifier(u64 token) const
{
if (!token)
return false;
return false;
}
bool SyntaxHighlighter::is_navigatable(void*) const
bool SyntaxHighlighter::is_navigatable(u64) const
{
return false;
}
@ -35,8 +37,8 @@ bool SyntaxHighlighter::is_navigatable(void*) const
void SyntaxHighlighter::rehighlight(Palette const& palette)
{
dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "(HTML::SyntaxHighlighter) starting rehighlight");
(void)palette;
auto text = m_client->get_text();
clear_nested_token_pairs();
Vector<GUI::TextDocumentSpan> spans;
auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, AugmentedTokenKind kind) {
@ -51,7 +53,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
{ end_line, end_column },
},
move(attributes),
(void*)kind,
static_cast<u64>(kind),
false);
};
@ -61,6 +63,9 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
Javascript,
CSS,
} state { State::HTML };
StringBuilder substring_builder;
GUI::TextPosition substring_start_position;
for (;;) {
auto token = tokenizer.next_token();
if (!token.has_value() || token.value().is_end_of_file())
@ -71,19 +76,41 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
if (token->tag_name() == "script"sv) {
tokenizer.switch_to(HTMLTokenizer::State::ScriptData);
state = State::Javascript;
substring_start_position = { token->end_position().line, token->end_position().column };
} else if (token->tag_name() == "style"sv) {
tokenizer.switch_to(HTMLTokenizer::State::RAWTEXT);
state = State::CSS;
substring_start_position = { token->end_position().line, token->end_position().column };
}
} else if (token->is_end_tag()) {
if (token->tag_name().is_one_of("script"sv, "style"sv)) {
if (state == State::Javascript) {
// FIXME: Highlight javascript code here instead.
Syntax::ProxyHighlighterClient proxy_client {
*m_client,
substring_start_position,
static_cast<u64>(AugmentedTokenKind::__Count) + first_free_token_kind_serial_value(),
substring_builder.string_view()
};
{
JS::SyntaxHighlighter highlighter;
highlighter.attach(proxy_client);
highlighter.rehighlight(palette);
highlighter.detach();
register_nested_token_pairs(proxy_client.corrected_token_pairs(highlighter.matching_token_pairs()));
}
spans.append(proxy_client.corrected_spans());
substring_builder.clear();
} else if (state == State::CSS) {
// FIXME: Highlight CSS code here instead.
substring_builder.clear();
}
state = State::HTML;
}
} else if (state != State::HTML) {
VERIFY(token->is_character());
substring_builder.append_code_point(token->code_point());
continue;
}
size_t token_start_offset = token->is_end_tag() ? 1 : 0;
@ -135,7 +162,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
if constexpr (SYNTAX_HIGHLIGHTING_DEBUG) {
dbgln("(HTML::SyntaxHighlighter) list of all spans:");
for (auto& span : spans)
dbgln("{}, {}", span.range, span.attributes.color);
dbgln("{}, {} - {}", span.range, span.attributes.color, span.data);
dbgln("(HTML::SyntaxHighlighter) end of list");
}
@ -145,16 +172,16 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
m_client->do_update();
}
Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
{
static Vector<MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({ (void*)AugmentedTokenKind::OpenTag, (void*)AugmentedTokenKind::CloseTag });
pairs.append({ static_cast<u64>(AugmentedTokenKind::OpenTag), static_cast<u64>(AugmentedTokenKind::CloseTag) });
}
return pairs;
}
bool SyntaxHighlighter::token_types_equal(void* token0, void* token1) const
bool SyntaxHighlighter::token_types_equal(u64 token0, u64 token1) const
{
return token0 == token1;
}

8
Userland/Libraries/LibWeb/HTML/SyntaxHighlighter/SyntaxHighlighter.h
View file

@ -15,15 +15,15 @@ public:
SyntaxHighlighter() = default;
virtual ~SyntaxHighlighter() override = default;
virtual bool is_identifier(void*) const override;
virtual bool is_navigatable(void*) const override;
virtual bool is_identifier(u64) const override;
virtual bool is_navigatable(u64) const override;
virtual Syntax::Language language() const override { return Syntax::Language::HTML; }
virtual void rehighlight(Palette const&) override;
protected:
virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
virtual bool token_types_equal(void*, void*) const override;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
virtual bool token_types_equal(u64, u64) const override;
size_t m_line { 1 };
size_t m_column { 0 };

30
Userland/Shell/SyntaxHighlighter.cpp
View file

@ -68,7 +68,7 @@ private:
GUI::TextDocumentSpan span;
set_offset_range_start(span.range, node->position().start_line);
set_offset_range_end(span.range, node->position().end_line);
span.data = (void*)static_cast<size_t>(node->kind());
span.data = static_cast<u64>(node->kind());
span.is_skippable = false;
m_spans.append(move(span));
@ -143,12 +143,12 @@ private:
auto& start_span = span_for_node(node);
start_span.attributes.color = m_palette.syntax_punctuation();
start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 });
start_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::OpenParen);
start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
auto& end_span = span_for_node(node);
end_span.attributes.color = m_palette.syntax_punctuation();
set_offset_range_start(end_span.range, node->position().end_line, 1);
end_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::CloseParen);
end_span.data = static_cast<u64>(AugmentedTokenKind::CloseParen);
}
virtual void visit(const AST::CloseFdRedirection* node) override
{
@ -289,12 +289,12 @@ private:
auto& start_span = span_for_node(node);
start_span.attributes.color = m_palette.syntax_punctuation();
start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 });
start_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::OpenParen);
start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
auto& end_span = span_for_node(node);
end_span.attributes.color = m_palette.syntax_punctuation();
set_offset_range_start(end_span.range, node->position().end_line, 1);
end_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::CloseParen);
end_span.data = static_cast<u64>(AugmentedTokenKind::CloseParen);
}
}
virtual void visit(const AST::IfCond* node) override
@ -328,7 +328,7 @@ private:
auto& start_span = span_for_node(node);
start_span.attributes.color = m_palette.syntax_punctuation();
start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 2 });
start_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::OpenParen);
start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
// Function name
auto& name_span = span_for_node(node);
@ -340,7 +340,7 @@ private:
auto& end_span = span_for_node(node);
end_span.attributes.color = m_palette.syntax_punctuation();
set_offset_range_start(end_span.range, node->position().end_line, 1);
end_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::CloseParen);
end_span.data = static_cast<u64>(AugmentedTokenKind::CloseParen);
}
virtual void visit(const AST::Join* node) override
@ -500,7 +500,7 @@ private:
start_span.range.set_start({ decl.name->position().end_line.line_number, decl.name->position().end_line.line_column });
start_span.range.set_end({ decl.value->position().start_line.line_number, decl.value->position().start_line.line_column + 1 });
start_span.attributes.color = m_palette.syntax_punctuation();
start_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::OpenParen);
start_span.data = static_cast<u64>(AugmentedTokenKind::OpenParen);
}
}
virtual void visit(const AST::WriteAppendRedirection* node) override
@ -518,7 +518,7 @@ private:
bool m_is_first_in_command { false };
};
bool SyntaxHighlighter::is_identifier(void* token) const
bool SyntaxHighlighter::is_identifier(u64 token) const
{
if (!token)
return false;
@ -529,7 +529,7 @@ bool SyntaxHighlighter::is_identifier(void* token) const
|| kind == (size_t)AST::Node::Kind::Tilde;
}
bool SyntaxHighlighter::is_navigatable(void*) const
bool SyntaxHighlighter::is_navigatable(u64) const
{
return false;
}
@ -551,7 +551,7 @@ void SyntaxHighlighter::rehighlight(const Palette& palette)
if constexpr (SYNTAX_HIGHLIGHTING_DEBUG) {
for (auto& span : spans) {
dbgln("Kind {}, range {}.", reinterpret_cast<size_t>(span.data), span.range);
dbgln("Kind {}, range {}.", span.data, span.range);
}
}
@ -561,19 +561,19 @@ void SyntaxHighlighter::rehighlight(const Palette& palette)
m_client->do_update();
}
Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs() const
Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token_pairs_impl() const
{
static Vector<MatchingTokenPair> pairs;
if (pairs.is_empty()) {
pairs.append({
(void*)static_cast<size_t>(AugmentedTokenKind::OpenParen),
(void*)static_cast<size_t>(AugmentedTokenKind::CloseParen),
static_cast<u64>(AugmentedTokenKind::OpenParen),
static_cast<u64>(AugmentedTokenKind::CloseParen),
});
}
return pairs;
}
bool SyntaxHighlighter::token_types_equal(void* token0, void* token1) const
bool SyntaxHighlighter::token_types_equal(u64 token0, u64 token1) const
{
return token0 == token1;
}

8
Userland/Shell/SyntaxHighlighter.h
View file

@ -15,15 +15,15 @@ public:
SyntaxHighlighter() { }
virtual ~SyntaxHighlighter() override;
virtual bool is_identifier(void*) const override;
virtual bool is_navigatable(void*) const override;
virtual bool is_identifier(u64) const override;
virtual bool is_navigatable(u64) const override;
virtual Syntax::Language language() const override { return Syntax::Language::Shell; }
virtual void rehighlight(const Palette&) override;
protected:
virtual Vector<MatchingTokenPair> matching_token_pairs() const override;
virtual bool token_types_equal(void*, void*) const override;
virtual Vector<MatchingTokenPair> matching_token_pairs_impl() const override;
virtual bool token_types_equal(u64, u64) const override;
};
}