LibWeb: Move Token and Tokenizer into Parser namespace

Userland/Libraries/LibWeb/CSS/Number.h

@@ -7,14 +7,13 @@
 #pragma once
 
 #include <AK/Types.h>
+#include <LibWeb/Forward.h>
 #include <math.h>
 
 namespace Web::CSS {
 
-class Tokenizer;
-
 class Number {
-    friend class Tokenizer;
+    friend class Parser::Tokenizer;
 
 public:
     enum class Type {

Userland/Libraries/LibWeb/CSS/Parser/Token.cpp

@@ -9,7 +9,7 @@
 #include <LibWeb/CSS/Parser/Token.h>
 #include <LibWeb/CSS/Serialize.h>
 
-namespace Web::CSS {
+namespace Web::CSS::Parser {
 
 String Token::to_string() const
 {

Userland/Libraries/LibWeb/CSS/Parser/Token.h

@@ -12,7 +12,7 @@
 #include <AK/Utf8View.h>
 #include <LibWeb/CSS/Number.h>
 
-namespace Web::CSS {
+namespace Web::CSS::Parser {
 
 class Token {
     friend class Tokenizer;

Userland/Libraries/LibWeb/CSS/Parser/Tokenizer.cpp

@@ -13,6 +13,8 @@
 #include <LibWeb/CSS/Parser/Tokenizer.h>
 #include <math.h>
 
+namespace Web::CSS::Parser {
+
 // U+FFFD REPLACEMENT CHARACTER (�)
 #define REPLACEMENT_CHARACTER 0xFFFD
 static constexpr u32 TOKENIZER_EOF = 0xFFFFFFFF;
@@ -190,8 +192,6 @@ static inline bool is_E(u32 code_point)
     return code_point == 0x45;
 }
 
-namespace Web::CSS {
-
 Tokenizer::Tokenizer(StringView input, String const& encoding)
 {
     auto* decoder = TextCodec::decoder_for(encoding);

Userland/Libraries/LibWeb/CSS/Parser/Tokenizer.h

@@ -14,7 +14,7 @@
 #include <LibWeb/CSS/Parser/Token.h>
 #include <LibWeb/Forward.h>
 
-namespace Web::CSS {
+namespace Web::CSS::Parser {
 
 class U32Twin {
 public:

Userland/Libraries/LibWeb/CSS/StyleComputer.cpp

@@ -557,7 +557,7 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
                     return false;
 
                 auto const& custom_property_name_token = var_contents.first();
-                if (!custom_property_name_token.is(Token::Type::Ident))
+                if (!custom_property_name_token.is(Parser::Token::Type::Ident))
                     return false;
                 auto custom_property_name = custom_property_name_token.token().ident();
                 if (!custom_property_name.starts_with("--"))
@@ -582,7 +582,7 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
                 }
 
                 // Use the provided fallback value, if any.
-                if (var_contents.size() > 2 && var_contents[1].is(Token::Type::Comma)) {
+                if (var_contents.size() > 2 && var_contents[1].is(Parser::Token::Type::Comma)) {
                     if (!expand_unresolved_values(element, property_name, dependencies, var_contents, dest, 2))
                         return false;
                     continue;
@@ -594,7 +594,7 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
                     return false;
 
                 auto const& attr_name_token = attr_contents.first();
-                if (!attr_name_token.is(Token::Type::Ident))
+                if (!attr_name_token.is(Parser::Token::Type::Ident))
                     return false;
                 auto attr_name = attr_name_token.token().ident();
 
@@ -602,13 +602,13 @@ bool StyleComputer::expand_unresolved_values(DOM::Element& element, StringView p
                 // 1. If the attr() function has a substitution value, replace the attr() function by the substitution value.
                 if (!attr_value.is_null()) {
                     // FIXME: attr() should also accept an optional type argument, not just strings.
-                    dest.empend(Token::of_string(attr_value));
+                    dest.empend(Parser::Token::of_string(attr_value));
                     continue;
                 }
 
                 // 2. Otherwise, if the attr() function has a fallback value as its last argument, replace the attr() function by the fallback value.
                 //    If there are any var() or attr() references in the fallback, substitute them as well.
-                if (attr_contents.size() > 2 && attr_contents[1].is(Token::Type::Comma)) {
+                if (attr_contents.size() > 2 && attr_contents[1].is(Parser::Token::Type::Comma)) {
                     if (!expand_unresolved_values(element, property_name, dependencies, attr_contents, dest, 2))
                         return false;
                     continue;

Userland/Libraries/LibWeb/CSS/SyntaxHighlighter/SyntaxHighlighter.cpp

@@ -12,7 +12,7 @@ namespace Web::CSS {
 
 bool SyntaxHighlighter::is_identifier(u64 token) const
 {
-    return static_cast<CSS::Token::Type>(token) == CSS::Token::Type::Ident;
+    return static_cast<CSS::Parser::Token::Type>(token) == CSS::Parser::Token::Type::Ident;
 }
 
 bool SyntaxHighlighter::is_navigatable(u64) const
@@ -27,7 +27,7 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
 
     Vector<GUI::TextDocumentSpan> spans;
 
-    auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, CSS::Token::Type type) {
+    auto highlight = [&](auto start_line, auto start_column, auto end_line, auto end_column, Gfx::TextAttributes attributes, CSS::Parser::Token::Type type) {
         if (start_line > end_line || (start_line == end_line && start_column >= end_column)) {
             dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "(CSS::SyntaxHighlighter) discarding ({}-{}) to ({}-{}) because it has zero or negative length", start_line, start_column, end_line, end_column);
             return;
@@ -43,85 +43,85 @@ void SyntaxHighlighter::rehighlight(Palette const& palette)
             false);
     };
 
-    CSS::Tokenizer tokenizer { text, "utf-8" };
+    CSS::Parser::Tokenizer tokenizer { text, "utf-8" };
     auto tokens = tokenizer.parse();
     for (auto const& token : tokens) {
-        if (token.is(Token::Type::EndOfFile))
+        if (token.is(Parser::Token::Type::EndOfFile))
             break;
 
         switch (token.type()) {
-        case Token::Type::Ident:
+        case Parser::Token::Type::Ident:
             highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_identifier(), {} }, token.type());
             break;
 
-        case Token::Type::String:
+        case Parser::Token::Type::String:
             highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_string(), {} }, token.type());
             break;
 
-        case Token::Type::Whitespace:
+        case Parser::Token::Type::Whitespace:
             // CSS doesn't produce comment tokens, they're just included as part of whitespace.
             highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type());
             break;
 
-        case Token::Type::AtKeyword:
+        case Parser::Token::Type::AtKeyword:
             highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_keyword(), {} }, token.type());
             break;
 
-        case Token::Type::Function:
+        case Parser::Token::Type::Function:
             // Function tokens include the opening '(', so we split that into two tokens for highlighting purposes.
             highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column - 1, { palette.syntax_keyword(), {} }, token.type());
-            highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Token::Type::OpenParen);
+            highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Parser::Token::Type::OpenParen);
             break;
 
-        case Token::Type::Url:
+        case Parser::Token::Type::Url:
             // An Url token is a `url()` function with its parameter string unquoted.
             // url
             highlight(token.start_position().line, token.start_position().column, token.start_position().line, token.start_position().column + 3, { palette.syntax_keyword(), {} }, token.type());
             // (
-            highlight(token.start_position().line, token.start_position().column + 3, token.start_position().line, token.start_position().column + 4, { palette.syntax_punctuation(), {} }, Token::Type::OpenParen);
+            highlight(token.start_position().line, token.start_position().column + 3, token.start_position().line, token.start_position().column + 4, { palette.syntax_punctuation(), {} }, Parser::Token::Type::OpenParen);
             // <string>
-            highlight(token.start_position().line, token.start_position().column + 4, token.end_position().line, token.end_position().column - 1, { palette.syntax_string(), {} }, Token::Type::String);
+            highlight(token.start_position().line, token.start_position().column + 4, token.end_position().line, token.end_position().column - 1, { palette.syntax_string(), {} }, Parser::Token::Type::String);
             // )
-            highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Token::Type::CloseParen);
+            highlight(token.end_position().line, token.end_position().column - 1, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, Parser::Token::Type::CloseParen);
             break;
 
-        case Token::Type::Number:
-        case Token::Type::Dimension:
-        case Token::Type::Percentage:
+        case Parser::Token::Type::Number:
+        case Parser::Token::Type::Dimension:
+        case Parser::Token::Type::Percentage:
             highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type());
             break;
 
-        case Token::Type::Delim:
-        case Token::Type::Colon:
-        case Token::Type::Comma:
-        case Token::Type::Semicolon:
-        case Token::Type::OpenCurly:
-        case Token::Type::OpenParen:
-        case Token::Type::OpenSquare:
-        case Token::Type::CloseCurly:
-        case Token::Type::CloseParen:
-        case Token::Type::CloseSquare:
+        case Parser::Token::Type::Delim:
+        case Parser::Token::Type::Colon:
+        case Parser::Token::Type::Comma:
+        case Parser::Token::Type::Semicolon:
+        case Parser::Token::Type::OpenCurly:
+        case Parser::Token::Type::OpenParen:
+        case Parser::Token::Type::OpenSquare:
+        case Parser::Token::Type::CloseCurly:
+        case Parser::Token::Type::CloseParen:
+        case Parser::Token::Type::CloseSquare:
             highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_punctuation(), {} }, token.type());
             break;
 
-        case Token::Type::CDO:
-        case Token::Type::CDC:
+        case Parser::Token::Type::CDO:
+        case Parser::Token::Type::CDC:
             highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_comment(), {} }, token.type());
             break;
 
-        case Token::Type::Hash:
+        case Parser::Token::Type::Hash:
             // FIXME: Hash tokens can be ID selectors or colors, we don't know which without parsing properly.
             highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { palette.syntax_number(), {} }, token.type());
             break;
 
-        case Token::Type::Invalid:
-        case Token::Type::BadUrl:
-        case Token::Type::BadString:
+        case Parser::Token::Type::Invalid:
+        case Parser::Token::Type::BadUrl:
+        case Parser::Token::Type::BadString:
             // FIXME: Error highlighting color in palette?
             highlight(token.start_position().line, token.start_position().column, token.end_position().line, token.end_position().column, { Color(Color::NamedColor::Red), {}, false, true }, token.type());
             break;
 
-        case Token::Type::EndOfFile:
+        case Parser::Token::Type::EndOfFile:
         default:
             break;
         }
@@ -144,10 +144,10 @@ Vector<Syntax::Highlighter::MatchingTokenPair> SyntaxHighlighter::matching_token
 {
     static Vector<Syntax::Highlighter::MatchingTokenPair> pairs;
     if (pairs.is_empty()) {
-        pairs.append({ static_cast<u64>(CSS::Token::Type::OpenCurly), static_cast<u64>(CSS::Token::Type::CloseCurly) });
-        pairs.append({ static_cast<u64>(CSS::Token::Type::OpenParen), static_cast<u64>(CSS::Token::Type::CloseParen) });
-        pairs.append({ static_cast<u64>(CSS::Token::Type::OpenSquare), static_cast<u64>(CSS::Token::Type::CloseSquare) });
-        pairs.append({ static_cast<u64>(CSS::Token::Type::CDO), static_cast<u64>(CSS::Token::Type::CDC) });
+        pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenCurly), static_cast<u64>(CSS::Parser::Token::Type::CloseCurly) });
+        pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenParen), static_cast<u64>(CSS::Parser::Token::Type::CloseParen) });
+        pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::OpenSquare), static_cast<u64>(CSS::Parser::Token::Type::CloseSquare) });
+        pairs.append({ static_cast<u64>(CSS::Parser::Token::Type::CDO), static_cast<u64>(CSS::Parser::Token::Type::CDC) });
     }
     return pairs;
 }

Userland/Libraries/LibWeb/Forward.h

@@ -106,6 +106,8 @@ class DeclarationOrAtRule;
 class Function;
 class Parser;
 class StyleRule;
+class Token;
+class Tokenizer;
 }
 
 namespace Web::DOM {