LibCpp: Add lexer option to ignore whitespace tokens

Userland/Libraries/LibCpp/Lexer.cpp

@@ -222,6 +222,8 @@ Vector<Token> Lexer::lex()
         token_start_position = m_position;
     };
     auto commit_token = [&](auto type) {
+        if (m_options.ignore_whitespace && type == Token::Type::Whitespace)
+            return;
         tokens.empend(type, token_start_position, m_previous_position, m_input.substring_view(token_start_index, m_index - token_start_index));
     };
 

Userland/Libraries/LibCpp/Lexer.h

@@ -18,6 +18,8 @@ public:
 
     Vector<Token> lex();
 
+    void set_ignore_whitespace(bool value) { m_options.ignore_whitespace = value; }
+
 private:
     char peek(size_t offset = 0) const;
     char consume();
@@ -26,6 +28,10 @@ private:
     size_t m_index { 0 };
     Position m_previous_position { 0, 0 };
     Position m_position { 0, 0 };
+
+    struct Options {
+        bool ignore_whitespace { false };
+    } m_options;
 };
 
 }