LibCpp: Add lex_iterable() method to the Lexer

This allows us to collect the tokens iteratively instead of having to
lex the whole program and then get a tokens vector.

Userland/Libraries/LibCpp/Lexer.cpp

@@ -6,6 +6,7 @@
 
 #include "Lexer.h"
 #include <AK/CharacterTypes.h>
+#include <AK/Function.h>
 #include <AK/HashTable.h>
 #include <AK/StdLibExtras.h>
 #include <AK/String.h>
@@ -205,15 +206,13 @@ static bool is_known_type(StringView const& string)
     return types.contains(string);
 }
 
-Vector<Token> Lexer::lex()
+void Lexer::lex_impl(Function<void(Token)> callback)
 {
-    Vector<Token> tokens;
-
     size_t token_start_index = 0;
     Position token_start_position;
 
     auto emit_single_char_token = [&](auto type) {
-        tokens.empend(type, m_position, m_position, m_input.substring_view(m_index, 1));
+        callback(Token(type, m_position, m_position, m_input.substring_view(m_index, 1)));
         consume();
     };
 
@@ -224,7 +223,7 @@ Vector<Token> Lexer::lex()
     auto commit_token = [&](auto type) {
         if (m_options.ignore_whitespace && type == Token::Type::Whitespace)
             return;
-        tokens.empend(type, token_start_position, m_previous_position, m_input.substring_view(token_start_index, m_index - token_start_index));
+        callback(Token(type, token_start_position, m_previous_position, m_input.substring_view(token_start_index, m_index - token_start_index)));
     };
 
     auto emit_token_equals = [&](auto type, auto equals_type) {
@@ -784,6 +783,14 @@ Vector<Token> Lexer::lex()
         dbgln("Unimplemented token character: {}", ch);
         emit_single_char_token(Token::Type::Unknown);
     }
+}
+
+Vector<Token> Lexer::lex()
+{
+    Vector<Token> tokens;
+    lex_impl([&](auto token) {
+        tokens.append(move(token));
+    });
     return tokens;
 }
 

Userland/Libraries/LibCpp/Lexer.h

@@ -17,12 +17,15 @@ public:
     explicit Lexer(StringView const&, size_t start_line = 0);
 
     Vector<Token> lex();
+    template<typename Callback>
+    void lex_iterable(Callback);
 
     void set_ignore_whitespace(bool value) { m_options.ignore_whitespace = value; }
 
 private:
     char peek(size_t offset = 0) const;
     char consume();
+    void lex_impl(Function<void(Token)>);
 
     StringView m_input;
     size_t m_index { 0 };
@@ -34,4 +37,10 @@ private:
     } m_options;
 };
 
+template<typename Callback>
+void Lexer::lex_iterable(Callback callback)
+{
+    return lex_impl(move(callback));
+}
+
 }