AK: Standardize the behaviour of GenericLexer::consume_until overloads

Before this commit all consume_until overloads aside from the Predicate
one would consume (and ignore) the stop char/string, while the
Predicate overload would not, in order to keep behaviour consistent,
the other overloads no longer consume the stop char/string as well.

AK/GenericLexer.cpp

@@ -53,7 +53,6 @@ StringView GenericLexer::consume_line()
 }
 
 // Consume and return characters until `stop` is peek'd
-// The `stop` character is ignored, as it is user-defined
 StringView GenericLexer::consume_until(char stop)
 {
     size_t start = m_index;
@@ -61,15 +60,12 @@ StringView GenericLexer::consume_until(char stop)
         m_index++;
     size_t length = m_index - start;
 
-    ignore();
-
     if (length == 0)
         return {};
     return m_input.substring_view(start, length);
 }
 
 // Consume and return characters until the string `stop` is found
-// The `stop` string is ignored, as it is user-defined
 StringView GenericLexer::consume_until(const char* stop)
 {
     size_t start = m_index;
@@ -77,15 +73,12 @@ StringView GenericLexer::consume_until(const char* stop)
         m_index++;
     size_t length = m_index - start;
 
-    ignore(__builtin_strlen(stop));
-
     if (length == 0)
         return {};
     return m_input.substring_view(start, length);
 }
 
 // Consume and return characters until the string `stop` is found
-// The `stop` string is ignored, as it is user-defined
 StringView GenericLexer::consume_until(StringView stop)
 {
     size_t start = m_index;
@@ -93,8 +86,6 @@ StringView GenericLexer::consume_until(StringView stop)
         m_index++;
     size_t length = m_index - start;
 
-    ignore(stop.length());
-
     if (length == 0)
         return {};
     return m_input.substring_view(start, length);

Kernel/FileSystem/VirtualFileSystem.cpp

@@ -865,7 +865,7 @@ ErrorOr<NonnullRefPtr<Custody>> VirtualFileSystem::resolve_path_without_veil(Str
         if (path_lexer.is_eof())
             extra_iteration = false;
         auto part = path_lexer.consume_until('/');
-        path_lexer.consume_specific('/');
+        path_lexer.ignore();
 
         Custody& parent = custody;
         auto parent_metadata = parent.inode().metadata();

Meta/Lagom/Tools/CodeGenerators/LibWeb/WrapperGenerator.cpp

@@ -230,6 +230,7 @@ static NonnullOwnPtr<Interface> parse_interface(StringView filename, StringView
 
             if (lexer.consume_specific("//")) {
                 lexer.consume_until('\n');
+                lexer.ignore();
                 consumed = true;
             }
         }
@@ -276,7 +277,9 @@ static NonnullOwnPtr<Interface> parse_interface(StringView filename, StringView
     while (lexer.consume_specific("#import")) {
         consume_whitespace();
         assert_specific('<');
-        imports.append(resolve_import(lexer.consume_until('>')));
+        auto path = lexer.consume_until('>');
+        lexer.ignore();
+        imports.append(resolve_import(path));
         consume_whitespace();
     }
 

Meta/Lagom/Tools/CodeGenerators/StateMachineGenerator/main.cpp

@@ -88,10 +88,12 @@ parse_state_machine(StringView input)
                 num = 16 * num + get_hex_value(c);
         } else {
             lexer.consume_specific('\'');
-            if (lexer.next_is('\\'))
+            if (lexer.next_is('\\')) {
                 num = (int)lexer.consume_escaped_character('\\');
-            else
+            } else {
                 num = lexer.consume_until('\'').to_int().value();
+                lexer.ignore();
+            }
             lexer.consume_specific('\'');
         }
         return num;

Userland/Libraries/LibC/scanf.cpp

@@ -506,6 +506,7 @@ extern "C" int vsscanf(const char* input, const char* format, va_list ap)
             case '[':
                 format_lexer.consume();
                 scanlist = format_lexer.consume_until(']');
+                format_lexer.ignore();
                 if (scanlist.starts_with('^')) {
                     scanlist = scanlist.substring_view(1);
                     invert_scanlist = true;

Userland/Libraries/LibCpp/Preprocessor.cpp

@@ -94,6 +94,7 @@ void Preprocessor::handle_preprocessor_statement(StringView line)
     lexer.consume_specific('#');
     consume_whitespace(lexer);
     auto keyword = lexer.consume_until(' ');
+    lexer.ignore();
     if (keyword.is_empty() || keyword.is_null() || keyword.is_whitespace())
         return;
 
@@ -165,6 +166,7 @@ void Preprocessor::handle_preprocessor_keyword(StringView keyword, GenericLexer&
         ++m_current_depth;
         if (m_state == State::Normal) {
             auto key = line_lexer.consume_until(' ');
+            line_lexer.ignore();
             if (m_definitions.contains(key)) {
                 m_depths_of_taken_branches.append(m_current_depth - 1);
                 return;
@@ -180,6 +182,7 @@ void Preprocessor::handle_preprocessor_keyword(StringView keyword, GenericLexer&
         ++m_current_depth;
         if (m_state == State::Normal) {
             auto key = line_lexer.consume_until(' ');
+            line_lexer.ignore();
             if (!m_definitions.contains(key)) {
                 m_depths_of_taken_branches.append(m_current_depth - 1);
                 return;
@@ -353,10 +356,12 @@ Optional<Preprocessor::Definition> Preprocessor::create_definition(StringView li
 
 String Preprocessor::remove_escaped_newlines(StringView value)
 {
+    static constexpr auto escaped_newline = "\\\n"sv;
     AK::StringBuilder processed_value;
     GenericLexer lexer { value };
     while (!lexer.is_eof()) {
-        processed_value.append(lexer.consume_until("\\\n"sv));
+        processed_value.append(lexer.consume_until(escaped_newline));
+        lexer.ignore(escaped_newline.length());
     }
     return processed_value.to_string();
 }

Userland/Utilities/lsof.cpp

@@ -31,6 +31,7 @@ static bool parse_name(StringView name, OpenFile& file)
 {
     GenericLexer lexer(name);
     auto component1 = lexer.consume_until(':');
+    lexer.ignore();
 
     if (lexer.tell_remaining() == 0) {
         file.name = component1;
@@ -50,6 +51,7 @@ static bool parse_name(StringView name, OpenFile& file)
             }
 
             auto component3 = lexer.consume_until(')');
+            lexer.ignore();
             if (lexer.tell_remaining() != 0) {
                 dbgln("parse_name: expected EOF");
                 return false;