LibJS: Allow invalid string in tagged template literals

Since tagged template literals can inspect the raw string it is not a
syntax error to have invalid escapes. However the cooked value should be
`undefined`.
We accomplish this by tracking whether parse_string_literal
fails and then using a NullLiteral (since UndefinedLiteral is not a
thing) and finally converting null in tagged template execution to
undefined.

Userland/Libraries/LibJS/AST.cpp

@@ -3545,6 +3545,11 @@ Completion TaggedTemplateLiteral::execute(Interpreter& interpreter, GlobalObject
         // tag`${foo}`             -> "", foo, ""                -> tag(["", ""], foo)
         // tag`foo${bar}baz${qux}` -> "foo", bar, "baz", qux, "" -> tag(["foo", "baz", ""], bar, qux)
         if (i % 2 == 0) {
+            // If the string contains invalid escapes we get a null expression here, which we then convert
+            // to the expected `undefined` TV.
+            if (value.is_nullish())
+                value = js_undefined();
+
             strings->indexed_properties().append(value);
         } else {
             arguments.append(value);

Userland/Libraries/LibJS/Parser.cpp

@@ -1811,16 +1811,18 @@ NonnullRefPtr<ArrayExpression> Parser::parse_array_expression()
     return create_ast_node<ArrayExpression>({ m_state.current_token.filename(), rule_start.position(), position() }, move(elements));
 }
 
-NonnullRefPtr<StringLiteral> Parser::parse_string_literal(Token const& token, bool in_template_literal)
+NonnullRefPtr<StringLiteral> Parser::parse_string_literal(Token const& token, StringLiteralType string_literal_type, bool* contains_invalid_escape)
 {
     auto rule_start = push_start();
     auto status = Token::StringValueStatus::Ok;
     auto string = token.string_value(status);
+    // NOTE: Tagged templates should not fail on invalid strings as their raw contents can still be accessed.
     if (status != Token::StringValueStatus::Ok) {
         String message;
         if (status == Token::StringValueStatus::LegacyOctalEscapeSequence) {
             m_state.string_legacy_octal_escape_sequence_in_scope = true;
-            if (in_template_literal)
+            // It is a Syntax Error if the [Tagged] parameter was not set and Template{Head, Middle, Tail} Contains NotEscapeSequence.
+            if (string_literal_type != StringLiteralType::Normal)
                 message = "Octal escape sequence not allowed in template literal";
             else if (m_state.strict_mode)
                 message = "Octal escape sequence in string literal not allowed in strict mode";
@@ -1833,11 +1835,17 @@ NonnullRefPtr<StringLiteral> Parser::parse_string_literal(Token const& token, bo
             VERIFY_NOT_REACHED();
         }
 
-        if (!message.is_empty())
-            syntax_error(message, Position { token.line_number(), token.line_column() });
+        if (!message.is_empty()) {
+            if (contains_invalid_escape != nullptr) {
+                VERIFY(string_literal_type == StringLiteralType::TaggedTemplate);
+                *contains_invalid_escape = true;
+            } else {
+                syntax_error(message, Position { token.line_number(), token.line_column() });
+            }
+        }
     }
 
-    auto is_use_strict_directive = !in_template_literal && (token.value() == "'use strict'" || token.value() == "\"use strict\"");
+    auto is_use_strict_directive = string_literal_type == StringLiteralType::Normal && (token.value() == "'use strict'" || token.value() == "\"use strict\"");
 
     return create_ast_node<StringLiteral>({ m_state.current_token.filename(), rule_start.position(), position() }, string, is_use_strict_directive);
 }
@@ -1863,7 +1871,15 @@ NonnullRefPtr<TemplateLiteral> Parser::parse_template_literal(bool is_tagged)
     while (!done() && !match(TokenType::TemplateLiteralEnd) && !match(TokenType::UnterminatedTemplateLiteral)) {
         if (match(TokenType::TemplateLiteralString)) {
             auto token = consume();
-            expressions.append(parse_string_literal(token, true));
+            bool contains_invalid_escape = false;
+            auto parsed_string_value = parse_string_literal(token,
+                is_tagged ? StringLiteralType::TaggedTemplate : StringLiteralType::NonTaggedTemplate,
+                is_tagged ? &contains_invalid_escape : nullptr);
+            // An invalid string leads to a cooked value of `undefined` but still gives the raw string.
+            if (contains_invalid_escape)
+                expressions.append(create_ast_node<NullLiteral>({ m_state.current_token.filename(), rule_start.position(), position() }));
+            else
+                expressions.append(move(parsed_string_value));
             if (is_tagged)
                 raw_strings.append(create_ast_node<StringLiteral>({ m_state.current_token.filename(), rule_start.position(), position() }, token.raw_template_value()));
         } else if (match(TokenType::TemplateLiteralExprStart)) {
@@ -4011,7 +4027,7 @@ FlyString Parser::consume_string_value()
 {
     VERIFY(match(TokenType::StringLiteral));
     auto string_token = consume();
-    FlyString value = parse_string_literal(string_token, false)->value();
+    FlyString value = parse_string_literal(string_token)->value();
 
     // This also checks IsStringWellFormedUnicode which makes sure there is no unpaired surrogate
     // Surrogates are at least 3 bytes

Userland/Libraries/LibJS/Parser.h

@@ -142,7 +142,14 @@ public:
     NonnullRefPtr<RegExpLiteral> parse_regexp_literal();
     NonnullRefPtr<ObjectExpression> parse_object_expression();
     NonnullRefPtr<ArrayExpression> parse_array_expression();
-    NonnullRefPtr<StringLiteral> parse_string_literal(Token const& token, bool in_template_literal = false);
+
+    enum class StringLiteralType {
+        Normal,
+        NonTaggedTemplate,
+        TaggedTemplate
+    };
+
+    NonnullRefPtr<StringLiteral> parse_string_literal(Token const& token, StringLiteralType string_literal_type = StringLiteralType::Normal, bool* contains_invalid_escape = nullptr);
     NonnullRefPtr<TemplateLiteral> parse_template_literal(bool is_tagged);
     ExpressionResult parse_secondary_expression(NonnullRefPtr<Expression>, int min_precedence, Associativity associate = Associativity::Right, ForbiddenTokens forbidden = {});
     NonnullRefPtr<Expression> parse_call_expression(NonnullRefPtr<Expression>);

Userland/Libraries/LibJS/Tests/tagged-template-literals.js

@@ -105,4 +105,29 @@ describe("tagged template literal functionality", () => {
         expect(raw[1]).toHaveLength(5);
         expect(raw[1]).toBe("\\nbar");
     });
+
+    test("invalid escapes give undefined cooked values but can be accesed in raw form", () => {
+        let calls = 0;
+        let lastValue = null;
+        function noCookedButRaw(values) {
+            ++calls;
+            expect(values).not.toBeNull();
+            expect(values.raw).toHaveLength(1);
+            expect(values.raw[0].length).toBeGreaterThan(0);
+            expect(values.raw[0].charAt(0)).toBe("\\");
+            expect(values[0]).toBeUndefined();
+            lastValue = values.raw[0];
+        }
+        noCookedButRaw`\u`;
+        expect(calls).toBe(1);
+        expect(lastValue).toBe("\\u");
+
+        noCookedButRaw`\01`;
+        expect(calls).toBe(2);
+        expect(lastValue).toBe("\\01");
+
+        noCookedButRaw`\u{10FFFFF}`;
+        expect(calls).toBe(3);
+        expect(lastValue).toBe("\\u{10FFFFF}");
+    });
 });

Userland/Libraries/LibJS/Tests/template-literals.js

@@ -63,3 +63,9 @@ test("line continuation in literals (not characters)", () => {
 test("reference error from expressions", () => {
     expect(() => `${b}`).toThrowWithMessage(ReferenceError, "'b' is not defined");
 });
+
+test("invalid escapes should give syntax error", () => {
+    expect("`\\u`").not.toEval();
+    expect("`\\01`").not.toEval();
+    expect("`\\u{10FFFFF}`").not.toEval();
+});