Shell: Add support for 'immediate' expressions as variable substitutions

This commit adds a few basic variable substitution operations:
- length
    Find the length of a string or a list
- length_across
    Find the lengths of things inside a list
- remove_{suffix,prefix}
    Remove a suffix or a prefix from all the passed values
- regex_replace
    Replace all matches of a given regex with a given template
- split
    Split the given string with the given delimiter (or to its
    code points if the delimiter is empty)
- concat_lists
    concatenates any given lists into one

Closes #4316 (the ancient version of this same feature)

Userland/Shell/AST.cpp

@@ -1706,6 +1706,98 @@ IfCond::~IfCond()
 {
 }
 
+void ImmediateExpression::dump(int level) const
+{
+    Node::dump(level);
+    print_indented("(function)", level + 1);
+    print_indented(m_function.name, level + 2);
+    print_indented("(arguments)", level + 1);
+    for (auto& argument : arguments())
+        argument.dump(level + 2);
+}
+
+RefPtr<Value> ImmediateExpression::run(RefPtr<Shell> shell)
+{
+    auto node = shell->run_immediate_function(m_function.name, *this, arguments());
+    if (node)
+        return node->run(shell);
+
+    return create<ListValue>({});
+}
+
+void ImmediateExpression::highlight_in_editor(Line::Editor& editor, Shell& shell, HighlightMetadata metadata)
+{
+    // '${' - FIXME: This could also be '$\\\n{'
+    editor.stylize({ m_position.start_offset, m_position.start_offset + 2 }, { Line::Style::Foreground(Line::Style::XtermColor::Green) });
+
+    // Function name
+    Line::Style function_style { Line::Style::Foreground(Line::Style::XtermColor::Red) };
+    if (shell.has_immediate_function(function_name()))
+        function_style = { Line::Style::Foreground(Line::Style::XtermColor::Green) };
+    editor.stylize({ m_function.position.start_offset, m_function.position.end_offset }, move(function_style));
+
+    // Arguments
+    for (auto& argument : m_arguments) {
+        metadata.is_first_in_list = false;
+        argument.highlight_in_editor(editor, shell, metadata);
+    }
+
+    // Closing brace
+    if (m_closing_brace_position.has_value())
+        editor.stylize({ m_closing_brace_position->start_offset, m_closing_brace_position->end_offset }, { Line::Style::Foreground(Line::Style::XtermColor::Green) });
+}
+
+Vector<Line::CompletionSuggestion> ImmediateExpression::complete_for_editor(Shell& shell, size_t offset, const HitTestResult& hit_test_result)
+{
+    auto matching_node = hit_test_result.matching_node;
+    if (!matching_node || matching_node != this)
+        return {};
+
+    auto corrected_offset = offset - m_function.position.start_offset;
+
+    if (corrected_offset > m_function.name.length())
+        return {};
+
+    return shell.complete_immediate_function_name(m_function.name, corrected_offset);
+}
+
+HitTestResult ImmediateExpression::hit_test_position(size_t offset) const
+{
+    if (!position().contains(offset))
+        return {};
+
+    if (m_function.position.contains(offset))
+        return { this, this, this };
+
+    for (auto& argument : m_arguments) {
+        if (auto result = argument.hit_test_position(offset); result.matching_node)
+            return result;
+    }
+
+    return {};
+}
+
+ImmediateExpression::ImmediateExpression(Position position, NameWithPosition function, NonnullRefPtrVector<AST::Node> arguments, Optional<Position> closing_brace_position)
+    : Node(move(position))
+    , m_arguments(move(arguments))
+    , m_function(move(function))
+    , m_closing_brace_position(move(closing_brace_position))
+{
+    if (m_is_syntax_error)
+        return;
+
+    for (auto& argument : m_arguments) {
+        if (argument.is_syntax_error()) {
+            set_is_syntax_error(argument.syntax_error_node());
+            return;
+        }
+    }
+}
+
+ImmediateExpression::~ImmediateExpression()
+{
+}
+
 void Join::dump(int level) const
 {
     Node::dump(level);
@@ -2754,6 +2846,26 @@ SyntaxError::~SyntaxError()
 {
 }
 
+void SyntheticNode::dump(int level) const
+{
+    Node::dump(level);
+}
+
+RefPtr<Value> SyntheticNode::run(RefPtr<Shell>)
+{
+    return m_value;
+}
+
+void SyntheticNode::highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata)
+{
+}
+
+SyntheticNode::SyntheticNode(Position position, NonnullRefPtr<Value> value)
+    : Node(move(position))
+    , m_value(move(value))
+{
+}
+
 void Tilde::dump(int level) const
 {
     Node::dump(level);
@@ -2946,6 +3058,8 @@ Vector<AST::Command> Value::resolve_as_commands(RefPtr<Shell> shell)
 
 ListValue::ListValue(Vector<String> values)
 {
+    if (values.is_empty())
+        return;
     m_contained_values.ensure_capacity(values.size());
     for (auto& str : values)
         m_contained_values.append(adopt(*new StringValue(move(str))));
@@ -3024,6 +3138,14 @@ Vector<String> StringValue::resolve_as_list(RefPtr<Shell>)
     return { m_string };
 }
 
+NonnullRefPtr<Value> StringValue::resolve_without_cast(RefPtr<Shell> shell)
+{
+    if (is_list())
+        return create<AST::ListValue>(resolve_as_list(shell));
+
+    return *this;
+}
+
 GlobValue::~GlobValue()
 {
 }

Userland/Shell/AST.h

@@ -73,6 +73,11 @@ struct Position {
     bool contains(size_t offset) const { return start_offset <= offset && offset <= end_offset; }
 };
 
+struct NameWithPosition {
+    String name;
+    Position position;
+};
+
 struct FdRedirection;
 struct Rewiring : public RefCounted<Rewiring> {
     int old_fd { -1 };
@@ -330,6 +335,7 @@ public:
     virtual ~StringValue();
     virtual bool is_string() const override { return m_split.is_null(); }
     virtual bool is_list() const override { return !m_split.is_null(); }
+    NonnullRefPtr<Value> resolve_without_cast(RefPtr<Shell>) override;
     StringValue(String string, String split_by = {}, bool keep_empty = false)
         : m_string(move(string))
         , m_split(move(split_by))
@@ -472,6 +478,7 @@ public:
         Glob,
         HistoryEvent,
         IfCond,
+        ImmediateExpression,
         Join,
         Juxtaposition,
         ListConcatenate,
@@ -488,6 +495,7 @@ public:
         StringPartCompose,
         Subshell,
         SyntaxError,
+        SyntheticValue,
         Tilde,
         VariableDeclarations,
         WriteAppendRedirection,
@@ -813,10 +821,6 @@ private:
 
 class FunctionDeclaration final : public Node {
 public:
-    struct NameWithPosition {
-        String name;
-        Position position;
-    };
     FunctionDeclaration(Position, NameWithPosition name, Vector<NameWithPosition> argument_names, RefPtr<AST::Node> body);
     virtual ~FunctionDeclaration();
     virtual void visit(NodeVisitor& visitor) override { visitor.visit(this); }
@@ -994,6 +998,31 @@ private:
     Optional<Position> m_else_position;
 };
 
+class ImmediateExpression final : public Node {
+public:
+    ImmediateExpression(Position, NameWithPosition function, NonnullRefPtrVector<AST::Node> arguments, Optional<Position> closing_brace_position);
+    virtual ~ImmediateExpression();
+    virtual void visit(NodeVisitor& visitor) override { visitor.visit(this); }
+
+    const NonnullRefPtrVector<Node>& arguments() const { return m_arguments; }
+    const auto& function() const { return m_function; }
+    const String& function_name() const { return m_function.name; }
+    const Position& function_position() const { return m_function.position; }
+    bool has_closing_brace() const { return m_closing_brace_position.has_value(); }
+
+private:
+    NODE(ImmediateExpression);
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(RefPtr<Shell>) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+    Vector<Line::CompletionSuggestion> complete_for_editor(Shell&, size_t, const HitTestResult&) override;
+    virtual HitTestResult hit_test_position(size_t) const override;
+
+    NonnullRefPtrVector<AST::Node> m_arguments;
+    NameWithPosition m_function;
+    Optional<Position> m_closing_brace_position;
+};
+
 class Join final : public Node {
 public:
     Join(Position, NonnullRefPtr<Node>, NonnullRefPtr<Node>);
@@ -1301,6 +1330,23 @@ private:
     bool m_is_continuable { false };
 };
 
+class SyntheticNode final : public Node {
+public:
+    SyntheticNode(Position, NonnullRefPtr<Value>);
+    virtual ~SyntheticNode() = default;
+    virtual void visit(NodeVisitor& visitor) override { visitor.visit(this); }
+
+    const Value& value() const { return m_value; }
+
+private:
+    NODE(SyntheticValue);
+    virtual void dump(int level) const override;
+    virtual RefPtr<Value> run(RefPtr<Shell>) override;
+    virtual void highlight_in_editor(Line::Editor&, Shell&, HighlightMetadata = {}) override;
+
+    NonnullRefPtr<Value> m_value;
+};
+
 class Tilde final : public Node {
 public:
     Tilde(Position, String);

Userland/Shell/CMakeLists.txt

@@ -1,20 +1,21 @@
 set(SOURCES
-    AST.cpp
-    Builtin.cpp
-    Formatter.cpp
-    Job.cpp
-    NodeVisitor.cpp
-    Parser.cpp
-    Shell.cpp
-    SyntaxHighlighter.cpp
-)
+        AST.cpp
+        Builtin.cpp
+        Formatter.cpp
+        ImmediateFunctions.cpp
+        Job.cpp
+        NodeVisitor.cpp
+        Parser.cpp
+        Shell.cpp
+        SyntaxHighlighter.cpp
+        )
 
 serenity_lib(LibShell shell)
-target_link_libraries(LibShell LibCore LibLine LibSyntax)
+target_link_libraries(LibShell LibCore LibLine LibSyntax LibRegex)
 
 set(SOURCES
-    main.cpp
-)
+        main.cpp
+        )
 
 serenity_bin(Shell)
 target_link_libraries(Shell LibShell)
@@ -22,5 +23,5 @@ target_link_libraries(Shell LibShell)
 install(DIRECTORY Tests/ DESTINATION usr/Tests/Shell
         PATTERN "Tests/*"
         PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
-                   GROUP_EXECUTE GROUP_READ
-                   WORLD_EXECUTE WORLD_READ)
+        GROUP_EXECUTE GROUP_READ
+        WORLD_EXECUTE WORLD_READ)

Userland/Shell/Formatter.cpp

@@ -33,7 +33,7 @@ namespace Shell {
 
 String Formatter::format()
 {
-    auto node = Parser(m_source).parse();
+    auto node = m_root_node ? m_root_node : Parser(m_source).parse();
     if (m_cursor >= 0)
         m_output_cursor = m_cursor;
 
@@ -472,6 +472,27 @@ void Formatter::visit(const AST::IfCond* node)
     visited(node);
 }
 
+void Formatter::visit(const AST::ImmediateExpression* node)
+{
+    will_visit(node);
+    test_and_update_output_cursor(node);
+
+    current_builder().append("${");
+    TemporaryChange<const AST::Node*> parent { m_parent_node, node };
+
+    current_builder().append(node->function_name());
+
+    for (auto& node : node->arguments()) {
+        current_builder().append(' ');
+        node.visit(*this);
+    }
+
+    if (node->has_closing_brace())
+        current_builder().append('}');
+
+    visited(node);
+}
+
 void Formatter::visit(const AST::Join* node)
 {
     will_visit(node);

Userland/Shell/Formatter.h

@@ -49,6 +49,12 @@ public:
         m_trivia = m_source.substring_view(m_source.length() - offset, offset);
     }
 
+    explicit Formatter(const AST::Node& node)
+        : m_cursor(-1)
+        , m_root_node(node)
+    {
+    }
+
     String format();
     size_t cursor() const { return m_output_cursor; }
 
@@ -74,6 +80,7 @@ private:
     virtual void visit(const AST::HistoryEvent*) override;
     virtual void visit(const AST::Execute*) override;
     virtual void visit(const AST::IfCond*) override;
+    virtual void visit(const AST::ImmediateExpression*) override;
     virtual void visit(const AST::Join*) override;
     virtual void visit(const AST::MatchExpr*) override;
     virtual void visit(const AST::Or*) override;
@@ -119,6 +126,7 @@ private:
     StringView m_source;
     size_t m_output_cursor { 0 };
     ssize_t m_cursor { -1 };
+    RefPtr<AST::Node> m_root_node;
     AST::Node* m_hit_node { nullptr };
 
     const AST::Node* m_parent_node { nullptr };

Userland/Shell/Forward.h

@@ -57,6 +57,7 @@ class Glob;
 class HistoryEvent;
 class Execute;
 class IfCond;
+class ImmediateExpression;
 class Join;
 class MatchExpr;
 class Or;
@@ -72,6 +73,7 @@ class Juxtaposition;
 class StringLiteral;
 class StringPartCompose;
 class SyntaxError;
+class SyntheticNode;
 class Tilde;
 class VariableDeclarations;
 class WriteAppendRedirection;

Userland/Shell/ImmediateFunctions.cpp

@@ -0,0 +1,431 @@
+/*
+ * Copyright (c) 2021, The SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "Formatter.h"
+#include "Shell.h"
+#include <LibRegex/Regex.h>
+
+namespace Shell {
+
+RefPtr<AST::Node> Shell::immediate_length_impl(AST::ImmediateExpression& invoking_node, const NonnullRefPtrVector<AST::Node>& arguments, bool across)
+{
+    auto name = across ? "length_across" : "length";
+    if (arguments.size() < 1 || arguments.size() > 2) {
+        raise_error(ShellError::EvaluatedSyntaxError, String::formatted("Expected one or two arguments to `{}'", name), invoking_node.position());
+        return nullptr;
+    }
+
+    enum {
+        Infer,
+        String,
+        List,
+    } mode { Infer };
+
+    bool is_inferred = false;
+
+    const AST::Node* expr_node;
+    if (arguments.size() == 2) {
+        // length string <expr>
+        // length list <expr>
+
+        auto& mode_arg = arguments.first();
+        if (!mode_arg.is_bareword()) {
+            raise_error(ShellError::EvaluatedSyntaxError, String::formatted("Expected a bareword (either 'string' or 'list') in the two-argument form of the `{}' immediate", name), mode_arg.position());
+            return nullptr;
+        }
+
+        const auto& mode_name = static_cast<const AST::BarewordLiteral&>(mode_arg).text();
+        if (mode_name == "list") {
+            mode = List;
+        } else if (mode_name == "string") {
+            mode = String;
+        } else if (mode_name == "infer") {
+            mode = Infer;
+        } else {
+            raise_error(ShellError::EvaluatedSyntaxError, String::formatted("Expected either 'string' or 'list' (and not {}) in the two-argument form of the `{}' immediate", mode_name, name), mode_arg.position());
+            return nullptr;
+        }
+
+        expr_node = &arguments[1];
+    } else {
+        expr_node = &arguments[0];
+    }
+
+    if (mode == Infer) {
+        is_inferred = true;
+        if (expr_node->is_list())
+            mode = List;
+        else if (expr_node->is_simple_variable()) // "Look inside" variables
+            mode = const_cast<AST::Node*>(expr_node)->run(this)->resolve_without_cast(this)->is_list_without_resolution() ? List : String;
+        else if (is<AST::ImmediateExpression>(expr_node))
+            mode = List;
+        else
+            mode = String;
+    }
+
+    auto value_with_number = [&](auto number) -> NonnullRefPtr<AST::Node> {
+        return AST::create<AST::BarewordLiteral>(invoking_node.position(), String::number(number));
+    };
+
+    auto do_across = [&](StringView mode_name, auto& values) {
+        if (is_inferred)
+            mode_name = "infer";
+        // Translate to a list of applications of `length <mode_name>`
+        Vector<NonnullRefPtr<AST::Node>> resulting_nodes;
+        resulting_nodes.ensure_capacity(values.size());
+        for (auto& entry : values) {
+            // ImmediateExpression(length <mode_name> <entry>)
+            resulting_nodes.unchecked_append(AST::create<AST::ImmediateExpression>(
+                expr_node->position(),
+                AST::NameWithPosition { "length", invoking_node.function_position() },
+                NonnullRefPtrVector<AST::Node> { Vector<NonnullRefPtr<AST::Node>> {
+                    static_cast<NonnullRefPtr<AST::Node>>(AST::create<AST::BarewordLiteral>(expr_node->position(), mode_name)),
+                    AST::create<AST::SyntheticNode>(expr_node->position(), NonnullRefPtr<AST::Value>(entry)),
+                } },
+                expr_node->position()));
+        }
+
+        return AST::create<AST::ListConcatenate>(invoking_node.position(), move(resulting_nodes));
+    };
+
+    switch (mode) {
+    default:
+    case Infer:
+        VERIFY_NOT_REACHED();
+    case List: {
+        auto value = (const_cast<AST::Node*>(expr_node))->run(this);
+        if (!value)
+            return value_with_number(0);
+
+        value = value->resolve_without_cast(this);
+
+        if (auto list = dynamic_cast<AST::ListValue*>(value.ptr())) {
+            if (across)
+                return do_across("list", list->values());
+
+            return value_with_number(list->values().size());
+        }
+
+        auto list = value->resolve_as_list(this);
+        if (!across)
+            return value_with_number(list.size());
+
+        dbgln("List has {} entries", list.size());
+        auto values = AST::create<AST::ListValue>(move(list));
+        return do_across("list", values->values());
+    }
+    case String: {
+        // 'across' will only accept lists, and '!across' will only accept non-lists here.
+        if (expr_node->is_list()) {
+            if (!across) {
+            raise_no_list_allowed:;
+                Formatter formatter { *expr_node };
+
+                if (is_inferred) {
+                    raise_error(ShellError::EvaluatedSyntaxError,
+                        String::formatted("Could not infer expression type, please explicitly use `{0} string' or `{0} list'", name),
+                        invoking_node.position());
+                    return nullptr;
+                }
+
+                auto source = formatter.format();
+                raise_error(ShellError::EvaluatedSyntaxError,
+                    source.is_empty()
+                        ? "Invalid application of `length' to a list"
+                        : String::formatted("Invalid application of `length' to a list\nperhaps you meant `{1}length \"{0}\"{2}' or `{1}length_across {0}{2}'?", source, "\x1b[32m", "\x1b[0m"),
+                    expr_node->position());
+                return nullptr;
+            }
+        }
+
+        auto value = (const_cast<AST::Node*>(expr_node))->run(this);
+        if (!value)
+            return value_with_number(0);
+
+        value = value->resolve_without_cast(*this);
+
+        if (auto list = dynamic_cast<AST::ListValue*>(value.ptr())) {
+            if (!across)
+                goto raise_no_list_allowed;
+
+            return do_across("string", list->values());
+        }
+
+        if (across && !value->is_list()) {
+            Formatter formatter { *expr_node };
+
+            auto source = formatter.format();
+            raise_error(ShellError::EvaluatedSyntaxError,
+                String::formatted("Invalid application of `length_across' to a non-list\nperhaps you meant `{1}length {0}{2}'?", source, "\x1b[32m", "\x1b[0m"),
+                expr_node->position());
+            return nullptr;
+        }
+
+        // Evaluate the nodes and substitute with the lengths.
+        auto list = value->resolve_as_list(this);
+
+        if (!expr_node->is_list()) {
+            if (list.size() == 1) {
+                if (across)
+                    goto raise_no_list_allowed;
+
+                // This is the normal case, the expression is a normal non-list expression.
+                return value_with_number(list.first().length());
+            }
+
+            // This can be hit by asking for the length of a command list (e.g. `(>/dev/null)`)
+            // raise an error about misuse of command lists for now.
+            // FIXME: What's the length of `(>/dev/null)` supposed to be?
+            raise_error(ShellError::EvaluatedSyntaxError, "Length of meta value (or command list) requested, this is currently not supported.", expr_node->position());
+            return nullptr;
+        }
+
+        auto values = AST::create<AST::ListValue>(move(list));
+        return do_across("string", values->values());
+    }
+    }
+}
+
+RefPtr<AST::Node> Shell::immediate_length(AST::ImmediateExpression& invoking_node, const NonnullRefPtrVector<AST::Node>& arguments)
+{
+    return immediate_length_impl(invoking_node, arguments, false);
+}
+
+RefPtr<AST::Node> Shell::immediate_length_across(AST::ImmediateExpression& invoking_node, const NonnullRefPtrVector<AST::Node>& arguments)
+{
+    return immediate_length_impl(invoking_node, arguments, true);
+}
+
+RefPtr<AST::Node> Shell::immediate_regex_replace(AST::ImmediateExpression& invoking_node, const NonnullRefPtrVector<AST::Node>& arguments)
+{
+    if (arguments.size() != 3) {
+        raise_error(ShellError::EvaluatedSyntaxError, "Expected exactly 3 arguments to regex_replace", invoking_node.position());
+        return nullptr;
+    }
+
+    auto pattern = const_cast<AST::Node&>(arguments[0]).run(this);
+    auto replacement = const_cast<AST::Node&>(arguments[1]).run(this);
+    auto value = const_cast<AST::Node&>(arguments[2]).run(this)->resolve_without_cast(this);
+
+    if (!pattern->is_string()) {
+        raise_error(ShellError::EvaluatedSyntaxError, "Expected the regex_replace pattern to be a string", arguments[0].position());
+        return nullptr;
+    }
+
+    if (!replacement->is_string()) {
+        raise_error(ShellError::EvaluatedSyntaxError, "Expected the regex_replace replacement string to be a string", arguments[1].position());
+        return nullptr;
+    }
+
+    if (!value->is_string()) {
+        raise_error(ShellError::EvaluatedSyntaxError, "Expected the regex_replace target value to be a string", arguments[2].position());
+        return nullptr;
+    }
+
+    Regex<PosixExtendedParser> re { pattern->resolve_as_list(this).first() };
+    auto result = re.replace(value->resolve_as_list(this)[0], replacement->resolve_as_list(this)[0], PosixFlags::Global | PosixFlags::Multiline | PosixFlags::Unicode);
+
+    return AST::create<AST::StringLiteral>(invoking_node.position(), move(result));
+}
+
+RefPtr<AST::Node> Shell::immediate_remove_suffix(AST::ImmediateExpression& invoking_node, const NonnullRefPtrVector<AST::Node>& arguments)
+{
+    if (arguments.size() != 2) {
+        raise_error(ShellError::EvaluatedSyntaxError, "Expected exactly 2 arguments to remove_suffix", invoking_node.position());
+        return nullptr;
+    }
+
+    auto suffix = const_cast<AST::Node&>(arguments[0]).run(this);
+    auto value = const_cast<AST::Node&>(arguments[1]).run(this)->resolve_without_cast(this);
+
+    if (!suffix->is_string()) {
+        raise_error(ShellError::EvaluatedSyntaxError, "Expected the remove_suffix suffix string to be a string", arguments[0].position());
+        return nullptr;
+    }
+
+    auto suffix_str = suffix->resolve_as_list(this)[0];
+    auto values = value->resolve_as_list(this);
+
+    Vector<NonnullRefPtr<AST::Node>> nodes;
+
+    for (auto& value_str : values) {
+        StringView removed { value_str };
+
+        if (value_str.ends_with(suffix_str))
+            removed = removed.substring_view(0, value_str.length() - suffix_str.length());
+        nodes.append(AST::create<AST::StringLiteral>(invoking_node.position(), removed));
+    }
+
+    return AST::create<AST::ListConcatenate>(invoking_node.position(), move(nodes));
+}
+
+RefPtr<AST::Node> Shell::immediate_remove_prefix(AST::ImmediateExpression& invoking_node, const NonnullRefPtrVector<AST::Node>& arguments)
+{
+    if (arguments.size() != 2) {
+        raise_error(ShellError::EvaluatedSyntaxError, "Expected exactly 2 arguments to remove_prefix", invoking_node.position());
+        return nullptr;
+    }
+
+    auto prefix = const_cast<AST::Node&>(arguments[0]).run(this);
+    auto value = const_cast<AST::Node&>(arguments[1]).run(this)->resolve_without_cast(this);
+
+    if (!prefix->is_string()) {
+        raise_error(ShellError::EvaluatedSyntaxError, "Expected the remove_prefix prefix string to be a string", arguments[0].position());
+        return nullptr;
+    }
+
+    auto prefix_str = prefix->resolve_as_list(this)[0];
+    auto values = value->resolve_as_list(this);
+
+    Vector<NonnullRefPtr<AST::Node>> nodes;
+
+    for (auto& value_str : values) {
+        StringView removed { value_str };
+
+        if (value_str.starts_with(prefix_str))
+            removed = removed.substring_view(prefix_str.length());
+        nodes.append(AST::create<AST::StringLiteral>(invoking_node.position(), removed));
+    }
+
+    return AST::create<AST::ListConcatenate>(invoking_node.position(), move(nodes));
+}
+
+RefPtr<AST::Node> Shell::immediate_split(AST::ImmediateExpression& invoking_node, const NonnullRefPtrVector<AST::Node>& arguments)
+{
+    if (arguments.size() != 2) {
+        raise_error(ShellError::EvaluatedSyntaxError, "Expected exactly 2 arguments to split", invoking_node.position());
+        return nullptr;
+    }
+
+    auto delimiter = const_cast<AST::Node&>(arguments[0]).run(this);
+    auto value = const_cast<AST::Node&>(arguments[1]).run(this)->resolve_without_cast(this);
+
+    if (!delimiter->is_string()) {
+        raise_error(ShellError::EvaluatedSyntaxError, "Expected the split delimiter string to be a string", arguments[0].position());
+        return nullptr;
+    }
+
+    auto delimiter_str = delimiter->resolve_as_list(this)[0];
+
+    auto transform = [&](const auto& values) {
+        // Translate to a list of applications of `split <delimiter>`
+        Vector<NonnullRefPtr<AST::Node>> resulting_nodes;
+        resulting_nodes.ensure_capacity(values.size());
+        for (auto& entry : values) {
+            // ImmediateExpression(split <delimiter> <entry>)
+            resulting_nodes.unchecked_append(AST::create<AST::ImmediateExpression>(
+                arguments[1].position(),
+                invoking_node.function(),
+                NonnullRefPtrVector<AST::Node> { Vector<NonnullRefPtr<AST::Node>> {
+                    arguments[0],
+                    AST::create<AST::SyntheticNode>(arguments[1].position(), NonnullRefPtr<AST::Value>(entry)),
+                } },
+                arguments[1].position()));
+        }
+
+        return AST::create<AST::ListConcatenate>(invoking_node.position(), move(resulting_nodes));
+    };
+
+    if (auto list = dynamic_cast<AST::ListValue*>(value.ptr())) {
+        return transform(list->values());
+    }
+
+    // Otherwise, just resolve to a list and transform that.
+    auto list = value->resolve_as_list(this);
+    if (!value->is_list()) {
+        if (list.is_empty())
+            return AST::create<AST::ListConcatenate>(invoking_node.position(), NonnullRefPtrVector<AST::Node> {});
+
+        auto& value = list.first();
+        Vector<String> split_strings;
+        if (delimiter_str.is_empty()) {
+            StringBuilder builder;
+            for (auto code_point : Utf8View { value }) {
+                builder.append_code_point(code_point);
+                split_strings.append(builder.build());
+                builder.clear();
+            }
+        } else {
+            auto split = StringView { value }.split_view(delimiter_str, options.inline_exec_keep_empty_segments);
+            split_strings.ensure_capacity(split.size());
+            for (auto& entry : split)
+                split_strings.append(entry);
+        }
+        return AST::create<AST::SyntheticNode>(invoking_node.position(), AST::create<AST::ListValue>(move(split_strings)));
+    }
+
+    return transform(AST::create<AST::ListValue>(list)->values());
+}
+
+RefPtr<AST::Node> Shell::immediate_concat_lists(AST::ImmediateExpression& invoking_node, const NonnullRefPtrVector<AST::Node>& arguments)
+{
+    NonnullRefPtrVector<AST::Node> result;
+
+    for (auto& argument : arguments) {
+        if (auto* list = dynamic_cast<const AST::ListConcatenate*>(&argument)) {
+            result.append(list->list());
+        } else {
+            auto list_of_values = const_cast<AST::Node&>(argument).run(this)->resolve_without_cast(this);
+            if (auto* list = dynamic_cast<AST::ListValue*>(list_of_values.ptr())) {
+                for (auto& entry : static_cast<Vector<NonnullRefPtr<AST::Value>>&>(list->values()))
+                    result.append(AST::create<AST::SyntheticNode>(argument.position(), entry));
+            } else {
+                auto values = list_of_values->resolve_as_list(this);
+                for (auto& entry : values)
+                    result.append(AST::create<AST::StringLiteral>(argument.position(), entry));
+            }
+        }
+    }
+
+    return AST::create<AST::ListConcatenate>(invoking_node.position(), move(result));
+}
+
+RefPtr<AST::Node> Shell::run_immediate_function(StringView str, AST::ImmediateExpression& invoking_node, const NonnullRefPtrVector<AST::Node>& arguments)
+{
+#define __ENUMERATE_SHELL_IMMEDIATE_FUNCTION(name) \
+    if (str == #name)                              \
+        return immediate_##name(invoking_node, arguments);
+
+    ENUMERATE_SHELL_IMMEDIATE_FUNCTIONS()
+
+#undef __ENUMERATE_SHELL_IMMEDIATE_FUNCTION
+    raise_error(ShellError::EvaluatedSyntaxError, String::formatted("Unknown immediate function {}", str), invoking_node.position());
+    return nullptr;
+}
+
+bool Shell::has_immediate_function(const StringView& str)
+{
+#define __ENUMERATE_SHELL_IMMEDIATE_FUNCTION(name) \
+    if (str == #name)                              \
+        return true;
+
+    ENUMERATE_SHELL_IMMEDIATE_FUNCTIONS()
+
+#undef __ENUMERATE_SHELL_IMMEDIATE_FUNCTION
+
+    return false;
+}
+}

Userland/Shell/NodeVisitor.cpp

@@ -139,6 +139,12 @@ void NodeVisitor::visit(const AST::IfCond* node)
         node->false_branch()->visit(*this);
 }
 
+void NodeVisitor::visit(const AST::ImmediateExpression* node)
+{
+    for (auto& node : node->arguments())
+        node.visit(*this);
+}
+
 void NodeVisitor::visit(const AST::Join* node)
 {
     node->left()->visit(*this);
@@ -224,6 +230,10 @@ void NodeVisitor::visit(const AST::SyntaxError*)
 {
 }
 
+void NodeVisitor::visit(const AST::SyntheticNode*)
+{
+}
+
 void NodeVisitor::visit(const AST::Tilde*)
 {
 }

Userland/Shell/NodeVisitor.h

@@ -53,6 +53,7 @@ public:
     virtual void visit(const AST::HistoryEvent*);
     virtual void visit(const AST::Execute*);
     virtual void visit(const AST::IfCond*);
+    virtual void visit(const AST::ImmediateExpression*);
     virtual void visit(const AST::Join*);
     virtual void visit(const AST::MatchExpr*);
     virtual void visit(const AST::Or*);
@@ -68,6 +69,7 @@ public:
     virtual void visit(const AST::StringLiteral*);
     virtual void visit(const AST::StringPartCompose*);
     virtual void visit(const AST::SyntaxError*);
+    virtual void visit(const AST::SyntheticNode*);
     virtual void visit(const AST::Tilde*);
     virtual void visit(const AST::VariableDeclarations*);
     virtual void visit(const AST::WriteAppendRedirection*);

Userland/Shell/Parser.cpp

@@ -88,8 +88,8 @@ bool Parser::expect(const StringView& expected)
     if (expected.length() + m_offset > m_input.length())
         return false;
 
-    for (size_t i = 0; i < expected.length(); ++i) {
-        if (peek() != expected[i]) {
+    for (auto& c : expected) {
+        if (peek() != c) {
             restore_to(offset_at_start, line_at_start);
             return false;
         }
@@ -353,7 +353,7 @@ RefPtr<AST::Node> Parser::parse_function_decl()
     if (!expect('('))
         return restore();
 
-    Vector<AST::FunctionDeclaration::NameWithPosition> arguments;
+    Vector<AST::NameWithPosition> arguments;
     for (;;) {
         consume_while(is_whitespace);
 
@@ -380,7 +380,7 @@ RefPtr<AST::Node> Parser::parse_function_decl()
         }
         if (!expect('{')) {
             return create<AST::FunctionDeclaration>(
-                AST::FunctionDeclaration::NameWithPosition {
+                AST::NameWithPosition {
                     move(function_name),
                     { pos_before_name.offset, pos_after_name.offset, pos_before_name.line, pos_after_name.line } },
                 move(arguments),
@@ -404,7 +404,7 @@ RefPtr<AST::Node> Parser::parse_function_decl()
                 body = move(syntax_error);
 
             return create<AST::FunctionDeclaration>(
-                AST::FunctionDeclaration::NameWithPosition {
+                AST::NameWithPosition {
                     move(function_name),
                     { pos_before_name.offset, pos_after_name.offset, pos_before_name.line, pos_after_name.line } },
                 move(arguments),
@@ -413,7 +413,7 @@ RefPtr<AST::Node> Parser::parse_function_decl()
     }
 
     return create<AST::FunctionDeclaration>(
-        AST::FunctionDeclaration::NameWithPosition {
+        AST::NameWithPosition {
             move(function_name),
             { pos_before_name.offset, pos_after_name.offset, pos_before_name.line, pos_after_name.line } },
         move(arguments),
@@ -1101,6 +1101,9 @@ RefPtr<AST::Node> Parser::parse_expression()
         if (auto variable = parse_variable())
             return read_concat(variable.release_nonnull());
 
+        if (auto immediate = parse_immediate_expression())
+            return read_concat(immediate.release_nonnull());
+
         if (auto inline_exec = parse_evaluate())
             return read_concat(inline_exec.release_nonnull());
     }
@@ -1266,29 +1269,26 @@ RefPtr<AST::Node> Parser::parse_doublequoted_string_inner()
         }
         if (peek() == '$') {
             auto string_literal = create<AST::StringLiteral>(builder.to_string()); // String Literal
-            if (auto variable = parse_variable()) {
+            auto read_concat = [&](auto&& node) {
                 auto inner = create<AST::StringPartCompose>(
                     move(string_literal),
-                    variable.release_nonnull()); // Compose String Variable
+                    move(node)); // Compose String Node
 
                 if (auto string = parse_doublequoted_string_inner()) {
                     return create<AST::StringPartCompose>(move(inner), string.release_nonnull()); // Compose Composition Composition
                 }
 
                 return inner;
-            }
+            };
 
-            if (auto evaluate = parse_evaluate()) {
-                auto composition = create<AST::StringPartCompose>(
-                    move(string_literal),
-                    evaluate.release_nonnull()); // Compose String Sequence
+            if (auto variable = parse_variable())
+                return read_concat(variable.release_nonnull());
 
-                if (auto string = parse_doublequoted_string_inner()) {
-                    return create<AST::StringPartCompose>(move(composition), string.release_nonnull()); // Compose Composition Composition
-                }
+            if (auto immediate = parse_immediate_expression())
+                return read_concat(immediate.release_nonnull());
 
-                return composition;
-            }
+            if (auto evaluate = parse_evaluate())
+                return read_concat(evaluate.release_nonnull());
         }
 
         builder.append(consume());
@@ -1364,6 +1364,75 @@ RefPtr<AST::Node> Parser::parse_evaluate()
     return inner;
 }
 
+RefPtr<AST::Node> Parser::parse_immediate_expression()
+{
+    auto rule_start = push_start();
+    if (at_end())
+        return nullptr;
+
+    if (peek() != '$')
+        return nullptr;
+
+    consume();
+
+    if (peek() != '{') {
+        restore_to(*rule_start);
+        return nullptr;
+    }
+
+    consume();
+    consume_while(is_whitespace);
+
+    auto function_name_start_offset = current_position();
+    auto function_name = consume_while(is_word_character);
+    auto function_name_end_offset = current_position();
+    AST::Position function_position {
+        function_name_start_offset.offset,
+        function_name_end_offset.offset,
+        function_name_start_offset.line,
+        function_name_end_offset.line,
+    };
+
+    consume_while(is_whitespace);
+
+    NonnullRefPtrVector<AST::Node> arguments;
+    do {
+        auto expr = parse_expression();
+        if (!expr)
+            break;
+        arguments.append(expr.release_nonnull());
+    } while (!consume_while(is_whitespace).is_empty());
+
+    auto ending_brace_start_offset = current_position();
+    if (peek() == '}')
+        consume();
+
+    auto ending_brace_end_offset = current_position();
+
+    auto ending_brace_position = ending_brace_start_offset.offset == ending_brace_end_offset.offset
+        ? Optional<AST::Position> {}
+        : Optional<AST::Position> {
+              AST::Position {
+                  ending_brace_start_offset.offset,
+                  ending_brace_end_offset.offset,
+                  ending_brace_start_offset.line,
+                  ending_brace_end_offset.line,
+              }
+          };
+
+    auto node = create<AST::ImmediateExpression>(
+        AST::NameWithPosition { function_name, move(function_position) },
+        move(arguments),
+        ending_brace_position);
+
+    if (!ending_brace_position.has_value())
+        node->set_is_syntax_error(create<AST::SyntaxError>("Expected a closing brace '}' to end an immediate expression", true));
+    else if (node->function_name().is_empty())
+        node->set_is_syntax_error(create<AST::SyntaxError>("Expected an immediate function name"));
+
+    return node;
+}
+
 RefPtr<AST::Node> Parser::parse_history_designator()
 {
     auto rule_start = push_start();

Userland/Shell/Parser.h

@@ -98,6 +98,7 @@ private:
     RefPtr<AST::Node> parse_glob();
     RefPtr<AST::Node> parse_brace_expansion();
     RefPtr<AST::Node> parse_brace_expansion_spec();
+    RefPtr<AST::Node> parse_immediate_expression();
 
     template<typename A, typename... Args>
     NonnullRefPtr<A> create(Args... args);
@@ -238,6 +239,7 @@ list_expression :: ' '* expression (' '+ list_expression)?
 expression :: evaluate expression?
             | string_composite expression?
             | comment expression?
+            | immediate_expression expression?
             | history_designator expression?
             | '(' list_expression ')' expression?
 
@@ -268,6 +270,10 @@ variable :: '$' identifier
 
 comment :: '#' [^\n]*
 
+immediate_expression :: '$' '{' immediate_function expression* '}'
+
+immediate_function :: identifier       { predetermined list of names, see Shell.h:ENUMERATE_SHELL_IMMEDIATE_FUNCTIONS }
+
 history_designator :: '!' event_selector (':' word_selector_composite)?
 
 event_selector :: '!'                  {== '-0'}

Userland/Shell/Shell.cpp

@@ -1518,6 +1518,26 @@ Vector<Line::CompletionSuggestion> Shell::complete_option(const String& program_
     return suggestions;
 }
 
+Vector<Line::CompletionSuggestion> Shell::complete_immediate_function_name(const String& name, size_t offset)
+{
+    Vector<Line::CompletionSuggestion> suggestions;
+
+#define __ENUMERATE_SHELL_IMMEDIATE_FUNCTION(fn_name)                            \
+    if (auto name_view = StringView { #fn_name }; name_view.starts_with(name)) { \
+        suggestions.append({ name_view, " " });                                  \
+        suggestions.last().input_offset = offset;                                \
+    }
+
+    ENUMERATE_SHELL_IMMEDIATE_FUNCTIONS();
+
+#undef __ENUMERATE_SHELL_IMMEDIATE_FUNCTION
+
+    if (m_editor)
+        m_editor->suggest(offset);
+
+    return suggestions;
+}
+
 void Shell::bring_cursor_to_beginning_of_a_line() const
 {
     struct winsize ws;

Userland/Shell/Shell.h

@@ -71,6 +71,15 @@
     __ENUMERATE_SHELL_OPTION(inline_exec_keep_empty_segments, false, "Keep empty segments in inline execute $(...)") \
     __ENUMERATE_SHELL_OPTION(verbose, false, "Announce every command that is about to be executed")
 
+#define ENUMERATE_SHELL_IMMEDIATE_FUNCTIONS()           \
+    __ENUMERATE_SHELL_IMMEDIATE_FUNCTION(concat_lists)  \
+    __ENUMERATE_SHELL_IMMEDIATE_FUNCTION(length)        \
+    __ENUMERATE_SHELL_IMMEDIATE_FUNCTION(length_across) \
+    __ENUMERATE_SHELL_IMMEDIATE_FUNCTION(remove_suffix) \
+    __ENUMERATE_SHELL_IMMEDIATE_FUNCTION(remove_prefix) \
+    __ENUMERATE_SHELL_IMMEDIATE_FUNCTION(regex_replace) \
+    __ENUMERATE_SHELL_IMMEDIATE_FUNCTION(split)
+
 namespace Shell {
 
 class Shell;
@@ -100,6 +109,8 @@ public:
     bool run_file(const String&, bool explicitly_invoked = true);
     bool run_builtin(const AST::Command&, const NonnullRefPtrVector<AST::Rewiring>&, int& retval);
     bool has_builtin(const StringView&) const;
+    RefPtr<AST::Node> run_immediate_function(StringView name, AST::ImmediateExpression& invoking_node, const NonnullRefPtrVector<AST::Node>&);
+    static bool has_immediate_function(const StringView&);
     void block_on_job(RefPtr<Job>);
     void block_on_pipeline(RefPtr<AST::Pipeline>);
     String prompt() const;
@@ -173,6 +184,7 @@ public:
     Vector<Line::CompletionSuggestion> complete_variable(const String&, size_t offset);
     Vector<Line::CompletionSuggestion> complete_user(const String&, size_t offset);
     Vector<Line::CompletionSuggestion> complete_option(const String&, const String&, size_t offset);
+    Vector<Line::CompletionSuggestion> complete_immediate_function_name(const String&, size_t offset);
 
     void restore_ios();
 
@@ -285,6 +297,15 @@ private:
 
     virtual void custom_event(Core::CustomEvent&) override;
 
+#define __ENUMERATE_SHELL_IMMEDIATE_FUNCTION(name) \
+    RefPtr<AST::Node> immediate_##name(AST::ImmediateExpression& invoking_node, const NonnullRefPtrVector<AST::Node>&);
+
+    ENUMERATE_SHELL_IMMEDIATE_FUNCTIONS();
+
+#undef __ENUMERATE_SHELL_IMMEDIATE_FUNCTION
+
+    RefPtr<AST::Node> immediate_length_impl(AST::ImmediateExpression& invoking_node, const NonnullRefPtrVector<AST::Node>&, bool across);
+
 #define __ENUMERATE_SHELL_BUILTIN(builtin) \
     int builtin_##builtin(int argc, const char** argv);
 

Userland/Shell/SyntaxHighlighter.cpp

@@ -307,6 +307,31 @@ private:
             else_span.attributes.color = m_palette.syntax_keyword();
         }
     }
+
+    virtual void visit(const AST::ImmediateExpression* node) override
+    {
+        TemporaryChange first { m_is_first_in_command, false };
+        NodeVisitor::visit(node);
+
+        // ${
+        auto& start_span = span_for_node(node);
+        start_span.attributes.color = m_palette.syntax_punctuation();
+        start_span.range.set_end({ node->position().start_line.line_number, node->position().start_line.line_column + 1 });
+        start_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::OpenParen);
+
+        // Function name
+        auto& name_span = span_for_node(node);
+        name_span.attributes.color = m_palette.syntax_preprocessor_statement(); // Closest thing we have to this
+        set_offset_range_start(name_span.range, node->function_position().start_line);
+        set_offset_range_end(name_span.range, node->function_position().end_line);
+
+        // }
+        auto& end_span = span_for_node(node);
+        end_span.attributes.color = m_palette.syntax_punctuation();
+        set_offset_range_start(end_span.range, node->position().end_line, 1);
+        end_span.data = (void*)static_cast<size_t>(AugmentedTokenKind::CloseParen);
+    }
+
     virtual void visit(const AST::Join* node) override
     {
         NodeVisitor::visit(node);

Userland/Shell/Tests/immediate.sh

@@ -0,0 +1,85 @@
+#!/bin/sh
+
+source $(dirname "$0")/test-commons.inc
+
+# Length
+
+if test "${length foo}" -ne 3 {
+  fail invalid length for bareword
+}
+
+if test "${length "foo"}" -ne 3 {
+  fail invalid length for literal string
+}
+
+if test "${length foobar}" -ne 6 {
+  fail length string always returns 3...\?
+}
+if test "${length string foo}" -ne 3 {
+  fail invalid length for bareword with explicit string mode
+}
+
+if test "${length list foo}" -ne 1 {
+  fail invalid length for bareword with explicit list mode
+}
+
+if test "${length (1 2 3 4)}" -ne 4 {
+  fail invalid length for list
+}
+
+if test "${length list (1 2 3 4)}" -ne 4 {
+  fail invalid length for list with explicit list mode
+}
+
+if test "${length_across (1 2 3 4)}" != "1 1 1 1" {
+  fail invalid length_across for list
+}
+
+if test "${length_across list ((1 2 3) (4 5))}" != "3 2" {
+  fail invalid length_across for list with explicit list mode
+}
+
+if test "${length_across string (foo test)}" != "3 4" {
+  fail invalid length_across for list of strings
+}
+
+# remove_suffix and remove_prefix
+if test "${remove_suffix .txt foo.txt}" != "foo" {
+  fail remove_suffix did not remove suffix from a single entry
+}
+
+if test "${remove_suffix .txt (foo.txt bar.txt)}" != "foo bar" {
+  fail remove_suffix did not remove suffix from a list
+}
+
+if test "${remove_prefix fo foo.txt}" != "o.txt" {
+  fail remove_prefix did not remove prefix from a single entry
+}
+
+if test "${remove_prefix x (xfoo.txt xbar.txt)}" != "foo.txt bar.txt" {
+  fail remove_prefix did not remove prefix from a list
+}
+
+# regex_replace
+if test "${regex_replace a e wall}" != "well" {
+  fail regex_replace did not replace single character
+}
+
+if test "${regex_replace a e waaaall}" != "weeeell" {
+  fail regex_replace did not replace multiple characters
+}
+
+if test "${regex_replace '(.)l' 'e\1' hall}" != "heal" {
+  fail regex_replace did not replace with pattern
+}
+
+# split
+
+if test "${split 'x' "fooxbarxbaz"}" != "foo bar baz" {
+  fail split could not split correctly
+}
+
+if test "${split '' "abc"}" != "a b c" {
+  fail split count not split to all characters
+}
+pass

Userland/Shell/Tests/test-commons.inc

@@ -4,3 +4,8 @@ fail() {
     echo "FA""IL:" $*
     exit 1
 }
+
+pass() {
+    echo "PA""SS"
+    exit 0
+}