LibJS: Implement ImportCall and HostImportModuleDynamically

This allows us to load modules from scripts.
This can be dangerous as it can load arbitrary files. Because of that it
fails and throws by default. Currently, only js and JavaScriptTestRunner
enable the default hook.

This also adds tests to test-js which test module code. Because we
form a spec perspective can't "enter" a module this is the easiest way
to run tests without having to modify test-js to have special cases for
modules.
To specify modules in test-js we use the extension '.mjs' this is to
ensure the files are not executed. We do still want to lint these files
so the prettier scripts have changed to look for '.mjs' files as well.

Meta/lint-prettier.sh

@@ -10,12 +10,12 @@ if [ "$#" -eq "0" ]; then
         git ls-files \
             --exclude-from .prettierignore \
             -- \
-            '*.js'
+            '*.js' '*.mjs'
     )
 else
     files=()
     for file in "$@"; do
-        if [[ "${file}" == *".js" ]]; then
+        if [[ "${file}" == *".js" ]] || [[ "${file}" == *".mjs" ]]; then
             files+=("${file}")
         fi
     done
@@ -34,5 +34,5 @@ if (( ${#files[@]} )); then
 
     prettier --check "${files[@]}"
 else
-    echo "No .js files to check."
+    echo "No .js or .mjs files to check."
 fi

Userland/Libraries/LibJS/AST.cpp

@@ -3190,7 +3190,39 @@ void ImportCall::dump(int indent) const
 Completion ImportCall::execute(Interpreter& interpreter, GlobalObject& global_object) const
 {
     InterpreterNodeScope node_scope { interpreter, *this };
-    return interpreter.vm().throw_completion<InternalError>(global_object, ErrorType::NotImplemented, "'import(...)' in modules");
+    // 1. Let referencingScriptOrModule be ! GetActiveScriptOrModule().
+    auto referencing_script_or_module = interpreter.vm().get_active_script_or_module();
+
+    if (m_options)
+        return interpreter.vm().throw_completion<InternalError>(global_object, ErrorType::NotImplemented, "import call with assertions/options");
+
+    // 2. Let argRef be the result of evaluating AssignmentExpression.
+    // 3. Let specifier be ? GetValue(argRef).
+    auto specifier = TRY(m_specifier->execute(interpreter, global_object));
+
+    // 4. Let promiseCapability be ! NewPromiseCapability(%Promise%).
+    auto promise_capability = MUST(new_promise_capability(global_object, global_object.promise_constructor()));
+
+    VERIFY(!interpreter.exception());
+    // 5. Let specifierString be ToString(specifier).
+    auto specifier_string = specifier->to_string(global_object);
+
+    // 6. IfAbruptRejectPromise(specifierString, promiseCapability).
+    // Note: Since we have to use completions and not ThrowCompletionOr's in AST we have to do this manually.
+    if (specifier_string.is_throw_completion()) {
+        // FIXME: We shouldn't have to clear this exception
+        interpreter.vm().clear_exception();
+        (void)TRY(call(global_object, promise_capability.reject, js_undefined(), *specifier_string.throw_completion().value()));
+        return Value { promise_capability.promise };
+    }
+
+    ModuleRequest request { specifier_string.release_value() };
+
+    // 7. Perform ! HostImportModuleDynamically(referencingScriptOrModule, specifierString, promiseCapability).
+    interpreter.vm().host_import_module_dynamically(referencing_script_or_module, request, promise_capability);
+
+    // 8. Return promiseCapability.[[Promise]].
+    return Value { promise_capability.promise };
 }
 
 // 13.2.3.1 Runtime Semantics: Evaluation, https://tc39.es/ecma262/#sec-literals-runtime-semantics-evaluation

Userland/Libraries/LibJS/Runtime/ErrorTypes.h

@@ -29,6 +29,7 @@
     M(DescWriteNonWritable, "Cannot write to non-writable property '{}'")                                                               \
     M(DetachedArrayBuffer, "ArrayBuffer is detached")                                                                                   \
     M(DivisionByZero, "Division by zero")                                                                                               \
+    M(DynamicImportNotAllowed, "Dynamic Imports are not allowed")                                                                       \
     M(FinalizationRegistrySameTargetAndValue, "Target and held value must not be the same")                                             \
     M(GetCapabilitiesExecutorCalledMultipleTimes, "GetCapabilitiesExecutor was called multiple times")                                  \
     M(GlobalEnvironmentAlreadyHasBinding, "Global environment already has binding '{}'")                                                \
@@ -68,6 +69,7 @@
     M(MissingRequiredProperty, "Required property {} is missing or undefined")                                                          \
     M(ModuleNoEnvironment, "Cannot find module environment for imported binding")                                                       \
     M(ModuleNotFound, "Cannot find/open module: '{}'")                                                                                  \
+    M(ModuleNotFoundNoReferencingScript, "Cannot resolve module {} without any active script or module")                                \
     M(NegativeExponent, "Exponent must be positive")                                                                                    \
     M(NonExtensibleDefine, "Cannot define property {} on non-extensible object")                                                        \
     M(NotAConstructor, "{} is not a constructor")                                                                                       \

Userland/Libraries/LibJS/Runtime/VM.cpp

@@ -50,6 +50,37 @@ VM::VM(OwnPtr<CustomData> custom_data)
         return resolve_imported_module(move(referencing_script_or_module), specifier);
     };
 
+    host_import_module_dynamically = [&](ScriptOrModule, ModuleRequest const&, PromiseCapability promise_capability) {
+        // By default, we throw on dynamic imports this is to prevent arbitrary file access by scripts.
+        VERIFY(current_realm());
+        auto& global_object = current_realm()->global_object();
+        auto* promise = Promise::create(global_object);
+
+        // If you are here because you want to enable dynamic module importing make sure it won't be a security problem
+        // by checking the default implementation of HostImportModuleDynamically and creating your own hook or calling
+        // vm.enable_default_host_import_module_dynamically_hook().
+        promise->reject(Error::create(global_object, ErrorType::DynamicImportNotAllowed.message()));
+
+        promise->perform_then(
+            NativeFunction::create(global_object, "", [](auto&, auto&) -> ThrowCompletionOr<Value> {
+                VERIFY_NOT_REACHED();
+            }),
+            NativeFunction::create(global_object, "", [reject = make_handle(promise_capability.reject)](auto& vm, auto& global_object) -> ThrowCompletionOr<Value> {
+                auto error = vm.argument(0);
+
+                // a. Perform ! Call(promiseCapability.[[Reject]], undefined, « error »).
+                MUST(JS::call(global_object, reject.cell(), js_undefined(), error));
+
+                // b. Return undefined.
+                return js_undefined();
+            }),
+            {});
+    };
+
+    host_finish_dynamic_import = [&](ScriptOrModule referencing_script_or_module, ModuleRequest const& specifier, PromiseCapability promise_capability, Promise* promise) {
+        return finish_dynamic_import(move(referencing_script_or_module), specifier, promise_capability, promise);
+    };
+
 #define __JS_ENUMERATE(SymbolName, snake_name) \
     m_well_known_symbol_##snake_name = js_symbol(*this, "Symbol." #SymbolName, false);
     JS_ENUMERATE_WELL_KNOWN_SYMBOLS
@@ -60,6 +91,13 @@ VM::~VM()
 {
 }
 
+void VM::enable_default_host_import_module_dynamically_hook()
+{
+    host_import_module_dynamically = [&](ScriptOrModule referencing_script_or_module, ModuleRequest const& specifier, PromiseCapability promise_capability) {
+        return import_module_dynamically(move(referencing_script_or_module), specifier, promise_capability);
+    };
+}
+
 Interpreter& VM::interpreter()
 {
     VERIFY(!m_interpreters.is_empty());
@@ -848,4 +886,127 @@ ThrowCompletionOr<NonnullRefPtr<Module>> VM::resolve_imported_module(ScriptOrMod
     return module;
 }
 
+// 16.2.1.8 HostImportModuleDynamically ( referencingScriptOrModule, specifier, promiseCapability ), https://tc39.es/ecma262/#sec-hostimportmoduledynamically
+void VM::import_module_dynamically(ScriptOrModule referencing_script_or_module, ModuleRequest const& specifier, PromiseCapability promise_capability)
+{
+    auto& global_object = current_realm()->global_object();
+
+    // Success path:
+    //  - At some future time, the host environment must perform FinishDynamicImport(referencingScriptOrModule, specifier, promiseCapability, promise),
+    //    where promise is a Promise resolved with undefined.
+    //  - Any subsequent call to HostResolveImportedModule after FinishDynamicImport has completed,
+    //    given the arguments referencingScriptOrModule and specifier, must complete normally.
+    //  - The completion value of any subsequent call to HostResolveImportedModule after FinishDynamicImport has completed,
+    //    given the arguments referencingScriptOrModule and specifier, must be a module which has already been evaluated,
+    //    i.e. whose Evaluate concrete method has already been called and returned a normal completion.
+    // Failure path:
+    //  - At some future time, the host environment must perform
+    //    FinishDynamicImport(referencingScriptOrModule, specifier, promiseCapability, promise),
+    //    where promise is a Promise rejected with an error representing the cause of failure.
+
+    auto* promise = Promise::create(global_object);
+
+    ScopeGuard finish_dynamic_import = [&] {
+        host_finish_dynamic_import(referencing_script_or_module, specifier, promise_capability, promise);
+    };
+
+    // Generally within ECMA262 we always get a referencing_script_or_moulde. However, ShadowRealm gives an explicit null.
+    // To get around this is we attempt to get the active script_or_module otherwise we might start loading "random" files from the working directory.
+    if (referencing_script_or_module.has<Empty>()) {
+        referencing_script_or_module = get_active_script_or_module();
+
+        // If there is no ScriptOrModule in any of the execution contexts
+        if (referencing_script_or_module.has<Empty>()) {
+            // Throw an error for now
+            promise->reject(InternalError::create(global_object, String::formatted(ErrorType::ModuleNotFoundNoReferencingScript.message(), specifier.module_specifier)));
+            return;
+        }
+    }
+
+    VERIFY(!exception());
+    // Note: If host_resolve_imported_module returns a module it has been loaded successfully and the next call in finish_dynamic_import will retrieve it again.
+    auto module_or_error = host_resolve_imported_module(referencing_script_or_module, specifier);
+    dbgln_if(JS_MODULE_DEBUG, "[JS MODULE] HostImportModuleDynamically(..., {}) -> {}", specifier.module_specifier, module_or_error.is_error() ? "failed" : "passed");
+    if (module_or_error.is_throw_completion()) {
+        // Note: We should not leak the exception thrown in host_resolve_imported_module.
+        clear_exception();
+        promise->reject(*module_or_error.throw_completion().value());
+    } else {
+        // Note: If you are here because this VERIFY is failing overwrite host_import_module_dynamically
+        //       because this is LibJS internal logic which won't always work
+        auto module = module_or_error.release_value();
+        VERIFY(is<SourceTextModule>(*module));
+        auto& source_text_module = static_cast<SourceTextModule&>(*module);
+
+        auto evaluated_or_error = link_and_eval_module(source_text_module);
+
+        if (evaluated_or_error.is_throw_completion()) {
+            // Note: Again we don't want to leak the exception from link_and_eval_module.
+            clear_exception();
+            promise->reject(*evaluated_or_error.throw_completion().value());
+        } else {
+            VERIFY(!exception());
+            promise->fulfill(js_undefined());
+        }
+    }
+
+    // It must return NormalCompletion(undefined).
+    // Note: Just return void always since the resulting value cannot be accessed by user code.
+}
+
+// 16.2.1.9 FinishDynamicImport ( referencingScriptOrModule, specifier, promiseCapability, innerPromise ), https://tc39.es/ecma262/#sec-finishdynamicimport
+void VM::finish_dynamic_import(ScriptOrModule referencing_script_or_module, ModuleRequest const& specifier, PromiseCapability promise_capability, Promise* inner_promise)
+{
+    dbgln_if(JS_MODULE_DEBUG, "[JS MODULE] finish_dynamic_import on {}", specifier.module_specifier);
+
+    // 1. Let fulfilledClosure be a new Abstract Closure with parameters (result) that captures referencingScriptOrModule, specifier, and promiseCapability and performs the following steps when called:
+    auto fulfilled_closure = [referencing_script_or_module, specifier, promise_capability](VM& vm, GlobalObject& global_object) -> ThrowCompletionOr<Value> {
+        auto result = vm.argument(0);
+        // a. Assert: result is undefined.
+        VERIFY(result.is_undefined());
+        // b. Let moduleRecord be ! HostResolveImportedModule(referencingScriptOrModule, specifier).
+        auto module_record = MUST(vm.host_resolve_imported_module(referencing_script_or_module, specifier));
+
+        // c. Assert: Evaluate has already been invoked on moduleRecord and successfully completed.
+        // Note: If HostResolveImportedModule returns a module evaluate will have been called on it.
+
+        // d. Let namespace be GetModuleNamespace(moduleRecord).
+        auto namespace_ = module_record->get_module_namespace(vm);
+
+        VERIFY(!vm.exception());
+        // e. If namespace is an abrupt completion, then
+        if (namespace_.is_throw_completion()) {
+            // i. Perform ! Call(promiseCapability.[[Reject]], undefined, « namespace.[[Value]] »).
+            MUST(JS::call(global_object, promise_capability.reject, js_undefined(), *namespace_.throw_completion().value()));
+        }
+        // f. Else,
+        else {
+            // i. Perform ! Call(promiseCapability.[[Resolve]], undefined, « namespace.[[Value]] »).
+            MUST(JS::call(global_object, promise_capability.resolve, js_undefined(), namespace_.release_value()));
+        }
+        // g. Return undefined.
+        return js_undefined();
+    };
+
+    // 2. Let onFulfilled be ! CreateBuiltinFunction(fulfilledClosure, 0, "", « »).
+    auto* on_fulfilled = NativeFunction::create(current_realm()->global_object(), "", move(fulfilled_closure));
+
+    // 3. Let rejectedClosure be a new Abstract Closure with parameters (error) that captures promiseCapability and performs the following steps when called:
+    auto rejected_closure = [promise_capability](VM& vm, GlobalObject& global_object) -> ThrowCompletionOr<Value> {
+        auto error = vm.argument(0);
+        // a. Perform ! Call(promiseCapability.[[Reject]], undefined, « error »).
+        MUST(JS::call(global_object, promise_capability.reject, js_undefined(), error));
+        // b. Return undefined.
+        return js_undefined();
+    };
+
+    // 4. Let onRejected be ! CreateBuiltinFunction(rejectedClosure, 0, "", « »).
+    auto* on_rejected = NativeFunction::create(current_realm()->global_object(), "", move(rejected_closure));
+
+    // 5. Perform ! PerformPromiseThen(innerPromise, onFulfilled, onRejected).
+    inner_promise->perform_then(on_fulfilled, on_rejected, {});
+
+    VERIFY(!exception());
+}
+
 }

Userland/Libraries/LibJS/Runtime/VM.h

@@ -251,6 +251,8 @@ public:
     Function<HashMap<PropertyKey, Value>(SourceTextModule const&)> host_get_import_meta_properties;
     Function<void(Object*, SourceTextModule const&)> host_finalize_import_meta;
 
+    void enable_default_host_import_module_dynamically_hook();
+
 private:
     explicit VM(OwnPtr<CustomData>);
 
@@ -262,6 +264,9 @@ private:
     ThrowCompletionOr<NonnullRefPtr<Module>> resolve_imported_module(ScriptOrModule referencing_script_or_module, ModuleRequest const& specifier);
     ThrowCompletionOr<void> link_and_eval_module(SourceTextModule& module);
 
+    void import_module_dynamically(ScriptOrModule referencing_script_or_module, ModuleRequest const& specifier, PromiseCapability promise_capability);
+    void finish_dynamic_import(ScriptOrModule referencing_script_or_module, ModuleRequest const& specifier, PromiseCapability promise_capability, Promise* inner_promise);
+
     Exception* m_exception { nullptr };
 
     HashMap<String, PrimitiveString*> m_string_cache;

Userland/Libraries/LibJS/Tests/modules/basic-export-types.mjs

@@ -0,0 +1,13 @@
+export const constValue = 1;
+
+export let letValue = 2;
+
+export var varValue = 3;
+
+const namedConstValue = 4;
+let namedLetValue = 5;
+var namedVarValue = 6;
+
+export { namedConstValue, namedLetValue, namedVarValue };
+
+export const passed = true;

Userland/Libraries/LibJS/Tests/modules/basic-modules.js

@@ -0,0 +1,144 @@
+// Because you can't easily load modules directly we load them via here and check
+// if they passed by checking the result
+
+function expectModulePassed(filename) {
+    if (!filename.endsWith(".mjs") || !filename.startsWith("./")) {
+        throw new ExpectationError(
+            "Expected module name to start with './' " +
+                "and end with '.mjs' but got '" +
+                filename +
+                "'"
+        );
+    }
+
+    async function getModule() {
+        return import(filename);
+    }
+
+    let moduleLoaded = false;
+    let moduleResult = null;
+    let thrownError = null;
+
+    getModule()
+        .then(result => {
+            moduleLoaded = true;
+            moduleResult = result;
+            expect(moduleResult).toHaveProperty("passed", true);
+        })
+        .catch(error => {
+            thrownError = error;
+        });
+
+    runQueuedPromiseJobs();
+
+    if (thrownError) {
+        throw thrownError;
+    }
+
+    expect(moduleLoaded).toBeTrue();
+
+    return moduleResult;
+}
+
+describe("testing behavior", () => {
+    // To ensure the other tests are interpreter correctly we first test the underlying
+    // mechanisms so these tests don't use expectModulePassed.
+
+    test("can load a module", () => {
+        let passed = false;
+        let error = null;
+
+        import("./empty.mjs")
+            .then(() => {
+                passed = true;
+            })
+            .catch(err => {
+                error = err;
+            });
+
+        runQueuedPromiseJobs();
+        if (error) throw error;
+
+        expect(passed).toBeTrue();
+    });
+
+    test("can load a module twice", () => {
+        let passed = false;
+        let error = null;
+
+        import("./empty.mjs")
+            .then(() => {
+                passed = true;
+            })
+            .catch(err => {
+                error = err;
+            });
+
+        runQueuedPromiseJobs();
+        if (error) throw error;
+
+        expect(passed).toBeTrue();
+    });
+
+    test("can retrieve exported value", () => {
+        async function getValue(filename) {
+            const imported = await import(filename);
+            expect(imported).toHaveProperty("passed", true);
+        }
+
+        let passed = false;
+        let error = null;
+
+        getValue("./single-const-export.mjs")
+            .then(obj => {
+                passed = true;
+            })
+            .catch(err => {
+                error = err;
+            });
+
+        runQueuedPromiseJobs();
+
+        if (error) throw error;
+
+        expect(passed).toBeTrue();
+    });
+
+    test("expectModulePassed works", () => {
+        expectModulePassed("./single-const-export.mjs");
+    });
+});
+
+describe("in- and exports", () => {
+    test("variable and lexical declarations", () => {
+        const result = expectModulePassed("./basic-export-types.mjs");
+        expect(result).not.toHaveProperty("default", null);
+        expect(result).toHaveProperty("constValue", 1);
+        expect(result).toHaveProperty("letValue", 2);
+        expect(result).toHaveProperty("varValue", 3);
+
+        expect(result).toHaveProperty("namedConstValue", 1 + 3);
+        expect(result).toHaveProperty("namedLetValue", 2 + 3);
+        expect(result).toHaveProperty("namedVarValue", 3 + 3);
+    });
+
+    test("default exports", () => {
+        const result = expectModulePassed("./module-with-default.mjs");
+        expect(result).toHaveProperty("defaultValue");
+        expect(result.default).toBe(result.defaultValue);
+    });
+
+    test("declaration exports which can be used in the module it self", () => {
+        expectModulePassed("./declarations-tests.mjs");
+    });
+});
+
+describe("loops", () => {
+    test("import and export from own file", () => {
+        expectModulePassed("./loop-self.mjs");
+    });
+
+    test("import something which imports a cycle", () => {
+        expectModulePassed("./loop-entry.mjs");
+    });
+});

Userland/Libraries/LibJS/Tests/modules/declarations-tests.mjs

@@ -0,0 +1,28 @@
+export function returnsOne() {
+    return 1;
+}
+
+export class hasStaticFieldTwo {
+    static two = 2;
+}
+
+const expectedValue = 10;
+const didNotHoistClass = (() => {
+    try {
+        new ShouldNotBeHoisted();
+    } catch (e) {
+        if (e instanceof ReferenceError) return 4;
+    }
+    return 0;
+})();
+
+export const passed =
+    returnsOne() + hasStaticFieldTwo.two + shouldBeHoisted() + didNotHoistClass === expectedValue;
+
+export function shouldBeHoisted() {
+    return 3;
+}
+
+export class ShouldNotBeHoisted {
+    static no = 5;
+}

Userland/Libraries/LibJS/Tests/modules/loop-a.mjs

@@ -0,0 +1,3 @@
+export { bValue } from "./loop-b.mjs";
+
+export const aValue = 1;

Userland/Libraries/LibJS/Tests/modules/loop-b.mjs

@@ -0,0 +1,3 @@
+import "./loop-a.mjs";
+
+export const bValue = 2;

Userland/Libraries/LibJS/Tests/modules/loop-entry.mjs

@@ -0,0 +1,3 @@
+import { aValue, bValue } from "./loop-a.mjs";
+
+export const passed = aValue < bValue;

Userland/Libraries/LibJS/Tests/modules/loop-self.mjs

@@ -0,0 +1,5 @@
+import { value as importValue } from "./loop-self.mjs";
+
+export const value = "loop de loop whooo";
+
+export const passed = value === importValue;

Userland/Libraries/LibJS/Tests/modules/module-with-default.mjs

@@ -0,0 +1,7 @@
+const value = "Well hello importer :^)";
+
+export const defaultValue = value;
+
+export default value;
+
+export const passed = true;

Userland/Libraries/LibJS/Tests/modules/single-const-export.mjs

@@ -0,0 +1 @@
+export const passed = true;

Userland/Libraries/LibTest/JavaScriptTestRunnerMain.cpp

@@ -182,8 +182,10 @@ int main(int argc, char** argv)
     if (g_main_hook)
         g_main_hook();
 
-    if (!g_vm)
+    if (!g_vm) {
         g_vm = JS::VM::create();
+        g_vm->enable_default_host_import_module_dynamically_hook();
+    }
 
     Test::JS::TestRunner test_runner(test_root, common_path, print_times, print_progress, print_json);
     test_runner.run(test_glob);

Userland/Utilities/js.cpp

@@ -1290,6 +1290,8 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
     bool syntax_highlight = !disable_syntax_highlight;
 
     vm = JS::VM::create();
+    vm->enable_default_host_import_module_dynamically_hook();
+
     // NOTE: These will print out both warnings when using something like Promise.reject().catch(...) -
     // which is, as far as I can tell, correct - a promise is created, rejected without handler, and a
     // handler then attached to it. The Node.js REPL doesn't warn in this case, so it's something we