LibCodeComprehension: Re-organize code comprehension related code

This moves all code comprehension-related code to a new library,
LibCodeComprehension.

This also moves some types related to code comprehension tasks (such as
autocomplete, find declaration) out of LibGUI and into
LibCodeComprehension.

Meta/build-root-filesystem.sh

@@ -160,7 +160,7 @@ mkdir -p mnt/home/anon/Tests/cpp-tests/
 cp "$SERENITY_SOURCE_DIR"/README.md mnt/home/anon/
 cp -r "$SERENITY_SOURCE_DIR"/Userland/Libraries/LibJS/Tests mnt/home/anon/Tests/js-tests
 cp -r "$SERENITY_SOURCE_DIR"/Userland/Libraries/LibWeb/Tests mnt/home/anon/Tests/web-tests
-cp -r "$SERENITY_SOURCE_DIR"/Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests mnt/home/anon/Tests/cpp-tests/comprehension
+cp -r "$SERENITY_SOURCE_DIR"/Userland/Libraries/LibCodeComprehension/Cpp/Tests mnt/home/anon/Tests/cpp-tests/comprehension
 cp -r "$SERENITY_SOURCE_DIR"/Userland/Libraries/LibCpp/Tests/parser mnt/home/anon/Tests/cpp-tests/parser
 cp -r "$SERENITY_SOURCE_DIR"/Userland/Libraries/LibCpp/Tests/preprocessor mnt/home/anon/Tests/cpp-tests/preprocessor
 cp -r "$SERENITY_SOURCE_DIR"/Userland/Libraries/LibWasm/Tests mnt/home/anon/Tests/wasm-tests

Meta/check-newlines-at-eof.py

@@ -22,7 +22,7 @@ def run():
             "CMake*.txt",
             "**/CMake*.txt",
             ":!:Kernel/FileSystem/ext2_fs.h",
-            ':!:Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests/*',
+            ':!:Userland/Libraries/LibCodeComprehension/Cpp/Tests/*',
             ':!:Userland/Libraries/LibCpp/Tests/parser/*',
             ':!:Userland/Libraries/LibCpp/Tests/preprocessor/*'
         ],

Meta/check-style.py

@@ -23,7 +23,7 @@ LICENSE_HEADER_CHECK_EXCLUDES = {
     'AK/Checked.h',
     'AK/Function.h',
     'Userland/Libraries/LibC/elf.h',
-    'Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests/',
+    'Userland/Libraries/LibCodeComprehension/Cpp/Tests/',
     'Userland/Libraries/LibCpp/Tests/parser/',
     'Userland/Libraries/LibCpp/Tests/preprocessor/'
 }

Meta/lint-clang-format.sh

@@ -12,7 +12,7 @@ if [ "$#" -eq "1" ]; then
             '*.h' \
             ':!:Base' \
             ':!:Kernel/FileSystem/ext2_fs.h' \
-            ':!:Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests/*' \
+            ':!:Userland/Libraries/LibCodeComprehension/Cpp/Tests/*' \
             ':!:Userland/Libraries/LibCpp/Tests/parser/*' \
             ':!:Userland/Libraries/LibCpp/Tests/preprocessor/*'
     )

Userland/DevTools/HackStudio/AutoCompleteResponse.h

@@ -16,7 +16,7 @@
 namespace IPC {
 
 template<>
-inline bool encode(Encoder& encoder, const GUI::AutocompleteProvider::Entry& response)
+inline bool encode(Encoder& encoder, CodeComprehension::AutocompleteResultEntry const& response)
 {
     encoder << response.completion;
     encoder << response.partial_input_length;
@@ -27,7 +27,7 @@ inline bool encode(Encoder& encoder, const GUI::AutocompleteProvider::Entry& res
 }
 
 template<>
-inline ErrorOr<void> decode(Decoder& decoder, GUI::AutocompleteProvider::Entry& response)
+inline ErrorOr<void> decode(Decoder& decoder, CodeComprehension::AutocompleteResultEntry& response)
 {
     TRY(decoder.decode(response.completion));
     TRY(decoder.decode(response.partial_input_length));
@@ -38,7 +38,7 @@ inline ErrorOr<void> decode(Decoder& decoder, GUI::AutocompleteProvider::Entry&
 }
 
 template<>
-inline bool encode(Encoder& encoder, const GUI::AutocompleteProvider::ProjectLocation& location)
+inline bool encode(Encoder& encoder, CodeComprehension::ProjectLocation const& location)
 {
     encoder << location.file;
     encoder << location.line;
@@ -47,7 +47,7 @@ inline bool encode(Encoder& encoder, const GUI::AutocompleteProvider::ProjectLoc
 }
 
 template<>
-inline ErrorOr<void> decode(Decoder& decoder, GUI::AutocompleteProvider::ProjectLocation& location)
+inline ErrorOr<void> decode(Decoder& decoder, CodeComprehension::ProjectLocation& location)
 {
     TRY(decoder.decode(location.file));
     TRY(decoder.decode(location.line));
@@ -56,7 +56,7 @@ inline ErrorOr<void> decode(Decoder& decoder, GUI::AutocompleteProvider::Project
 }
 
 template<>
-inline bool encode(Encoder& encoder, const GUI::AutocompleteProvider::Declaration& declaration)
+inline bool encode(Encoder& encoder, CodeComprehension::Declaration const& declaration)
 {
     encoder << declaration.name;
     if (!encode(encoder, declaration.position))
@@ -67,7 +67,7 @@ inline bool encode(Encoder& encoder, const GUI::AutocompleteProvider::Declaratio
 }
 
 template<>
-inline ErrorOr<void> decode(Decoder& decoder, GUI::AutocompleteProvider::Declaration& declaration)
+inline ErrorOr<void> decode(Decoder& decoder, CodeComprehension::Declaration& declaration)
 {
     TRY(decoder.decode(declaration.name));
     TRY(decoder.decode(declaration.position));
@@ -77,7 +77,7 @@ inline ErrorOr<void> decode(Decoder& decoder, GUI::AutocompleteProvider::Declara
 }
 
 template<>
-inline bool encode(Encoder& encoder, Cpp::Parser::TodoEntry const& entry)
+inline bool encode(Encoder& encoder, CodeComprehension::TodoEntry const& entry)
 {
     encoder << entry.content;
     encoder << entry.filename;
@@ -87,7 +87,7 @@ inline bool encode(Encoder& encoder, Cpp::Parser::TodoEntry const& entry)
 }
 
 template<>
-inline ErrorOr<void> decode(Decoder& decoder, Cpp::Parser::TodoEntry& entry)
+inline ErrorOr<void> decode(Decoder& decoder, CodeComprehension::TodoEntry& entry)
 {
     TRY(decoder.decode(entry.content));
     TRY(decoder.decode(entry.filename));
@@ -97,7 +97,7 @@ inline ErrorOr<void> decode(Decoder& decoder, Cpp::Parser::TodoEntry& entry)
 }
 
 template<>
-inline bool encode(Encoder& encoder, const GUI::AutocompleteProvider::TokenInfo& location)
+inline bool encode(Encoder& encoder, CodeComprehension::TokenInfo const& location)
 {
     encoder << (u32)location.type;
     static_assert(sizeof(location.type) == sizeof(u32));
@@ -109,13 +109,13 @@ inline bool encode(Encoder& encoder, const GUI::AutocompleteProvider::TokenInfo&
 }
 
 template<>
-inline ErrorOr<void> decode(Decoder& decoder, GUI::AutocompleteProvider::TokenInfo& entry)
+inline ErrorOr<void> decode(Decoder& decoder, CodeComprehension::TokenInfo& entry)
 {
     u32 semantic_type { 0 };
     static_assert(sizeof(semantic_type) == sizeof(entry.type));
 
     TRY(decoder.decode(semantic_type));
-    entry.type = static_cast<GUI::AutocompleteProvider::TokenInfo::SemanticType>(semantic_type);
+    entry.type = static_cast<CodeComprehension::TokenInfo::SemanticType>(semantic_type);
     TRY(decoder.decode(entry.start_line));
     TRY(decoder.decode(entry.start_column));
     TRY(decoder.decode(entry.end_line));

Userland/DevTools/HackStudio/ClassViewWidget.cpp

@@ -101,16 +101,16 @@ ClassViewModel::ClassViewModel()
 {
     m_root_scope.clear();
     ProjectDeclarations::the().for_each_declared_symbol([this](auto& decl) {
-        if (decl.type == GUI::AutocompleteProvider::DeclarationType::Class
-            || decl.type == GUI::AutocompleteProvider::DeclarationType::Struct
-            || decl.type == GUI::AutocompleteProvider::DeclarationType::Member
-            || decl.type == GUI::AutocompleteProvider::DeclarationType::Namespace) {
+        if (decl.type == CodeComprehension::DeclarationType::Class
+            || decl.type == CodeComprehension::DeclarationType::Struct
+            || decl.type == CodeComprehension::DeclarationType::Member
+            || decl.type == CodeComprehension::DeclarationType::Namespace) {
             add_declaration(decl);
         }
     });
 }
 
-static ClassViewNode& add_child_node(NonnullOwnPtrVector<ClassViewNode>& children, NonnullOwnPtr<ClassViewNode>&& node_ptr, ClassViewNode* parent, const GUI::AutocompleteProvider::Declaration* declaration)
+static ClassViewNode& add_child_node(NonnullOwnPtrVector<ClassViewNode>& children, NonnullOwnPtr<ClassViewNode>&& node_ptr, ClassViewNode* parent, CodeComprehension::Declaration const* declaration)
 {
     node_ptr->parent = parent;
     node_ptr->declaration = declaration;
@@ -127,7 +127,7 @@ static ClassViewNode& add_child_node(NonnullOwnPtrVector<ClassViewNode>& childre
     return children.at(inserted_index);
 }
 
-void ClassViewModel::add_declaration(const GUI::AutocompleteProvider::Declaration& decl)
+void ClassViewModel::add_declaration(CodeComprehension::Declaration const& decl)
 {
     ClassViewNode* parent = nullptr;
     auto scope_parts = decl.scope.view().split_view("::");

Userland/DevTools/HackStudio/ClassViewWidget.h

@@ -31,7 +31,7 @@ private:
 // This is currently achieved with the on_update callback of ProjectDeclarations.
 struct ClassViewNode {
     StringView name;
-    const GUI::AutocompleteProvider::Declaration* declaration { nullptr };
+    CodeComprehension::Declaration const* declaration { nullptr };
     NonnullOwnPtrVector<ClassViewNode> children;
     ClassViewNode* parent { nullptr };
 
@@ -50,7 +50,7 @@ public:
 
 private:
     explicit ClassViewModel();
-    void add_declaration(const GUI::AutocompleteProvider::Declaration&);
+    void add_declaration(CodeComprehension::Declaration const&);
     NonnullOwnPtrVector<ClassViewNode> m_root_scope;
 };
 

Userland/DevTools/HackStudio/Editor.cpp

@@ -497,7 +497,7 @@ Optional<Editor::AutoCompleteRequestData> Editor::get_autocomplete_request_data(
     return Editor::AutoCompleteRequestData { cursor() };
 }
 
-void Editor::LanguageServerAidedAutocompleteProvider::provide_completions(Function<void(Vector<Entry>)> callback)
+void Editor::LanguageServerAidedAutocompleteProvider::provide_completions(Function<void(Vector<CodeComprehension::AutocompleteResultEntry>)> callback)
 {
     auto& editor = static_cast<Editor&>(*m_editor).wrapper().editor();
     auto data = editor.get_autocomplete_request_data();
@@ -655,7 +655,7 @@ void Editor::set_language_client_for(CodeDocument const& document)
         m_language_client = get_language_client<LanguageClients::Shell::ConnectionToServer>(project().root_path());
 
     if (m_language_client) {
-        m_language_client->on_tokens_info_result = [this](Vector<GUI::AutocompleteProvider::TokenInfo> const& tokens_info) {
+        m_language_client->on_tokens_info_result = [this](Vector<CodeComprehension::TokenInfo> const& tokens_info) {
             on_tokens_info_result(tokens_info);
         };
     }
@@ -728,7 +728,7 @@ void Editor::on_token_info_timer_tick()
     m_language_client->get_tokens_info(code_document().file_path());
 }
 
-void Editor::on_tokens_info_result(Vector<GUI::AutocompleteProvider::TokenInfo> const& tokens_info)
+void Editor::on_tokens_info_result(Vector<CodeComprehension::TokenInfo> const& tokens_info)
 {
     auto highlighter = syntax_highlighter();
     if (highlighter && highlighter->is_cpp_semantic_highlighter()) {

Userland/DevTools/HackStudio/Editor.h

@@ -92,7 +92,7 @@ private:
         virtual ~LanguageServerAidedAutocompleteProvider() override { }
 
     private:
-        virtual void provide_completions(Function<void(Vector<Entry>)> callback) override;
+        virtual void provide_completions(Function<void(Vector<CodeComprehension::AutocompleteResultEntry>)> callback) override;
         LanguageClient& m_language_client;
     };
 
@@ -104,7 +104,7 @@ private:
     void set_autocomplete_provider_for(CodeDocument const&);
     void handle_function_parameters_hint_request();
     void on_token_info_timer_tick();
-    void on_tokens_info_result(Vector<GUI::AutocompleteProvider::TokenInfo> const& tokens_info);
+    void on_tokens_info_result(Vector<CodeComprehension::TokenInfo> const& tokens_info);
     void create_tokens_info_timer();
     ErrorOr<void> initialize_documentation_tooltip();
     ErrorOr<void> initialize_parameters_hint_tooltip();

Userland/DevTools/HackStudio/LanguageClient.cpp

@@ -14,7 +14,7 @@
 
 namespace HackStudio {
 
-void ConnectionToServer::auto_complete_suggestions(Vector<GUI::AutocompleteProvider::Entry> const& suggestions)
+void ConnectionToServer::auto_complete_suggestions(Vector<CodeComprehension::AutocompleteResultEntry> const& suggestions)
 {
     if (!m_current_language_client) {
         dbgln("Language Server connection has no attached language client");
@@ -23,7 +23,7 @@ void ConnectionToServer::auto_complete_suggestions(Vector<GUI::AutocompleteProvi
     m_current_language_client->provide_autocomplete_suggestions(suggestions);
 }
 
-void ConnectionToServer::declaration_location(const GUI::AutocompleteProvider::ProjectLocation& location)
+void ConnectionToServer::declaration_location(CodeComprehension::ProjectLocation const& location)
 {
     if (!m_current_language_client) {
         dbgln("Language Server connection has no attached language client");
@@ -43,7 +43,7 @@ void ConnectionToServer::parameters_hint_result(Vector<String> const& params, in
     m_current_language_client->parameters_hint_result(params, static_cast<size_t>(argument_index));
 }
 
-void ConnectionToServer::tokens_info_result(Vector<GUI::AutocompleteProvider::TokenInfo> const& tokens_info)
+void ConnectionToServer::tokens_info_result(Vector<CodeComprehension::TokenInfo> const& tokens_info)
 {
     if (!m_current_language_client) {
         dbgln("Language Server connection has no attached language client");
@@ -93,10 +93,10 @@ void LanguageClient::request_autocomplete(String const& path, size_t cursor_line
     if (!m_connection_wrapper.connection())
         return;
     set_active_client();
-    m_connection_wrapper.connection()->async_auto_complete_suggestions(GUI::AutocompleteProvider::ProjectLocation { path, cursor_line, cursor_column });
+    m_connection_wrapper.connection()->async_auto_complete_suggestions(CodeComprehension::ProjectLocation { path, cursor_line, cursor_column });
 }
 
-void LanguageClient::provide_autocomplete_suggestions(Vector<GUI::AutocompleteProvider::Entry> const& suggestions) const
+void LanguageClient::provide_autocomplete_suggestions(Vector<CodeComprehension::AutocompleteResultEntry> const& suggestions) const
 {
     if (on_autocomplete_suggestions)
         on_autocomplete_suggestions(suggestions);
@@ -120,12 +120,12 @@ bool LanguageClient::is_active_client() const
 
 HashMap<String, NonnullOwnPtr<ConnectionToServerWrapper>> ConnectionToServerInstances::s_instance_for_language;
 
-void ConnectionToServer::declarations_in_document(String const& filename, Vector<GUI::AutocompleteProvider::Declaration> const& declarations)
+void ConnectionToServer::declarations_in_document(String const& filename, Vector<CodeComprehension::Declaration> const& declarations)
 {
     ProjectDeclarations::the().set_declared_symbols(filename, declarations);
 }
 
-void ConnectionToServer::todo_entries_in_document(String const& filename, Vector<Cpp::Parser::TodoEntry> const& todo_entries)
+void ConnectionToServer::todo_entries_in_document(String const& filename, Vector<CodeComprehension::TodoEntry> const& todo_entries)
 {
     ToDoEntries::the().set_entries(filename, move(todo_entries));
 }
@@ -135,7 +135,7 @@ void LanguageClient::search_declaration(String const& path, size_t line, size_t
     if (!m_connection_wrapper.connection())
         return;
     set_active_client();
-    m_connection_wrapper.connection()->async_find_declaration(GUI::AutocompleteProvider::ProjectLocation { path, line, column });
+    m_connection_wrapper.connection()->async_find_declaration(CodeComprehension::ProjectLocation { path, line, column });
 }
 
 void LanguageClient::get_parameters_hint(String const& path, size_t line, size_t column)
@@ -143,7 +143,7 @@ void LanguageClient::get_parameters_hint(String const& path, size_t line, size_t
     if (!m_connection_wrapper.connection())
         return;
     set_active_client();
-    m_connection_wrapper.connection()->async_get_parameters_hint(GUI::AutocompleteProvider::ProjectLocation { path, line, column });
+    m_connection_wrapper.connection()->async_get_parameters_hint(CodeComprehension::ProjectLocation { path, line, column });
 }
 
 void LanguageClient::get_tokens_info(String const& filename)

Userland/DevTools/HackStudio/LanguageClient.h

@@ -46,12 +46,12 @@ public:
     LanguageClient const* active_client() const { return !m_current_language_client ? nullptr : m_current_language_client.ptr(); }
 
 protected:
-    virtual void auto_complete_suggestions(Vector<GUI::AutocompleteProvider::Entry> const&) override;
-    virtual void declaration_location(GUI::AutocompleteProvider::ProjectLocation const&) override;
-    virtual void declarations_in_document(String const&, Vector<GUI::AutocompleteProvider::Declaration> const&) override;
-    virtual void todo_entries_in_document(String const&, Vector<Cpp::Parser::TodoEntry> const&) override;
+    virtual void auto_complete_suggestions(Vector<CodeComprehension::AutocompleteResultEntry> const&) override;
+    virtual void declaration_location(CodeComprehension::ProjectLocation const&) override;
+    virtual void declarations_in_document(String const&, Vector<CodeComprehension::Declaration> const&) override;
+    virtual void todo_entries_in_document(String const&, Vector<CodeComprehension::TodoEntry> const&) override;
     virtual void parameters_hint_result(Vector<String> const&, int index) override;
-    virtual void tokens_info_result(Vector<GUI::AutocompleteProvider::TokenInfo> const&) override;
+    virtual void tokens_info_result(Vector<CodeComprehension::TokenInfo> const&) override;
     void set_wrapper(ConnectionToServerWrapper& wrapper) { m_wrapper = &wrapper; }
 
     String m_project_path;
@@ -137,15 +137,15 @@ public:
     virtual void get_parameters_hint(String const& path, size_t line, size_t column);
     virtual void get_tokens_info(String const& filename);
 
-    void provide_autocomplete_suggestions(Vector<GUI::AutocompleteProvider::Entry> const&) const;
+    void provide_autocomplete_suggestions(Vector<CodeComprehension::AutocompleteResultEntry> const&) const;
     void declaration_found(String const& file, size_t line, size_t column) const;
     void parameters_hint_result(Vector<String> const& params, size_t argument_index) const;
 
     // Callbacks that get called when the result of a language server query is ready
-    Function<void(Vector<GUI::AutocompleteProvider::Entry>)> on_autocomplete_suggestions;
+    Function<void(Vector<CodeComprehension::AutocompleteResultEntry>)> on_autocomplete_suggestions;
     Function<void(String const&, size_t, size_t)> on_declaration_found;
     Function<void(Vector<String> const&, size_t)> on_function_parameters_hint_result;
-    Function<void(Vector<GUI::AutocompleteProvider::TokenInfo> const&)> on_tokens_info_result;
+    Function<void(Vector<CodeComprehension::TokenInfo> const&)> on_tokens_info_result;
 
 private:
     ConnectionToServerWrapper& m_connection_wrapper;

Userland/DevTools/HackStudio/LanguageServers/CMakeLists.txt

@@ -2,7 +2,6 @@ compile_ipc(LanguageServer.ipc LanguageServerEndpoint.h)
 compile_ipc(LanguageClient.ipc LanguageClientEndpoint.h)
 
 set(SOURCES
-        CodeComprehensionEngine.cpp
         ConnectionFromClient.cpp
         FileDB.cpp)
 set(GENERATED_SOURCES
@@ -10,7 +9,7 @@ set(GENERATED_SOURCES
         LanguageServerEndpoint.h)
 
 serenity_lib(LibLanguageServer language_server)
-target_link_libraries(LibLanguageServer LibC)
+target_link_libraries(LibLanguageServer LibCodeComprehension LibC)
 
 add_subdirectory(Cpp)
 add_subdirectory(Shell)

Userland/DevTools/HackStudio/LanguageServers/CodeComprehensionEngine.h

@@ -1,55 +0,0 @@
-/*
- * Copyright (c) 2021, Itamar S. <itamar8910@gmail.com>
- * Copyright (c) 2022, the SerenityOS developers.
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#pragma once
-
-#include "../AutoCompleteResponse.h"
-#include "FileDB.h"
-#include <LibGUI/AutocompleteProvider.h>
-#include <LibGUI/TextPosition.h>
-
-namespace LanguageServers {
-
-class ConnectionFromClient;
-
-class CodeComprehensionEngine {
-public:
-    CodeComprehensionEngine(FileDB const& filedb, bool store_all_declarations = false);
-    virtual ~CodeComprehensionEngine() = default;
-
-    virtual Vector<GUI::AutocompleteProvider::Entry> get_suggestions(String const& file, const GUI::TextPosition& autocomplete_position) = 0;
-
-    // TODO: In the future we can pass the range that was edited and only re-parse what we have to.
-    virtual void on_edit([[maybe_unused]] String const& file) {};
-    virtual void file_opened([[maybe_unused]] String const& file) {};
-
-    virtual Optional<GUI::AutocompleteProvider::ProjectLocation> find_declaration_of(String const&, const GUI::TextPosition&) { return {}; }
-
-    struct FunctionParamsHint {
-        Vector<String> params;
-        size_t current_index { 0 };
-    };
-    virtual Optional<FunctionParamsHint> get_function_params_hint(String const&, const GUI::TextPosition&) { return {}; }
-
-    virtual Vector<GUI::AutocompleteProvider::TokenInfo> get_tokens_info(String const&) { return {}; }
-
-public:
-    Function<void(String const&, Vector<GUI::AutocompleteProvider::Declaration>&&)> set_declarations_of_document_callback;
-    Function<void(String const&, Vector<Cpp::Parser::TodoEntry>&&)> set_todo_entries_of_document_callback;
-
-protected:
-    FileDB const& filedb() const { return m_filedb; }
-    void set_declarations_of_document(String const&, Vector<GUI::AutocompleteProvider::Declaration>&&);
-    void set_todo_entries_of_document(String const&, Vector<Cpp::Parser::TodoEntry>&&);
-    HashMap<String, Vector<GUI::AutocompleteProvider::Declaration>> const& all_declarations() const { return m_all_declarations; }
-
-private:
-    HashMap<String, Vector<GUI::AutocompleteProvider::Declaration>> m_all_declarations;
-    FileDB const& m_filedb;
-    bool m_store_all_declarations { false };
-};
-}

Userland/DevTools/HackStudio/LanguageServers/ConnectionFromClient.cpp

@@ -66,25 +66,25 @@ void ConnectionFromClient::file_edit_remove_text(String const& filename, i32 sta
     m_autocomplete_engine->on_edit(filename);
 }
 
-void ConnectionFromClient::auto_complete_suggestions(GUI::AutocompleteProvider::ProjectLocation const& location)
+void ConnectionFromClient::auto_complete_suggestions(CodeComprehension::ProjectLocation const& location)
 {
     dbgln_if(LANGUAGE_SERVER_DEBUG, "AutoCompleteSuggestions for: {} {}:{}", location.file, location.line, location.column);
 
-    auto document = m_filedb.get(location.file);
+    auto document = m_filedb.get_document(location.file);
     if (!document) {
         dbgln("file {} has not been opened", location.file);
         return;
     }
 
     GUI::TextPosition autocomplete_position = { (size_t)location.line, (size_t)max(location.column, location.column - 1) };
-    Vector<GUI::AutocompleteProvider::Entry> suggestions = m_autocomplete_engine->get_suggestions(location.file, autocomplete_position);
+    Vector<CodeComprehension::AutocompleteResultEntry> suggestions = m_autocomplete_engine->get_suggestions(location.file, autocomplete_position);
     async_auto_complete_suggestions(move(suggestions));
 }
 
 void ConnectionFromClient::set_file_content(String const& filename, String const& content)
 {
     dbgln_if(LANGUAGE_SERVER_DEBUG, "SetFileContent: {}", filename);
-    auto document = m_filedb.get(filename);
+    auto document = m_filedb.get_document(filename);
     if (!document) {
         m_filedb.add(filename, content);
         VERIFY(m_filedb.is_open(filename));
@@ -95,10 +95,10 @@ void ConnectionFromClient::set_file_content(String const& filename, String const
     m_autocomplete_engine->on_edit(filename);
 }
 
-void ConnectionFromClient::find_declaration(GUI::AutocompleteProvider::ProjectLocation const& location)
+void ConnectionFromClient::find_declaration(CodeComprehension::ProjectLocation const& location)
 {
     dbgln_if(LANGUAGE_SERVER_DEBUG, "FindDeclaration: {} {}:{}", location.file, location.line, location.column);
-    auto document = m_filedb.get(location.file);
+    auto document = m_filedb.get_document(location.file);
     if (!document) {
         dbgln("file {} has not been opened", location.file);
         return;
@@ -112,13 +112,13 @@ void ConnectionFromClient::find_declaration(GUI::AutocompleteProvider::ProjectLo
     }
 
     dbgln_if(LANGUAGE_SERVER_DEBUG, "declaration location: {} {}:{}", decl_location.value().file, decl_location.value().line, decl_location.value().column);
-    async_declaration_location(GUI::AutocompleteProvider::ProjectLocation { decl_location.value().file, decl_location.value().line, decl_location.value().column });
+    async_declaration_location(CodeComprehension::ProjectLocation { decl_location.value().file, decl_location.value().line, decl_location.value().column });
 }
 
-void ConnectionFromClient::get_parameters_hint(GUI::AutocompleteProvider::ProjectLocation const& location)
+void ConnectionFromClient::get_parameters_hint(CodeComprehension::ProjectLocation const& location)
 {
     dbgln_if(LANGUAGE_SERVER_DEBUG, "GetParametersHint: {} {}:{}", location.file, location.line, location.column);
-    auto document = m_filedb.get(location.file);
+    auto document = m_filedb.get_document(location.file);
     if (!document) {
         dbgln("file {} has not been opened", location.file);
         return;
@@ -143,7 +143,7 @@ void ConnectionFromClient::get_parameters_hint(GUI::AutocompleteProvider::Projec
 void ConnectionFromClient::get_tokens_info(String const& filename)
 {
     dbgln_if(LANGUAGE_SERVER_DEBUG, "GetTokenInfo: {}", filename);
-    auto document = m_filedb.get(filename);
+    auto document = m_filedb.get_document(filename);
     if (!document) {
         dbgln("file {} has not been opened", filename);
         return;

Userland/DevTools/HackStudio/LanguageServers/ConnectionFromClient.h

@@ -8,10 +8,10 @@
 #pragma once
 
 #include "../AutoCompleteResponse.h"
-#include "CodeComprehensionEngine.h"
 #include "FileDB.h"
 #include <AK/HashMap.h>
 #include <AK/LexicalPath.h>
+#include <LibCodeComprehension/CodeComprehensionEngine.h>
 #include <LibIPC/ConnectionFromClient.h>
 
 #include <Userland/DevTools/HackStudio/LanguageServers/LanguageClientEndpoint.h>
@@ -32,13 +32,13 @@ protected:
     virtual void file_edit_insert_text(String const&, String const&, i32, i32) override;
     virtual void file_edit_remove_text(String const&, i32, i32, i32, i32) override;
     virtual void set_file_content(String const&, String const&) override;
-    virtual void auto_complete_suggestions(GUI::AutocompleteProvider::ProjectLocation const&) override;
-    virtual void find_declaration(GUI::AutocompleteProvider::ProjectLocation const&) override;
-    virtual void get_parameters_hint(GUI::AutocompleteProvider::ProjectLocation const&) override;
+    virtual void auto_complete_suggestions(CodeComprehension::ProjectLocation const&) override;
+    virtual void find_declaration(CodeComprehension::ProjectLocation const&) override;
+    virtual void get_parameters_hint(CodeComprehension::ProjectLocation const&) override;
     virtual void get_tokens_info(String const&) override;
 
     FileDB m_filedb;
-    OwnPtr<CodeComprehensionEngine> m_autocomplete_engine;
+    OwnPtr<CodeComprehension::CodeComprehensionEngine> m_autocomplete_engine;
 };
 
 }

Userland/DevTools/HackStudio/LanguageServers/Cpp/CMakeLists.txt

@@ -4,8 +4,6 @@ serenity_component(
 )
 
 set(SOURCES
-    CppComprehensionEngine.cpp
-    Tests.cpp
     main.cpp
 )
 
@@ -17,4 +15,4 @@ serenity_bin(CppLanguageServer)
 
 # We link with LibGUI because we use GUI::TextDocument to update
 # the content of files according to the edit actions we receive over IPC.
-target_link_libraries(CppLanguageServer LibIPC LibCpp LibGUI LibLanguageServer LibMain)
+target_link_libraries(CppLanguageServer LibIPC LibCpp LibGUI LibLanguageServer LibCppComprehension LibMain)

Userland/DevTools/HackStudio/LanguageServers/Cpp/ConnectionFromClient.h

@@ -6,8 +6,8 @@
 
 #pragma once
 
-#include "CppComprehensionEngine.h"
 #include <DevTools/HackStudio/LanguageServers/ConnectionFromClient.h>
+#include <LibCodeComprehension/Cpp/CppComprehensionEngine.h>
 
 namespace LanguageServers::Cpp {
 
@@ -18,11 +18,11 @@ private:
     ConnectionFromClient(NonnullOwnPtr<Core::Stream::LocalSocket> socket)
         : LanguageServers::ConnectionFromClient(move(socket))
     {
-        m_autocomplete_engine = make<CppComprehensionEngine>(m_filedb);
-        m_autocomplete_engine->set_declarations_of_document_callback = [this](String const& filename, Vector<GUI::AutocompleteProvider::Declaration>&& declarations) {
+        m_autocomplete_engine = adopt_own(*new CodeComprehension::Cpp::CppComprehensionEngine(m_filedb));
+        m_autocomplete_engine->set_declarations_of_document_callback = [this](String const& filename, Vector<CodeComprehension::Declaration>&& declarations) {
             async_declarations_in_document(filename, move(declarations));
         };
-        m_autocomplete_engine->set_todo_entries_of_document_callback = [this](String const& filename, Vector<Cpp::Parser::TodoEntry>&& todo_entries) {
+        m_autocomplete_engine->set_todo_entries_of_document_callback = [this](String const& filename, Vector<CodeComprehension::TodoEntry>&& todo_entries) {
             async_todo_entries_in_document(filename, move(todo_entries));
         };
     }

Userland/DevTools/HackStudio/LanguageServers/Cpp/main.cpp

@@ -5,7 +5,6 @@
  */
 
 #include "ConnectionFromClient.h"
-#include "Tests.h"
 #include <LibCore/ArgsParser.h>
 #include <LibCore/EventLoop.h>
 #include <LibCore/LocalServer.h>
@@ -13,23 +12,7 @@
 #include <LibIPC/SingleServer.h>
 #include <LibMain/Main.h>
 
-static ErrorOr<int> mode_server();
-
-ErrorOr<int> serenity_main(Main::Arguments arguments)
-{
-    bool tests = false;
-
-    Core::ArgsParser parser;
-    parser.add_option(tests, "Run tests", "tests", 't');
-    parser.parse(arguments);
-
-    if (tests)
-        return run_tests();
-
-    return mode_server();
-}
-
-ErrorOr<int> mode_server()
+ErrorOr<int> serenity_main(Main::Arguments)
 {
     Core::EventLoop event_loop;
     TRY(Core::System::pledge("stdio unix recvfd rpath"));

Userland/DevTools/HackStudio/LanguageServers/FileDB.cpp

@@ -12,7 +12,7 @@
 
 namespace LanguageServers {
 
-RefPtr<const GUI::TextDocument> FileDB::get(String const& filename) const
+RefPtr<const GUI::TextDocument> FileDB::get_document(String const& filename) const
 {
     auto absolute_path = to_absolute_path(filename);
     auto document_optional = m_open_files.get(absolute_path);
@@ -22,29 +22,25 @@ RefPtr<const GUI::TextDocument> FileDB::get(String const& filename) const
     return *document_optional.value();
 }
 
-RefPtr<GUI::TextDocument> FileDB::get(String const& filename)
+RefPtr<GUI::TextDocument> FileDB::get_document(String const& filename)
 {
-    auto document = reinterpret_cast<FileDB const*>(this)->get(filename);
+    auto document = reinterpret_cast<FileDB const*>(this)->get_document(filename);
     if (document.is_null())
         return nullptr;
     return adopt_ref(*const_cast<GUI::TextDocument*>(document.leak_ref()));
 }
 
-RefPtr<const GUI::TextDocument> FileDB::get_or_create_from_filesystem(String const& filename) const
+Optional<String> FileDB::get_or_read_from_filesystem(StringView filename) const
 {
     auto absolute_path = to_absolute_path(filename);
-    auto document = get(absolute_path);
+    auto document = get_document(absolute_path);
     if (document)
-        return document;
-    return create_from_filesystem(absolute_path);
-}
+        return document->text();
 
-RefPtr<GUI::TextDocument> FileDB::get_or_create_from_filesystem(String const& filename)
-{
-    auto document = reinterpret_cast<FileDB const*>(this)->get_or_create_from_filesystem(filename);
-    if (document.is_null())
-        return nullptr;
-    return adopt_ref(*const_cast<GUI::TextDocument*>(document.leak_ref()));
+    document = create_from_filesystem(absolute_path);
+    if (document)
+        return document->text();
+    return {};
 }
 
 bool FileDB::is_open(String const& filename) const
@@ -123,7 +119,7 @@ RefPtr<GUI::TextDocument> FileDB::create_from_file(Core::File& file) const
 void FileDB::on_file_edit_insert_text(String const& filename, String const& inserted_text, size_t start_line, size_t start_column)
 {
     VERIFY(is_open(filename));
-    auto document = get(filename);
+    auto document = get_document(filename);
     VERIFY(document);
     GUI::TextPosition start_position { start_line, start_column };
     document->insert_at(start_position, inserted_text, &s_default_document_client);
@@ -136,7 +132,7 @@ void FileDB::on_file_edit_remove_text(String const& filename, size_t start_line,
     // TODO: If file is not open - need to get its contents
     // Otherwise- somehow verify that respawned language server is synced with all file contents
     VERIFY(is_open(filename));
-    auto document = get(filename);
+    auto document = get_document(filename);
     VERIFY(document);
     GUI::TextPosition start_position { start_line, start_column };
     GUI::TextRange range {

Userland/DevTools/HackStudio/LanguageServers/FileDB.h

@@ -9,22 +9,22 @@
 #include <AK/HashMap.h>
 #include <AK/NonnullRefPtr.h>
 #include <AK/String.h>
+#include <LibCodeComprehension/FileDB.h>
 #include <LibGUI/TextDocument.h>
 
 namespace LanguageServers {
 
-class FileDB final {
+class FileDB final : public CodeComprehension::FileDB {
 public:
-    RefPtr<const GUI::TextDocument> get(String const& filename) const;
-    RefPtr<GUI::TextDocument> get(String const& filename);
-    RefPtr<const GUI::TextDocument> get_or_create_from_filesystem(String const& filename) const;
-    RefPtr<GUI::TextDocument> get_or_create_from_filesystem(String const& filename);
+    FileDB() = default;
+    virtual Optional<String> get_or_read_from_filesystem(StringView filename) const override;
+
+    RefPtr<const GUI::TextDocument> get_document(String const& filename) const;
+    RefPtr<GUI::TextDocument> get_document(String const& filename);
+
     bool add(String const& filename, int fd);
     bool add(String const& filename, String const& content);
 
-    void set_project_root(String const& root_path) { m_project_root = root_path; }
-    String const& project_root() const { return m_project_root; }
-
     void on_file_edit_insert_text(String const& filename, String const& inserted_text, size_t start_line, size_t start_column);
     void on_file_edit_remove_text(String const& filename, size_t start_line, size_t start_column, size_t end_line, size_t end_column);
     String to_absolute_path(String const& filename) const;

Userland/DevTools/HackStudio/LanguageServers/LanguageClient.ipc

@@ -1,9 +1,9 @@
 endpoint LanguageClient
 {
-    auto_complete_suggestions(Vector<GUI::AutocompleteProvider::Entry> suggestions) =|
-    declaration_location(GUI::AutocompleteProvider::ProjectLocation location) =|
-    declarations_in_document(String filename, Vector<GUI::AutocompleteProvider::Declaration> declarations) =|
-    todo_entries_in_document(String filename, Vector<Cpp::Parser::TodoEntry> todo_entries) =|
+    auto_complete_suggestions(Vector<CodeComprehension::AutocompleteResultEntry> suggestions) =|
+    declaration_location(CodeComprehension::ProjectLocation location) =|
+    declarations_in_document(String filename, Vector<CodeComprehension::Declaration> declarations) =|
+    todo_entries_in_document(String filename, Vector<CodeComprehension::TodoEntry> todo_entries) =|
     parameters_hint_result(Vector<String> params, int current_index) =|
-    tokens_info_result(Vector<GUI::AutocompleteProvider::TokenInfo> tokens_info) =|
+    tokens_info_result(Vector<CodeComprehension::TokenInfo> tokens_info) =|
 }

Userland/DevTools/HackStudio/LanguageServers/LanguageServer.ipc

@@ -7,9 +7,9 @@ endpoint LanguageServer
     file_edit_remove_text(String filename, i32 start_line, i32 start_column, i32 end_line, i32 end_column) =|
     set_file_content(String filename, String content) =|
 
-    auto_complete_suggestions(GUI::AutocompleteProvider::ProjectLocation location) =|
-    find_declaration(GUI::AutocompleteProvider::ProjectLocation location) =|
-    get_parameters_hint(GUI::AutocompleteProvider::ProjectLocation location) =|
+    auto_complete_suggestions(CodeComprehension::ProjectLocation location) =|
+    find_declaration(CodeComprehension::ProjectLocation location) =|
+    get_parameters_hint(CodeComprehension::ProjectLocation location) =|
     get_tokens_info(String filename) =|
 
 }

Userland/DevTools/HackStudio/LanguageServers/Shell/CMakeLists.txt

@@ -4,7 +4,6 @@ serenity_component(
 )
 
 set(SOURCES
-    ShellComprehensionEngine.cpp
     main.cpp
 )
 
@@ -16,4 +15,4 @@ serenity_bin(ShellLanguageServer)
 
 # We link with LibGUI because we use GUI::TextDocument to update
 # the content of files according to the edit actions we receive over IPC.
-target_link_libraries(ShellLanguageServer LibIPC LibShell LibGUI LibLanguageServer LibMain)
+target_link_libraries(ShellLanguageServer LibIPC LibShell LibGUI LibLanguageServer LibShellComprehension LibMain)

Userland/DevTools/HackStudio/LanguageServers/Shell/ConnectionFromClient.h

@@ -6,8 +6,8 @@
 
 #pragma once
 
-#include "ShellComprehensionEngine.h"
 #include <DevTools/HackStudio/LanguageServers/ConnectionFromClient.h>
+#include <LibCodeComprehension/Shell/ShellComprehensionEngine.h>
 #include <LibCpp/Parser.h>
 
 namespace LanguageServers::Shell {
@@ -19,11 +19,11 @@ private:
     ConnectionFromClient(NonnullOwnPtr<Core::Stream::LocalSocket> socket)
         : LanguageServers::ConnectionFromClient(move(socket))
     {
-        m_autocomplete_engine = make<ShellComprehensionEngine>(m_filedb);
-        m_autocomplete_engine->set_declarations_of_document_callback = [this](String const& filename, Vector<GUI::AutocompleteProvider::Declaration>&& declarations) {
+        m_autocomplete_engine = make<CodeComprehension::Shell::ShellComprehensionEngine>(m_filedb);
+        m_autocomplete_engine->set_declarations_of_document_callback = [this](String const& filename, Vector<CodeComprehension::Declaration>&& declarations) {
             async_declarations_in_document(filename, move(declarations));
         };
-        m_autocomplete_engine->set_todo_entries_of_document_callback = [this](String const& filename, Vector<Cpp::Parser::TodoEntry>&& todo_entries) {
+        m_autocomplete_engine->set_todo_entries_of_document_callback = [this](String const& filename, Vector<CodeComprehension::TodoEntry>&& todo_entries) {
             async_todo_entries_in_document(filename, move(todo_entries));
         };
     }

Userland/DevTools/HackStudio/Locator.cpp

@@ -22,13 +22,13 @@ class LocatorSuggestionModel final : public GUI::Model {
 public:
     struct Suggestion {
         static Suggestion create_filename(String const& filename);
-        static Suggestion create_symbol_declaration(const GUI::AutocompleteProvider::Declaration&);
+        static Suggestion create_symbol_declaration(CodeComprehension::Declaration const&);
 
         bool is_filename() const { return as_filename.has_value(); }
         bool is_symbol_declaration() const { return as_symbol_declaration.has_value(); }
 
         Optional<String> as_filename;
-        Optional<GUI::AutocompleteProvider::Declaration> as_symbol_declaration;
+        Optional<CodeComprehension::Declaration> as_symbol_declaration;
     };
 
     explicit LocatorSuggestionModel(Vector<Suggestion>&& suggestions)
@@ -88,7 +88,7 @@ LocatorSuggestionModel::Suggestion LocatorSuggestionModel::Suggestion::create_fi
     s.as_filename = filename;
     return s;
 }
-LocatorSuggestionModel::Suggestion LocatorSuggestionModel::Suggestion::create_symbol_declaration(const GUI::AutocompleteProvider::Declaration& decl)
+LocatorSuggestionModel::Suggestion LocatorSuggestionModel::Suggestion::create_symbol_declaration(CodeComprehension::Declaration const& decl)
 {
     LocatorSuggestionModel::Suggestion s;
     s.as_symbol_declaration = decl;

Userland/DevTools/HackStudio/ProjectDeclarations.cpp

@@ -11,14 +11,14 @@ HackStudio::ProjectDeclarations& HackStudio::ProjectDeclarations::the()
     static ProjectDeclarations s_instance;
     return s_instance;
 }
-void HackStudio::ProjectDeclarations::set_declared_symbols(String const& filename, Vector<GUI::AutocompleteProvider::Declaration> const& declarations)
+void HackStudio::ProjectDeclarations::set_declared_symbols(String const& filename, Vector<CodeComprehension::Declaration> const& declarations)
 {
     m_document_to_declarations.set(filename, declarations);
     if (on_update)
         on_update();
 }
 
-Optional<GUI::Icon> HackStudio::ProjectDeclarations::get_icon_for(GUI::AutocompleteProvider::DeclarationType type)
+Optional<GUI::Icon> HackStudio::ProjectDeclarations::get_icon_for(CodeComprehension::DeclarationType type)
 {
     static GUI::Icon struct_icon(Gfx::Bitmap::try_load_from_file("/res/icons/hackstudio/Struct.png").release_value_but_fixme_should_propagate_errors());
     static GUI::Icon class_icon(Gfx::Bitmap::try_load_from_file("/res/icons/hackstudio/Class.png").release_value_but_fixme_should_propagate_errors());
@@ -28,19 +28,19 @@ Optional<GUI::Icon> HackStudio::ProjectDeclarations::get_icon_for(GUI::Autocompl
     static GUI::Icon member_icon(Gfx::Bitmap::try_load_from_file("/res/icons/hackstudio/Member.png").release_value_but_fixme_should_propagate_errors());
     static GUI::Icon namespace_icon(Gfx::Bitmap::try_load_from_file("/res/icons/hackstudio/Namespace.png").release_value_but_fixme_should_propagate_errors());
     switch (type) {
-    case GUI::AutocompleteProvider::DeclarationType::Struct:
+    case CodeComprehension::DeclarationType::Struct:
         return struct_icon;
-    case GUI::AutocompleteProvider::DeclarationType::Class:
+    case CodeComprehension::DeclarationType::Class:
         return class_icon;
-    case GUI::AutocompleteProvider::DeclarationType::Function:
+    case CodeComprehension::DeclarationType::Function:
         return function_icon;
-    case GUI::AutocompleteProvider::DeclarationType::Variable:
+    case CodeComprehension::DeclarationType::Variable:
         return variable_icon;
-    case GUI::AutocompleteProvider::DeclarationType::PreprocessorDefinition:
+    case CodeComprehension::DeclarationType::PreprocessorDefinition:
         return preprocessor_icon;
-    case GUI::AutocompleteProvider::DeclarationType::Member:
+    case CodeComprehension::DeclarationType::Member:
         return member_icon;
-    case GUI::AutocompleteProvider::DeclarationType::Namespace:
+    case CodeComprehension::DeclarationType::Namespace:
         return namespace_icon;
     default:
         return {};

Userland/DevTools/HackStudio/ProjectDeclarations.h

@@ -23,15 +23,15 @@ public:
     template<typename Func>
     void for_each_declared_symbol(Func);
 
-    void set_declared_symbols(String const& filename, Vector<GUI::AutocompleteProvider::Declaration> const&);
+    void set_declared_symbols(String const& filename, Vector<CodeComprehension::Declaration> const&);
 
-    static Optional<GUI::Icon> get_icon_for(GUI::AutocompleteProvider::DeclarationType);
+    static Optional<GUI::Icon> get_icon_for(CodeComprehension::DeclarationType);
 
     Function<void()> on_update = nullptr;
 
 private:
     ProjectDeclarations() = default;
-    HashMap<String, Vector<GUI::AutocompleteProvider::Declaration>> m_document_to_declarations;
+    HashMap<String, Vector<CodeComprehension::Declaration>> m_document_to_declarations;
 };
 
 template<typename Func>

Userland/DevTools/HackStudio/ToDoEntries.cpp

@@ -14,16 +14,16 @@ ToDoEntries& HackStudio::ToDoEntries::the()
     return s_instance;
 }
 
-void ToDoEntries::set_entries(String const& filename, Vector<Cpp::Parser::TodoEntry> const&& entries)
+void ToDoEntries::set_entries(String const& filename, Vector<CodeComprehension::TodoEntry> const&& entries)
 {
     m_document_to_entries.set(filename, move(entries));
     if (on_update)
         on_update();
 }
 
-Vector<Cpp::Parser::TodoEntry> ToDoEntries::get_entries()
+Vector<CodeComprehension::TodoEntry> ToDoEntries::get_entries()
 {
-    Vector<Cpp::Parser::TodoEntry> ret;
+    Vector<CodeComprehension::TodoEntry> ret;
     for (auto& it : m_document_to_entries) {
         for (auto& entry : it.value)
             ret.append({ entry.content, it.key, entry.line, entry.column });

Userland/DevTools/HackStudio/ToDoEntries.h

@@ -20,9 +20,9 @@ class ToDoEntries {
 public:
     static ToDoEntries& the();
 
-    void set_entries(String const& filename, Vector<Cpp::Parser::TodoEntry> const&& entries);
+    void set_entries(String const& filename, Vector<CodeComprehension::TodoEntry> const&& entries);
 
-    Vector<Cpp::Parser::TodoEntry> get_entries();
+    Vector<CodeComprehension::TodoEntry> get_entries();
 
     void clear_entries();
 
@@ -30,7 +30,7 @@ public:
 
 private:
     ToDoEntries() = default;
-    HashMap<String, Vector<Cpp::Parser::TodoEntry>> m_document_to_entries;
+    HashMap<String, Vector<CodeComprehension::TodoEntry>> m_document_to_entries;
 };
 
 }

Userland/DevTools/HackStudio/ToDoEntriesWidget.cpp

@@ -22,7 +22,7 @@ public:
         __Count
     };
 
-    explicit ToDoEntriesModel(Vector<Cpp::Parser::TodoEntry> const&& matches)
+    explicit ToDoEntriesModel(Vector<CodeComprehension::TodoEntry> const&& matches)
         : m_matches(move(matches))
     {
     }
@@ -81,7 +81,7 @@ public:
     }
 
 private:
-    Vector<Cpp::Parser::TodoEntry> m_matches;
+    Vector<CodeComprehension::TodoEntry> m_matches;
 };
 
 void ToDoEntriesWidget::refresh()
@@ -103,7 +103,7 @@ ToDoEntriesWidget::ToDoEntriesWidget()
     m_result_view = add<GUI::TableView>();
 
     m_result_view->on_activation = [](auto& index) {
-        auto& match = *(Cpp::Parser::TodoEntry const*)index.internal_data();
+        auto& match = *(CodeComprehension::TodoEntry const*)index.internal_data();
         open_file(match.filename, match.line, match.column);
     };
 }

Userland/Libraries/CMakeLists.txt

@@ -3,6 +3,7 @@ add_subdirectory(LibAudio)
 add_subdirectory(LibC)
 add_subdirectory(LibCards)
 add_subdirectory(LibChess)
+add_subdirectory(LibCodeComprehension)
 add_subdirectory(LibCompress)
 add_subdirectory(LibConfig)
 add_subdirectory(LibCore)

Userland/Libraries/LibCodeComprehension/CMakeLists.txt

@@ -0,0 +1,10 @@
+set(SOURCES
+    CodeComprehensionEngine.cpp
+    FileDB.cpp
+)
+
+serenity_lib(LibCodeComprehension codecomprehension)
+target_link_libraries(LibCodeComprehension LibC)
+
+add_subdirectory(Cpp)
+add_subdirectory(Shell)

Userland/DevTools/HackStudio/LanguageServers/CodeComprehensionEngine.cpp → Userland/Libraries/LibCodeComprehension/CodeComprehensionEngine.cpp

@@ -7,7 +7,7 @@
 
 #include "CodeComprehensionEngine.h"
 
-namespace LanguageServers {
+namespace CodeComprehension {
 
 CodeComprehensionEngine::CodeComprehensionEngine(FileDB const& filedb, bool should_store_all_declarations)
     : m_filedb(filedb)
@@ -15,7 +15,7 @@ CodeComprehensionEngine::CodeComprehensionEngine(FileDB const& filedb, bool shou
 {
 }
 
-void CodeComprehensionEngine::set_declarations_of_document(String const& filename, Vector<GUI::AutocompleteProvider::Declaration>&& declarations)
+void CodeComprehensionEngine::set_declarations_of_document(String const& filename, Vector<Declaration>&& declarations)
 {
     // Callback may not be configured if we're running tests
     if (!set_declarations_of_document_callback)
@@ -31,7 +31,7 @@ void CodeComprehensionEngine::set_declarations_of_document(String const& filenam
     set_declarations_of_document_callback(filename, move(declarations));
 }
 
-void CodeComprehensionEngine::set_todo_entries_of_document(String const& filename, Vector<Cpp::Parser::TodoEntry>&& todo_entries)
+void CodeComprehensionEngine::set_todo_entries_of_document(String const& filename, Vector<TodoEntry>&& todo_entries)
 {
     // Callback may not be configured if we're running tests
     if (!set_todo_entries_of_document_callback)

Userland/Libraries/LibCodeComprehension/CodeComprehensionEngine.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2021, Itamar S. <itamar8910@gmail.com>
+ * Copyright (c) 2022, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include "FileDB.h"
+#include "Types.h"
+#include <AK/Function.h>
+#include <AK/HashMap.h>
+#include <AK/Vector.h>
+#include <LibGUI/TextPosition.h>
+
+namespace CodeComprehension {
+
+class CodeComprehensionEngine {
+    AK_MAKE_NONCOPYABLE(CodeComprehensionEngine);
+    AK_MAKE_NONMOVABLE(CodeComprehensionEngine);
+
+public:
+    CodeComprehensionEngine(FileDB const& filedb, bool store_all_declarations = false);
+    virtual ~CodeComprehensionEngine() = default;
+
+    virtual Vector<AutocompleteResultEntry> get_suggestions(String const& file, GUI::TextPosition const& autocomplete_position) = 0;
+
+    // TODO: In the future we can pass the range that was edited and only re-parse what we have to.
+    virtual void on_edit([[maybe_unused]] String const& file) {};
+    virtual void file_opened([[maybe_unused]] String const& file) {};
+
+    virtual Optional<ProjectLocation> find_declaration_of(String const&, GUI::TextPosition const&) { return {}; }
+
+    struct FunctionParamsHint {
+        Vector<String> params;
+        size_t current_index { 0 };
+    };
+    virtual Optional<FunctionParamsHint> get_function_params_hint(String const&, GUI::TextPosition const&) { return {}; }
+
+    virtual Vector<TokenInfo> get_tokens_info(String const&) { return {}; }
+
+    Function<void(String const&, Vector<Declaration>&&)> set_declarations_of_document_callback;
+    Function<void(String const&, Vector<TodoEntry>&&)> set_todo_entries_of_document_callback;
+
+protected:
+    FileDB const& filedb() const { return m_filedb; }
+    void set_declarations_of_document(String const&, Vector<Declaration>&&);
+    void set_todo_entries_of_document(String const&, Vector<TodoEntry>&&);
+    HashMap<String, Vector<Declaration>> const& all_declarations() const { return m_all_declarations; }
+
+private:
+    HashMap<String, Vector<Declaration>> m_all_declarations;
+    FileDB const& m_filedb;
+    bool m_store_all_declarations { false };
+};
+}

Userland/Libraries/LibCodeComprehension/Cpp/CMakeLists.txt

@@ -0,0 +1,20 @@
+set(SOURCES
+    CppComprehensionEngine.cpp
+)
+
+serenity_lib(LibCppComprehension cppcomprehension)
+target_link_libraries(LibCppComprehension LibCodeComprehension LibC)
+
+serenity_component(
+    CppComprehensionTests
+    TARGETS CppComprehensionTests
+)
+
+set(SOURCES
+    CppComprehensionEngine.cpp
+    Tests.cpp
+)
+
+serenity_bin(CppComprehensionTests)
+
+target_link_libraries(CppComprehensionTests LibCodeComprehension LibCpp LibRegex LibMain)

Userland/Libraries/LibCodeComprehension/Cpp/ConnectionFromClient.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2020, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include "CppComprehensionEngine.h"
+#include <DevTools/HackStudio/LanguageServers/ConnectionFromClient.h>
+
+namespace LanguageServers::Cpp {
+
+class ConnectionFromClient final : public LanguageServers::ConnectionFromClient {
+    C_OBJECT(ConnectionFromClient);
+
+private:
+    ConnectionFromClient(NonnullOwnPtr<Core::Stream::LocalSocket> socket)
+        : LanguageServers::ConnectionFromClient(move(socket))
+    {
+        m_autocomplete_engine = make<CodeComprehension::Cpp::CppComprehensionEngine>(m_filedb);
+        m_autocomplete_engine->set_declarations_of_document_callback = [this](String const& filename, Vector<CodeComprehension::Declaration>&& declarations) {
+            async_declarations_in_document(filename, move(declarations));
+        };
+        m_autocomplete_engine->set_todo_entries_of_document_callback = [this](String const& filename, Vector<CodeComprehension::TodoEntry>&& todo_entries) {
+            async_todo_entries_in_document(filename, move(todo_entries));
+        };
+    }
+
+    virtual ~ConnectionFromClient() override = default;
+};
+}

Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.cpp → Userland/Libraries/LibCodeComprehension/Cpp/CppComprehensionEngine.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Itamar S. <itamar8910@gmail.com>
+ * Copyright (c) 2021-2022, Itamar S. <itamar8910@gmail.com>
  *
  * SPDX-License-Identifier: BSD-2-Clause
  */
@@ -18,7 +18,7 @@
 #include <LibRegex/Regex.h>
 #include <Userland/DevTools/HackStudio/LanguageServers/ConnectionFromClient.h>
 
-namespace LanguageServers::Cpp {
+namespace CodeComprehension::Cpp {
 
 CppComprehensionEngine::CppComprehensionEngine(FileDB const& filedb)
     : CodeComprehensionEngine(filedb, true)
@@ -50,10 +50,10 @@ OwnPtr<CppComprehensionEngine::DocumentData> CppComprehensionEngine::create_docu
     }
     m_unfinished_documents.set(file);
     ScopeGuard mark_finished([&file, this]() { m_unfinished_documents.remove(file); });
-    auto document = filedb().get_or_create_from_filesystem(file);
-    if (!document)
+    auto document = filedb().get_or_read_from_filesystem(file);
+    if (!document.has_value())
         return {};
-    return create_document_data(document->text(), file);
+    return create_document_data(move(document.value()), file);
 }
 
 void CppComprehensionEngine::set_document_data(String const& file, OwnPtr<DocumentData>&& data)
@@ -61,7 +61,7 @@ void CppComprehensionEngine::set_document_data(String const& file, OwnPtr<Docume
     m_documents.set(filedb().to_absolute_path(file), move(data));
 }
 
-Vector<GUI::AutocompleteProvider::Entry> CppComprehensionEngine::get_suggestions(String const& file, const GUI::TextPosition& autocomplete_position)
+Vector<CodeComprehension::AutocompleteResultEntry> CppComprehensionEngine::get_suggestions(String const& file, const GUI::TextPosition& autocomplete_position)
 {
     Cpp::Position position { autocomplete_position.line(), autocomplete_position.column() > 0 ? autocomplete_position.column() - 1 : 0 };
 
@@ -102,7 +102,7 @@ Vector<GUI::AutocompleteProvider::Entry> CppComprehensionEngine::get_suggestions
     return {};
 }
 
-Optional<Vector<GUI::AutocompleteProvider::Entry>> CppComprehensionEngine::try_autocomplete_name(DocumentData const& document, ASTNode const& node, Optional<Token> containing_token) const
+Optional<Vector<CodeComprehension::AutocompleteResultEntry>> CppComprehensionEngine::try_autocomplete_name(DocumentData const& document, ASTNode const& node, Optional<Token> containing_token) const
 {
     auto partial_text = String::empty();
     if (containing_token.has_value() && containing_token.value().type() != Token::Type::ColonColon) {
@@ -111,7 +111,7 @@ Optional<Vector<GUI::AutocompleteProvider::Entry>> CppComprehensionEngine::try_a
     return autocomplete_name(document, node, partial_text);
 }
 
-Optional<Vector<GUI::AutocompleteProvider::Entry>> CppComprehensionEngine::try_autocomplete_property(DocumentData const& document, ASTNode const& node, Optional<Token> containing_token) const
+Optional<Vector<CodeComprehension::AutocompleteResultEntry>> CppComprehensionEngine::try_autocomplete_property(DocumentData const& document, ASTNode const& node, Optional<Token> containing_token) const
 {
     if (!containing_token.has_value())
         return {};
@@ -131,7 +131,7 @@ Optional<Vector<GUI::AutocompleteProvider::Entry>> CppComprehensionEngine::try_a
     return autocomplete_property(document, parent, partial_text);
 }
 
-Vector<GUI::AutocompleteProvider::Entry> CppComprehensionEngine::autocomplete_name(DocumentData const& document, ASTNode const& node, String const& partial_text) const
+Vector<CodeComprehension::AutocompleteResultEntry> CppComprehensionEngine::autocomplete_name(DocumentData const& document, ASTNode const& node, String const& partial_text) const
 {
     auto reference_scope = scope_of_reference_to_symbol(node);
     auto current_scope = scope_of_node(node);
@@ -163,7 +163,7 @@ Vector<GUI::AutocompleteProvider::Entry> CppComprehensionEngine::autocomplete_na
         return IterationDecision::Continue;
     });
 
-    Vector<GUI::AutocompleteProvider::Entry> suggestions;
+    Vector<CodeComprehension::AutocompleteResultEntry> suggestions;
     for (auto& symbol : matches) {
         suggestions.append({ symbol.name.name, partial_text.length() });
     }
@@ -206,7 +206,7 @@ Vector<StringView> CppComprehensionEngine::scope_of_reference_to_symbol(ASTNode
     return scope_parts;
 }
 
-Vector<GUI::AutocompleteProvider::Entry> CppComprehensionEngine::autocomplete_property(DocumentData const& document, MemberExpression const& parent, const String partial_text) const
+Vector<CodeComprehension::AutocompleteResultEntry> CppComprehensionEngine::autocomplete_property(DocumentData const& document, MemberExpression const& parent, const String partial_text) const
 {
     VERIFY(parent.object());
     auto type = type_of(document, *parent.object());
@@ -215,7 +215,7 @@ Vector<GUI::AutocompleteProvider::Entry> CppComprehensionEngine::autocomplete_pr
         return {};
     }
 
-    Vector<GUI::AutocompleteProvider::Entry> suggestions;
+    Vector<CodeComprehension::AutocompleteResultEntry> suggestions;
     for (auto& prop : properties_of_type(document, type)) {
         if (prop.name.name.starts_with(partial_text)) {
             suggestions.append({ prop.name.name, partial_text.length() });
@@ -331,7 +331,7 @@ Vector<CppComprehensionEngine::Symbol> CppComprehensionEngine::properties_of_typ
     return properties;
 }
 
-CppComprehensionEngine::Symbol CppComprehensionEngine::Symbol::create(StringView name, Vector<StringView> const& scope, NonnullRefPtr<Declaration> declaration, IsLocal is_local)
+CppComprehensionEngine::Symbol CppComprehensionEngine::Symbol::create(StringView name, Vector<StringView> const& scope, NonnullRefPtr<Cpp::Declaration> declaration, IsLocal is_local)
 {
     return { { name, scope }, move(declaration), is_local == IsLocal::Yes };
 }
@@ -401,7 +401,7 @@ void CppComprehensionEngine::file_opened([[maybe_unused]] String const& file)
     get_or_create_document_data(file);
 }
 
-Optional<GUI::AutocompleteProvider::ProjectLocation> CppComprehensionEngine::find_declaration_of(String const& filename, const GUI::TextPosition& identifier_position)
+Optional<CodeComprehension::ProjectLocation> CppComprehensionEngine::find_declaration_of(String const& filename, const GUI::TextPosition& identifier_position)
 {
     auto const* document_ptr = get_or_create_document_data(filename);
     if (!document_ptr)
@@ -410,13 +410,13 @@ Optional<GUI::AutocompleteProvider::ProjectLocation> CppComprehensionEngine::fin
     auto const& document = *document_ptr;
     auto decl = find_declaration_of(document, identifier_position);
     if (decl) {
-        return GUI::AutocompleteProvider::ProjectLocation { decl->filename(), decl->start().line, decl->start().column };
+        return CodeComprehension::ProjectLocation { decl->filename(), decl->start().line, decl->start().column };
     }
 
     return find_preprocessor_definition(document, identifier_position);
 }
 
-RefPtr<Declaration> CppComprehensionEngine::find_declaration_of(DocumentData const& document, const GUI::TextPosition& identifier_position)
+RefPtr<Cpp::Declaration> CppComprehensionEngine::find_declaration_of(DocumentData const& document, const GUI::TextPosition& identifier_position)
 {
     auto node = document.parser().node_at(Cpp::Position { identifier_position.line(), identifier_position.column() });
     if (!node) {
@@ -426,13 +426,13 @@ RefPtr<Declaration> CppComprehensionEngine::find_declaration_of(DocumentData con
     return find_declaration_of(document, *node);
 }
 
-Optional<GUI::AutocompleteProvider::ProjectLocation> CppComprehensionEngine::find_preprocessor_definition(DocumentData const& document, const GUI::TextPosition& text_position)
+Optional<CodeComprehension::ProjectLocation> CppComprehensionEngine::find_preprocessor_definition(DocumentData const& document, const GUI::TextPosition& text_position)
 {
     Position cpp_position { text_position.line(), text_position.column() };
     auto substitution = find_preprocessor_substitution(document, cpp_position);
     if (!substitution.has_value())
         return {};
-    return GUI::AutocompleteProvider::ProjectLocation { substitution->defined_value.filename, substitution->defined_value.line, substitution->defined_value.column };
+    return CodeComprehension::ProjectLocation { substitution->defined_value.filename, substitution->defined_value.line, substitution->defined_value.column };
 }
 
 Optional<Cpp::Preprocessor::Substitution> CppComprehensionEngine::find_preprocessor_substitution(DocumentData const& document, Cpp::Position const& cpp_position)
@@ -467,11 +467,11 @@ static Optional<TargetDeclaration> get_target_declaration(ASTNode const& node)
     }
 
     if (node.is_declaration()) {
-        return get_target_declaration(node, verify_cast<Declaration>(node).full_name());
+        return get_target_declaration(node, verify_cast<Cpp::Declaration>(node).full_name());
     }
 
     if (node.is_type() && node.parent() && node.parent()->is_declaration()) {
-        return get_target_declaration(*node.parent(), verify_cast<Declaration>(node.parent())->full_name());
+        return get_target_declaration(*node.parent(), verify_cast<Cpp::Declaration>(node.parent())->full_name());
     }
 
     dbgln("get_target_declaration: Invalid argument node of type: {}", node.class_name());
@@ -487,7 +487,7 @@ static Optional<TargetDeclaration> get_target_declaration(ASTNode const& node, S
             return TargetDeclaration { TargetDeclaration::Type::Scope, name };
         }
         if (name_node.parent() && name_node.parent()->is_declaration()) {
-            auto declaration = verify_cast<Declaration>(name_node.parent());
+            auto declaration = verify_cast<Cpp::Declaration>(name_node.parent());
             if (declaration->is_struct_or_class() || declaration->is_enum()) {
                 return TargetDeclaration { TargetDeclaration::Type::Type, name };
             }
@@ -509,7 +509,7 @@ static Optional<TargetDeclaration> get_target_declaration(ASTNode const& node, S
 
     return TargetDeclaration { TargetDeclaration::Type::Variable, name };
 }
-RefPtr<Declaration> CppComprehensionEngine::find_declaration_of(DocumentData const& document_data, ASTNode const& node) const
+RefPtr<Cpp::Declaration> CppComprehensionEngine::find_declaration_of(DocumentData const& document_data, ASTNode const& node) const
 {
     dbgln_if(CPP_LANGUAGE_SERVER_DEBUG, "find_declaration_of: {} ({})", document_data.parser().text_of_node(node), node.class_name());
 
@@ -524,7 +524,7 @@ RefPtr<Declaration> CppComprehensionEngine::find_declaration_of(DocumentData con
         bool match_function = target_decl.value().type == TargetDeclaration::Function && symbol.declaration->is_function();
         bool match_variable = target_decl.value().type == TargetDeclaration::Variable && symbol.declaration->is_variable_declaration();
         bool match_type = target_decl.value().type == TargetDeclaration::Type && (symbol.declaration->is_struct_or_class() || symbol.declaration->is_enum());
-        bool match_property = target_decl.value().type == TargetDeclaration::Property && symbol.declaration->parent()->is_declaration() && verify_cast<Declaration>(symbol.declaration->parent())->is_struct_or_class();
+        bool match_property = target_decl.value().type == TargetDeclaration::Property && symbol.declaration->parent()->is_declaration() && verify_cast<Cpp::Declaration>(symbol.declaration->parent())->is_struct_or_class();
         bool match_parameter = target_decl.value().type == TargetDeclaration::Variable && symbol.declaration->is_parameter();
         bool match_scope = target_decl.value().type == TargetDeclaration::Scope && (symbol.declaration->is_namespace() || symbol.declaration->is_struct_or_class());
 
@@ -578,14 +578,14 @@ void CppComprehensionEngine::update_declared_symbols(DocumentData& document)
         document.m_symbols.set(symbol.name, move(symbol));
     }
 
-    Vector<GUI::AutocompleteProvider::Declaration> declarations;
+    Vector<CodeComprehension::Declaration> declarations;
     for (auto& symbol_entry : document.m_symbols) {
         auto& symbol = symbol_entry.value;
         declarations.append({ symbol.name.name, { document.filename(), symbol.declaration->start().line, symbol.declaration->start().column }, type_of_declaration(symbol.declaration), symbol.name.scope_as_string() });
     }
 
     for (auto& definition : document.preprocessor().definitions()) {
-        declarations.append({ definition.key, { document.filename(), definition.value.line, definition.value.column }, GUI::AutocompleteProvider::DeclarationType::PreprocessorDefinition, {} });
+        declarations.append({ definition.key, { document.filename(), definition.value.line, definition.value.column }, CodeComprehension::DeclarationType::PreprocessorDefinition, {} });
     }
     set_declarations_of_document(document.filename(), move(declarations));
 }
@@ -595,24 +595,24 @@ void CppComprehensionEngine::update_todo_entries(DocumentData& document)
     set_todo_entries_of_document(document.filename(), document.parser().get_todo_entries());
 }
 
-GUI::AutocompleteProvider::DeclarationType CppComprehensionEngine::type_of_declaration(Declaration const& decl)
+CodeComprehension::DeclarationType CppComprehensionEngine::type_of_declaration(Cpp::Declaration const& decl)
 {
     if (decl.is_struct())
-        return GUI::AutocompleteProvider::DeclarationType::Struct;
+        return CodeComprehension::DeclarationType::Struct;
     if (decl.is_class())
-        return GUI::AutocompleteProvider::DeclarationType::Class;
+        return CodeComprehension::DeclarationType::Class;
     if (decl.is_function())
-        return GUI::AutocompleteProvider::DeclarationType::Function;
+        return CodeComprehension::DeclarationType::Function;
     if (decl.is_variable_declaration())
-        return GUI::AutocompleteProvider::DeclarationType::Variable;
+        return CodeComprehension::DeclarationType::Variable;
     if (decl.is_namespace())
-        return GUI::AutocompleteProvider::DeclarationType::Namespace;
+        return CodeComprehension::DeclarationType::Namespace;
     if (decl.is_member())
-        return GUI::AutocompleteProvider::DeclarationType::Member;
-    return GUI::AutocompleteProvider::DeclarationType::Variable;
+        return CodeComprehension::DeclarationType::Member;
+    return CodeComprehension::DeclarationType::Variable;
 }
 
-OwnPtr<CppComprehensionEngine::DocumentData> CppComprehensionEngine::create_document_data(String&& text, String const& filename)
+OwnPtr<CppComprehensionEngine::DocumentData> CppComprehensionEngine::create_document_data(String text, String const& filename)
 {
     auto document_data = make<DocumentData>();
     document_data->m_filename = filename;
@@ -669,7 +669,7 @@ Vector<StringView> CppComprehensionEngine::scope_of_node(ASTNode const& node) co
     if (!parent->is_declaration())
         return parent_scope;
 
-    auto& parent_decl = static_cast<Declaration&>(*parent);
+    auto& parent_decl = static_cast<Cpp::Declaration&>(*parent);
 
     StringView containing_scope;
     if (parent_decl.is_namespace())
@@ -683,7 +683,7 @@ Vector<StringView> CppComprehensionEngine::scope_of_node(ASTNode const& node) co
     return parent_scope;
 }
 
-Optional<Vector<GUI::AutocompleteProvider::Entry>> CppComprehensionEngine::try_autocomplete_include(DocumentData const&, Token include_path_token, Cpp::Position const& cursor_position) const
+Optional<Vector<CodeComprehension::AutocompleteResultEntry>> CppComprehensionEngine::try_autocomplete_include(DocumentData const&, Token include_path_token, Cpp::Position const& cursor_position) const
 {
     VERIFY(include_path_token.type() == Token::Type::IncludePath);
     auto partial_include = include_path_token.text().trim_whitespace();
@@ -726,7 +726,7 @@ Optional<Vector<GUI::AutocompleteProvider::Entry>> CppComprehensionEngine::try_a
     dbgln_if(CPP_LANGUAGE_SERVER_DEBUG, "searching path: {}, partial_basename: {}", full_dir, partial_basename);
 
     Core::DirIterator it(full_dir, Core::DirIterator::Flags::SkipDots);
-    Vector<GUI::AutocompleteProvider::Entry> options;
+    Vector<CodeComprehension::AutocompleteResultEntry> options;
 
     auto prefix = include_type == System ? "<" : "\"";
     auto suffix = include_type == System ? ">" : "\"";
@@ -739,21 +739,21 @@ Optional<Vector<GUI::AutocompleteProvider::Entry>> CppComprehensionEngine::try_a
         if (Core::File::is_directory(LexicalPath::join(full_dir, path).string())) {
             // FIXME: Don't dismiss the autocomplete when filling these suggestions.
             auto completion = String::formatted("{}{}{}/", prefix, include_dir, path);
-            options.empend(completion, include_dir.length() + partial_basename.length() + 1, GUI::AutocompleteProvider::Language::Cpp, path, GUI::AutocompleteProvider::Entry::HideAutocompleteAfterApplying::No);
+            options.empend(completion, include_dir.length() + partial_basename.length() + 1, CodeComprehension::Language::Cpp, path, CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying::No);
         } else if (path.ends_with(".h")) {
             // FIXME: Place the cursor after the trailing > or ", even if it was
             //        already typed.
             auto completion = String::formatted("{}{}{}{}", prefix, include_dir, path, already_has_suffix ? "" : suffix);
-            options.empend(completion, include_dir.length() + partial_basename.length() + 1, GUI::AutocompleteProvider::Language::Cpp, path);
+            options.empend(completion, include_dir.length() + partial_basename.length() + 1, CodeComprehension::Language::Cpp, path);
         }
     }
 
     return options;
 }
 
-RefPtr<Declaration> CppComprehensionEngine::find_declaration_of(CppComprehensionEngine::DocumentData const& document, CppComprehensionEngine::SymbolName const& target_symbol_name) const
+RefPtr<Cpp::Declaration> CppComprehensionEngine::find_declaration_of(CppComprehensionEngine::DocumentData const& document, CppComprehensionEngine::SymbolName const& target_symbol_name) const
 {
-    RefPtr<Declaration> target_declaration;
+    RefPtr<Cpp::Declaration> target_declaration;
     for_each_available_symbol(document, [&](Symbol const& symbol) {
         if (symbol.name == target_symbol_name) {
             target_declaration = symbol.declaration;
@@ -929,7 +929,7 @@ Optional<CppComprehensionEngine::FunctionParamsHint> CppComprehensionEngine::get
     return hint;
 }
 
-Vector<GUI::AutocompleteProvider::TokenInfo> CppComprehensionEngine::get_tokens_info(String const& filename)
+Vector<CodeComprehension::TokenInfo> CppComprehensionEngine::get_tokens_info(String const& filename)
 {
     dbgln_if(CPP_LANGUAGE_SERVER_DEBUG, "CppComprehensionEngine::get_tokens_info: {}", filename);
 
@@ -939,73 +939,73 @@ Vector<GUI::AutocompleteProvider::TokenInfo> CppComprehensionEngine::get_tokens_
 
     auto const& document = *document_ptr;
 
-    Vector<GUI::AutocompleteProvider::TokenInfo> tokens_info;
+    Vector<CodeComprehension::TokenInfo> tokens_info;
     size_t i = 0;
     for (auto const& token : document.preprocessor().unprocessed_tokens()) {
 
         tokens_info.append({ get_token_semantic_type(document, token),
             token.start().line, token.start().column, token.end().line, token.end().column });
         ++i;
-        dbgln_if(CPP_LANGUAGE_SERVER_DEBUG, "{}: {}", token.text(), GUI::AutocompleteProvider::TokenInfo::type_to_string(tokens_info.last().type));
+        dbgln_if(CPP_LANGUAGE_SERVER_DEBUG, "{}: {}", token.text(), CodeComprehension::TokenInfo::type_to_string(tokens_info.last().type));
     }
     return tokens_info;
 }
 
-GUI::AutocompleteProvider::TokenInfo::SemanticType CppComprehensionEngine::get_token_semantic_type(DocumentData const& document, Token const& token)
+CodeComprehension::TokenInfo::SemanticType CppComprehensionEngine::get_token_semantic_type(DocumentData const& document, Token const& token)
 {
     using GUI::AutocompleteProvider;
     switch (token.type()) {
     case Cpp::Token::Type::Identifier:
         return get_semantic_type_for_identifier(document, token.start());
     case Cpp::Token::Type::Keyword:
-        return AutocompleteProvider::TokenInfo::SemanticType::Keyword;
+        return CodeComprehension::TokenInfo::SemanticType::Keyword;
     case Cpp::Token::Type::KnownType:
-        return AutocompleteProvider::TokenInfo::SemanticType::Type;
+        return CodeComprehension::TokenInfo::SemanticType::Type;
     case Cpp::Token::Type::DoubleQuotedString:
     case Cpp::Token::Type::SingleQuotedString:
     case Cpp::Token::Type::RawString:
-        return AutocompleteProvider::TokenInfo::SemanticType::String;
+        return CodeComprehension::TokenInfo::SemanticType::String;
     case Cpp::Token::Type::Integer:
     case Cpp::Token::Type::Float:
-        return AutocompleteProvider::TokenInfo::SemanticType::Number;
+        return CodeComprehension::TokenInfo::SemanticType::Number;
     case Cpp::Token::Type::IncludePath:
-        return AutocompleteProvider::TokenInfo::SemanticType::IncludePath;
+        return CodeComprehension::TokenInfo::SemanticType::IncludePath;
     case Cpp::Token::Type::EscapeSequence:
-        return AutocompleteProvider::TokenInfo::SemanticType::Keyword;
+        return CodeComprehension::TokenInfo::SemanticType::Keyword;
     case Cpp::Token::Type::PreprocessorStatement:
     case Cpp::Token::Type::IncludeStatement:
-        return AutocompleteProvider::TokenInfo::SemanticType::PreprocessorStatement;
+        return CodeComprehension::TokenInfo::SemanticType::PreprocessorStatement;
     case Cpp::Token::Type::Comment:
-        return AutocompleteProvider::TokenInfo::SemanticType::Comment;
+        return CodeComprehension::TokenInfo::SemanticType::Comment;
     default:
-        return AutocompleteProvider::TokenInfo::SemanticType::Unknown;
+        return CodeComprehension::TokenInfo::SemanticType::Unknown;
     }
 }
 
-GUI::AutocompleteProvider::TokenInfo::SemanticType CppComprehensionEngine::get_semantic_type_for_identifier(DocumentData const& document, Position position)
+CodeComprehension::TokenInfo::SemanticType CppComprehensionEngine::get_semantic_type_for_identifier(DocumentData const& document, Position position)
 {
     if (find_preprocessor_substitution(document, position).has_value())
-        return GUI::AutocompleteProvider::TokenInfo::SemanticType::PreprocessorMacro;
+        return CodeComprehension::TokenInfo::SemanticType::PreprocessorMacro;
 
     auto decl = find_declaration_of(document, GUI::TextPosition { position.line, position.column });
     if (!decl)
-        return GUI::AutocompleteProvider::TokenInfo::SemanticType::Identifier;
+        return CodeComprehension::TokenInfo::SemanticType::Identifier;
 
     if (decl->is_function())
-        return GUI::AutocompleteProvider::TokenInfo::SemanticType::Function;
+        return CodeComprehension::TokenInfo::SemanticType::Function;
     if (decl->is_parameter())
-        return GUI::AutocompleteProvider::TokenInfo::SemanticType::Parameter;
+        return CodeComprehension::TokenInfo::SemanticType::Parameter;
     if (decl->is_variable_declaration()) {
         if (decl->is_member())
-            return GUI::AutocompleteProvider::TokenInfo::SemanticType::Member;
-        return GUI::AutocompleteProvider::TokenInfo::SemanticType::Variable;
+            return CodeComprehension::TokenInfo::SemanticType::Member;
+        return CodeComprehension::TokenInfo::SemanticType::Variable;
     }
     if (decl->is_struct_or_class() || decl->is_enum())
-        return GUI::AutocompleteProvider::TokenInfo::SemanticType::CustomType;
+        return CodeComprehension::TokenInfo::SemanticType::CustomType;
     if (decl->is_namespace())
-        return GUI::AutocompleteProvider::TokenInfo::SemanticType::Namespace;
+        return CodeComprehension::TokenInfo::SemanticType::Namespace;
 
-    return GUI::AutocompleteProvider::TokenInfo::SemanticType::Identifier;
+    return CodeComprehension::TokenInfo::SemanticType::Identifier;
 }
 
 }

Userland/DevTools/HackStudio/LanguageServers/Cpp/CppComprehensionEngine.h → Userland/Libraries/LibCodeComprehension/Cpp/CppComprehensionEngine.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Itamar S. <itamar8910@gmail.com>
+ * Copyright (c) 2021-2022, Itamar S. <itamar8910@gmail.com>
  *
  * SPDX-License-Identifier: BSD-2-Clause
  */
@@ -10,14 +10,14 @@
 #include <AK/String.h>
 #include <AK/Vector.h>
 #include <DevTools/HackStudio/AutoCompleteResponse.h>
-#include <DevTools/HackStudio/LanguageServers/CodeComprehensionEngine.h>
 #include <DevTools/HackStudio/LanguageServers/FileDB.h>
 #include <LibCpp/AST.h>
 #include <LibCpp/Parser.h>
 #include <LibCpp/Preprocessor.h>
 #include <LibGUI/TextPosition.h>
+#include <Libraries/LibCodeComprehension/CodeComprehensionEngine.h>
 
-namespace LanguageServers::Cpp {
+namespace CodeComprehension::Cpp {
 
 using namespace ::Cpp;
 
@@ -25,12 +25,12 @@ class CppComprehensionEngine : public CodeComprehensionEngine {
 public:
     CppComprehensionEngine(FileDB const& filedb);
 
-    virtual Vector<GUI::AutocompleteProvider::Entry> get_suggestions(String const& file, const GUI::TextPosition& autocomplete_position) override;
+    virtual Vector<CodeComprehension::AutocompleteResultEntry> get_suggestions(String const& file, GUI::TextPosition const& autocomplete_position) override;
     virtual void on_edit(String const& file) override;
     virtual void file_opened([[maybe_unused]] String const& file) override;
-    virtual Optional<GUI::AutocompleteProvider::ProjectLocation> find_declaration_of(String const& filename, const GUI::TextPosition& identifier_position) override;
-    virtual Optional<FunctionParamsHint> get_function_params_hint(String const&, const GUI::TextPosition&) override;
-    virtual Vector<GUI::AutocompleteProvider::TokenInfo> get_tokens_info(String const& filename) override;
+    virtual Optional<CodeComprehension::ProjectLocation> find_declaration_of(String const& filename, GUI::TextPosition const& identifier_position) override;
+    virtual Optional<FunctionParamsHint> get_function_params_hint(String const&, GUI::TextPosition const&) override;
+    virtual Vector<CodeComprehension::TokenInfo> get_tokens_info(String const& filename) override;
 
 private:
     struct SymbolName {
@@ -47,7 +47,7 @@ private:
 
     struct Symbol {
         SymbolName name;
-        NonnullRefPtr<Declaration> declaration;
+        NonnullRefPtr<Cpp::Declaration> declaration;
 
         // Local symbols are symbols that should not appear in a global symbol search.
         // For example, a variable that is declared inside a function will have is_local = true.
@@ -57,7 +57,7 @@ private:
             No,
             Yes
         };
-        static Symbol create(StringView name, Vector<StringView> const& scope, NonnullRefPtr<Declaration>, IsLocal is_local);
+        static Symbol create(StringView name, Vector<StringView> const& scope, NonnullRefPtr<Cpp::Declaration>, IsLocal is_local);
     };
 
     friend Traits<SymbolName>;
@@ -95,15 +95,15 @@ private:
         HashTable<String> m_available_headers;
     };
 
-    Vector<GUI::AutocompleteProvider::Entry> autocomplete_property(DocumentData const&, MemberExpression const&, const String partial_text) const;
-    Vector<GUI::AutocompleteProvider::Entry> autocomplete_name(DocumentData const&, ASTNode const&, String const& partial_text) const;
+    Vector<CodeComprehension::AutocompleteResultEntry> autocomplete_property(DocumentData const&, MemberExpression const&, const String partial_text) const;
+    Vector<AutocompleteResultEntry> autocomplete_name(DocumentData const&, ASTNode const&, String const& partial_text) const;
     String type_of(DocumentData const&, Expression const&) const;
     String type_of_property(DocumentData const&, Identifier const&) const;
     String type_of_variable(Identifier const&) const;
     bool is_property(ASTNode const&) const;
-    RefPtr<Declaration> find_declaration_of(DocumentData const&, ASTNode const&) const;
-    RefPtr<Declaration> find_declaration_of(DocumentData const&, SymbolName const&) const;
-    RefPtr<Declaration> find_declaration_of(DocumentData const&, const GUI::TextPosition& identifier_position);
+    RefPtr<Cpp::Declaration> find_declaration_of(DocumentData const&, ASTNode const&) const;
+    RefPtr<Cpp::Declaration> find_declaration_of(DocumentData const&, SymbolName const&) const;
+    RefPtr<Cpp::Declaration> find_declaration_of(DocumentData const&, const GUI::TextPosition& identifier_position);
 
     enum class RecurseIntoScopes {
         No,
@@ -122,17 +122,17 @@ private:
     String document_path_from_include_path(StringView include_path) const;
     void update_declared_symbols(DocumentData&);
     void update_todo_entries(DocumentData&);
-    GUI::AutocompleteProvider::DeclarationType type_of_declaration(Declaration const&);
+    CodeComprehension::DeclarationType type_of_declaration(Cpp::Declaration const&);
     Vector<StringView> scope_of_node(ASTNode const&) const;
     Vector<StringView> scope_of_reference_to_symbol(ASTNode const&) const;
 
-    Optional<GUI::AutocompleteProvider::ProjectLocation> find_preprocessor_definition(DocumentData const&, const GUI::TextPosition&);
+    Optional<CodeComprehension::ProjectLocation> find_preprocessor_definition(DocumentData const&, const GUI::TextPosition&);
     Optional<Cpp::Preprocessor::Substitution> find_preprocessor_substitution(DocumentData const&, Cpp::Position const&);
 
-    OwnPtr<DocumentData> create_document_data(String&& text, String const& filename);
-    Optional<Vector<GUI::AutocompleteProvider::Entry>> try_autocomplete_property(DocumentData const&, ASTNode const&, Optional<Token> containing_token) const;
-    Optional<Vector<GUI::AutocompleteProvider::Entry>> try_autocomplete_name(DocumentData const&, ASTNode const&, Optional<Token> containing_token) const;
-    Optional<Vector<GUI::AutocompleteProvider::Entry>> try_autocomplete_include(DocumentData const&, Token include_path_token, Cpp::Position const& cursor_position) const;
+    OwnPtr<DocumentData> create_document_data(String text, String const& filename);
+    Optional<Vector<CodeComprehension::AutocompleteResultEntry>> try_autocomplete_property(DocumentData const&, ASTNode const&, Optional<Token> containing_token) const;
+    Optional<Vector<CodeComprehension::AutocompleteResultEntry>> try_autocomplete_name(DocumentData const&, ASTNode const&, Optional<Token> containing_token) const;
+    Optional<Vector<CodeComprehension::AutocompleteResultEntry>> try_autocomplete_include(DocumentData const&, Token include_path_token, Cpp::Position const& cursor_position) const;
     static bool is_symbol_available(Symbol const&, Vector<StringView> const& current_scope, Vector<StringView> const& reference_scope);
     Optional<FunctionParamsHint> get_function_params_hint(DocumentData const&, FunctionCall&, size_t argument_index);
 
@@ -142,8 +142,8 @@ private:
     template<typename Func>
     void for_each_included_document_recursive(DocumentData const&, Func) const;
 
-    GUI::AutocompleteProvider::TokenInfo::SemanticType get_token_semantic_type(DocumentData const&, Token const&);
-    GUI::AutocompleteProvider::TokenInfo::SemanticType get_semantic_type_for_identifier(DocumentData const&, Position);
+    CodeComprehension::TokenInfo::SemanticType get_token_semantic_type(DocumentData const&, Token const&);
+    CodeComprehension::TokenInfo::SemanticType get_semantic_type_for_identifier(DocumentData const&, Position);
 
     HashMap<String, OwnPtr<DocumentData>> m_documents;
 
@@ -189,8 +189,8 @@ void CppComprehensionEngine::for_each_included_document_recursive(DocumentData c
 namespace AK {
 
 template<>
-struct Traits<LanguageServers::Cpp::CppComprehensionEngine::SymbolName> : public GenericTraits<LanguageServers::Cpp::CppComprehensionEngine::SymbolName> {
-    static unsigned hash(LanguageServers::Cpp::CppComprehensionEngine::SymbolName const& key)
+struct Traits<CodeComprehension::Cpp::CppComprehensionEngine::SymbolName> : public GenericTraits<CodeComprehension::Cpp::CppComprehensionEngine::SymbolName> {
+    static unsigned hash(CodeComprehension::Cpp::CppComprehensionEngine::SymbolName const& key)
     {
         unsigned hash = 0;
         hash = pair_int_hash(hash, string_hash(key.name.characters_without_null_termination(), key.name.length()));

Userland/DevTools/HackStudio/LanguageServers/Cpp/Tests.cpp → Userland/Libraries/LibCodeComprehension/Cpp/Tests.cpp

@@ -9,9 +9,7 @@
 #include "CppComprehensionEngine.h"
 #include <AK/LexicalPath.h>
 #include <LibCore/File.h>
-
-using namespace LanguageServers;
-using namespace LanguageServers::Cpp;
+#include <LibMain/Main.h>
 
 static bool s_some_test_failed = false;
 
@@ -37,6 +35,28 @@ static bool s_some_test_failed = false;
 
 constexpr char TESTS_ROOT_DIR[] = "/home/anon/Tests/cpp-tests/comprehension";
 
+class FileDB : public CodeComprehension::FileDB {
+public:
+    FileDB() = default;
+
+    void add(String filename, String content)
+    {
+        m_map.set(filename, content);
+    }
+
+    virtual Optional<String> get_or_read_from_filesystem(StringView filename) const override
+    {
+        String target_filename = filename;
+        if (!project_root().is_null() && filename.starts_with(project_root())) {
+            target_filename = LexicalPath::relative_path(filename, project_root());
+        }
+        return m_map.get(target_filename);
+    }
+
+private:
+    HashMap<String, String> m_map;
+};
+
 static void test_complete_local_args();
 static void test_complete_local_vars();
 static void test_complete_type();
@@ -59,7 +79,7 @@ static void add_file(FileDB& filedb, String const& name)
 {
     auto file = Core::File::open(LexicalPath::join(TESTS_ROOT_DIR, name).string(), Core::OpenMode::ReadOnly);
     VERIFY(!file.is_error());
-    filedb.add(name, file.value()->fd());
+    filedb.add(name, String::copy(file.value()->read_all()));
 }
 
 void test_complete_local_args()
@@ -67,7 +87,7 @@ void test_complete_local_args()
     I_TEST(Complete Local Args)
     FileDB filedb;
     add_file(filedb, "complete_local_args.cpp");
-    CppComprehensionEngine engine(filedb);
+    CodeComprehension::Cpp::CppComprehensionEngine engine(filedb);
     auto suggestions = engine.get_suggestions("complete_local_args.cpp", { 2, 6 });
     if (suggestions.size() != 2)
         FAIL(bad size);
@@ -83,7 +103,7 @@ void test_complete_local_vars()
     I_TEST(Complete Local Vars)
     FileDB filedb;
     add_file(filedb, "complete_local_vars.cpp");
-    CppComprehensionEngine autocomplete(filedb);
+    CodeComprehension::Cpp::CppComprehensionEngine autocomplete(filedb);
     auto suggestions = autocomplete.get_suggestions("complete_local_vars.cpp", { 3, 7 });
     if (suggestions.size() != 1)
         FAIL(bad size);
@@ -99,7 +119,7 @@ void test_complete_type()
     I_TEST(Complete Type)
     FileDB filedb;
     add_file(filedb, "complete_type.cpp");
-    CppComprehensionEngine autocomplete(filedb);
+    CodeComprehension::Cpp::CppComprehensionEngine autocomplete(filedb);
     auto suggestions = autocomplete.get_suggestions("complete_type.cpp", { 5, 7 });
     if (suggestions.size() != 1)
         FAIL(bad size);
@@ -115,7 +135,7 @@ void test_find_variable_definition()
     I_TEST(Find Variable Declaration)
     FileDB filedb;
     add_file(filedb, "find_variable_declaration.cpp");
-    CppComprehensionEngine engine(filedb);
+    CodeComprehension::Cpp::CppComprehensionEngine engine(filedb);
     auto position = engine.find_declaration_of("find_variable_declaration.cpp", { 2, 5 });
     if (!position.has_value())
         FAIL("declaration not found");
@@ -127,12 +147,12 @@ void test_find_variable_definition()
 
 void test_complete_includes()
 {
-    I_TEST(Complete Type)
+    I_TEST(Complete include statements)
     FileDB filedb;
     filedb.set_project_root(TESTS_ROOT_DIR);
     add_file(filedb, "complete_includes.cpp");
     add_file(filedb, "sample_header.h");
-    CppComprehensionEngine autocomplete(filedb);
+    CodeComprehension::Cpp::CppComprehensionEngine autocomplete(filedb);
 
     auto suggestions = autocomplete.get_suggestions("complete_includes.cpp", { 0, 22 });
     if (suggestions.size() != 1)
@@ -157,7 +177,7 @@ void test_parameters_hint()
     FileDB filedb;
     filedb.set_project_root(TESTS_ROOT_DIR);
     add_file(filedb, "parameters_hint1.cpp");
-    CppComprehensionEngine engine(filedb);
+    CodeComprehension::Cpp::CppComprehensionEngine engine(filedb);
 
     auto result = engine.get_function_params_hint("parameters_hint1.cpp", { 4, 9 });
     if (!result.has_value())
@@ -179,3 +199,8 @@ void test_parameters_hint()
 
     PASS;
 }
+
+ErrorOr<int> serenity_main(Main::Arguments)
+{
+    return run_tests();
+}

Userland/Libraries/LibCodeComprehension/FileDB.cpp

@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2022, Itamar S. <itamar8910@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "FileDB.h"
+#include <AK/LexicalPath.h>
+
+namespace CodeComprehension {
+
+String FileDB::to_absolute_path(StringView filename) const
+{
+    if (LexicalPath { filename }.is_absolute()) {
+        return filename;
+    }
+    if (m_project_root.is_null())
+        return filename;
+    return LexicalPath { String::formatted("{}/{}", m_project_root, filename) }.string();
+}
+
+}

Userland/Libraries/LibCodeComprehension/FileDB.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2022, Itamar S. <itamar8910@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/String.h>
+#include <AK/StringView.h>
+
+namespace CodeComprehension {
+
+class FileDB {
+    AK_MAKE_NONCOPYABLE(FileDB);
+    AK_MAKE_NONMOVABLE(FileDB);
+
+public:
+    virtual ~FileDB() = default;
+
+    virtual Optional<String> get_or_read_from_filesystem(StringView filename) const = 0;
+    void set_project_root(StringView project_root) { m_project_root = project_root; }
+    String const& project_root() const { return m_project_root; }
+    String to_absolute_path(StringView filename) const;
+
+protected:
+    FileDB() = default;
+
+private:
+    String m_project_root;
+};
+
+}

Userland/Libraries/LibCodeComprehension/Shell/CMakeLists.txt

@@ -0,0 +1,6 @@
+set(SOURCES
+    ShellComprehensionEngine.cpp
+)
+
+serenity_lib(LibShellComprehension shellcomprehension)
+target_link_libraries(LibShellComprehension LibCodeComprehension LibC)

Userland/Libraries/LibCodeComprehension/Shell/ConnectionFromClient.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2020, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include "ShellComprehensionEngine.h"
+#include <DevTools/HackStudio/LanguageServers/ConnectionFromClient.h>
+#include <LibCpp/Parser.h>
+
+namespace LanguageServers::Shell {
+
+class ConnectionFromClient final : public LanguageServers::ConnectionFromClient {
+    C_OBJECT(ConnectionFromClient);
+
+private:
+    ConnectionFromClient(NonnullOwnPtr<Core::Stream::LocalSocket> socket)
+        : LanguageServers::ConnectionFromClient(move(socket))
+    {
+        m_autocomplete_engine = make<ShellComprehensionEngine>(m_filedb);
+        m_autocomplete_engine->set_declarations_of_document_callback = [this](String const& filename, Vector<CodeComprehension::Declaration>&& declarations) {
+            async_declarations_in_document(filename, move(declarations));
+        };
+        m_autocomplete_engine->set_todo_entries_of_document_callback = [this](String const& filename, Vector<CodeComprehension::TodoEntry>&& todo_entries) {
+            async_todo_entries_in_document(filename, move(todo_entries));
+        };
+    }
+    virtual ~ConnectionFromClient() override = default;
+};
+}

Userland/DevTools/HackStudio/LanguageServers/Shell/ShellComprehensionEngine.cpp → Userland/Libraries/LibCodeComprehension/Shell/ShellComprehensionEngine.cpp

@@ -8,9 +8,8 @@
 #include <AK/Assertions.h>
 #include <AK/HashTable.h>
 #include <LibRegex/Regex.h>
-#include <Userland/DevTools/HackStudio/LanguageServers/ConnectionFromClient.h>
 
-namespace LanguageServers::Shell {
+namespace CodeComprehension::Shell {
 
 RefPtr<::Shell::Shell> ShellComprehensionEngine::s_shell {};
 
@@ -38,11 +37,12 @@ ShellComprehensionEngine::DocumentData const& ShellComprehensionEngine::get_docu
 
 OwnPtr<ShellComprehensionEngine::DocumentData> ShellComprehensionEngine::create_document_data_for(String const& file)
 {
-    auto document = filedb().get(file);
-    if (!document)
+    auto document = filedb().get_or_read_from_filesystem(file);
+    if (!document.has_value())
         return {};
-    auto content = document->text();
-    auto document_data = make<DocumentData>(document->text(), file);
+
+    auto content = document.value();
+    auto document_data = make<DocumentData>(move(content), file);
     for (auto& path : document_data->sourced_paths())
         get_or_create_document_data(path);
 
@@ -133,7 +133,7 @@ size_t ShellComprehensionEngine::resolve(ShellComprehensionEngine::DocumentData
     return offset;
 }
 
-Vector<GUI::AutocompleteProvider::Entry> ShellComprehensionEngine::get_suggestions(String const& file, const GUI::TextPosition& position)
+Vector<CodeComprehension::AutocompleteResultEntry> ShellComprehensionEngine::get_suggestions(String const& file, const GUI::TextPosition& position)
 {
     dbgln_if(SH_LANGUAGE_SERVER_DEBUG, "ShellComprehensionEngine position {}:{}", position.line(), position.column());
 
@@ -147,7 +147,7 @@ Vector<GUI::AutocompleteProvider::Entry> ShellComprehensionEngine::get_suggestio
     }
 
     auto completions = const_cast<::Shell::AST::Node*>(document.node.ptr())->complete_for_editor(shell(), offset_in_file, hit_test);
-    Vector<GUI::AutocompleteProvider::Entry> entries;
+    Vector<CodeComprehension::AutocompleteResultEntry> entries;
     for (auto& completion : completions)
         entries.append({ completion.text_string, completion.input_offset });
 
@@ -164,7 +164,7 @@ void ShellComprehensionEngine::file_opened([[maybe_unused]] String const& file)
     set_document_data(file, create_document_data_for(file));
 }
 
-Optional<GUI::AutocompleteProvider::ProjectLocation> ShellComprehensionEngine::find_declaration_of(String const& filename, const GUI::TextPosition& identifier_position)
+Optional<CodeComprehension::ProjectLocation> ShellComprehensionEngine::find_declaration_of(String const& filename, const GUI::TextPosition& identifier_position)
 {
     dbgln_if(SH_LANGUAGE_SERVER_DEBUG, "find_declaration_of({}, {}:{})", filename, identifier_position.line(), identifier_position.column());
     auto const& document = get_or_create_document_data(filename);
@@ -213,7 +213,7 @@ void ShellComprehensionEngine::update_declared_symbols(DocumentData const& docum
 
                 if (!name.is_empty()) {
                     dbgln("Found variable {}", name);
-                    declarations.append({ move(name), { filename, entry.name->position().start_line.line_number, entry.name->position().start_line.line_column }, GUI::AutocompleteProvider::DeclarationType::Variable, {} });
+                    declarations.append({ move(name), { filename, entry.name->position().start_line.line_number, entry.name->position().start_line.line_column }, CodeComprehension::DeclarationType::Variable, {} });
                 }
             }
             ::Shell::AST::NodeVisitor::visit(node);
@@ -222,11 +222,11 @@ void ShellComprehensionEngine::update_declared_symbols(DocumentData const& docum
         void visit(const ::Shell::AST::FunctionDeclaration* node) override
         {
             dbgln("Found function {}", node->name().name);
-            declarations.append({ node->name().name, { filename, node->position().start_line.line_number, node->position().start_line.line_column }, GUI::AutocompleteProvider::DeclarationType::Function, {} });
+            declarations.append({ node->name().name, { filename, node->position().start_line.line_number, node->position().start_line.line_column }, CodeComprehension::DeclarationType::Function, {} });
         }
 
         String const& filename;
-        Vector<GUI::AutocompleteProvider::Declaration> declarations;
+        Vector<CodeComprehension::Declaration> declarations;
     } visitor { document.filename };
 
     document.node->visit(visitor);

Userland/DevTools/HackStudio/LanguageServers/Shell/ShellComprehensionEngine.h → Userland/Libraries/LibCodeComprehension/Shell/ShellComprehensionEngine.h

@@ -6,18 +6,18 @@
 
 #pragma once
 
-#include <DevTools/HackStudio/LanguageServers/CodeComprehensionEngine.h>
+#include <LibCodeComprehension/CodeComprehensionEngine.h>
 #include <Shell/Shell.h>
 
-namespace LanguageServers::Shell {
+namespace CodeComprehension::Shell {
 
 class ShellComprehensionEngine : public CodeComprehensionEngine {
 public:
     ShellComprehensionEngine(FileDB const& filedb);
-    virtual Vector<GUI::AutocompleteProvider::Entry> get_suggestions(String const& file, const GUI::TextPosition& position) override;
+    virtual Vector<CodeComprehension::AutocompleteResultEntry> get_suggestions(String const& file, const GUI::TextPosition& position) override;
     virtual void on_edit(String const& file) override;
     virtual void file_opened([[maybe_unused]] String const& file) override;
-    virtual Optional<GUI::AutocompleteProvider::ProjectLocation> find_declaration_of(String const& filename, const GUI::TextPosition& identifier_position) override;
+    virtual Optional<CodeComprehension::ProjectLocation> find_declaration_of(String const& filename, const GUI::TextPosition& identifier_position) override;
 
 private:
     struct DocumentData {

Userland/Libraries/LibCodeComprehension/Shell/main.cpp

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2020, the SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "ConnectionFromClient.h"
+#include <LibCore/EventLoop.h>
+#include <LibCore/LocalServer.h>
+#include <LibCore/System.h>
+#include <LibIPC/SingleServer.h>
+#include <LibMain/Main.h>
+
+ErrorOr<int> serenity_main(Main::Arguments)
+{
+    Core::EventLoop event_loop;
+    TRY(Core::System::pledge("stdio unix rpath recvfd"));
+
+    auto client = TRY(IPC::take_over_accepted_client_from_system_server<LanguageServers::Shell::ConnectionFromClient>());
+
+    TRY(Core::System::pledge("stdio rpath recvfd"));
+    TRY(Core::System::unveil("/etc/passwd", "r"));
+
+    return event_loop.exec();
+}

Userland/Libraries/LibCodeComprehension/Types.h

@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2022, Itamar S. <itamar8910@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/String.h>
+
+#pragma once
+
+namespace CodeComprehension {
+
+enum class Language {
+    Unspecified,
+    Cpp,
+};
+
+struct AutocompleteResultEntry {
+    String completion;
+    size_t partial_input_length { 0 };
+    // TODO: Actually assign the value of this field in more places (when applicable).
+    Language language { Language::Unspecified };
+    String display_text {};
+
+    enum class HideAutocompleteAfterApplying {
+        No,
+        Yes,
+    };
+    HideAutocompleteAfterApplying hide_autocomplete_after_applying { HideAutocompleteAfterApplying::Yes };
+};
+
+struct ProjectLocation {
+    String file;
+    size_t line { 0 };
+    size_t column { 0 };
+
+    bool operator==(ProjectLocation const& other) const
+    {
+        return file == other.file && line == other.line && column == other.column;
+    }
+};
+
+enum class DeclarationType {
+    Function,
+    Struct,
+    Class,
+    Variable,
+    PreprocessorDefinition,
+    Namespace,
+    Member,
+};
+
+struct Declaration {
+    String name;
+    ProjectLocation position;
+    DeclarationType type;
+    String scope;
+
+    bool operator==(Declaration const& other) const
+    {
+        return name == other.name && position == other.position && type == other.type && scope == other.scope;
+    }
+};
+
+#define FOR_EACH_SEMANTIC_TYPE        \
+    __SEMANTIC(Unknown)               \
+    __SEMANTIC(Regular)               \
+    __SEMANTIC(Keyword)               \
+    __SEMANTIC(Type)                  \
+    __SEMANTIC(Identifier)            \
+    __SEMANTIC(String)                \
+    __SEMANTIC(Number)                \
+    __SEMANTIC(IncludePath)           \
+    __SEMANTIC(PreprocessorStatement) \
+    __SEMANTIC(Comment)               \
+    __SEMANTIC(Whitespace)            \
+    __SEMANTIC(Function)              \
+    __SEMANTIC(Variable)              \
+    __SEMANTIC(CustomType)            \
+    __SEMANTIC(Namespace)             \
+    __SEMANTIC(Member)                \
+    __SEMANTIC(Parameter)             \
+    __SEMANTIC(PreprocessorMacro)
+
+struct TokenInfo {
+
+    enum class SemanticType : u32 {
+#define __SEMANTIC(x) x,
+        FOR_EACH_SEMANTIC_TYPE
+#undef __SEMANTIC
+
+    } type { SemanticType::Unknown };
+    size_t start_line { 0 };
+    size_t start_column { 0 };
+    size_t end_line { 0 };
+    size_t end_column { 0 };
+
+    static constexpr char const* type_to_string(SemanticType t)
+    {
+        switch (t) {
+#define __SEMANTIC(x)     \
+    case SemanticType::x: \
+        return #x;
+            FOR_EACH_SEMANTIC_TYPE
+#undef __SEMANTIC
+        }
+        VERIFY_NOT_REACHED();
+    };
+};
+
+struct TodoEntry {
+    String content;
+    String filename;
+    size_t line { 0 };
+    size_t column { 0 };
+};
+
+}

Userland/Libraries/LibCpp/Parser.cpp

@@ -1012,9 +1012,9 @@ void Parser::print_tokens() const
     }
 }
 
-Vector<Parser::TodoEntry> Parser::get_todo_entries() const
+Vector<CodeComprehension::TodoEntry> Parser::get_todo_entries() const
 {
-    Vector<TodoEntry> ret;
+    Vector<CodeComprehension::TodoEntry> ret;
     for (auto& token : m_tokens) {
         if (token.type() == Token::Type::Comment) {
             if (token.text().contains("TODO")) {

Userland/Libraries/LibCpp/Parser.h

@@ -10,6 +10,7 @@
 #include "AST.h"
 #include "Preprocessor.h"
 #include <AK/Noncopyable.h>
+#include <LibCodeComprehension/Types.h>
 #include <LibCpp/Lexer.h>
 
 namespace Cpp {
@@ -35,13 +36,7 @@ public:
     Vector<Token> const& tokens() const { return m_tokens; }
     Vector<String> const& errors() const { return m_errors; }
 
-    struct TodoEntry {
-        String content;
-        String filename;
-        size_t line { 0 };
-        size_t column { 0 };
-    };
-    Vector<TodoEntry> get_todo_entries() const;
+    Vector<CodeComprehension::TodoEntry> get_todo_entries() const;
 
     Vector<Token> tokens_in_range(Position start, Position end) const;
 

Userland/Libraries/LibCpp/SemanticSyntaxHighlighter.cpp

@@ -15,7 +15,7 @@ namespace Cpp {
 
 void SemanticSyntaxHighlighter::rehighlight(Palette const& palette)
 {
-    Vector<GUI::AutocompleteProvider::TokenInfo> new_tokens_info;
+    Vector<CodeComprehension::TokenInfo> new_tokens_info;
     auto text = m_client->get_text();
     {
         Threading::MutexLocker locker(m_lock);
@@ -39,7 +39,7 @@ void SemanticSyntaxHighlighter::rehighlight(Palette const& palette)
         //        An improvement over this could be only including the tokens that are in edited text ranges in the diff.
         auto diff_hunks = Diff::from_text(previous.view(), current.view());
         for (auto& token : current_tokens) {
-            new_tokens_info.append(GUI::AutocompleteProvider::TokenInfo { GUI::AutocompleteProvider::TokenInfo::SemanticType::Unknown,
+            new_tokens_info.append(CodeComprehension::TokenInfo { CodeComprehension::TokenInfo::SemanticType::Unknown,
                 token.start().line, token.start().column, token.end().line, token.end().column });
         }
         size_t previous_token_index = 0;
@@ -67,47 +67,47 @@ void SemanticSyntaxHighlighter::rehighlight(Palette const& palette)
     update_spans(new_tokens_info, palette);
 }
 
-static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, GUI::AutocompleteProvider::TokenInfo::SemanticType type)
+static Syntax::TextStyle style_for_token_type(Gfx::Palette const& palette, CodeComprehension::TokenInfo::SemanticType type)
 {
     switch (type) {
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Unknown:
+    case CodeComprehension::TokenInfo::SemanticType::Unknown:
         return { palette.base_text(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Keyword:
+    case CodeComprehension::TokenInfo::SemanticType::Keyword:
         return { palette.syntax_keyword(), true };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Type:
+    case CodeComprehension::TokenInfo::SemanticType::Type:
         return { palette.syntax_type(), true };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Identifier:
+    case CodeComprehension::TokenInfo::SemanticType::Identifier:
         return { palette.syntax_identifier(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::String:
+    case CodeComprehension::TokenInfo::SemanticType::String:
         return { palette.syntax_string(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Number:
+    case CodeComprehension::TokenInfo::SemanticType::Number:
         return { palette.syntax_number(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::IncludePath:
+    case CodeComprehension::TokenInfo::SemanticType::IncludePath:
         return { palette.syntax_preprocessor_value(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::PreprocessorStatement:
+    case CodeComprehension::TokenInfo::SemanticType::PreprocessorStatement:
         return { palette.syntax_preprocessor_statement(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Comment:
+    case CodeComprehension::TokenInfo::SemanticType::Comment:
         return { palette.syntax_comment(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Function:
+    case CodeComprehension::TokenInfo::SemanticType::Function:
         return { palette.syntax_function(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Variable:
+    case CodeComprehension::TokenInfo::SemanticType::Variable:
         return { palette.syntax_variable(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::CustomType:
+    case CodeComprehension::TokenInfo::SemanticType::CustomType:
         return { palette.syntax_custom_type(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Namespace:
+    case CodeComprehension::TokenInfo::SemanticType::Namespace:
         return { palette.syntax_namespace(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Member:
+    case CodeComprehension::TokenInfo::SemanticType::Member:
         return { palette.syntax_member(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::Parameter:
+    case CodeComprehension::TokenInfo::SemanticType::Parameter:
         return { palette.syntax_parameter(), false };
-    case GUI::AutocompleteProvider::TokenInfo::SemanticType::PreprocessorMacro:
+    case CodeComprehension::TokenInfo::SemanticType::PreprocessorMacro:
         return { palette.syntax_preprocessor_value(), false };
     default:
         VERIFY_NOT_REACHED();
         return { palette.base_text(), false };
     }
 }
-void SemanticSyntaxHighlighter::update_spans(Vector<GUI::AutocompleteProvider::TokenInfo> const& tokens_info, Gfx::Palette const& pallete)
+void SemanticSyntaxHighlighter::update_spans(Vector<CodeComprehension::TokenInfo> const& tokens_info, Gfx::Palette const& pallete)
 {
     Vector<GUI::TextDocumentSpan> spans;
     for (auto& token : tokens_info) {
@@ -118,7 +118,7 @@ void SemanticSyntaxHighlighter::update_spans(Vector<GUI::AutocompleteProvider::T
         auto style = style_for_token_type(pallete, token.type);
         span.attributes.color = style.color;
         span.attributes.bold = style.bold;
-        span.is_skippable = token.type == GUI::AutocompleteProvider::TokenInfo::SemanticType::Whitespace;
+        span.is_skippable = token.type == CodeComprehension::TokenInfo::SemanticType::Whitespace;
         span.data = static_cast<u64>(token.type);
         spans.append(span);
     }
@@ -130,7 +130,7 @@ void SemanticSyntaxHighlighter::update_spans(Vector<GUI::AutocompleteProvider::T
     m_client->do_update();
 }
 
-void SemanticSyntaxHighlighter::update_tokens_info(Vector<GUI::AutocompleteProvider::TokenInfo> tokens_info)
+void SemanticSyntaxHighlighter::update_tokens_info(Vector<CodeComprehension::TokenInfo> tokens_info)
 {
     {
         Threading::MutexLocker locker(m_lock);
@@ -145,22 +145,21 @@ void SemanticSyntaxHighlighter::update_tokens_info(Vector<GUI::AutocompleteProvi
 
 bool SemanticSyntaxHighlighter::is_identifier(u64 token_type) const
 {
-    using GUI::AutocompleteProvider;
-    auto type = static_cast<AutocompleteProvider::TokenInfo::SemanticType>(token_type);
-
-    return type == AutocompleteProvider::TokenInfo::SemanticType::Identifier
-        || type == AutocompleteProvider::TokenInfo::SemanticType::Function
-        || type == AutocompleteProvider::TokenInfo::SemanticType::Variable
-        || type == AutocompleteProvider::TokenInfo::SemanticType::CustomType
-        || type == AutocompleteProvider::TokenInfo::SemanticType::Namespace
-        || type == AutocompleteProvider::TokenInfo::SemanticType::Member
-        || type == AutocompleteProvider::TokenInfo::SemanticType::Parameter
-        || type == AutocompleteProvider::TokenInfo::SemanticType::PreprocessorMacro;
+    auto type = static_cast<CodeComprehension::TokenInfo::SemanticType>(token_type);
+
+    return type == CodeComprehension::TokenInfo::SemanticType::Identifier
+        || type == CodeComprehension::TokenInfo::SemanticType::Function
+        || type == CodeComprehension::TokenInfo::SemanticType::Variable
+        || type == CodeComprehension::TokenInfo::SemanticType::CustomType
+        || type == CodeComprehension::TokenInfo::SemanticType::Namespace
+        || type == CodeComprehension::TokenInfo::SemanticType::Member
+        || type == CodeComprehension::TokenInfo::SemanticType::Parameter
+        || type == CodeComprehension::TokenInfo::SemanticType::PreprocessorMacro;
 }
 
 bool SemanticSyntaxHighlighter::is_navigatable(u64 token_type) const
 {
-    return static_cast<GUI::AutocompleteProvider::TokenInfo::SemanticType>(token_type) == GUI::AutocompleteProvider::TokenInfo::SemanticType::IncludePath;
+    return static_cast<CodeComprehension::TokenInfo::SemanticType>(token_type) == CodeComprehension::TokenInfo::SemanticType::IncludePath;
 }
 
 }

Userland/Libraries/LibCpp/SemanticSyntaxHighlighter.h

@@ -27,7 +27,7 @@ public:
     virtual Syntax::Language language() const override { return Syntax::Language::Cpp; }
     virtual void rehighlight(Palette const&) override;
 
-    void update_tokens_info(Vector<GUI::AutocompleteProvider::TokenInfo>);
+    void update_tokens_info(Vector<CodeComprehension::TokenInfo>);
 
     virtual bool is_cpp_semantic_highlighter() const override { return true; }
 
@@ -36,10 +36,10 @@ protected:
     virtual bool token_types_equal(u64 token1, u64 token2) const override { return m_simple_syntax_highlighter.token_types_equal(token1, token2); };
 
 private:
-    void update_spans(Vector<GUI::AutocompleteProvider::TokenInfo> const&, Gfx::Palette const&);
+    void update_spans(Vector<CodeComprehension::TokenInfo> const&, Gfx::Palette const&);
 
     Cpp::SyntaxHighlighter m_simple_syntax_highlighter;
-    Vector<GUI::AutocompleteProvider::TokenInfo> m_tokens_info;
+    Vector<CodeComprehension::TokenInfo> m_tokens_info;
     String m_saved_tokens_text;
     Vector<Token> m_saved_tokens;
     Threading::Mutex m_lock;

Userland/Libraries/LibGUI/AutocompleteProvider.cpp

@@ -19,7 +19,7 @@ namespace GUI {
 
 class AutocompleteSuggestionModel final : public GUI::Model {
 public:
-    explicit AutocompleteSuggestionModel(Vector<AutocompleteProvider::Entry>&& suggestions)
+    explicit AutocompleteSuggestionModel(Vector<CodeComprehension::AutocompleteResultEntry>&& suggestions)
         : m_suggestions(move(suggestions))
     {
     }
@@ -50,13 +50,13 @@ public:
                     return suggestion.completion;
             }
             if (index.column() == Column::Icon) {
-                if (suggestion.language == GUI::AutocompleteProvider::Language::Cpp) {
+                if (suggestion.language == CodeComprehension::Language::Cpp) {
                     if (!s_cpp_identifier_icon) {
                         s_cpp_identifier_icon = Gfx::Bitmap::try_load_from_file("/res/icons/16x16/completion/cpp-identifier.png").release_value_but_fixme_should_propagate_errors();
                     }
                     return *s_cpp_identifier_icon;
                 }
-                if (suggestion.language == GUI::AutocompleteProvider::Language::Unspecified) {
+                if (suggestion.language == CodeComprehension::Language::Unspecified) {
                     if (!s_unspecified_identifier_icon) {
                         s_unspecified_identifier_icon = Gfx::Bitmap::try_load_from_file("/res/icons/16x16/completion/unspecified-identifier.png").release_value_but_fixme_should_propagate_errors();
                     }
@@ -73,15 +73,15 @@ public:
             return suggestion.completion;
 
         if ((int)role == InternalRole::HideAutocompleteAfterApplying)
-            return suggestion.hide_autocomplete_after_applying == AutocompleteProvider::Entry::HideAutocompleteAfterApplying::Yes;
+            return suggestion.hide_autocomplete_after_applying == CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying::Yes;
 
         return {};
     }
 
-    void set_suggestions(Vector<AutocompleteProvider::Entry>&& suggestions) { m_suggestions = move(suggestions); }
+    void set_suggestions(Vector<CodeComprehension::AutocompleteResultEntry>&& suggestions) { m_suggestions = move(suggestions); }
 
 private:
-    Vector<AutocompleteProvider::Entry> m_suggestions;
+    Vector<CodeComprehension::AutocompleteResultEntry> m_suggestions;
 };
 
 AutocompleteBox::AutocompleteBox(TextEditor& editor)
@@ -109,7 +109,7 @@ AutocompleteBox::AutocompleteBox(TextEditor& editor)
     m_no_suggestions_view = main_widget.add<GUI::Label>("No suggestions");
 }
 
-void AutocompleteBox::update_suggestions(Vector<AutocompleteProvider::Entry>&& suggestions)
+void AutocompleteBox::update_suggestions(Vector<CodeComprehension::AutocompleteResultEntry>&& suggestions)
 {
     // FIXME: There's a potential race here if, after the user selected an autocomplete suggestion,
     // the LanguageServer sends an update and this function is executed before AutocompleteBox::apply_suggestion()
@@ -182,9 +182,9 @@ void AutocompleteBox::previous_suggestion()
     }
 }
 
-AutocompleteProvider::Entry::HideAutocompleteAfterApplying AutocompleteBox::apply_suggestion()
+CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying AutocompleteBox::apply_suggestion()
 {
-    auto hide_when_done = AutocompleteProvider::Entry::HideAutocompleteAfterApplying::Yes;
+    auto hide_when_done = CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying::Yes;
 
     if (m_editor.is_null())
         return hide_when_done;
@@ -202,7 +202,7 @@ AutocompleteProvider::Entry::HideAutocompleteAfterApplying AutocompleteBox::appl
     auto hide_after_applying = suggestion_index.data((GUI::ModelRole)AutocompleteSuggestionModel::InternalRole::HideAutocompleteAfterApplying).to_bool();
 
     if (!hide_after_applying)
-        hide_when_done = AutocompleteProvider::Entry::HideAutocompleteAfterApplying::No;
+        hide_when_done = CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying::No;
 
     VERIFY(completion.length() >= partial_length);
     if (!m_editor->has_selection()) {
@@ -219,14 +219,4 @@ AutocompleteProvider::Entry::HideAutocompleteAfterApplying AutocompleteBox::appl
     return hide_when_done;
 }
 
-bool AutocompleteProvider::Declaration::operator==(AutocompleteProvider::Declaration const& other) const
-{
-    return name == other.name && position == other.position && type == other.type && scope == other.scope;
-}
-
-bool AutocompleteProvider::ProjectLocation::operator==(ProjectLocation const& other) const
-{
-    return file == other.file && line == other.line && column == other.column;
-}
-
 }

Userland/Libraries/LibGUI/AutocompleteProvider.h

@@ -6,6 +6,7 @@
 
 #pragma once
 
+#include <LibCodeComprehension/Types.h>
 #include <LibGUI/Forward.h>
 #include <LibGUI/Label.h>
 #include <LibGUI/TableView.h>
@@ -22,98 +23,7 @@ class AutocompleteProvider {
 public:
     virtual ~AutocompleteProvider() = default;
 
-    enum class Language {
-        Unspecified,
-        Cpp,
-    };
-
-    struct Entry {
-        String completion;
-        size_t partial_input_length { 0 };
-        Language language { Language::Unspecified };
-        String display_text {};
-
-        enum class HideAutocompleteAfterApplying {
-            No,
-            Yes,
-        };
-        HideAutocompleteAfterApplying hide_autocomplete_after_applying { HideAutocompleteAfterApplying::Yes };
-    };
-
-    struct ProjectLocation {
-        String file;
-        size_t line { 0 };
-        size_t column { 0 };
-
-        bool operator==(ProjectLocation const&) const;
-    };
-
-    enum class DeclarationType {
-        Function,
-        Struct,
-        Class,
-        Variable,
-        PreprocessorDefinition,
-        Namespace,
-        Member,
-    };
-
-    struct Declaration {
-        String name;
-        ProjectLocation position;
-        DeclarationType type;
-        String scope;
-
-        bool operator==(Declaration const&) const;
-    };
-
-    virtual void provide_completions(Function<void(Vector<Entry>)>) = 0;
-
-#define FOR_EACH_SEMANTIC_TYPE        \
-    __SEMANTIC(Unknown)               \
-    __SEMANTIC(Regular)               \
-    __SEMANTIC(Keyword)               \
-    __SEMANTIC(Type)                  \
-    __SEMANTIC(Identifier)            \
-    __SEMANTIC(String)                \
-    __SEMANTIC(Number)                \
-    __SEMANTIC(IncludePath)           \
-    __SEMANTIC(PreprocessorStatement) \
-    __SEMANTIC(Comment)               \
-    __SEMANTIC(Whitespace)            \
-    __SEMANTIC(Function)              \
-    __SEMANTIC(Variable)              \
-    __SEMANTIC(CustomType)            \
-    __SEMANTIC(Namespace)             \
-    __SEMANTIC(Member)                \
-    __SEMANTIC(Parameter)             \
-    __SEMANTIC(PreprocessorMacro)
-
-    struct TokenInfo {
-
-        enum class SemanticType : u32 {
-#define __SEMANTIC(x) x,
-            FOR_EACH_SEMANTIC_TYPE
-#undef __SEMANTIC
-
-        } type { SemanticType::Unknown };
-        size_t start_line { 0 };
-        size_t start_column { 0 };
-        size_t end_line { 0 };
-        size_t end_column { 0 };
-
-        static constexpr char const* type_to_string(SemanticType t)
-        {
-            switch (t) {
-#define __SEMANTIC(x)     \
-    case SemanticType::x: \
-        return #x;
-                FOR_EACH_SEMANTIC_TYPE
-#undef __SEMANTIC
-            }
-            VERIFY_NOT_REACHED();
-        };
-    };
+    virtual void provide_completions(Function<void(Vector<CodeComprehension::AutocompleteResultEntry>)>) = 0;
 
     void attach(TextEditor& editor)
     {
@@ -133,7 +43,7 @@ public:
     explicit AutocompleteBox(TextEditor&);
     ~AutocompleteBox() = default;
 
-    void update_suggestions(Vector<AutocompleteProvider::Entry>&& suggestions);
+    void update_suggestions(Vector<CodeComprehension::AutocompleteResultEntry>&& suggestions);
     bool is_visible() const;
     void show(Gfx::IntPoint suggestion_box_location);
     void close();
@@ -141,7 +51,7 @@ public:
     bool has_suggestions() { return m_suggestion_view->model()->row_count() > 0; }
     void next_suggestion();
     void previous_suggestion();
-    AutocompleteProvider::Entry::HideAutocompleteAfterApplying apply_suggestion();
+    CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying apply_suggestion();
 
 private:
     WeakPtr<TextEditor> m_editor;
@@ -149,5 +59,4 @@ private:
     RefPtr<GUI::TableView> m_suggestion_view;
     RefPtr<GUI::Label> m_no_suggestions_view;
 };
-
 }

Userland/Libraries/LibGUI/GML/AutocompleteProvider.cpp

@@ -11,7 +11,7 @@
 
 namespace GUI::GML {
 
-void AutocompleteProvider::provide_completions(Function<void(Vector<Entry>)> callback)
+void AutocompleteProvider::provide_completions(Function<void(Vector<CodeComprehension::AutocompleteResultEntry>)> callback)
 {
     auto cursor = m_editor->cursor();
     auto text = m_editor->text();
@@ -121,7 +121,7 @@ void AutocompleteProvider::provide_completions(Function<void(Vector<Entry>)> cal
         return fuzzy_str_builder.build();
     };
 
-    Vector<AutocompleteProvider::Entry> class_entries, identifier_entries;
+    Vector<CodeComprehension::AutocompleteResultEntry> class_entries, identifier_entries;
 
     auto register_layouts_matching_pattern = [&](String pattern, size_t partial_input_length) {
         Core::ObjectClassRegistration::for_each([&](const Core::ObjectClassRegistration& registration) {
@@ -146,15 +146,15 @@ void AutocompleteProvider::provide_completions(Function<void(Vector<Entry>)> cal
             if (auto instance = registration->construct()) {
                 for (auto& it : instance->properties()) {
                     if (!it.value->is_readonly() && it.key.matches(pattern))
-                        identifier_entries.empend(String::formatted("{}: ", it.key), partial_input_length, Language::Unspecified, it.key);
+                        identifier_entries.empend(String::formatted("{}: ", it.key), partial_input_length, CodeComprehension::Language::Unspecified, it.key);
                 }
             }
         }
 
         if (can_have_declared_layout(class_names.last()) && "layout"sv.matches(pattern))
-            identifier_entries.empend("layout: ", partial_input_length, Language::Unspecified, "layout", AutocompleteProvider::Entry::HideAutocompleteAfterApplying::No);
+            identifier_entries.empend("layout: ", partial_input_length, CodeComprehension::Language::Unspecified, "layout", CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying::No);
         if (class_names.last() == "GUI::ScrollableContainerWidget" && "content_widget"sv.matches(pattern))
-            identifier_entries.empend("content_widget: ", partial_input_length, Language::Unspecified, "content_widget", AutocompleteProvider::Entry::HideAutocompleteAfterApplying::No);
+            identifier_entries.empend("content_widget: ", partial_input_length, CodeComprehension::Language::Unspecified, "content_widget", CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying::No);
     };
 
     auto register_properties_and_widgets_matching_pattern = [&](String pattern, size_t partial_input_length) {
@@ -235,7 +235,7 @@ void AutocompleteProvider::provide_completions(Function<void(Vector<Entry>)> cal
     quick_sort(class_entries, [](auto& a, auto& b) { return a.completion < b.completion; });
     quick_sort(identifier_entries, [](auto& a, auto& b) { return a.completion < b.completion; });
 
-    Vector<GUI::AutocompleteProvider::Entry> entries;
+    Vector<CodeComprehension::AutocompleteResultEntry> entries;
     entries.extend(move(identifier_entries));
     entries.extend(move(class_entries));
 

Userland/Libraries/LibGUI/GML/AutocompleteProvider.h

@@ -22,7 +22,7 @@ private:
         return class_name.is_one_of("GUI::Widget", "GUI::Frame");
     }
 
-    virtual void provide_completions(Function<void(Vector<Entry>)> callback) override;
+    virtual void provide_completions(Function<void(Vector<CodeComprehension::AutocompleteResultEntry>)> callback) override;
 };
 
 }

Userland/Libraries/LibGUI/TextEditor.cpp

@@ -757,7 +757,7 @@ void TextEditor::keydown_event(KeyEvent& event)
 {
     if (m_autocomplete_box && m_autocomplete_box->is_visible() && (event.key() == KeyCode::Key_Return || event.key() == KeyCode::Key_Tab)) {
         TemporaryChange change { m_should_keep_autocomplete_box, true };
-        if (m_autocomplete_box->apply_suggestion() == AutocompleteProvider::Entry::HideAutocompleteAfterApplying::Yes)
+        if (m_autocomplete_box->apply_suggestion() == CodeComprehension::AutocompleteResultEntry::HideAutocompleteAfterApplying::Yes)
             hide_autocomplete();
         else
             try_update_autocomplete();

Userland/Shell/CMakeLists.txt

@@ -17,7 +17,7 @@ set(SOURCES
         )
 
 serenity_lib(LibShell shell)
-target_link_libraries(LibShell LibCore LibLine LibSyntax LibRegex)
+target_link_libraries(LibShell LibCore LibLine LibSyntax LibRegex LibCodeComprehension)
 
 set(SOURCES
         main.cpp