HackStudio: Restart LanguageServer if it crashes

We now restart the language server transparently if it crashes.
If the language server crashes too frequently (current threshold is
twice within 3 seconds), we give up and will not attempt to restart it
again. HackStudio will still work fine, but features that depend on the
language server will not function.

To support this change we use a new class, 'ServerConnectionWrapper',
that holds the actual ServerConnection and is responsible for restarting
the language-server if it crashes.

Closes #5574.

Userland/DevTools/HackStudio/Editor.cpp

@@ -426,6 +426,11 @@ void Editor::set_document(GUI::TextDocument& doc)
 
     if (m_language_client) {
         set_autocomplete_provider(make<LanguageServerAidedAutocompleteProvider>(*m_language_client));
+        // NOTE:
+        // When a file is opened for the first time in HackStudio, its content is already synced with the filesystem.
+        // Otherwise, if the file has already been opened before in some Editor instance, it should exist in the LanguageServer's
+        // FileDB, and the LanguageServer should already have its up-to-date content.
+        // So it's OK to just pass an fd here (rather than the TextDocument's content).
         int fd = open(code_document.file_path().characters(), O_RDONLY | O_NOCTTY);
         if (fd < 0) {
             perror("open");

Userland/DevTools/HackStudio/LanguageClient.cpp

@@ -36,67 +36,67 @@ namespace HackStudio {
 
 void ServerConnection::handle(const Messages::LanguageClient::AutoCompleteSuggestions& message)
 {
-    if (!m_language_client) {
+    if (!m_current_language_client) {
         dbgln("Language Server connection has no attached language client");
         return;
     }
-    m_language_client->provide_autocomplete_suggestions(message.suggestions());
+    m_current_language_client->provide_autocomplete_suggestions(message.suggestions());
 }
 
 void ServerConnection::handle(const Messages::LanguageClient::DeclarationLocation& message)
 {
-    if (!m_language_client) {
+    if (!m_current_language_client) {
         dbgln("Language Server connection has no attached language client");
         return;
     }
-    m_language_client->declaration_found(message.location().file, message.location().line, message.location().column);
+    m_current_language_client->declaration_found(message.location().file, message.location().line, message.location().column);
 }
 
 void ServerConnection::die()
 {
-    dbgln("ServerConnection::die()");
-    if (!m_language_client)
-        return;
-    m_language_client->on_server_crash();
+    VERIFY(m_wrapper);
+    // Wrapper destructs us here
+    m_wrapper->on_crash();
 }
 
 void LanguageClient::open_file(const String& path, int fd)
 {
-    if (!m_server_connection)
+    if (!m_connection_wrapper.connection())
         return;
-    m_server_connection->post_message(Messages::LanguageServer::FileOpened(path, fd));
+    m_connection_wrapper.connection()->post_message(Messages::LanguageServer::FileOpened(path, fd));
 }
 
 void LanguageClient::set_file_content(const String& path, const String& content)
 {
-    if (!m_server_connection)
+    if (!m_connection_wrapper.connection())
         return;
-    m_server_connection->post_message(Messages::LanguageServer::SetFileContent(path, content));
+    m_connection_wrapper.connection()->post_message(Messages::LanguageServer::SetFileContent(path, content));
 }
 
 void LanguageClient::insert_text(const String& path, const String& text, size_t line, size_t column)
 {
-    if (!m_server_connection)
+    if (!m_connection_wrapper.connection())
         return;
-    m_server_connection->post_message(Messages::LanguageServer::FileEditInsertText(path, text, line, column));
+    //    set_active_client();
+    m_connection_wrapper.connection()->post_message(Messages::LanguageServer::FileEditInsertText(path, text, line, column));
 }
 
 void LanguageClient::remove_text(const String& path, size_t from_line, size_t from_column, size_t to_line, size_t to_column)
 {
-    if (!m_server_connection)
+    if (!m_connection_wrapper.connection())
         return;
-    m_server_connection->post_message(Messages::LanguageServer::FileEditRemoveText(path, from_line, from_column, to_line, to_column));
+    m_connection_wrapper.connection()->post_message(Messages::LanguageServer::FileEditRemoveText(path, from_line, from_column, to_line, to_column));
 }
 
 void LanguageClient::request_autocomplete(const String& path, size_t cursor_line, size_t cursor_column)
 {
-    if (!m_server_connection)
+    if (!m_connection_wrapper.connection())
         return;
     set_active_client();
-    m_server_connection->post_message(Messages::LanguageServer::AutoCompleteSuggestions(GUI::AutocompleteProvider::ProjectLocation { path, cursor_line, cursor_column }));
+    m_connection_wrapper.connection()->post_message(Messages::LanguageServer::AutoCompleteSuggestions(GUI::AutocompleteProvider::ProjectLocation { path, cursor_line, cursor_column }));
 }
 
-void LanguageClient::provide_autocomplete_suggestions(const Vector<GUI::AutocompleteProvider::Entry>& suggestions)
+void LanguageClient::provide_autocomplete_suggestions(const Vector<GUI::AutocompleteProvider::Entry>& suggestions) const
 {
     if (on_autocomplete_suggestions)
         on_autocomplete_suggestions(suggestions);
@@ -106,64 +106,143 @@ void LanguageClient::provide_autocomplete_suggestions(const Vector<GUI::Autocomp
 
 void LanguageClient::set_autocomplete_mode(const String& mode)
 {
-    if (!m_server_connection)
+    if (!m_connection_wrapper.connection())
         return;
-    m_server_connection->post_message(Messages::LanguageServer::SetAutoCompleteMode(mode));
+    m_connection_wrapper.connection()->post_message(Messages::LanguageServer::SetAutoCompleteMode(mode));
 }
 
 void LanguageClient::set_active_client()
 {
-    if (!m_server_connection)
+    if (!m_connection_wrapper.connection())
         return;
-    m_server_connection->attach(*this);
+    m_connection_wrapper.set_active_client(*this);
 }
 
-void LanguageClient::on_server_crash()
+HashMap<String, NonnullOwnPtr<ServerConnectionWrapper>> ServerConnectionInstances::s_instance_for_language;
+
+void ServerConnection::handle(const Messages::LanguageClient::DeclarationsInDocument& message)
 {
-    VERIFY(m_server_connection);
-    auto project_path = m_server_connection->project_path();
-    ServerConnection::remove_instance_for_language(project_path);
-    m_server_connection = nullptr;
+    locator().set_declared_symbols(message.filename(), message.declarations());
+}
 
-    auto notification = GUI::Notification::construct();
+void LanguageClient::search_declaration(const String& path, size_t line, size_t column)
+{
+    if (!m_connection_wrapper.connection())
+        return;
+    set_active_client();
+    m_connection_wrapper.connection()->post_message(Messages::LanguageServer::FindDeclaration(GUI::AutocompleteProvider::ProjectLocation { path, line, column }));
+}
+
+void LanguageClient::declaration_found(const String& file, size_t line, size_t column) const
+{
+    if (!on_declaration_found) {
+        dbgln("on_declaration_found callback is not set");
+        return;
+    }
+    on_declaration_found(file, line, column);
+}
+
+void ServerConnectionInstances::set_instance_for_language(const String& language_name, NonnullOwnPtr<ServerConnectionWrapper>&& connection_wrapper)
+{
+    s_instance_for_language.set(language_name, move(connection_wrapper));
+}
+
+void ServerConnectionInstances::remove_instance_for_language(const String& language_name)
+{
+    s_instance_for_language.remove(language_name);
+}
+
+ServerConnectionWrapper* ServerConnectionInstances::get_instance_wrapper(const String& language_name)
+{
+    if (auto instance = s_instance_for_language.get(language_name); instance.has_value()) {
+        return const_cast<ServerConnectionWrapper*>(instance.value());
+    }
+    return nullptr;
+}
 
+void ServerConnectionWrapper::on_crash()
+{
+    show_crash_notification();
+    m_connection.clear();
+
+    static constexpr int max_crash_frequency_seconds = 3;
+    if (m_last_crash_timer.is_valid() && m_last_crash_timer.elapsed() / 1000 < max_crash_frequency_seconds) {
+        dbgln("LanguageServer crash frequency is too high");
+        m_respawn_allowed = false;
+
+        show_frequenct_crashes_notification();
+    } else {
+        m_last_crash_timer.start();
+        try_respawn_connection();
+    }
+}
+void ServerConnectionWrapper::show_frequenct_crashes_notification() const
+{
+    auto notification = GUI::Notification::construct();
+    notification->set_icon(Gfx::Bitmap::load_from_file("/res/icons/32x32/app-hack-studio.png"));
+    notification->set_title("LanguageServer Crashes too much!");
+    notification->set_text("LanguageServer aided features will not be available in this session");
+    notification->show();
+}
+void ServerConnectionWrapper::show_crash_notification() const
+{
+    auto notification = GUI::Notification::construct();
     notification->set_icon(Gfx::Bitmap::load_from_file("/res/icons/32x32/app-hack-studio.png"));
     notification->set_title("Oops!");
-    notification->set_text(String::formatted("LanguageServer for {} crashed", project_path));
+    notification->set_text(String::formatted("LanguageServer has crashed"));
     notification->show();
 }
 
-HashMap<String, NonnullRefPtr<ServerConnection>> ServerConnection::s_instance_for_language;
+ServerConnectionWrapper::ServerConnectionWrapper(const String& language_name, Function<NonnullRefPtr<ServerConnection>()> connection_creator)
+    : m_language(language_from_name(language_name))
+    , m_connection_creator(move(connection_creator))
+{
+    create_connection();
+}
 
-void ServerConnection::set_instance_for_project(const String& language_name, NonnullRefPtr<ServerConnection>&& instance)
+void ServerConnectionWrapper::create_connection()
 {
-    s_instance_for_language.set(language_name, move(instance));
+    VERIFY(m_connection.is_null());
+    m_connection = m_connection_creator();
+    m_connection->set_wrapper(*this);
+    m_connection->handshake();
 }
 
-void ServerConnection::remove_instance_for_language(const String& language_name)
+ServerConnection* ServerConnectionWrapper::connection()
 {
-    s_instance_for_language.remove(language_name);
+    return m_connection.ptr();
 }
-void ServerConnection::handle(const Messages::LanguageClient::DeclarationsInDocument& message)
+
+void ServerConnectionWrapper::attach(LanguageClient& client)
 {
-    locator().set_declared_symbols(message.filename(), message.declarations());
+    m_connection->m_current_language_client = &client;
 }
 
-void LanguageClient::search_declaration(const String& path, size_t line, size_t column)
+void ServerConnectionWrapper::detach()
 {
-    if (!m_server_connection)
-        return;
-    set_active_client();
-    m_server_connection->post_message(Messages::LanguageServer::FindDeclaration(GUI::AutocompleteProvider::ProjectLocation { path, line, column }));
+    m_connection->m_current_language_client.clear();
 }
 
-void LanguageClient::declaration_found(const String& file, size_t line, size_t column)
+void ServerConnectionWrapper::set_active_client(LanguageClient& client)
 {
-    if (!on_declaration_found) {
-        dbgln("on_declaration_found callback is not set");
+    m_connection->m_current_language_client = &client;
+}
+
+void ServerConnectionWrapper::try_respawn_connection()
+{
+    if (!m_respawn_allowed)
         return;
-    }
-    on_declaration_found(file, line, column);
+
+    dbgln("Respawning ServerConnection");
+    create_connection();
+
+    // After respawning the language-server, we have to flush the content of the project files
+    // so the server's FileDB will be up-to-date.
+    project().for_each_text_file([this](const ProjectFile& file) {
+        if (file.code_document().language() != m_language)
+            return;
+        m_connection->post_message(Messages::LanguageServer::SetFileContent(file.code_document().file_path(), file.document().text()));
+    });
 }
 
 }

Userland/DevTools/HackStudio/LanguageClient.h

@@ -27,11 +27,13 @@
 #pragma once
 
 #include "AutoCompleteResponse.h"
+#include "Language.h"
 #include <AK/Forward.h>
 #include <AK/LexicalPath.h>
 #include <AK/Types.h>
 #include <AK/WeakPtr.h>
 #include <AK/Weakable.h>
+#include <LibCore/ElapsedTimer.h>
 #include <LibIPC/ServerConnection.h>
 
 #include <DevTools/HackStudio/LanguageServers/LanguageClientEndpoint.h>
@@ -40,10 +42,13 @@
 namespace HackStudio {
 
 class LanguageClient;
+class ServerConnectionWrapper;
 
 class ServerConnection
     : public IPC::ServerConnection<LanguageClientEndpoint, LanguageServerEndpoint>
     , public LanguageClientEndpoint {
+    friend class ServerConnectionWrapper;
+
 public:
     ServerConnection(const StringView& socket, const String& project_path)
         : IPC::ServerConnection<LanguageClientEndpoint, LanguageServerEndpoint>(*this, socket)
@@ -51,74 +56,90 @@ public:
         m_project_path = project_path;
     }
 
-    void attach(LanguageClient& client)
-    {
-        m_language_client = &client;
-    }
-
-    void detach()
-    {
-        m_language_client = nullptr;
-    }
-
     virtual void handshake() override
     {
         send_sync<Messages::LanguageServer::Greet>(m_project_path);
     }
 
-    WeakPtr<LanguageClient> language_client() { return m_language_client; }
+    WeakPtr<LanguageClient> language_client() { return m_current_language_client; }
     const String& project_path() const { return m_project_path; }
 
-    template<typename LanguageServerType>
-    static NonnullRefPtr<ServerConnection> get_or_create(const String& project_path)
-    {
-        auto key = LanguageServerType::language_name();
-        if (auto instance = s_instance_for_language.get(key); instance.has_value()) {
-            return *instance.value();
-        }
-
-        auto connection = LanguageServerType::construct(project_path);
-        connection->handshake();
-        set_instance_for_project(LanguageServerType::language_name(), *connection);
-        return *connection;
-    }
-
-    static void set_instance_for_project(const String& language_name, NonnullRefPtr<ServerConnection>&&);
-    static void remove_instance_for_language(const String& language_name);
-
     virtual void die() override;
 
 protected:
     virtual void handle(const Messages::LanguageClient::AutoCompleteSuggestions&) override;
     virtual void handle(const Messages::LanguageClient::DeclarationLocation&) override;
     virtual void handle(const Messages::LanguageClient::DeclarationsInDocument&) override;
+    void set_wrapper(ServerConnectionWrapper& wrapper) { m_wrapper = &wrapper; }
 
     String m_project_path;
-    WeakPtr<LanguageClient> m_language_client;
+    WeakPtr<LanguageClient> m_current_language_client;
+    ServerConnectionWrapper* m_wrapper { nullptr };
+};
+
+class ServerConnectionWrapper {
+    AK_MAKE_NONCOPYABLE(ServerConnectionWrapper);
+
+public:
+    explicit ServerConnectionWrapper(const String& language_name, Function<NonnullRefPtr<ServerConnection>()> connection_creator);
+    ~ServerConnectionWrapper() = default;
+
+    template<typename LanguageServerType>
+    static ServerConnectionWrapper& get_or_create(const String& project_path);
+
+    ServerConnection* connection();
+    void on_crash();
+    void try_respawn_connection();
+
+    void attach(LanguageClient& client);
+    void detach();
+    void set_active_client(LanguageClient& client);
+
+private:
+    void create_connection();
+    void show_crash_notification() const;
+    void show_frequenct_crashes_notification() const;
+
+    Language m_language;
+    Function<NonnullRefPtr<ServerConnection>()> m_connection_creator;
+    RefPtr<ServerConnection> m_connection;
+
+    Core::ElapsedTimer m_last_crash_timer;
+    bool m_respawn_allowed { true };
+};
+
+class ServerConnectionInstances {
+public:
+    static void set_instance_for_language(const String& language_name, NonnullOwnPtr<ServerConnectionWrapper>&& connection_wrapper);
+    static void remove_instance_for_language(const String& language_name);
+
+    static ServerConnectionWrapper* get_instance_wrapper(const String& language_name);
 
 private:
-    static HashMap<String, NonnullRefPtr<ServerConnection>> s_instance_for_language;
+    static HashMap<String, NonnullOwnPtr<ServerConnectionWrapper>> s_instance_for_language;
 };
 
 class LanguageClient : public Weakable<LanguageClient> {
 public:
-    explicit LanguageClient(NonnullRefPtr<ServerConnection>&& connection)
-        : m_server_connection(move(connection))
+    explicit LanguageClient(ServerConnectionWrapper& connection_wrapper)
+        : m_connection_wrapper(connection_wrapper)
     {
-        m_previous_client = m_server_connection->language_client();
-        VERIFY(m_previous_client.ptr() != this);
-        m_server_connection->attach(*this);
+        if (m_connection_wrapper.connection()) {
+            m_previous_client = m_connection_wrapper.connection()->language_client();
+            VERIFY(m_previous_client.ptr() != this);
+            m_connection_wrapper.attach(*this);
+        }
     }
 
     virtual ~LanguageClient()
     {
-        // m_server_connection is nullified if the server crashes
-        if (m_server_connection)
-            m_server_connection->detach();
+        // m_connection_wrapper is nullified if the server crashes
+        if (m_connection_wrapper.connection())
+            m_connection_wrapper.detach();
 
         VERIFY(m_previous_client.ptr() != this);
-        if (m_previous_client)
-            m_server_connection->attach(*m_previous_client);
+        if (m_previous_client && m_connection_wrapper.connection())
+            m_connection_wrapper.set_active_client(*m_previous_client);
     }
 
     void set_active_client();
@@ -130,22 +151,34 @@ public:
     virtual void set_autocomplete_mode(const String& mode);
     virtual void search_declaration(const String& path, size_t line, size_t column);
 
-    void provide_autocomplete_suggestions(const Vector<GUI::AutocompleteProvider::Entry>&);
-    void declaration_found(const String& file, size_t line, size_t column);
-    void on_server_crash();
+    void provide_autocomplete_suggestions(const Vector<GUI::AutocompleteProvider::Entry>&) const;
+    void declaration_found(const String& file, size_t line, size_t column) const;
 
     Function<void(Vector<GUI::AutocompleteProvider::Entry>)> on_autocomplete_suggestions;
     Function<void(const String&, size_t, size_t)> on_declaration_found;
 
 private:
-    WeakPtr<ServerConnection> m_server_connection;
+    ServerConnectionWrapper& m_connection_wrapper;
     WeakPtr<LanguageClient> m_previous_client;
 };
 
 template<typename ServerConnectionT>
 static inline NonnullOwnPtr<LanguageClient> get_language_client(const String& project_path)
 {
-    return make<LanguageClient>(ServerConnection::get_or_create<ServerConnectionT>(project_path));
+    return make<LanguageClient>(ServerConnectionWrapper::get_or_create<ServerConnectionT>(project_path));
+}
+
+template<typename LanguageServerType>
+ServerConnectionWrapper& ServerConnectionWrapper::get_or_create(const String& project_path)
+{
+    auto* wrapper = ServerConnectionInstances::get_instance_wrapper(LanguageServerType::language_name());
+    if (wrapper)
+        return *wrapper;
+
+    auto connection_wrapper_ptr = make<ServerConnectionWrapper>(LanguageServerType::language_name(), [project_path]() { return LanguageServerType::construct(project_path); });
+    auto& connection_wrapper = *connection_wrapper_ptr;
+    ServerConnectionInstances::set_instance_for_language(LanguageServerType::language_name(), move(connection_wrapper_ptr));
+    return connection_wrapper;
 }
 
 }