WindowServer: Add support for alpha channel based hit testing

This enables implementing non-rectangular window shapes, including
non-rectangular window frames.

Userland/Libraries/LibGUI/Window.cpp

@@ -147,6 +147,7 @@ void Window::show()
         m_frameless,
         m_accessory,
         m_opacity_when_windowless,
+        m_alpha_hit_threshold,
         m_base_size,
         m_size_increment,
         m_resize_aspect_ratio,
@@ -673,6 +674,20 @@ void Window::set_opacity(float opacity)
     WindowServerConnection::the().send_sync<Messages::WindowServer::SetWindowOpacity>(m_window_id, opacity);
 }
 
+void Window::set_alpha_hit_threshold(float threshold)
+{
+    if (threshold < 0.0f)
+        threshold = 0.0f;
+    else if (threshold > 1.0f)
+        threshold = 1.0f;
+    if (m_alpha_hit_threshold == threshold)
+        return;
+    m_alpha_hit_threshold = threshold;
+    if (!is_visible())
+        return;
+    WindowServerConnection::the().send_sync<Messages::WindowServer::SetWindowAlphaHitThreshold>(m_window_id, threshold);
+}
+
 void Window::set_hovered_widget(Widget* widget)
 {
     if (widget == m_hovered_widget)

Userland/Libraries/LibGUI/Window.h

@@ -72,6 +72,9 @@ public:
     void set_opacity(float);
     float opacity() const { return m_opacity_when_windowless; }
 
+    void set_alpha_hit_threshold(float);
+    float alpha_hit_threshold() const { return m_alpha_hit_threshold; }
+
     WindowType window_type() const { return m_window_type; }
     void set_window_type(WindowType);
 
@@ -238,6 +241,7 @@ private:
     RefPtr<Gfx::Bitmap> m_custom_cursor;
     int m_window_id { 0 };
     float m_opacity_when_windowless { 1.0f };
+    float m_alpha_hit_threshold { 0.0f };
     RefPtr<Widget> m_main_widget;
     WeakPtr<Widget> m_focused_widget;
     WeakPtr<Widget> m_global_cursor_tracking_widget;

Userland/Libraries/LibGfx/ClassicWindowTheme.h

@@ -52,6 +52,7 @@ public:
     {
         return compute_frame_colors(state, palette).uses_alpha();
     }
+    virtual float frame_alpha_hit_threshold(WindowState) const override { return 1.0f; }
 
 private:
     struct FrameColors {

Userland/Libraries/LibGfx/WindowTheme.h

@@ -63,6 +63,7 @@ public:
     virtual Vector<IntRect> layout_buttons(WindowType, const IntRect& window_rect, const Palette&, size_t buttons) const = 0;
     virtual bool is_simple_rect_frame() const = 0;
     virtual bool frame_uses_alpha(WindowState, const Palette&) const = 0;
+    virtual float frame_alpha_hit_threshold(WindowState) const = 0;
 
 protected:
     WindowTheme() { }

Userland/Services/WindowServer/ClientConnection.cpp

@@ -463,6 +463,7 @@ OwnPtr<Messages::WindowServer::CreateWindowResponse> ClientConnection::handle(co
         window->recalculate_rect();
     }
     window->set_opacity(message.opacity());
+    window->set_alpha_hit_threshold(message.alpha_hit_threshold());
     window->set_size_increment(message.size_increment());
     window->set_base_size(message.base_size());
     window->set_resize_aspect_ratio(message.resize_aspect_ratio());
@@ -636,6 +637,17 @@ OwnPtr<Messages::WindowServer::SetWindowHasAlphaChannelResponse> ClientConnectio
     return make<Messages::WindowServer::SetWindowHasAlphaChannelResponse>();
 }
 
+OwnPtr<Messages::WindowServer::SetWindowAlphaHitThresholdResponse> ClientConnection::handle(const Messages::WindowServer::SetWindowAlphaHitThreshold& message)
+{
+    auto it = m_windows.find(message.window_id());
+    if (it == m_windows.end()) {
+        did_misbehave("SetWindowAlphaHitThreshold: Bad window ID");
+        return {};
+    }
+    it->value->set_alpha_hit_threshold(message.threshold());
+    return make<Messages::WindowServer::SetWindowAlphaHitThresholdResponse>();
+}
+
 void ClientConnection::handle(const Messages::WindowServer::WM_SetActiveWindow& message)
 {
     auto* client = ClientConnection::from_client_id(message.client_id());

Userland/Services/WindowServer/ClientConnection.h

@@ -120,6 +120,7 @@ private:
     virtual void handle(const Messages::WindowServer::WM_StartWindowResize&) override;
     virtual void handle(const Messages::WindowServer::WM_PopupWindowMenu&) override;
     virtual OwnPtr<Messages::WindowServer::SetWindowHasAlphaChannelResponse> handle(const Messages::WindowServer::SetWindowHasAlphaChannel&) override;
+    virtual OwnPtr<Messages::WindowServer::SetWindowAlphaHitThresholdResponse> handle(const Messages::WindowServer::SetWindowAlphaHitThreshold&) override;
     virtual OwnPtr<Messages::WindowServer::MoveWindowToFrontResponse> handle(const Messages::WindowServer::MoveWindowToFront&) override;
     virtual OwnPtr<Messages::WindowServer::SetFullscreenResponse> handle(const Messages::WindowServer::SetFullscreen&) override;
     virtual void handle(const Messages::WindowServer::AsyncSetWallpaper&) override;

Userland/Services/WindowServer/Window.cpp

@@ -873,4 +873,20 @@ bool Window::is_descendant_of(Window& window) const
     return false;
 }
 
+bool Window::hit_test(const Gfx::IntPoint& point, bool include_frame) const
+{
+    if (!frame().rect().contains(point))
+        return false;
+    if (!rect().contains(point)) {
+        if (include_frame)
+            return frame().hit_test(point);
+        return false;
+    }
+    u8 threshold = alpha_hit_threshold() * 255;
+    if (threshold == 0 || !m_backing_store || !m_backing_store->has_alpha_channel())
+        return true;
+    auto color = m_backing_store->get_pixel(point.translated(-rect().location()));
+    return color.alpha() >= threshold;
+}
+
 }

Userland/Services/WindowServer/Window.h

@@ -145,6 +145,13 @@ public:
     float opacity() const { return m_opacity; }
     void set_opacity(float);
 
+    float alpha_hit_threshold() const { return m_alpha_hit_threshold; }
+    void set_alpha_hit_threshold(float threshold)
+    {
+        m_alpha_hit_threshold = threshold;
+    }
+    bool hit_test(const Gfx::IntPoint&, bool include_frame = true) const;
+
     int x() const { return m_rect.x(); }
     int y() const { return m_rect.y(); }
     int width() const { return m_rect.width(); }
@@ -365,6 +372,7 @@ private:
     int m_window_id { -1 };
     i32 m_client_id { -1 };
     float m_opacity { 1 };
+    float m_alpha_hit_threshold { 0.0f };
     Gfx::IntSize m_size_increment;
     Gfx::IntSize m_base_size;
     NonnullRefPtr<Gfx::Bitmap> m_icon;

Userland/Services/WindowServer/WindowFrame.cpp

@@ -535,6 +535,40 @@ void WindowFrame::layout_buttons()
         m_buttons[i].set_relative_rect(button_rects[i]);
 }
 
+bool WindowFrame::hit_test(const Gfx::IntPoint& point) const
+{
+    if (m_window.is_frameless())
+        return false;
+    auto frame_rect = rect();
+    if (!frame_rect.contains(point))
+        return false;
+    auto window_rect = m_window.rect();
+    if (window_rect.contains(point))
+        return false;
+
+    u8 alpha_threshold = Gfx::WindowTheme::current().frame_alpha_hit_threshold(window_state_for_theme()) * 255;
+    if (alpha_threshold == 0)
+        return true;
+    u8 alpha = 0xff;
+    auto relative_point = point.translated(-render_rect().location());
+    if (point.y() < window_rect.y()) {
+        if (m_top_bottom)
+            alpha = m_top_bottom->get_pixel(relative_point).alpha();
+    } else if (point.y() > window_rect.bottom()) {
+        if (m_top_bottom)
+            alpha = m_top_bottom->get_pixel(relative_point.x(), m_bottom_y + point.y() - window_rect.bottom() - 1).alpha();
+    } else if (point.x() < window_rect.x()) {
+        if (m_left_right)
+            alpha = m_left_right->get_pixel(relative_point.x(), relative_point.y() - m_bottom_y).alpha();
+    } else if (point.x() > window_rect.right()) {
+        if (m_left_right)
+            alpha = m_left_right->get_pixel(m_right_x + point.x() - window_rect.right() - 1, relative_point.y() - m_bottom_y).alpha();
+    } else {
+        return false;
+    }
+    return alpha >= alpha_threshold;
+}
+
 void WindowFrame::on_mouse_event(const MouseEvent& event)
 {
     ASSERT(!m_window.is_fullscreen());

Userland/Services/WindowServer/WindowFrame.h

@@ -90,6 +90,8 @@ public:
 
     void theme_changed();
 
+    bool hit_test(const Gfx::IntPoint&) const;
+
 private:
     void paint_simple_rect_shadow(Gfx::Painter&, const Gfx::IntRect&, const Gfx::Bitmap&) const;
     void paint_notification_frame(Gfx::Painter&);

Userland/Services/WindowServer/WindowManager.cpp

@@ -726,7 +726,7 @@ bool WindowManager::process_ongoing_drag(MouseEvent& event, Window*& hovered_win
 
     hovered_window = nullptr;
     for_each_visible_window_from_front_to_back([&](auto& window) {
-        if (window.frame().rect().contains(event.position())) {
+        if (window.hit_test(event.position())) {
             hovered_window = &window;
             return IterationDecision::Break;
         }
@@ -940,7 +940,7 @@ void WindowManager::process_mouse_event(MouseEvent& event, Window*& hovered_wind
         }
 
         for_each_visible_window_from_front_to_back([&](auto& window) {
-            if (window.frame().rect().contains(event.position())) {
+            if (window.hit_test(event.position())) {
                 hovered_window = &window;
                 return IterationDecision::Break;
             }
@@ -977,7 +977,7 @@ void WindowManager::process_mouse_event(MouseEvent& event, Window*& hovered_wind
                 return;
             }
 
-            ASSERT(window.frame().rect().contains(event.position()));
+            ASSERT(window.hit_test(event.position()));
             if (event.type() == Event::MouseDown) {
                 // We're clicking on something that's blocked by a modal window.
                 // Flash the modal window to let the user know about it.
@@ -990,8 +990,12 @@ void WindowManager::process_mouse_event(MouseEvent& event, Window*& hovered_wind
                     set_active_window(&window);
             }
 
-            // Well okay, let's see if we're hitting the frame or the window inside the frame.
-            if (window.rect().contains(event.position())) {
+            if (window.frame().hit_test(event.position())) {
+                // We are hitting the frame, pass the event along to WindowFrame.
+                window.frame().on_mouse_event(event.translated(-window.frame().rect().location()));
+                event_window_with_frame = &window;
+            } else if (window.hit_test(event.position(), false)) {
+                // We are hitting the window content
                 hovered_window = &window;
                 if (!window.global_cursor_tracking() && !window.blocking_modal_window()) {
                     auto translated_event = event.translated(-window.position());
@@ -1001,20 +1005,14 @@ void WindowManager::process_mouse_event(MouseEvent& event, Window*& hovered_wind
                         m_active_input_tracking_window = window;
                     }
                 }
-                return;
             }
-
-            // We are hitting the frame, pass the event along to WindowFrame.
-            window.frame().on_mouse_event(event.translated(-window.frame().rect().location()));
-            event_window_with_frame = &window;
         };
 
         if (auto* fullscreen_window = active_fullscreen_window()) {
             process_mouse_event_for_window(*fullscreen_window);
         } else {
             for_each_visible_window_from_front_to_back([&](Window& window) {
-                auto window_frame_rect = window.frame().rect();
-                if (!window_frame_rect.contains(event.position()))
+                if (!window.hit_test(event.position()))
                     return IterationDecision::Continue;
                 process_mouse_event_for_window(window);
                 return IterationDecision::Break;

Userland/Services/WindowServer/WindowServer.ipc

@@ -41,6 +41,7 @@ endpoint WindowServer = 2
         bool frameless,
         bool accessory,
         float opacity,
+        float alpha_hit_threshold,
         Gfx::IntSize base_size,
         Gfx::IntSize size_increment,
         Optional<Gfx::IntSize> resize_aspect_ratio,
@@ -70,6 +71,8 @@ endpoint WindowServer = 2
     SetGlobalCursorTracking(i32 window_id, bool enabled) => ()
     SetWindowOpacity(i32 window_id, float opacity) => ()
 
+    SetWindowAlphaHitThreshold(i32 window_id, float threshold) => ()
+
     SetWindowBackingStore(i32 window_id, i32 bpp, i32 pitch, IPC::File anon_file, i32 serial, bool has_alpha_channel, Gfx::IntSize size, bool flush_immediately) => ()
 
     WM_SetActiveWindow(i32 client_id, i32 window_id) =|