LibGL: Implement `glStencil*` functions

This implements the context state for stencil testing functions and
operations. No rasterization is implemented.

Userland/Libraries/LibGL/CMakeLists.txt

@@ -11,6 +11,7 @@ set(SOURCES
     GLLights.cpp
     GLLists.cpp
     GLMat.cpp
+    GLStencil.cpp
     GLTexture.cpp
     GLUtils.cpp
     GLVert.cpp

Userland/Libraries/LibGL/GL/gl.h

@@ -192,7 +192,13 @@ extern "C" {
 #define GL_UNSIGNED_BYTE 0x1401
 
 // Stencil buffer operations
+#define GL_KEEP 0x1E00
 #define GL_REPLACE 0x1E01
+#define GL_INCR 0x1E02
+#define GL_INCR_WRAP 0x8507
+#define GL_DECR 0x1E03
+#define GL_DECR_WRAP 0x8508
+#define GL_INVERT 0x150A
 
 // Texture targets
 #define GL_TEXTURE_1D 0x0DE0
@@ -439,6 +445,10 @@ GLAPI void glPixelStorei(GLenum pname, GLint param);
 GLAPI void glScissor(GLint x, GLint y, GLsizei width, GLsizei height);
 GLAPI void glLightf(GLenum light, GLenum pname, GLfloat param);
 GLAPI void glLightfv(GLenum light, GLenum pname, GLfloat* param);
+GLAPI void glStencilFunc(GLenum func, GLint ref, GLuint mask);
+GLAPI void glStencilFuncSeparate(GLenum face, GLenum func, GLint ref, GLuint mask);
+GLAPI void glStencilOp(GLenum sfail, GLenum dpfail, GLenum dppass);
+GLAPI void glStencilOpSeparate(GLenum face, GLenum sfail, GLenum dpfail, GLenum dppass);
 
 #ifdef __cplusplus
 }

Userland/Libraries/LibGL/GLContext.h

@@ -91,6 +91,8 @@ public:
     virtual void gl_fogi(GLenum pname, GLint param) = 0;
     virtual void gl_pixel_storei(GLenum pname, GLint param) = 0;
     virtual void gl_scissor(GLint x, GLint y, GLsizei width, GLsizei height) = 0;
+    virtual void gl_stencil_func_separate(GLenum face, GLenum func, GLint ref, GLuint mask) = 0;
+    virtual void gl_stencil_op_separate(GLenum face, GLenum sfail, GLenum dpfail, GLenum dppass) = 0;
 
     virtual void present() = 0;
 };

Userland/Libraries/LibGL/GLStencil.cpp

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2021, Jelle Raaijmakers <jelle@gmta.nl>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "GL/gl.h"
+#include "GLContext.h"
+
+extern GL::GLContext* g_gl_context;
+
+void glStencilFunc(GLenum func, GLint ref, GLuint mask)
+{
+    g_gl_context->gl_stencil_func_separate(GL_FRONT_AND_BACK, func, ref, mask);
+}
+
+void glStencilFuncSeparate(GLenum face, GLenum func, GLint ref, GLuint mask)
+{
+    g_gl_context->gl_stencil_func_separate(face, func, ref, mask);
+}
+
+void glStencilOp(GLenum sfail, GLenum dpfail, GLenum dppass)
+{
+    g_gl_context->gl_stencil_op_separate(GL_FRONT_AND_BACK, sfail, dpfail, dppass);
+}
+
+void glStencilOpSeparate(GLenum face, GLenum sfail, GLenum dpfail, GLenum dppass)
+{
+    g_gl_context->gl_stencil_op_separate(face, sfail, dpfail, dppass);
+}

Userland/Libraries/LibGL/SoftwareGLContext.cpp

@@ -2172,6 +2172,74 @@ void SoftwareGLContext::gl_scissor(GLint x, GLint y, GLsizei width, GLsizei heig
     m_rasterizer.set_options(options);
 }
 
+void SoftwareGLContext::gl_stencil_func_separate(GLenum face, GLenum func, GLint ref, GLuint mask)
+{
+    APPEND_TO_CALL_LIST_AND_RETURN_IF_NEEDED(gl_stencil_func_separate, face, func, ref, mask);
+    RETURN_WITH_ERROR_IF(m_in_draw_state, GL_INVALID_OPERATION);
+
+    RETURN_WITH_ERROR_IF(!(face == GL_FRONT || face == GL_BACK || face == GL_FRONT_AND_BACK), GL_INVALID_ENUM);
+
+    RETURN_WITH_ERROR_IF(!(func == GL_NEVER
+                             || func == GL_LESS
+                             || func == GL_LEQUAL
+                             || func == GL_GREATER
+                             || func == GL_GEQUAL
+                             || func == GL_EQUAL
+                             || func == GL_NOTEQUAL
+                             || func == GL_ALWAYS),
+        GL_INVALID_ENUM);
+
+    // FIXME: "ref is clamped to the range 02^n - 1 , where n is the number of bitplanes in the stencil buffer"
+
+    StencilFunctionOptions new_options = { func, ref, mask };
+    if (face == GL_FRONT || face == GL_FRONT_AND_BACK)
+        m_stencil_frontfacing_func = new_options;
+    if (face == GL_BACK || face == GL_FRONT_AND_BACK)
+        m_stencil_backfacing_func = new_options;
+}
+
+void SoftwareGLContext::gl_stencil_op_separate(GLenum face, GLenum sfail, GLenum dpfail, GLenum dppass)
+{
+    APPEND_TO_CALL_LIST_AND_RETURN_IF_NEEDED(gl_stencil_op_separate, face, sfail, dpfail, dppass);
+    RETURN_WITH_ERROR_IF(m_in_draw_state, GL_INVALID_OPERATION);
+
+    RETURN_WITH_ERROR_IF(!(face == GL_FRONT || face == GL_BACK || face == GL_FRONT_AND_BACK), GL_INVALID_ENUM);
+
+    RETURN_WITH_ERROR_IF(!(sfail == GL_KEEP
+                             || sfail == GL_ZERO
+                             || sfail == GL_REPLACE
+                             || sfail == GL_INCR
+                             || sfail == GL_INCR_WRAP
+                             || sfail == GL_DECR
+                             || sfail == GL_DECR_WRAP
+                             || sfail == GL_INVERT),
+        GL_INVALID_ENUM);
+    RETURN_WITH_ERROR_IF(!(dpfail == GL_KEEP
+                             || dpfail == GL_ZERO
+                             || dpfail == GL_REPLACE
+                             || dpfail == GL_INCR
+                             || dpfail == GL_INCR_WRAP
+                             || dpfail == GL_DECR
+                             || dpfail == GL_DECR_WRAP
+                             || dpfail == GL_INVERT),
+        GL_INVALID_ENUM);
+    RETURN_WITH_ERROR_IF(!(dppass == GL_KEEP
+                             || dppass == GL_ZERO
+                             || dppass == GL_REPLACE
+                             || dppass == GL_INCR
+                             || dppass == GL_INCR_WRAP
+                             || dppass == GL_DECR
+                             || dppass == GL_DECR_WRAP
+                             || dppass == GL_INVERT),
+        GL_INVALID_ENUM);
+
+    StencilOperationOptions new_options = { sfail, dpfail, dppass };
+    if (face == GL_FRONT || face == GL_FRONT_AND_BACK)
+        m_stencil_frontfacing_op = new_options;
+    if (face == GL_BACK || face == GL_FRONT_AND_BACK)
+        m_stencil_backfacing_op = new_options;
+}
+
 void SoftwareGLContext::present()
 {
     m_rasterizer.blit_to(*m_frontbuffer);

Userland/Libraries/LibGL/SoftwareGLContext.h

@@ -102,6 +102,8 @@ public:
     virtual void gl_fogi(GLenum pname, GLint param) override;
     virtual void gl_pixel_storei(GLenum pname, GLint param) override;
     virtual void gl_scissor(GLint x, GLint y, GLsizei width, GLsizei height) override;
+    virtual void gl_stencil_func_separate(GLenum face, GLenum func, GLint ref, GLuint mask) override;
+    virtual void gl_stencil_op_separate(GLenum face, GLenum sfail, GLenum dpfail, GLenum dppass) override;
     virtual void present() override;
 
 private:
@@ -164,8 +166,25 @@ private:
     GLenum m_alpha_test_func = GL_ALWAYS;
     GLclampf m_alpha_test_ref_value = 0;
 
+    // Stencil configuration
     bool m_stencil_test_enabled { false };
 
+    struct StencilFunctionOptions {
+        GLenum func { GL_ALWAYS };
+        GLint reference_value { 0 };
+        GLuint mask { NumericLimits<GLuint>::max() };
+    };
+    StencilFunctionOptions m_stencil_backfacing_func;
+    StencilFunctionOptions m_stencil_frontfacing_func;
+
+    struct StencilOperationOptions {
+        GLenum op_fail { GL_KEEP };
+        GLenum op_depth_fail { GL_KEEP };
+        GLenum op_pass { GL_KEEP };
+    };
+    StencilOperationOptions m_stencil_backfacing_op;
+    StencilOperationOptions m_stencil_frontfacing_op;
+
     GLenum m_current_read_buffer = GL_BACK;
     GLenum m_current_draw_buffer = GL_BACK;
 
@@ -245,7 +264,9 @@ private:
             decltype(&SoftwareGLContext::gl_draw_elements),
             decltype(&SoftwareGLContext::gl_depth_range),
             decltype(&SoftwareGLContext::gl_polygon_offset),
-            decltype(&SoftwareGLContext::gl_scissor)>;
+            decltype(&SoftwareGLContext::gl_scissor),
+            decltype(&SoftwareGLContext::gl_stencil_func_separate),
+            decltype(&SoftwareGLContext::gl_stencil_op_separate)>;
 
         using ExtraSavedArguments = Variant<
             FloatMatrix4x4>;

Userland/Libraries/LibGL/SoftwareRasterizer.cpp

@@ -204,6 +204,8 @@ static void rasterize_triangle(const RasterizerOptions& options, Gfx::Bitmap& re
 
     FloatVector4 pixel_buffer[RASTERIZER_BLOCK_SIZE][RASTERIZER_BLOCK_SIZE];
 
+    // FIXME: implement stencil testing
+
     // Iterate over all blocks within the bounds of the triangle
     for (int by = by0; by < by1; by++) {
         for (int bx = bx0; bx < bx1; bx++) {