feat(lib/challenge): HTTP meta refresh challenge method (#623)

* feat(lib/challenge): HTTP meta refresh challenge method

Closes #95

This challenge method enables users that don't (or won't) support
JavaScript to pass Anubis challenges. It works by using HTML meta
refresh directives to ensure that the client is a browser.

This is OFF by default. In order to enable it, an administrator MUST
choose to make the default challenge method `metarefresh`.

TODO(Xe):

- [ ] Documentation on this challenge method
- [ ] Amend wording around Anubis being a proof of work proxy in the docs
- [ ] Add configuration file syntax for the default challenge method and settings
- [ ] Test with early customers

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix(lib/challenge/metarefresh): use this value of err

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs: add metarefresh challenge info, Web AI Firewall Utility

Signed-off-by: Xe Iaso <me@xeiaso.net>

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>

.github/actions/spelling/expect.txt

@@ -149,6 +149,7 @@ maintainership
 malware
 mcr
 memes
+metarefresh
 metrix
 mimi
 minica

README.md

@@ -43,7 +43,7 @@ Anubis is brought to you by sponsors and donors like:
 
 ## Overview
 
-Anubis [weighs the soul of your connection](https://en.wikipedia.org/wiki/Weighing_of_souls) using a proof-of-work challenge in order to protect upstream resources from scraper bots.
+Anubis is a Web AI Firewall Utility that [weighs the soul of your connection](https://en.wikipedia.org/wiki/Weighing_of_souls) using one or more challenges in order to protect upstream resources from scraper bots.
 
 This program is designed to help protect the small internet from the endless storm of requests that flood in from AI companies. Anubis is as lightweight as possible to ensure that everyone can afford to protect the communities closest to them.
 

docs/docs/CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Refactor challenge presentation logic to use a challenge registry
 - Allow challenge implementations to register HTTP routes
+- Implement a no-JS challenge method: [`metarefresh`](./admin/configuration/challenges/metarefresh.mdx) ([#95](https://github.com/TecharoHQ/anubis/issues/95))
 
 ## v1.19.1: Jenomis cen Lexentale - Echo 1
 

docs/docs/admin/configuration/challenges/_category_.json

@@ -0,0 +1,8 @@
+{
+  "label": "Challenges",
+  "position": 10,
+  "link": {
+    "type": "generated-index",
+    "description": "The different challenge methods that Anubis supports."
+  }
+}

docs/docs/admin/configuration/challenges/metarefresh.mdx

@@ -0,0 +1,19 @@
+# Meta Refresh (No JavaScript)
+
+The `metarefresh` challenge sends a browser a much simpler challenge that makes it refresh the page after a set period of time. This enables clients to pass challenges without executing JavaScript.
+
+To use it in your Anubis configuration:
+
+```yaml
+# Generic catchall rule
+- name: generic-browser
+  user_agent_regex: >-
+    Mozilla|Opera
+  action: CHALLENGE
+  challenge:
+    difficulty: 1 # Number of seconds to wait before refreshing the page
+    report_as: 4 # Unused by this challenge method
+    algorithm: metarefresh # Specify a non-JS challenge method
+```
+
+This is not enabled by default while this method is tested and its false positive rate is ascertained. Many modern scrapers use headless Google Chrome, so this will have a much higher false positive rate.

docs/docs/admin/configuration/challenges/proof-of-work.mdx

@@ -0,0 +1,5 @@
+# Proof of Work (JavaScript)
+
+When Anubis is configured to use the `fast` or `slow` challenge methods, clients will be sent a small [proof of work](https://en.wikipedia.org/wiki/Proof_of_work) challenge. In order to get a token used to access the upstream resource, clients must calculate a complicated math puzzle with JavaScript.
+
+A `fast` challenge uses a heavily optimized multithreaded implementation and a `slow` challenge uses a simplistic single-threaded implementation. The `slow` method is kept around for legacy compatibility.

docs/docs/index.mdx

@@ -60,7 +60,7 @@ Anubis is brought to you by sponsors and donors like:
 
 ## Overview
 
-Anubis [weighs the soul of your connection](https://en.wikipedia.org/wiki/Weighing_of_souls) using a proof-of-work challenge in order to protect upstream resources from scraper bots.
+Anubis is a Web AI Firewall Utility that [weighs the soul of your connection](https://en.wikipedia.org/wiki/Weighing_of_souls) using one or more challenges in order to protect upstream resources from scraper bots.
 
 This program is designed to help protect the small internet from the endless storm of requests that flood in from AI companies. Anubis is as lightweight as possible to ensure that everyone can afford to protect the communities closest to them.
 

lib/anubis.go

@@ -29,6 +29,7 @@ import (
 	"github.com/TecharoHQ/anubis/lib/policy/config"
 
 	// challenge implementations
+	_ "github.com/TecharoHQ/anubis/lib/challenge/metarefresh"
 	_ "github.com/TecharoHQ/anubis/lib/challenge/proofofwork"
 )
 

lib/challenge/metarefresh/metarefresh.go

@@ -0,0 +1,53 @@
+package metarefresh
+
+import (
+	"crypto/subtle"
+	"fmt"
+	"log/slog"
+	"net/http"
+
+	"github.com/TecharoHQ/anubis"
+	"github.com/TecharoHQ/anubis/lib/challenge"
+	"github.com/TecharoHQ/anubis/lib/policy"
+	"github.com/TecharoHQ/anubis/web"
+	"github.com/a-h/templ"
+)
+
+//go:generate go tool github.com/a-h/templ/cmd/templ generate
+
+func init() {
+	challenge.Register("metarefresh", &Impl{})
+}
+
+type Impl struct{}
+
+func (i *Impl) Setup(mux *http.ServeMux) {}
+
+func (i *Impl) Issue(r *http.Request, lg *slog.Logger, rule *policy.Bot, challenge string, ogTags map[string]string) (templ.Component, error) {
+	u, err := r.URL.Parse(anubis.BasePrefix + "/.within.website/x/cmd/anubis/api/pass-challenge")
+	if err != nil {
+		return nil, fmt.Errorf("can't render page: %w", err)
+	}
+
+	q := u.Query()
+	q.Set("redir", r.URL.String())
+	q.Set("challenge", challenge)
+	u.RawQuery = q.Encode()
+
+	component, err := web.BaseWithChallengeAndOGTags("Making sure you're not a bot!", page(challenge, u.String(), rule.Challenge.Difficulty), challenge, rule.Challenge, ogTags)
+	if err != nil {
+		return nil, fmt.Errorf("can't render page: %w", err)
+	}
+
+	return component, nil
+}
+
+func (i *Impl) Validate(r *http.Request, lg *slog.Logger, rule *policy.Bot, wantChallenge string) error {
+	gotChallenge := r.FormValue("challenge")
+
+	if subtle.ConstantTimeCompare([]byte(wantChallenge), []byte(gotChallenge)) != 1 {
+		return challenge.NewError("validate", "invalid response", fmt.Errorf("%w: wanted response %s but got %s", challenge.ErrFailed, wantChallenge, gotChallenge))
+	}
+
+	return nil
+}

lib/challenge/metarefresh/metarefresh.templ

@@ -0,0 +1,17 @@
+package metarefresh
+
+import (
+	"fmt"
+
+	"github.com/TecharoHQ/anubis"
+)
+
+templ page(challenge, redir string, difficulty int) {
+	<div class="centered-div">
+		<img id="image" style="width:100%;max-width:256px;" src={ anubis.BasePrefix + "/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" + anubis.Version }/>
+		<img style="display:none;" style="width:100%;max-width:256px;" src={ anubis.BasePrefix + "/.within.website/x/cmd/anubis/static/img/happy.webp?cacheBuster=" + anubis.Version }/>
+		<p id="status">Loading...</p>
+		<p>Please wait a moment while we ensure the security of your connection.</p>
+		<meta http-equiv="refresh" content={ fmt.Sprintf("%d; url=%s", difficulty, redir) }/>
+	</div>
+}

lib/challenge/metarefresh/metarefresh_templ.go

@@ -0,0 +1,85 @@
+// Code generated by templ - DO NOT EDIT.
+
+// templ: version: v0.3.887
+package metarefresh
+
+//lint:file-ignore SA4006 This context is only used if a nested component is present.
+
+import "github.com/a-h/templ"
+import templruntime "github.com/a-h/templ/runtime"
+
+import (
+	"fmt"
+
+	"github.com/TecharoHQ/anubis"
+)
+
+func page(challenge, redir string, difficulty int) templ.Component {
+	return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
+		templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
+		if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
+			return templ_7745c5c3_CtxErr
+		}
+		templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
+		if !templ_7745c5c3_IsBuffer {
+			defer func() {
+				templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
+				if templ_7745c5c3_Err == nil {
+					templ_7745c5c3_Err = templ_7745c5c3_BufErr
+				}
+			}()
+		}
+		ctx = templ.InitializeContext(ctx)
+		templ_7745c5c3_Var1 := templ.GetChildren(ctx)
+		if templ_7745c5c3_Var1 == nil {
+			templ_7745c5c3_Var1 = templ.NopComponent
+		}
+		ctx = templ.ClearChildren(ctx)
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"centered-div\"><img id=\"image\" style=\"width:100%;max-width:256px;\" src=\"")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		var templ_7745c5c3_Var2 string
+		templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(anubis.BasePrefix + "/.within.website/x/cmd/anubis/static/img/pensive.webp?cacheBuster=" + anubis.Version)
+		if templ_7745c5c3_Err != nil {
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `metarefresh.templ`, Line: 11, Col: 165}
+		}
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\"> <img style=\"display:none;\" style=\"width:100%;max-width:256px;\" src=\"")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		var templ_7745c5c3_Var3 string
+		templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(anubis.BasePrefix + "/.within.website/x/cmd/anubis/static/img/happy.webp?cacheBuster=" + anubis.Version)
+		if templ_7745c5c3_Err != nil {
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `metarefresh.templ`, Line: 12, Col: 174}
+		}
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "\"><p id=\"status\">Loading...</p><p>Please wait a moment while we ensure the security of your connection.</p><meta http-equiv=\"refresh\" content=\"")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		var templ_7745c5c3_Var4 string
+		templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d; url=%s", difficulty, redir))
+		if templ_7745c5c3_Err != nil {
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `metarefresh.templ`, Line: 15, Col: 83}
+		}
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "\"></div>")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		return nil
+	})
+}
+
+var _ = templruntime.GeneratedTemplate