data: disable generic-bot-catchall by default (#322)

Signed-off-by: Xe Iaso <me@xeiaso.net>

data/botPolicies.json

@@ -670,16 +670,6 @@
       "user_agent_regex": "HeadlessChromium",
       "action": "DENY"
     },
-    {
-      "name": "generic-bot-catchall",
-      "user_agent_regex": "(?i:bot|crawler)",
-      "action": "CHALLENGE",
-      "challenge": {
-        "difficulty": 16,
-        "report_as": 4,
-        "algorithm": "slow"
-      }
-    },
     {
       "name": "generic-browser",
       "user_agent_regex": "Mozilla|Opera",

data/botPolicies.yaml

@@ -645,14 +645,14 @@ bots:
   path_regex: ^/robots.txt$
   action: ALLOW
 
-# Punish any bot with "bot" in the user-agent string
-- name: generic-bot-catchall
-  user_agent_regex: (?i:bot|crawler)
-  action: CHALLENGE
-  challenge:
-    difficulty: 16  # impossible
-    report_as: 4    # lie to the operator
-    algorithm: slow # intentionally waste CPU cycles and time
+# # Punish any bot with "bot" in the user-agent string
+# - name: generic-bot-catchall
+#   user_agent_regex: (?i:bot|crawler)
+#   action: CHALLENGE
+#   challenge:
+#     difficulty: 16  # impossible
+#     report_as: 4    # lie to the operator
+#     algorithm: slow # intentionally waste CPU cycles and time
 
 - name: generic-browser
   user_agent_regex: >

docs/docs/CHANGELOG.md

@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added headers support to bot policy rules
 - Moved configuration file from JSON to YAML by default
 - Added documentation on how to use Anubis with Traefik in Docker
+- Disable `generic-bot-catchall` rule because of its high false positive rate in real-world scenarios
 
 ## v1.16.0
 

docs/docs/admin/policies.mdx

@@ -132,6 +132,8 @@ Rules can also have their own challenge settings. These are customized using the
 <Tabs>
 <TabItem value="json" label="JSON" default>
 
+This rule has been known to have a high false positive rate in testing. Please use this with care.
+
 ```json
 {
   "name": "generic-bot-catchall",
@@ -148,6 +150,8 @@ Rules can also have their own challenge settings. These are customized using the
 </TabItem>
 <TabItem value="yaml" label="YAML">
 
+This rule has been known to have a high false positive rate in testing. Please use this with care.
+
 ```yaml
 # Punish any bot with "bot" in the user-agent string
 - name: generic-bot-catchall