data/botPolicies: allow iMessage scraper by default (#178)

Signed-off-by: Xe Iaso <me@xeiaso.net>

data/botPolicies.json

@@ -343,6 +343,12 @@
         "5.102.173.71/32"
       ]
     },
+    {
+      "_comment": "This has been reverse-engineered through making iMessage's preview function hit a URL that prints the user-agent in the server logs.",
+      "name": "iMessage preview",
+      "user_agent_regex": ".*facebookexternalhit/1\\.1 Facebot Twitterbot/1\\.0$",
+      "action": "ALLOW"
+    },
     {
       "name": "us-artificial-intelligence-scraper",
       "user_agent_regex": "\\+https\\://github\\.com/US-Artificial-Intelligence/scraper",

docs/docs/CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- Allow iMessage's link preview fetcher through Anubis by default.
 - Added a periodic cleanup routine for the decaymap that removes expired entries, ensuring stale data is properly pruned.
 - Added a no-store Cache-Control header to the challenge page
 - Hide the directory listings for Anubis' internal static content

internal/test/playwright_test.go

@@ -63,6 +63,12 @@ var (
 			realIP:    "216.18.205.234",
 			userAgent: "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)",
 		},
+		{
+			name:      "iMessageScraper",
+			action:    actionAllow,
+			realIP:    placeholderIP,
+			userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0",
+		},
 		{
 			name:      "unknownAgent",
 			action:    actionAllow,