Added robots.txt plugin

Automatically generates the content, including the correct link to the sitemap.

configs/nginx.conf

@@ -5,16 +5,6 @@ location ~ ^/Themes/[^/]+/Assets/ {
     }
 }
 
-location /robots.txt {
-    # Allow access to the robots.txt file
-    try_files $uri =204;
-}
-
-location /sitemap.xml {
-    # Allow access to the sitemap.xml file
-    try_files $uri =204;
-}
-
 location / {
     try_files $uri $uri/ /index.php?$args;
 }

src/.htaccess

@@ -5,7 +5,4 @@ CGIPassAuth On
 RewriteCond %{REQUEST_FILENAME} -f
 RewriteRule ^Themes/[^/]+/Assets/.+$ - [L]
 
-# Allow access to the robots.txt file
-RewriteRule ^robots\.txt$ - [L]
-
 RewriteRule ^(.+)$ index.php [L,QSA]

src/Plugins/Robotstxt/RobotstxtPlugin.php

@@ -0,0 +1,25 @@
+<?php
+
+use AntCMS\AntPlugin;
+use AntCMS\AntConfig;
+use AntCMS\AntTools;
+
+class RobotstxtPlugin extends AntPlugin
+{
+    public function handlePluginRoute(array $route)
+    {
+        $protocol = AntConfig::currentConfig('forceHTTPS') ? 'https' : 'http';
+        $baseURL = AntConfig::currentConfig('baseURL');
+
+        $robotstxt = 'User-agent: *' . "\n";
+        $robotstxt.= 'Disallow: /plugin/' . "\n";
+        $robotstxt.= 'Sitemap: ' . $protocol . '://' . AntTools::repairURL($baseURL . '/sitemap.xml' . "\n");
+        header("Content-Type: text/plain");
+        echo $robotstxt;
+        exit;
+    }
+    public function getName()
+    {
+        return 'Robotstxt';
+    }
+}

src/index.php

@@ -54,11 +54,16 @@ if ($segments[0] === 'themes' && $segments[2] === 'assets') {
     exit;
 }
 
-if($segments[0] == 'sitemap.xml'){
+if ($segments[0] == 'sitemap.xml') {
     $segments[0] = 'plugin';
     $segments[1] = 'sitemap';
 }
 
+if ($segments[0] == 'robots.txt') {
+    $segments[0] = 'plugin';
+    $segments[1] = 'robotstxt';
+}
+
 if ($segments[0] === 'plugin') {
     $pluginName = $segments[1];
     $pluginLoader = new AntPluginLoader();

src/robots.txt

@@ -1,2 +0,0 @@
-User-agent: *
-Disallow: /plugin/