fix: add robots.txt setup for api-gateway and files server to disallow indexing

.pnp.cjs

@@ -2701,6 +2701,7 @@ const RAW_RUNTIME_STATE =
           ["eslint", "npm:8.25.0"],\
           ["eslint-plugin-prettier", "virtual:c66bf20e88479ada0172094776519a9f51acc4731d22079b60a295bcec7ea42d5545cbce58a77a50d932bf953298799135e99707486e343da6d99ba1d167bdbd#npm:4.2.1"],\
           ["express", "npm:4.18.2"],\
+          ["express-robots-txt", "virtual:04783e12400851b8a3d76e71495851cc94959db6e62f04cb0a31190080629440b182d8c8eb4d7f2b04e281912f2783a5fd4d2c3c6ab68d38b7097246c93f4c19#npm:1.0.0"],\
           ["helmet", "npm:6.0.0"],\
           ["inversify", "npm:6.0.1"],\
           ["inversify-express-utils", "npm:6.4.3"],\
@@ -2990,6 +2991,7 @@ const RAW_RUNTIME_STATE =
           ["eslint", "npm:8.25.0"],\
           ["eslint-plugin-prettier", "virtual:c66bf20e88479ada0172094776519a9f51acc4731d22079b60a295bcec7ea42d5545cbce58a77a50d932bf953298799135e99707486e343da6d99ba1d167bdbd#npm:4.2.1"],\
           ["express", "npm:4.18.2"],\
+          ["express-robots-txt", "virtual:04783e12400851b8a3d76e71495851cc94959db6e62f04cb0a31190080629440b182d8c8eb4d7f2b04e281912f2783a5fd4d2c3c6ab68d38b7097246c93f4c19#npm:1.0.0"],\
           ["express-winston", "virtual:b442cf0427cc365d1c137f7340f9b81f9b204561afe791a8564ae9590c3a7fc4b5f793aaf8817b946f75a3cb64d03ef8790eb847f8b576b41e700da7b00c240c#npm:4.2.0"],\
           ["helmet", "npm:6.0.0"],\
           ["inversify", "npm:6.0.1"],\
@@ -7316,6 +7318,28 @@ const RAW_RUNTIME_STATE =
         "linkType": "HARD"\
       }]\
     ]],\
+    ["express-robots-txt", [\
+      ["npm:1.0.0", {\
+        "packageLocation": "./.yarn/cache/express-robots-txt-npm-1.0.0-dcc8bd8f0a-54f066f6c3.zip/node_modules/express-robots-txt/",\
+        "packageDependencies": [\
+          ["express-robots-txt", "npm:1.0.0"]\
+        ],\
+        "linkType": "SOFT"\
+      }],\
+      ["virtual:04783e12400851b8a3d76e71495851cc94959db6e62f04cb0a31190080629440b182d8c8eb4d7f2b04e281912f2783a5fd4d2c3c6ab68d38b7097246c93f4c19#npm:1.0.0", {\
+        "packageLocation": "./.yarn/__virtual__/express-robots-txt-virtual-0a3eb9f2f5/0/cache/express-robots-txt-npm-1.0.0-dcc8bd8f0a-54f066f6c3.zip/node_modules/express-robots-txt/",\
+        "packageDependencies": [\
+          ["express-robots-txt", "virtual:04783e12400851b8a3d76e71495851cc94959db6e62f04cb0a31190080629440b182d8c8eb4d7f2b04e281912f2783a5fd4d2c3c6ab68d38b7097246c93f4c19#npm:1.0.0"],\
+          ["@types/express", "npm:4.17.14"],\
+          ["express", "npm:4.18.2"]\
+        ],\
+        "packagePeers": [\
+          "@types/express",\
+          "express"\
+        ],\
+        "linkType": "HARD"\
+      }]\
+    ]],\
     ["express-winston", [\
       ["npm:4.2.0", {\
         "packageLocation": "./.yarn/cache/express-winston-npm-4.2.0-e4cfb26486-2d4b37671d.zip/node_modules/express-winston/",\

packages/api-gateway/bin/server.ts

@@ -31,6 +31,8 @@ import helmet from 'helmet'
 import * as cors from 'cors'
 import { text, json, Request, Response, NextFunction, RequestHandler, ErrorRequestHandler } from 'express'
 import * as winston from 'winston'
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const robots = require('express-robots-txt')
 
 import { InversifyExpressServer } from 'inversify-express-utils'
 import { ContainerConfigLoader } from '../src/Bootstrap/Container'
@@ -78,6 +80,12 @@ void container.load().then((container) => {
       }),
     )
     app.use(cors())
+    app.use(
+      robots({
+        UserAgent: '*',
+        Disallow: '/',
+      }),
+    )
 
     if (env.get('SENTRY_DSN', true)) {
       Sentry.init({

packages/api-gateway/package.json

@@ -32,6 +32,7 @@
     "cors": "2.8.5",
     "dotenv": "^16.0.1",
     "express": "^4.18.2",
+    "express-robots-txt": "^1.0.0",
     "helmet": "^6.0.0",
     "inversify": "^6.0.1",
     "inversify-express-utils": "^6.4.3",

packages/files/bin/server.ts

@@ -12,6 +12,8 @@ import helmet from 'helmet'
 import * as cors from 'cors'
 import { urlencoded, json, raw, Request, Response, NextFunction, RequestHandler, ErrorRequestHandler } from 'express'
 import * as winston from 'winston'
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const robots = require('express-robots-txt')
 
 import { InversifyExpressServer } from 'inversify-express-utils'
 import { ContainerConfigLoader } from '../src/Bootstrap/Container'
@@ -65,6 +67,12 @@ void container.load().then((container) => {
         exposedHeaders: ['Content-Range', 'Accept-Ranges'],
       }),
     )
+    app.use(
+      robots({
+        UserAgent: '*',
+        Disallow: '/',
+      }),
+    )
 
     if (env.get('SENTRY_DSN', true)) {
       Sentry.init({

packages/files/package.json

@@ -39,6 +39,7 @@
     "dayjs": "^1.11.6",
     "dotenv": "^16.0.1",
     "express": "^4.18.2",
+    "express-robots-txt": "^1.0.0",
     "express-winston": "^4.0.5",
     "helmet": "^6.0.0",
     "inversify": "^6.0.1",

yarn.lock

@@ -1929,6 +1929,7 @@ __metadata:
     eslint: "npm:^8.14.0"
     eslint-plugin-prettier: "npm:^4.0.0"
     express: "npm:^4.18.2"
+    express-robots-txt: "npm:^1.0.0"
     helmet: "npm:^6.0.0"
     inversify: "npm:^6.0.1"
     inversify-express-utils: "npm:^6.4.3"
@@ -2216,6 +2217,7 @@ __metadata:
     eslint: "npm:^8.14.0"
     eslint-plugin-prettier: "npm:^4.0.0"
     express: "npm:^4.18.2"
+    express-robots-txt: "npm:^1.0.0"
     express-winston: "npm:^4.0.5"
     helmet: "npm:^6.0.0"
     inversify: "npm:^6.0.1"
@@ -5580,6 +5582,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"express-robots-txt@npm:^1.0.0":
+  version: 1.0.0
+  resolution: "express-robots-txt@npm:1.0.0"
+  peerDependencies:
+    express: ^4.12.1
+  checksum: 54f066f6c305694ea2082d2b0a46bab8dcbf2b478780cf3f1bc404a5d8c83a4e2d6f06e15b42f0aec2c8866ffb97150487e0bb9abdd9604fa78ea68950946b43
+  languageName: node
+  linkType: hard
+
 "express-winston@npm:^4.0.5":
   version: 4.2.0
   resolution: "express-winston@npm:4.2.0"