beenull/standardnotes-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix: add robots.txt setup for api-gateway and files server to disallow indexing

Browse source
This commit is contained in:
Karol Sójko 2023-01-13 10:03:03 +01:00
parent d1a4bd38e0
commit bb820437af
No known key found for this signature in database
GPG key ID: A50543BF560BDEB0
7 changed files with 53 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
.pnp.cjs generated
View file

@ -2701,6 +2701,7 @@ const RAW_RUNTIME_STATE =
["eslint", "npm:8.25.0"],\
["eslint-plugin-prettier", "virtual:c66bf20e88479ada0172094776519a9f51acc4731d22079b60a295bcec7ea42d5545cbce58a77a50d932bf953298799135e99707486e343da6d99ba1d167bdbd#npm:4.2.1"],\
["express", "npm:4.18.2"],\
["express-robots-txt", "virtual:04783e12400851b8a3d76e71495851cc94959db6e62f04cb0a31190080629440b182d8c8eb4d7f2b04e281912f2783a5fd4d2c3c6ab68d38b7097246c93f4c19#npm:1.0.0"],\
["helmet", "npm:6.0.0"],\
["inversify", "npm:6.0.1"],\
["inversify-express-utils", "npm:6.4.3"],\
@ -2990,6 +2991,7 @@ const RAW_RUNTIME_STATE =
["eslint", "npm:8.25.0"],\
["eslint-plugin-prettier", "virtual:c66bf20e88479ada0172094776519a9f51acc4731d22079b60a295bcec7ea42d5545cbce58a77a50d932bf953298799135e99707486e343da6d99ba1d167bdbd#npm:4.2.1"],\
["express", "npm:4.18.2"],\
["express-robots-txt", "virtual:04783e12400851b8a3d76e71495851cc94959db6e62f04cb0a31190080629440b182d8c8eb4d7f2b04e281912f2783a5fd4d2c3c6ab68d38b7097246c93f4c19#npm:1.0.0"],\
["express-winston", "virtual:b442cf0427cc365d1c137f7340f9b81f9b204561afe791a8564ae9590c3a7fc4b5f793aaf8817b946f75a3cb64d03ef8790eb847f8b576b41e700da7b00c240c#npm:4.2.0"],\
["helmet", "npm:6.0.0"],\
["inversify", "npm:6.0.1"],\
@ -7316,6 +7318,28 @@ const RAW_RUNTIME_STATE =
"linkType": "HARD"\
}]\
]],\
["express-robots-txt", [\
["npm:1.0.0", {\
"packageLocation": "./.yarn/cache/express-robots-txt-npm-1.0.0-dcc8bd8f0a-54f066f6c3.zip/node_modules/express-robots-txt/",\
"packageDependencies": [\
["express-robots-txt", "npm:1.0.0"]\
],\
"linkType": "SOFT"\
}],\
["virtual:04783e12400851b8a3d76e71495851cc94959db6e62f04cb0a31190080629440b182d8c8eb4d7f2b04e281912f2783a5fd4d2c3c6ab68d38b7097246c93f4c19#npm:1.0.0", {\
"packageLocation": "./.yarn/__virtual__/express-robots-txt-virtual-0a3eb9f2f5/0/cache/express-robots-txt-npm-1.0.0-dcc8bd8f0a-54f066f6c3.zip/node_modules/express-robots-txt/",\
"packageDependencies": [\
["express-robots-txt", "virtual:04783e12400851b8a3d76e71495851cc94959db6e62f04cb0a31190080629440b182d8c8eb4d7f2b04e281912f2783a5fd4d2c3c6ab68d38b7097246c93f4c19#npm:1.0.0"],\
["@types/express", "npm:4.17.14"],\
["express", "npm:4.18.2"]\
],\
"packagePeers": [\
"@types/express",\
"express"\
],\
"linkType": "HARD"\
}]\
]],\
["express-winston", [\
["npm:4.2.0", {\
"packageLocation": "./.yarn/cache/express-winston-npm-4.2.0-e4cfb26486-2d4b37671d.zip/node_modules/express-winston/",\

BIN
.yarn/cache/express-robots-txt-npm-1.0.0-dcc8bd8f0a-54f066f6c3.zip vendored Normal file
View file

Binary file not shown.

8
packages/api-gateway/bin/server.ts
View file

@ -31,6 +31,8 @@ import helmet from 'helmet'
import * as cors from 'cors'
import { text, json, Request, Response, NextFunction, RequestHandler, ErrorRequestHandler } from 'express'
import * as winston from 'winston'
// eslint-disable-next-line @typescript-eslint/no-var-requires
const robots = require('express-robots-txt')
import { InversifyExpressServer } from 'inversify-express-utils'
import { ContainerConfigLoader } from '../src/Bootstrap/Container'
@ -78,6 +80,12 @@ void container.load().then((container) => {
}),
)
app.use(cors())
app.use(
robots({
UserAgent: '*',
Disallow: '/',
}),
)
if (env.get('SENTRY_DSN', true)) {
Sentry.init({

1
packages/api-gateway/package.json
View file

@ -32,6 +32,7 @@
"cors": "2.8.5",
"dotenv": "^16.0.1",
"express": "^4.18.2",
"express-robots-txt": "^1.0.0",
"helmet": "^6.0.0",
"inversify": "^6.0.1",
"inversify-express-utils": "^6.4.3",

8
packages/files/bin/server.ts
View file

@ -12,6 +12,8 @@ import helmet from 'helmet'
import * as cors from 'cors'
import { urlencoded, json, raw, Request, Response, NextFunction, RequestHandler, ErrorRequestHandler } from 'express'
import * as winston from 'winston'
// eslint-disable-next-line @typescript-eslint/no-var-requires
const robots = require('express-robots-txt')
import { InversifyExpressServer } from 'inversify-express-utils'
import { ContainerConfigLoader } from '../src/Bootstrap/Container'
@ -65,6 +67,12 @@ void container.load().then((container) => {
exposedHeaders: ['Content-Range', 'Accept-Ranges'],
}),
)
app.use(
robots({
UserAgent: '*',
Disallow: '/',
}),
)
if (env.get('SENTRY_DSN', true)) {
Sentry.init({

1
packages/files/package.json
View file

@ -39,6 +39,7 @@
"dayjs": "^1.11.6",
"dotenv": "^16.0.1",
"express": "^4.18.2",
"express-robots-txt": "^1.0.0",
"express-winston": "^4.0.5",
"helmet": "^6.0.0",
"inversify": "^6.0.1",

11
yarn.lock
View file

@ -1929,6 +1929,7 @@ __metadata:
eslint: "npm:^8.14.0"
eslint-plugin-prettier: "npm:^4.0.0"
express: "npm:^4.18.2"
express-robots-txt: "npm:^1.0.0"
helmet: "npm:^6.0.0"
inversify: "npm:^6.0.1"
inversify-express-utils: "npm:^6.4.3"
@ -2216,6 +2217,7 @@ __metadata:
eslint: "npm:^8.14.0"
eslint-plugin-prettier: "npm:^4.0.0"
express: "npm:^4.18.2"
express-robots-txt: "npm:^1.0.0"
express-winston: "npm:^4.0.5"
helmet: "npm:^6.0.0"
inversify: "npm:^6.0.1"
@ -5580,6 +5582,15 @@ __metadata:
languageName: node
linkType: hard
"express-robots-txt@npm:^1.0.0":
version: 1.0.0
resolution: "express-robots-txt@npm:1.0.0"
peerDependencies:
express: ^4.12.1
checksum: 54f066f6c305694ea2082d2b0a46bab8dcbf2b478780cf3f1bc404a5d8c83a4e2d6f06e15b42f0aec2c8866ffb97150487e0bb9abdd9604fa78ea68950946b43
languageName: node
linkType: hard
"express-winston@npm:^4.0.5":
version: 4.2.0
resolution: "express-winston@npm:4.2.0"