beenull/sftpgo-mirror
1
0
Fork 0
mirror of https://github.com/drakkan/sftpgo.git synced 2024-11-22 15:40:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: disallow all crawlers with robots.txt (#833)

Browse source 
Signed-off-by: Tim Birkett <tim.birkett@sainsburys.co.uk>
This commit is contained in:
Tim Birkett 2022-05-13 08:23:43 +01:00 committed by GitHub
parent 1e0b3a2a8c
commit 7b1a0d3cd3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 13 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
httpd/httpd.go
View file

@ -86,6 +86,7 @@ const (
providerEventsPath = "/api/v2/events/provider" providerEventsPath = "/api/v2/events/provider"
sharesPath = "/api/v2/shares" sharesPath = "/api/v2/shares"
healthzPath = "/healthz" healthzPath = "/healthz"
robotsTxtPath = "/robots.txt"
webRootPathDefault = "/" webRootPathDefault = "/"
webBasePathDefault = "/web" webBasePathDefault = "/web"
webBasePathAdminDefault = "/web/admin" webBasePathAdminDefault = "/web/admin"

8
httpd/httpd_test.go
View file

@ -118,6 +118,7 @@ const (
providerEventsPath = "/api/v2/events/provider" providerEventsPath = "/api/v2/events/provider"
sharesPath = "/api/v2/shares" sharesPath = "/api/v2/shares"
healthzPath = "/healthz" healthzPath = "/healthz"
robotsTxtPath = "/robots.txt"
webBasePath = "/web" webBasePath = "/web"
webBasePathAdmin = "/web/admin" webBasePathAdmin = "/web/admin"
webAdminSetupPath = "/web/admin/setup" webAdminSetupPath = "/web/admin/setup"
@ -8910,6 +8911,13 @@ func TestHealthCheck(t *testing.T) {
assert.Equal(t, "ok", rr.Body.String()) assert.Equal(t, "ok", rr.Body.String())
} }
func TestRobotsTxtCheck(t *testing.T) {
req, _ := http.NewRequest(http.MethodGet, "/robots.txt", nil)
rr := executeRequest(req)
checkResponseCode(t, http.StatusOK, rr)
assert.Equal(t, "User-agent: *\nDisallow: /", rr.Body.String())
}
func TestGetWebRootMock(t *testing.T) { func TestGetWebRootMock(t *testing.T) {
req, _ := http.NewRequest(http.MethodGet, "/", nil) req, _ := http.NewRequest(http.MethodGet, "/", nil)
rr := executeRequest(req) rr := executeRequest(req)

4
httpd/server.go
View file

@ -1137,6 +1137,10 @@ func (s *httpdServer) initializeRouter() {
render.PlainText(w, r, "ok") render.PlainText(w, r, "ok")
}) })
s.router.Get(robotsTxtPath, func(w http.ResponseWriter, r *http.Request) {
render.PlainText(w, r, "User-agent: *\nDisallow: /")
})
// share API exposed to external users // share API exposed to external users
s.router.Get(sharesPath+"/{id}", s.downloadFromShare) s.router.Get(sharesPath+"/{id}", s.downloadFromShare)
s.router.Post(sharesPath+"/{id}", s.uploadFilesToShare) s.router.Post(sharesPath+"/{id}", s.uploadFilesToShare)