remove robots.txt endpoint

This reverts #833 because the contributor did not respond to our
request to sign the CLA

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>

internal/httpd/httpd.go

@@ -95,7 +95,6 @@ const (
 	rolesPath                             = "/api/v2/roles"
 	ipListsPath                           = "/api/v2/iplists"
 	healthzPath                           = "/healthz"
-	robotsTxtPath                         = "/robots.txt"
 	webRootPathDefault                    = "/"
 	webBasePathDefault                    = "/web"
 	webBasePathAdminDefault               = "/web/admin"

internal/httpd/httpd_test.go

@@ -131,7 +131,6 @@ const (
 	rolesPath                      = "/api/v2/roles"
 	ipListsPath                    = "/api/v2/iplists"
 	healthzPath                    = "/healthz"
-	robotsTxtPath                  = "/robots.txt"
 	webBasePath                    = "/web"
 	webBasePathAdmin               = "/web/admin"
 	webAdminSetupPath              = "/web/admin/setup"
@@ -12512,13 +12511,6 @@ func TestHealthCheck(t *testing.T) {
 	assert.Equal(t, "ok", rr.Body.String())
 }
 
-func TestRobotsTxtCheck(t *testing.T) {
-	req, _ := http.NewRequest(http.MethodGet, robotsTxtPath, nil)
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusOK, rr)
-	assert.Equal(t, "User-agent: *\nDisallow: /", rr.Body.String())
-}
-
 func TestGetWebRootMock(t *testing.T) {
 	req, _ := http.NewRequest(http.MethodGet, "/", nil)
 	rr := executeRequest(req)

internal/httpd/server.go

@@ -1284,10 +1284,6 @@ func (s *httpdServer) initializeRouter() {
 		render.PlainText(w, r, "ok")
 	})
 
-	s.router.Get(robotsTxtPath, func(w http.ResponseWriter, r *http.Request) {
-		render.PlainText(w, r, "User-agent: *\nDisallow: /")
-	})
-
 	if hasHTTPSRedirect {
 		if p := acme.GetHTTP01WebRoot(); p != "" {
 			serveStaticDir(s.router, acmeChallengeURI, p, true)