From 81c8e8d898a2360a034aea4e890d66fc8b9d7fb9 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sun, 15 Mar 2020 15:16:35 +0100 Subject: [PATCH] add profiler support profiling is now available via the HTTP base URL /debug/pprof/ examples, use this URL to start and download a 30 seconds CPU profile: /debug/pprof/profile?seconds=30 use this URL to profile used memory: /debug/pprof/heap?gc=1 use this URL to profile allocated memory: /debug/pprof/allocs?gc=1 Full docs here: https://golang.org/pkg/net/http/pprof/ --- cmd/portable.go | 1 + cmd/root.go | 14 ++++++++++++++ cmd/serve.go | 1 + cmd/start_windows.go | 1 + httpd/httpd.go | 5 +++-- httpd/httpd_test.go | 17 ++++++++++++----- httpd/router.go | 8 +++++++- service/service.go | 7 ++++--- sftpd/sftpd_test.go | 2 +- 9 files changed, 44 insertions(+), 12 deletions(-) diff --git a/cmd/portable.go b/cmd/portable.go index 1340437c..0cb72c1e 100644 --- a/cmd/portable.go +++ b/cmd/portable.go @@ -91,6 +91,7 @@ Please take a look at the usage below to customize the serving parameters`, LogMaxAge: defaultLogMaxAge, LogCompress: defaultLogCompress, LogVerbose: defaultLogVerbose, + Profiler: defaultProfiler, Shutdown: make(chan bool), PortableMode: 1, PortableUser: dataprovider.User{ diff --git a/cmd/root.go b/cmd/root.go index 4785edc4..714047a5 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -30,6 +30,8 @@ const ( logCompressKey = "log_compress" logVerboseFlag = "log-verbose" logVerboseKey = "log_verbose" + profilerFlag = "profiler" + profilerKey = "profiler" defaultConfigDir = "." defaultConfigName = config.DefaultConfigName defaultLogFile = "sftpgo.log" @@ -38,6 +40,7 @@ const ( defaultLogMaxAge = 28 defaultLogCompress = false defaultLogVerbose = true + defaultProfiler = false ) var ( @@ -49,6 +52,7 @@ var ( logMaxAge int logCompress bool logVerbose bool + profiler bool rootCmd = &cobra.Command{ Use: "sftpgo", @@ -135,6 +139,13 @@ func addServeFlags(cmd *cobra.Command) { cmd.Flags().BoolVarP(&logVerbose, logVerboseFlag, "v", viper.GetBool(logVerboseKey), "Enable verbose logs. "+ "This flag can be set using SFTPGO_LOG_VERBOSE env var too.") viper.BindPFlag(logVerboseKey, cmd.Flags().Lookup(logVerboseFlag)) + + viper.SetDefault(profilerKey, defaultProfiler) + viper.BindEnv(profilerKey, "SFTPGO_PROFILER") + cmd.Flags().BoolVarP(&profiler, profilerFlag, "p", viper.GetBool(profilerKey), "Enable the built-in profiler. "+ + "The profiler will be accessible via HTTP/HTTPS using the base URL \"/debug/pprof/\". "+ + "This flag can be set using SFTPGO_PROFILER env var too.") + viper.BindPFlag(profilerKey, cmd.Flags().Lookup(profilerFlag)) } func getCustomServeFlags() []string { @@ -170,5 +181,8 @@ func getCustomServeFlags() []string { if logCompress != defaultLogCompress { result = append(result, "--"+logCompressFlag+"=true") } + if profiler != defaultProfiler { + result = append(result, "--"+profilerFlag+"=true") + } return result } diff --git a/cmd/serve.go b/cmd/serve.go index c2d7ff07..aacba175 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -25,6 +25,7 @@ Please take a look at the usage below to customize the startup options`, LogMaxAge: logMaxAge, LogCompress: logCompress, LogVerbose: logVerbose, + Profiler: profiler, Shutdown: make(chan bool), } if err := service.Start(); err == nil { diff --git a/cmd/start_windows.go b/cmd/start_windows.go index 7384aef8..671fc41f 100644 --- a/cmd/start_windows.go +++ b/cmd/start_windows.go @@ -27,6 +27,7 @@ var ( LogMaxAge: logMaxAge, LogCompress: logCompress, LogVerbose: logVerbose, + Profiler: profiler, Shutdown: make(chan bool), } winService := service.WindowsService{ diff --git a/httpd/httpd.go b/httpd/httpd.go index d2326369..cf89ae06 100644 --- a/httpd/httpd.go +++ b/httpd/httpd.go @@ -30,6 +30,7 @@ const ( dumpDataPath = "/api/v1/dumpdata" loadDataPath = "/api/v1/loaddata" metricsPath = "/metrics" + pprofBasePath = "/debug" webBasePath = "/web" webUsersPath = "/web/users" webUserPath = "/web/user" @@ -85,7 +86,7 @@ func SetDataProvider(provider dataprovider.Provider) { } // Initialize the HTTP server -func (c Conf) Initialize(configDir string) error { +func (c Conf) Initialize(configDir string, profiler bool) error { var err error logger.Debug(logSender, "", "initializing HTTP server with config %+v", c) backupsPath = getConfigPath(c.BackupsPath, configDir) @@ -103,7 +104,7 @@ func (c Conf) Initialize(configDir string) error { certificateFile := getConfigPath(c.CertificateFile, configDir) certificateKeyFile := getConfigPath(c.CertificateKeyFile, configDir) loadTemplates(templatesPath) - initializeRouter(staticFilesPath) + initializeRouter(staticFilesPath, profiler) httpServer := &http.Server{ Addr: fmt.Sprintf("%s:%d", c.BindAddress, c.BindPort), Handler: router, diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index c70be603..d90927b9 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -49,6 +49,7 @@ const ( dumpDataPath = "/api/v1/dumpdata" loadDataPath = "/api/v1/loaddata" metricsPath = "/metrics" + pprofPath = "/debug/pprof/" webBasePath = "/web" webUsersPath = "/web/users" webUserPath = "/web/user" @@ -117,7 +118,7 @@ func TestMain(m *testing.M) { httpd.SetDataProvider(dataProvider) go func() { - if err := httpdConf.Initialize(configDir); err != nil { + if err := httpdConf.Initialize(configDir, true); err != nil { logger.Error(logSender, "", "could not start HTTP server: %v", err) } }() @@ -133,7 +134,7 @@ func TestMain(m *testing.M) { httpdConf.CertificateKeyFile = keyPath go func() { - if err := httpdConf.Initialize(configDir); err != nil { + if err := httpdConf.Initialize(configDir, true); err != nil { logger.Error(logSender, "", "could not start HTTPS server: %v", err) } }() @@ -157,7 +158,7 @@ func TestInitialization(t *testing.T) { httpdConf := config.GetHTTPDConfig() httpdConf.BackupsPath = "test_backups" httpdConf.AuthUserFile = "invalid file" - err := httpdConf.Initialize(configDir) + err := httpdConf.Initialize(configDir, true) if err == nil { t.Error("Inizialize must fail") } @@ -165,14 +166,14 @@ func TestInitialization(t *testing.T) { httpdConf.AuthUserFile = "" httpdConf.CertificateFile = "invalid file" httpdConf.CertificateKeyFile = "invalid file" - err = httpdConf.Initialize(configDir) + err = httpdConf.Initialize(configDir, true) if err == nil { t.Error("Inizialize must fail") } httpdConf.CertificateFile = "" httpdConf.CertificateKeyFile = "" httpdConf.TemplatesPath = "." - err = httpdConf.Initialize(configDir) + err = httpdConf.Initialize(configDir, true) if err == nil { t.Error("Inizialize must fail") } @@ -1681,6 +1682,12 @@ func TestMetricsMock(t *testing.T) { checkResponseCode(t, http.StatusOK, rr.Code) } +func TestPProfEndPointMock(t *testing.T) { + req, _ := http.NewRequest(http.MethodGet, pprofPath, nil) + rr := executeRequest(req) + checkResponseCode(t, http.StatusOK, rr.Code) +} + func TestGetWebRootMock(t *testing.T) { req, _ := http.NewRequest(http.MethodGet, "/", nil) rr := executeRequest(req) diff --git a/httpd/router.go b/httpd/router.go index 4c727639..778e3de7 100644 --- a/httpd/router.go +++ b/httpd/router.go @@ -18,13 +18,19 @@ func GetHTTPRouter() http.Handler { return router } -func initializeRouter(staticFilesPath string) { +func initializeRouter(staticFilesPath string, profiler bool) { router = chi.NewRouter() router.Use(middleware.RequestID) router.Use(middleware.RealIP) router.Use(logger.NewStructuredLogger(logger.GetLogger())) router.Use(middleware.Recoverer) + if profiler { + logger.InfoToConsole("enabling the built-in profiler") + logger.Info(logSender, "", "enabling the built-in profiler") + router.Mount(pprofBasePath, middleware.Profiler()) + } + router.NotFound(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, nil, "Not Found", http.StatusNotFound) })) diff --git a/service/service.go b/service/service.go index 6f8788a5..f1dddde2 100644 --- a/service/service.go +++ b/service/service.go @@ -41,6 +41,7 @@ type Service struct { LogVerbose bool PortableMode int PortableUser dataprovider.User + Profiler bool Shutdown chan bool } @@ -62,8 +63,8 @@ func (s *Service) Start() error { } version := utils.GetAppVersion() logger.Info(logSender, "", "starting SFTPGo %v, config dir: %v, config file: %v, log max size: %v log max backups: %v "+ - "log max age: %v log verbose: %v, log compress: %v", version.GetVersionAsString(), s.ConfigDir, s.ConfigFile, s.LogMaxSize, - s.LogMaxBackups, s.LogMaxAge, s.LogVerbose, s.LogCompress) + "log max age: %v log verbose: %v, log compress: %v, profile: %v", version.GetVersionAsString(), s.ConfigDir, s.ConfigFile, + s.LogMaxSize, s.LogMaxBackups, s.LogMaxAge, s.LogVerbose, s.LogCompress, s.Profiler) // in portable mode we don't read configuration from file if s.PortableMode != 1 { config.LoadConfig(s.ConfigDir, s.ConfigFile) @@ -105,7 +106,7 @@ func (s *Service) Start() error { httpd.SetDataProvider(dataProvider) go func() { - if err := httpdConf.Initialize(s.ConfigDir); err != nil { + if err := httpdConf.Initialize(s.ConfigDir, s.Profiler); err != nil { logger.Error(logSender, "", "could not start HTTP server: %v", err) logger.ErrorToConsole("could not start HTTP server: %v", err) } diff --git a/sftpd/sftpd_test.go b/sftpd/sftpd_test.go index defc60cf..9d8a3ac6 100644 --- a/sftpd/sftpd_test.go +++ b/sftpd/sftpd_test.go @@ -197,7 +197,7 @@ func TestMain(m *testing.M) { }() go func() { - if err := httpdConf.Initialize(configDir); err != nil { + if err := httpdConf.Initialize(configDir, false); err != nil { logger.Error(logSender, "", "could not start HTTP server: %v", err) } }()