From cc134cad9a853daf04409048f8d6d0daef1dad38 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sat, 2 Oct 2021 22:25:41 +0200 Subject: [PATCH] data retention: allow to notify results via e-mail --- Dockerfile | 1 + Dockerfile.alpine | 1 + Dockerfile.distroless | 1 + cmd/smtptest.go | 2 +- cmd/startsubsys.go | 2 +- common/dataretention.go | 118 ++++++++++++++++++-- common/dataretention_test.go | 80 +++++++++++++ common/protocol_test.go | 15 ++- config/config.go | 18 +-- docs/full-configuration.md | 3 +- docs/rest-api.md | 1 + go.mod | 5 +- go.sum | 10 +- httpd/api_retention.go | 16 ++- httpd/httpd_test.go | 27 ++++- httpd/internal_test.go | 40 +++++++ httpd/schema/openapi.yaml | 20 ++++ httpd/webadmin.go | 2 +- service/service.go | 2 +- sftpgo.json | 3 +- smtp/smtp.go | 55 ++++++++- templates/email/retention-check-report.html | 33 ++++++ util/util.go | 4 +- 23 files changed, 417 insertions(+), 42 deletions(-) create mode 100644 templates/email/retention-check-report.html diff --git a/Dockerfile b/Dockerfile index e3fe4b36..7e5b72b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,6 +48,7 @@ COPY --from=builder /workspace/sftpgo /usr/local/bin/ ENV SFTPGO_LOG_FILE_PATH="" # templates and static paths are inside the container ENV SFTPGO_HTTPD__TEMPLATES_PATH=/usr/share/sftpgo/templates +ENV SFTPGO_SMTP__TEMPLATES_PATH=/usr/share/sftpgo/templates ENV SFTPGO_HTTPD__STATIC_FILES_PATH=/usr/share/sftpgo/static # Modify the default configuration file diff --git a/Dockerfile.alpine b/Dockerfile.alpine index ee8d1f8e..b8f96665 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -53,6 +53,7 @@ COPY --from=builder /workspace/sftpgo /usr/local/bin/ ENV SFTPGO_LOG_FILE_PATH="" # templates and static paths are inside the container ENV SFTPGO_HTTPD__TEMPLATES_PATH=/usr/share/sftpgo/templates +ENV SFTPGO_SMTP__TEMPLATES_PATH=/usr/share/sftpgo/templates ENV SFTPGO_HTTPD__STATIC_FILES_PATH=/usr/share/sftpgo/static # Modify the default configuration file diff --git a/Dockerfile.distroless b/Dockerfile.distroless index b9833a19..f5d1b035 100644 --- a/Dockerfile.distroless +++ b/Dockerfile.distroless @@ -47,6 +47,7 @@ COPY --from=builder /etc/mime.types /etc/mime.types ENV SFTPGO_LOG_FILE_PATH="" # templates and static paths are inside the container ENV SFTPGO_HTTPD__TEMPLATES_PATH=/usr/share/sftpgo/templates +ENV SFTPGO_SMTP__TEMPLATES_PATH=/usr/share/sftpgo/templates ENV SFTPGO_HTTPD__STATIC_FILES_PATH=/usr/share/sftpgo/static # These env vars are required to avoid the following error when calling user.Current(): # unable to get the current user: user: Current requires cgo or $USER set in environment diff --git a/cmd/smtptest.go b/cmd/smtptest.go index 54fd0838..cf7cd4a8 100644 --- a/cmd/smtptest.go +++ b/cmd/smtptest.go @@ -29,7 +29,7 @@ If the SMTP configuration is correct you should receive this email.`, os.Exit(1) } smtpConfig := config.GetSMTPConfig() - err = smtpConfig.Initialize() + err = smtpConfig.Initialize(configDir) if err != nil { logger.ErrorToConsole("unable to initialize SMTP configuration: %v", err) os.Exit(1) diff --git a/cmd/startsubsys.go b/cmd/startsubsys.go index 31679f4d..21a4f208 100644 --- a/cmd/startsubsys.go +++ b/cmd/startsubsys.go @@ -87,7 +87,7 @@ Command-line flags should be specified in the Subsystem declaration. os.Exit(1) } smtpConfig := config.GetSMTPConfig() - err = smtpConfig.Initialize() + err = smtpConfig.Initialize(configDir) if err != nil { logger.Error(logSender, connectionID, "unable to initialize SMTP configuration: %v", err) os.Exit(1) diff --git a/common/dataretention.go b/common/dataretention.go index 2ddd41db..b44c1b70 100644 --- a/common/dataretention.go +++ b/common/dataretention.go @@ -1,6 +1,7 @@ package common import ( + "bytes" "fmt" "os" "path" @@ -9,9 +10,20 @@ import ( "github.com/drakkan/sftpgo/v2/dataprovider" "github.com/drakkan/sftpgo/v2/logger" + "github.com/drakkan/sftpgo/v2/smtp" "github.com/drakkan/sftpgo/v2/util" ) +// RetentionCheckNotification defines the supported notification methods for a retention check result +type RetentionCheckNotification = string + +const ( + // no notification, the check results are recorded in the logs + RetentionCheckNotificationNone = "None" + // notify results by email + RetentionCheckNotificationEmail = "Email" +) + var ( // RetentionChecks is the list of active quota scans RetentionChecks ActiveRetentionChecks @@ -33,9 +45,11 @@ func (c *ActiveRetentionChecks) Get() []RetentionCheck { foldersCopy := make([]FolderRetention, len(check.Folders)) copy(foldersCopy, check.Folders) checks = append(checks, RetentionCheck{ - Username: check.Username, - StartTime: check.StartTime, - Folders: foldersCopy, + Username: check.Username, + StartTime: check.StartTime, + Notification: check.Notification, + Email: check.Email, + Folders: foldersCopy, }) } return checks @@ -114,6 +128,16 @@ func (f *FolderRetention) isValid() error { return nil } +type folderRetentionCheckResult struct { + Path string + Retention int + DeletedFiles int + DeletedSize int64 + Elapsed time.Duration + Info string + Error string +} + // RetentionCheck defines an active retention check type RetentionCheck struct { // Username to which the retention check refers @@ -122,8 +146,13 @@ type RetentionCheck struct { StartTime int64 `json:"start_time"` // affected folders Folders []FolderRetention `json:"folders"` + // how cleanup results will be notified + Notification RetentionCheckNotification `json:"notification"` + // email to use if the notification method is set to email + Email string `json:"email,omitempty"` // Cleanup results - conn *BaseConnection + results []*folderRetentionCheckResult `json:"-"` + conn *BaseConnection } // Validate returns an error if the specified folders are not valid @@ -146,6 +175,17 @@ func (c *RetentionCheck) Validate() error { if nothingToDo { return util.NewValidationError("nothing to delete!") } + switch c.Notification { + case RetentionCheckNotificationEmail: + if !smtp.IsEnabled() { + return util.NewValidationError("in order to notify results via email you must configure an SMTP server") + } + if c.Email == "" { + return util.NewValidationError("in order to notify results via email you must add a valid email address to your profile") + } + default: + c.Notification = RetentionCheckNotificationNone + } return nil } @@ -180,7 +220,14 @@ func (c *RetentionCheck) removeFile(virtualPath string, info os.FileInfo) error func (c *RetentionCheck) cleanupFolder(folderPath string) error { cleanupPerms := []string{dataprovider.PermListItems, dataprovider.PermDelete} + startTime := time.Now() + result := &folderRetentionCheckResult{ + Path: folderPath, + } + c.results = append(c.results, result) if !c.conn.User.HasPerms(cleanupPerms, folderPath) { + result.Elapsed = time.Since(startTime) + result.Info = "data retention check skipped: no permissions" c.conn.Log(logger.LevelInfo, "user %#v does not have permissions to check retention on %#v, retention check skipped", c.conn.User, folderPath) return nil @@ -188,10 +235,15 @@ func (c *RetentionCheck) cleanupFolder(folderPath string) error { folderRetention, err := c.getFolderRetention(folderPath) if err != nil { + result.Elapsed = time.Since(startTime) + result.Error = "unable to get folder retention" c.conn.Log(logger.LevelError, "unable to get folder retention for path %#v", folderPath) return err } + result.Retention = folderRetention.Retention if folderRetention.Retention == 0 { + result.Elapsed = time.Since(startTime) + result.Info = "data retention check skipped: retention is set to 0" c.conn.Log(logger.LevelDebug, "retention check skipped for folder %#v, retention is set to 0", folderPath) return nil } @@ -199,19 +251,22 @@ func (c *RetentionCheck) cleanupFolder(folderPath string) error { folderPath, folderRetention.Retention, folderRetention.DeleteEmptyDirs, folderRetention.IgnoreUserPermissions) files, err := c.conn.ListDir(folderPath) if err != nil { + result.Elapsed = time.Since(startTime) if err == c.conn.GetNotExistError() { + result.Info = "data retention check skipped, folder does not exist" c.conn.Log(logger.LevelDebug, "folder %#v does not exist, retention check skipped", folderPath) return nil } - c.conn.Log(logger.LevelWarn, "unable to list directory %#v", folderPath) + result.Error = fmt.Sprintf("unable to list directory %#v", folderPath) + c.conn.Log(logger.LevelWarn, result.Error) return err } - deletedFiles := 0 - deletedSize := int64(0) for _, info := range files { virtualPath := path.Join(folderPath, info.Name()) if info.IsDir() { if err := c.cleanupFolder(virtualPath); err != nil { + result.Elapsed = time.Since(startTime) + result.Error = fmt.Sprintf("unable to check folder: %v", err) c.conn.Log(logger.LevelWarn, "unable to cleanup folder %#v: %v", virtualPath, err) return err } @@ -219,14 +274,16 @@ func (c *RetentionCheck) cleanupFolder(folderPath string) error { retentionTime := info.ModTime().Add(time.Duration(folderRetention.Retention) * time.Hour) if retentionTime.Before(time.Now()) { if err := c.removeFile(virtualPath, info); err != nil { + result.Elapsed = time.Since(startTime) + result.Error = fmt.Sprintf("unable to remove file %#v: %v", virtualPath, err) c.conn.Log(logger.LevelWarn, "unable to remove file %#v, retention %v: %v", virtualPath, retentionTime, err) return err } c.conn.Log(logger.LevelDebug, "removed file %#v, modification time: %v, retention: %v hours, retention time: %v", virtualPath, info.ModTime(), folderRetention.Retention, retentionTime) - deletedFiles++ - deletedSize += info.Size() + result.DeletedFiles++ + result.DeletedSize += info.Size() } } } @@ -234,8 +291,9 @@ func (c *RetentionCheck) cleanupFolder(folderPath string) error { if folderRetention.DeleteEmptyDirs { c.checkEmptyDirRemoval(folderPath) } + result.Elapsed = time.Since(startTime) c.conn.Log(logger.LevelDebug, "retention check completed for folder %#v, deleted files: %v, deleted size: %v bytes", - folderPath, deletedFiles, deletedSize) + folderPath, result.DeletedFiles, result.DeletedSize) return nil } @@ -256,14 +314,54 @@ func (c *RetentionCheck) Start() { defer RetentionChecks.remove(c.conn.User.Username) defer c.conn.CloseFS() //nolint:errcheck + startTime := time.Now() for _, folder := range c.Folders { if folder.Retention > 0 { if err := c.cleanupFolder(folder.Path); err != nil { c.conn.Log(logger.LevelWarn, "retention check failed, unable to cleanup folder %#v", folder.Path) + c.sendNotification(startTime, err) //nolint:errcheck return } } } c.conn.Log(logger.LevelInfo, "retention check completed") + c.sendNotification(startTime, nil) //nolint:errcheck +} + +func (c *RetentionCheck) sendNotification(startTime time.Time, err error) error { + switch c.Notification { + case RetentionCheckNotificationEmail: + body := new(bytes.Buffer) + data := make(map[string]interface{}) + data["Results"] = c.results + totalDeletedFiles := 0 + totalDeletedSize := int64(0) + for _, result := range c.results { + totalDeletedFiles += result.DeletedFiles + totalDeletedSize += result.DeletedSize + } + data["HumanizeSize"] = util.ByteCountIEC + data["TotalFiles"] = totalDeletedFiles + data["TotalSize"] = totalDeletedSize + data["Elapsed"] = time.Since(startTime) + data["Username"] = c.conn.User.Username + data["StartTime"] = util.GetTimeFromMsecSinceEpoch(c.StartTime) + if err == nil { + data["Status"] = "Succeeded" + } else { + data["Status"] = "Failed" + } + if err := smtp.RenderRetentionReportTemplate(body, data); err != nil { + c.conn.Log(logger.LevelWarn, "unable to render retention check template: %v", err) + return err + } + subject := fmt.Sprintf("Retention check completed for user %#v", c.conn.User.Username) + if err := smtp.SendEmail(c.Email, subject, body.String(), smtp.EmailContentTypeTextHTML); err != nil { + c.conn.Log(logger.LevelWarn, "unable to notify retention check result via email: %v", err) + return err + } + c.conn.Log(logger.LevelInfo, "retention check result successfully notified via email") + } + return nil } diff --git a/common/dataretention_test.go b/common/dataretention_test.go index e7d14aee..b5b4f277 100644 --- a/common/dataretention_test.go +++ b/common/dataretention_test.go @@ -1,14 +1,17 @@ package common import ( + "errors" "fmt" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/drakkan/sftpgo/v2/dataprovider" "github.com/drakkan/sftpgo/v2/sdk" + "github.com/drakkan/sftpgo/v2/smtp" ) func TestRetentionValidation(t *testing.T) { @@ -62,6 +65,83 @@ func TestRetentionValidation(t *testing.T) { } err = check.Validate() assert.NoError(t, err) + assert.Equal(t, RetentionCheckNotificationNone, check.Notification) + assert.Empty(t, check.Email) + + check.Notification = RetentionCheckNotificationEmail + err = check.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "you must configure an SMTP server") + + smtpCfg := smtp.Config{ + Host: "mail.example.com", + Port: 25, + TemplatesPath: "templates", + } + err = smtpCfg.Initialize("..") + require.NoError(t, err) + + err = check.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "you must add a valid email address") + + check.Email = "admin@example.com" + err = check.Validate() + assert.NoError(t, err) + + smtpCfg = smtp.Config{} + err = smtpCfg.Initialize("..") + require.NoError(t, err) +} + +func TestEmailNotifications(t *testing.T) { + smtpCfg := smtp.Config{ + Host: "127.0.0.1", + Port: 2525, + TemplatesPath: "templates", + } + err := smtpCfg.Initialize("..") + require.NoError(t, err) + + user := dataprovider.User{ + BaseUser: sdk.BaseUser{ + Username: "user1", + }, + } + user.Permissions = make(map[string][]string) + user.Permissions["/"] = []string{dataprovider.PermAny} + check := RetentionCheck{ + Notification: RetentionCheckNotificationEmail, + Email: "notification@example.com", + results: []*folderRetentionCheckResult{ + { + Path: "/", + Retention: 24, + DeletedFiles: 10, + DeletedSize: 32657, + Elapsed: 10 * time.Second, + }, + }, + } + conn := NewBaseConnection("", "", "", "", user) + conn.ID = fmt.Sprintf("retention_check_%v", user.Username) + check.conn = conn + err = check.sendNotification(time.Now(), nil) + assert.NoError(t, err) + err = check.sendNotification(time.Now(), errors.New("test error")) + assert.NoError(t, err) + + smtpCfg.Port = 2626 + err = smtpCfg.Initialize("..") + require.NoError(t, err) + err = check.sendNotification(time.Now(), nil) + assert.Error(t, err) + + smtpCfg = smtp.Config{} + err = smtpCfg.Initialize("..") + require.NoError(t, err) + err = check.sendNotification(time.Now(), nil) + assert.Error(t, err) } func TestRetentionPermissionsAndGetFolder(t *testing.T) { diff --git a/common/protocol_test.go b/common/protocol_test.go index b391130a..60e47dfb 100644 --- a/common/protocol_test.go +++ b/common/protocol_test.go @@ -19,15 +19,15 @@ import ( _ "github.com/go-sql-driver/mysql" _ "github.com/lib/pq" _ "github.com/mattn/go-sqlite3" + "github.com/mhale/smtpd" "github.com/pkg/sftp" "github.com/pquerna/otp" "github.com/pquerna/otp/totp" + "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/crypto/ssh" - "github.com/rs/zerolog" - "github.com/drakkan/sftpgo/v2/common" "github.com/drakkan/sftpgo/v2/config" "github.com/drakkan/sftpgo/v2/dataprovider" @@ -45,6 +45,7 @@ const ( httpAddr = "127.0.0.1:9999" httpProxyAddr = "127.0.0.1:7777" sftpServerAddr = "127.0.0.1:4022" + smtpServerAddr = "127.0.0.1:2525" defaultUsername = "test_common_sftp" defaultPassword = "test_password" defaultSFTPUsername = "test_common_sftpfs_user" @@ -162,8 +163,18 @@ func TestMain(m *testing.M) { } }() + go func() { + if err := smtpd.ListenAndServe(smtpServerAddr, func(remoteAddr net.Addr, from string, to []string, data []byte) error { + return nil + }, "SFTPGo test", "localhost"); err != nil { + logger.ErrorToConsole("could not start SMTP server: %v", err) + os.Exit(1) + } + }() + waitTCPListening(httpAddr) waitTCPListening(httpProxyAddr) + waitTCPListening(smtpServerAddr) waitTCPListening(sftpdConf.Bindings[0].GetAddress()) waitTCPListening(httpdConf.Bindings[0].GetAddress()) diff --git a/config/config.go b/config/config.go index 7137f5a0..4ccdc2a4 100644 --- a/config/config.go +++ b/config/config.go @@ -308,14 +308,15 @@ func Init() { }, PluginsConfig: nil, SMTPConfig: smtp.Config{ - Host: "", - Port: 25, - From: "", - User: "", - Password: "", - AuthType: 0, - Encryption: 0, - Domain: "", + Host: "", + Port: 25, + From: "", + User: "", + Password: "", + AuthType: 0, + Encryption: 0, + Domain: "", + TemplatesPath: "templates", }, } @@ -1176,6 +1177,7 @@ func setViperDefaults() { viper.SetDefault("smtp.auth_type", globalConf.SMTPConfig.AuthType) viper.SetDefault("smtp.encryption", globalConf.SMTPConfig.Encryption) viper.SetDefault("smtp.domain", globalConf.SMTPConfig.Domain) + viper.SetDefault("smtp.templates_path", globalConf.SMTPConfig.TemplatesPath) } func lookupBoolFromEnv(envName string) (bool, bool) { diff --git a/docs/full-configuration.md b/docs/full-configuration.md index ea1b68cd..450c199c 100644 --- a/docs/full-configuration.md +++ b/docs/full-configuration.md @@ -263,12 +263,13 @@ The configuration file contains the following sections: - **smtp**, SMTP configuration enables SFTPGo email sending capabilities - `host`, string. Location of SMTP email server. Leavy empty to disable email sending capabilities. Default: empty. - `port`, integer. Port of SMTP email server. - - `from`, string. From address, for example `SFTPGo `. Default: empty + - `from`, string. From address, for example `SFTPGo `. Many SMTP servers reject emails without a `From` header so, if not set, SFTPGo will try to use the username as fallback, this may or may not be appropriate. Default: empty - `user`, string. SMTP username. Default: empty - `password`, string. SMTP password. Leaving both username and password empty the SMTP authentication will be disabled. Default: empty - `auth_type`, integer. 0 means `Plain`, 1 means `Login`, 2 means `CRAM-MD5`. Default: `0`. - `encryption`, integer. 0 means no encryption, 1 means `TLS`, 2 means `STARTTLS`. Default: `0`. - `domain`, string. Domain to use for `HELO` command, if empty `localhost` will be used. Default: empty. + - `templates_path`, string. Path to the email templates. This can be an absolute path or a path relative to the config dir. Templates are searched within a subdirectory named "email" in the specified path. You can customize the email templates by simply specifying an alternate path and putting your custom templates there. - **plugins**, list of external plugins. Each plugin is configured using a struct with the following fields: - `type`, string. Defines the plugin type. Supported types: `notifier`, `kms`, `auth`. - `notifier_options`, struct. Defines the options for notifier plugins. diff --git a/docs/rest-api.md b/docs/rest-api.md index 81908149..13f90702 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -88,6 +88,7 @@ In the above example we asked to SFTPGo: - to exclude `/folder1/subfolder`, no files will be deleted here - to delete all the files with modification time older than 24 hours in `/folder2` +The check results can be, optionally, notified by e-mail. You can find an example script that shows how to manage data retention [here](../examples/data-retention). Checks the REST API schema for full details. :warning: Deleting files is an irreversible action, please make sure you fully understand what you are doing before using this feature, you may have users with overlapping home directories or virtual folders shared between multiple users, it is relatively easy to inadvertently delete files you need. diff --git a/go.mod b/go.mod index 92007e60..f582aafe 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/lithammer/shortuuid/v3 v3.0.7 github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-sqlite3 v1.14.8 + github.com/mhale/smtpd v0.8.0 github.com/miekg/dns v1.1.43 // indirect github.com/minio/sio v0.3.0 github.com/mitchellh/go-testing-interface v1.14.1 // indirect @@ -62,10 +63,10 @@ require ( gocloud.dev v0.24.0 golang.org/x/crypto v0.0.0-20210915214749-c084706c2272 golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6 - golang.org/x/sys v0.0.0-20211001092434-39dca1131b70 + golang.org/x/sys v0.0.0-20211002104244-808efd93c36d golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac google.golang.org/api v0.58.0 - google.golang.org/genproto v0.0.0-20210930144712-2e2e1008e8a3 // indirect + google.golang.org/genproto v0.0.0-20211001223012-bfb93cce50d9 // indirect google.golang.org/grpc v1.41.0 google.golang.org/protobuf v1.27.1 gopkg.in/natefinch/lumberjack.v2 v2.0.0 diff --git a/go.sum b/go.sum index 9590c8b3..607cb34c 100644 --- a/go.sum +++ b/go.sum @@ -595,6 +595,8 @@ github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxz github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mhale/smtpd v0.8.0 h1:5JvdsehCg33PQrZBvFyDMMUDQmvbzVpZgKob7eYBJc0= +github.com/mhale/smtpd v0.8.0/go.mod h1:MQl+y2hwIEQCXtNhe5+55n0GZOjSmeqORDIXbqUL3x4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -960,8 +962,8 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211001092434-39dca1131b70 h1:pGleJoyD1yA5HfvuaksHxD0404gsEkNDerKsQ0N0y1s= -golang.org/x/sys v0.0.0-20211001092434-39dca1131b70/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211002104244-808efd93c36d h1:SABT8Vei3iTiu+Gy8KOzpSNz+W1EQ5YBCRtiEETxF+0= +golang.org/x/sys v0.0.0-20211002104244-808efd93c36d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1162,8 +1164,8 @@ google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20210930144712-2e2e1008e8a3 h1:+F3FcO6LTrzNq5wp1Z6JtoBvnJzX6euyN70FoyMDXy4= -google.golang.org/genproto v0.0.0-20210930144712-2e2e1008e8a3/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211001223012-bfb93cce50d9 h1:eF1wcrhdz56Vugf8qNX5dD93ItkrhothojQyHXqloe0= +google.golang.org/genproto v0.0.0-20211001223012-bfb93cce50d9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= diff --git a/httpd/api_retention.go b/httpd/api_retention.go index 7d447139..1474916e 100644 --- a/httpd/api_retention.go +++ b/httpd/api_retention.go @@ -29,8 +29,22 @@ func startRetentionCheck(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, err, "", http.StatusBadRequest) return } + check.Notification = r.URL.Query().Get("notify") + if check.Notification == common.RetentionCheckNotificationEmail { + claims, err := getTokenClaims(r) + if err != nil { + sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest) + return + } + admin, err := dataprovider.AdminExists(claims.Username) + if err != nil { + sendAPIResponse(w, r, err, "", getRespStatus(err)) + return + } + check.Email = admin.Email + } if err := check.Validate(); err != nil { - sendAPIResponse(w, r, err, "Invalid folders to check", http.StatusBadRequest) + sendAPIResponse(w, r, err, "Invalid retention check", http.StatusBadRequest) return } c := common.RetentionChecks.Add(check, &user) diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index b4eae339..3c68c19d 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -1584,7 +1584,7 @@ func TestRetentionAPI(t *testing.T) { resp, err := httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusBadRequest) assert.NoError(t, err) - assert.Contains(t, string(resp), "Invalid folders to check") + assert.Contains(t, string(resp), "Invalid retention check") folderRetention[0].Retention = 24 _, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted) @@ -1621,7 +1621,13 @@ func TestRetentionAPI(t *testing.T) { c.Start() assert.Len(t, common.RetentionChecks.Get(), 0) - token, err := getJWTAPITokenFromTestServer(defaultTokenAuthUser, defaultTokenAuthPass) + admin := getTestAdmin() + admin.Username = altAdminUsername + admin.Password = altAdminPassword + admin, _, err = httpdtest.AddAdmin(admin, http.StatusCreated) + assert.NoError(t, err) + + token, err := getJWTAPITokenFromTestServer(altAdminUsername, altAdminPassword) assert.NoError(t, err) req, _ := http.NewRequest(http.MethodPost, retentionBasePath+"/"+user.Username+"/check", bytes.NewBuffer([]byte("invalid json"))) @@ -1629,6 +1635,23 @@ func TestRetentionAPI(t *testing.T) { rr := executeRequest(req) checkResponseCode(t, http.StatusBadRequest, rr) + asJSON, err := json.Marshal(folderRetention) + assert.NoError(t, err) + req, _ = http.NewRequest(http.MethodPost, retentionBasePath+"/"+user.Username+"/check?notify=Email", + bytes.NewBuffer(asJSON)) + setBearerForReq(req, token) + rr = executeRequest(req) + checkResponseCode(t, http.StatusBadRequest, rr) + assert.Contains(t, rr.Body.String(), "to notify results via email") + + _, err = httpdtest.RemoveAdmin(admin, http.StatusOK) + assert.NoError(t, err) + req, _ = http.NewRequest(http.MethodPost, retentionBasePath+"/"+user.Username+"/check?notify=Email", + bytes.NewBuffer(asJSON)) + setBearerForReq(req, token) + rr = executeRequest(req) + checkResponseCode(t, http.StatusNotFound, rr) + _, err = httpdtest.RemoveUser(user, http.StatusOK) assert.NoError(t, err) err = os.RemoveAll(user.GetHomeDir()) diff --git a/httpd/internal_test.go b/httpd/internal_test.go index d963900f..bfe634b7 100644 --- a/httpd/internal_test.go +++ b/httpd/internal_test.go @@ -480,6 +480,46 @@ func TestUpdateWebAdminInvalidClaims(t *testing.T) { assert.Contains(t, rr.Body.String(), "Invalid token claims") } +func TestRetentionInvalidTokenClaims(t *testing.T) { + username := "retentionuser" + user := dataprovider.User{ + BaseUser: sdk.BaseUser{ + Username: username, + Password: "pwd", + HomeDir: filepath.Join(os.TempDir(), username), + Status: 1, + }, + } + user.Permissions = make(map[string][]string) + user.Permissions["/"] = []string{dataprovider.PermAny} + user.Filters.AllowAPIKeyAuth = true + err := dataprovider.AddUser(&user) + assert.NoError(t, err) + folderRetention := []common.FolderRetention{ + { + Path: "/", + Retention: 0, + DeleteEmptyDirs: true, + }, + } + asJSON, err := json.Marshal(folderRetention) + assert.NoError(t, err) + req, _ := http.NewRequest(http.MethodPost, retentionBasePath+"/"+username+"/check?notify=Email", bytes.NewBuffer(asJSON)) + + rctx := chi.NewRouteContext() + rctx.URLParams.Add("username", username) + + req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)) + req = req.WithContext(context.WithValue(req.Context(), jwtauth.ErrorCtxKey, errors.New("error"))) + rr := httptest.NewRecorder() + startRetentionCheck(rr, req) + assert.Equal(t, http.StatusBadRequest, rr.Code) + assert.Contains(t, rr.Body.String(), "Invalid token claims") + + err = dataprovider.DeleteUser(username) + assert.NoError(t, err) +} + func TestCSRFToken(t *testing.T) { // invalid token err := verifyCSRFToken("token") diff --git a/httpd/schema/openapi.yaml b/httpd/schema/openapi.yaml index d0dfe45d..872b52ea 100644 --- a/httpd/schema/openapi.yaml +++ b/httpd/schema/openapi.yaml @@ -772,6 +772,11 @@ paths: required: true schema: type: string + - name: notify + in: query + description: 'specify how notify results' + schema: + $ref: '#/components/schemas/RetentionCheckNotification' post: tags: - data retention @@ -3223,6 +3228,15 @@ components: * `password-change-disabled` - changing password is not allowed * `api-key-auth-change-disabled` - enabling/disabling API key authentication is not allowed * `info-change-disabled` - changing info such as email and description is not allowed + RetentionCheckNotification: + type: string + enum: + - None + - Email + description: | + Options: + * `None` - no notification, the results are recorded in the logs + * `Email` - notify results by email. The admin starting the retention check must have an associated email address and the SMTP server must be configured for this to work APIKeyScope: type: integer enum: @@ -3991,6 +4005,12 @@ components: type: integer format: int64 description: check start time as unix timestamp in milliseconds + notification: + $ref: '#/components/schemas/RetentionCheckNotification' + email: + type: string + format: email + description: 'if the notification method is set to "Email", this is the e-mail address that receives the retention check report. This field is automatically set to the email address associated with the administrator starting the check' QuotaScan: type: object properties: diff --git a/httpd/webadmin.go b/httpd/webadmin.go index aa451cad..6981a539 100644 --- a/httpd/webadmin.go +++ b/httpd/webadmin.go @@ -296,7 +296,7 @@ func loadAdminTemplates(templatesPath string) { filepath.Join(templatesPath, templateAdminDir, templateSetup), } - fsBaseTpl := template.New("").Funcs(template.FuncMap{ + fsBaseTpl := template.New("fsBaseTemplate").Funcs(template.FuncMap{ "ListFSProviders": sdk.ListProviders, }) usersTmpl := util.LoadTemplate(nil, usersPaths...) diff --git a/service/service.go b/service/service.go index d31c229e..b7596bc0 100644 --- a/service/service.go +++ b/service/service.go @@ -111,7 +111,7 @@ func (s *Service) Start() error { os.Exit(1) } smtpConfig := config.GetSMTPConfig() - err = smtpConfig.Initialize() + err = smtpConfig.Initialize(s.ConfigDir) if err != nil { logger.Error(logSender, "", "unable to initialize SMTP configuration: %v", err) logger.ErrorToConsole("unable to initialize SMTP configuration: %v", err) diff --git a/sftpgo.json b/sftpgo.json index 21a5aea1..229b6562 100644 --- a/sftpgo.json +++ b/sftpgo.json @@ -262,7 +262,8 @@ "password": "", "auth_type": 0, "encryption": 0, - "domain": "" + "domain": "", + "templates_path": "templates" }, "plugins": [] } \ No newline at end of file diff --git a/smtp/smtp.go b/smtp/smtp.go index b5c65e40..c22b0574 100644 --- a/smtp/smtp.go +++ b/smtp/smtp.go @@ -2,13 +2,17 @@ package smtp import ( + "bytes" "errors" "fmt" + "html/template" + "path/filepath" "time" mail "github.com/xhit/go-simple-mail/v2" "github.com/drakkan/sftpgo/v2/logger" + "github.com/drakkan/sftpgo/v2/util" ) const ( @@ -24,18 +28,31 @@ const ( EmailContentTypeTextHTML ) -var ( - smtpServer *mail.SMTPServer - from string +const ( + templateEmailDir = "email" + templateRetentionCheckResult = "retention-check-report.html" ) +var ( + smtpServer *mail.SMTPServer + from string + emailTemplates = make(map[string]*template.Template) +) + +// IsEnabled returns true if an SMTP server is configured +func IsEnabled() bool { + return smtpServer != nil +} + // Config defines the SMTP configuration to use to send emails type Config struct { // Location of SMTP email server. Leavy empty to disable email sending capabilities Host string `json:"host" mapstructure:"host"` // Port of SMTP email server Port int `json:"port" mapstructure:"port"` - // From address, for example "SFTPGo " + // From address, for example "SFTPGo ". + // Many SMTP servers reject emails without a `From` header so, if not set, + // SFTPGo will try to use the username as fallback, this may or may not be appropriate From string `json:"from" mapstructure:"from"` // SMTP username User string `json:"user" mapstructure:"user"` @@ -52,10 +69,13 @@ type Config struct { Encryption int `json:"encryption" mapstructure:"encryption"` // Domain to use for HELO command, if empty localhost will be used Domain string `json:"domain" mapstructure:"domain"` + // Path to the email templates. This can be an absolute path or a path relative to the config dir. + // Templates are searched within a subdirectory named "email" in the specified path + TemplatesPath string `json:"templates_path" mapstructure:"templates_path"` } // Initialize initialized and validates the SMTP configuration -func (c *Config) Initialize() error { +func (c *Config) Initialize(configDir string) error { smtpServer = nil if c.Host == "" { logger.Debug(logSender, "", "configuration disabled, email capabilities will not be available") @@ -70,6 +90,14 @@ func (c *Config) Initialize() error { if c.Encryption < 0 || c.Encryption > 2 { return fmt.Errorf("smtp: invalid encryption %v", c.Encryption) } + templatesPath := c.TemplatesPath + if templatesPath == "" || !util.IsFileInputValid(templatesPath) { + return fmt.Errorf("smtp: invalid templates path %#v", templatesPath) + } + if !filepath.IsAbs(templatesPath) { + templatesPath = filepath.Join(configDir, templatesPath) + } + loadTemplates(filepath.Join(templatesPath, templateEmailDir)) from = c.From smtpServer = mail.NewSMTPClient() smtpServer.Host = c.Host @@ -114,6 +142,21 @@ func (c *Config) getAuthType() mail.AuthType { } } +func loadTemplates(templatesPath string) { + logger.Debug(logSender, "", "loading templates from %#v", templatesPath) + retentionCheckPath := filepath.Join(templatesPath, templateRetentionCheckResult) + retentionTmpl := util.LoadTemplate(nil, retentionCheckPath) + emailTemplates[templateRetentionCheckResult] = retentionTmpl +} + +// RenderRetentionReportTemplate executes the retention report template +func RenderRetentionReportTemplate(buf *bytes.Buffer, data interface{}) error { + if smtpServer == nil { + return errors.New("smtp: not configured") + } + return emailTemplates[templateRetentionCheckResult].Execute(buf, data) +} + // SendEmail tries to send an email using the specified parameters. func SendEmail(to, subject, body string, contentType EmailContentType) error { if smtpServer == nil { @@ -127,6 +170,8 @@ func SendEmail(to, subject, body string, contentType EmailContentType) error { email := mail.NewMSG() if from != "" { email.SetFrom(from) + } else { + email.SetFrom(smtpServer.Username) } email.AddTo(to).SetSubject(subject) switch contentType { diff --git a/templates/email/retention-check-report.html b/templates/email/retention-check-report.html new file mode 100644 index 00000000..cdeec51d --- /dev/null +++ b/templates/email/retention-check-report.html @@ -0,0 +1,33 @@ +Retention check report for user "{{.Username}}" +

+Status: {{.Status}} +
+Start time: {{.StartTime}} +
+Total files deleted: {{.TotalFiles}} +
+Total size deleted: {{call .HumanizeSize .TotalSize}} +
+Elapsed: {{.Elapsed}} +
+{{range .Results -}} +

+ Path: {{.Path}} + {{- if .Error}} +
+ Error: {{.Error}} + {{- end}} + {{- if .Info}} +
+ Info: {{.Info}} + {{- end}} +
+ Retention: {{.Retention}} hours +
+ Files deleted: {{.DeletedFiles}} +
+ Size deleted: {{call $.HumanizeSize .DeletedSize}} +
+ Elapsed: {{.Elapsed}} +

+{{end}} \ No newline at end of file diff --git a/util/util.go b/util/util.go index ea1bc7fc..9974bf67 100644 --- a/util/util.go +++ b/util/util.go @@ -354,8 +354,8 @@ func CleanPath(p string) string { } // LoadTemplate parses the given template paths. -// it behaves like template.Must but it writes a log before exiting -// you can optionally provide a base template (e.g. to define some custom functions) +// It behaves like template.Must but it writes a log before exiting. +// You can optionally provide a base template (e.g. to define some custom functions) func LoadTemplate(base *template.Template, paths ...string) *template.Template { var t *template.Template var err error