eventmanager: add data retention reports

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>

README.md

@@ -40,7 +40,7 @@ More [info](https://github.com/drakkan/sftpgo/issues/452).
 
 SFTPGo is an Open Source project and you can of course use it for free but please don't ask for free support as well.
 
-We will check the reported issues to see if you are experiencing a bug and if so we'll will fix it, but will only provide support to project sponsors/donors.
+We will check the reported issues to see if you are experiencing a bug and if so we'll will fix it, but will only provide support to project [sponsors/donors](#sponsors).
 
 If you report an invalid issue or ask for step-by-step support, your issue will remain open with no answer or will be closed as invalid without further explanation. Thanks for understanding.
 

docs/eventmanager.md

@@ -36,6 +36,7 @@ The following placeholders are supported:
 - `{{IP}}`. Client IP address.
 - `{{Timestamp}}`. Event timestamp as nanoseconds since epoch.
 - `{{ObjectData}}`. Provider object data serialized as JSON with sensitive fields removed.
+- `{{RetentionReports}}`. Data retention reports as zip compressed CSV files. Supported as email attachment, file path for multipart HTTP request and as single parameter for HTTP requests body.
 
 Event rules are based on the premise that an event occours. To each rule you can associate one or more actions.
 The following trigger events are supported:

go.mod

@@ -55,10 +55,10 @@ require (
 	github.com/shirou/gopsutil/v3 v3.22.8
 	github.com/spf13/afero v1.9.2
 	github.com/spf13/cobra v1.5.0
-	github.com/spf13/viper v1.12.0
+	github.com/spf13/viper v1.13.0
 	github.com/stretchr/testify v1.8.0
 	github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62
-	github.com/unrolled/secure v1.12.0
+	github.com/unrolled/secure v1.13.0
 	github.com/wagslane/go-password-validator v0.3.0
 	github.com/xhit/go-simple-mail/v2 v2.11.0
 	github.com/yl2chen/cidranger v1.0.3-0.20210928021809-d1cb2c52f37a
@@ -68,7 +68,7 @@ require (
 	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90
 	golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b
 	golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094
-	golang.org/x/sys v0.0.0-20220829200755-d48e67d00261
+	golang.org/x/sys v0.0.0-20220906135438-9e1f76180b77
 	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9
 	google.golang.org/api v0.94.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
@@ -77,7 +77,7 @@ require (
 require (
 	cloud.google.com/go v0.104.0 // indirect
 	cloud.google.com/go/compute v1.9.0 // indirect
-	cloud.google.com/go/iam v0.3.0 // indirect
+	cloud.google.com/go/iam v0.4.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect
 	github.com/ajg/form v1.5.1 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.7 // indirect

go.sum

@@ -54,8 +54,9 @@ cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1
 cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY=
 cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c=
 cloud.google.com/go/iam v0.1.1/go.mod h1:CKqrcnI/suGpybEHxZ7BMehL0oA4LpdyJdUlTl9jVMw=
-cloud.google.com/go/iam v0.3.0 h1:exkAomrVUuzx9kWFI1wm3KI0uoDeUFPB4kKGzx6x+Gc=
 cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY=
+cloud.google.com/go/iam v0.4.0 h1:YBYU00SCDzZJdHqVc4I5d6lsklcYIjQZa1YmEz4jlSE=
+cloud.google.com/go/iam v0.4.0/go.mod h1:cbaZxyScUhxl7ZAkNWiALgihfP75wS/fUsVNaa1r3vA=
 cloud.google.com/go/kms v1.1.0/go.mod h1:WdbppnCDMDpOvoYBMn1+gNmOeEoZYqAv+HeuKARGCXI=
 cloud.google.com/go/kms v1.4.0 h1:iElbfoE61VeLhnZcGOltqL8HIly8Nhbe5t6JlH9GXjo=
 cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA=
@@ -736,8 +737,8 @@ github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmq
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ=
+github.com/spf13/viper v1.13.0 h1:BWSJ/M+f+3nmdz9bxB+bWX28kkALN2ok11D0rSo8EJU=
-github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI=
+github.com/spf13/viper v1.13.0/go.mod h1:Icm2xNL3/8uyh/wFuB1jI7TiTNKp8632Nwegu+zgdYw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
@@ -765,8 +766,8 @@ github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 h1:PM5hJF7HVfNWmCjM
 github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208/go.mod h1:BzWtXXrXzZUvMacR0oF/fbDDgUPO8L36tDMmRAf14ns=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/unrolled/secure v1.12.0 h1:7k3jcgLwfjiKkhQde6VbQ3D4KDLtDBqDd/hs3PPANDY=
+github.com/unrolled/secure v1.13.0 h1:sdr3Phw2+f8Px8HE5sd1EHdj1aV3yUwed/uZXChLFsk=
-github.com/unrolled/secure v1.12.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
+github.com/unrolled/secure v1.13.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
 github.com/wagslane/go-password-validator v0.3.0 h1:vfxOPzGHkz5S146HDpavl0cw1DSVP061Ry2PX0/ON6I=
 github.com/wagslane/go-password-validator v0.3.0/go.mod h1:TI1XJ6T5fRdRnHqHt14pvy1tNVnrwe7m3/f1f2fDphQ=
 github.com/xhit/go-simple-mail/v2 v2.11.0 h1:o/056V50zfkO3Mm5tVdo9rG3ryg4ZmJ2XW5GMinHfVs=
@@ -976,8 +977,8 @@ golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY=
+golang.org/x/sys v0.0.0-20220906135438-9e1f76180b77 h1:C1tElbkWrsSkn3IRl1GCW/gETw1TywWIPgwZtXTZbYg=
-golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220906135438-9e1f76180b77/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

internal/common/dataretention.go

@@ -147,7 +147,7 @@ type RetentionCheck struct {
 	// email to use if the notification method is set to email
 	Email string `json:"email,omitempty"`
 	// Cleanup results
-	results []*folderRetentionCheckResult `json:"-"`
+	results []folderRetentionCheckResult `json:"-"`
 	conn    *BaseConnection
 }
 
@@ -223,10 +223,12 @@ func (c *RetentionCheck) removeFile(virtualPath string, info os.FileInfo) error
 func (c *RetentionCheck) cleanupFolder(folderPath string) error {
 	deleteFilesPerms := []string{dataprovider.PermDelete, dataprovider.PermDeleteFiles}
 	startTime := time.Now()
-	result := &folderRetentionCheckResult{
+	result := folderRetentionCheckResult{
 		Path: folderPath,
 	}
-	c.results = append(c.results, result)
+	defer func() {
+		c.results = append(c.results, result)
+	}()
 	if !c.conn.User.HasPerm(dataprovider.PermListItems, folderPath) || !c.conn.User.HasAnyPerm(deleteFilesPerms, folderPath) {
 		result.Elapsed = time.Since(startTime)
 		result.Info = "data retention check skipped: no permissions"
@@ -301,7 +303,15 @@ func (c *RetentionCheck) cleanupFolder(folderPath string) error {
 }
 
 func (c *RetentionCheck) checkEmptyDirRemoval(folderPath string) {
-	if folderPath != "/" && c.conn.User.HasAnyPerm([]string{
+	if folderPath == "/" {
+		return
+	}
+	for _, folder := range c.Folders {
+		if folderPath == folder.Path {
+			return
+		}
+	}
+	if c.conn.User.HasAnyPerm([]string{
 		dataprovider.PermDelete,
 		dataprovider.PermDeleteDirs,
 	}, path.Dir(folderPath),

internal/common/dataretention_test.go

@@ -131,7 +131,7 @@ func TestRetentionEmailNotifications(t *testing.T) {
 	check := RetentionCheck{
 		Notifications: []RetentionCheckNotification{RetentionCheckNotificationEmail},
 		Email:         "notification@example.com",
-		results: []*folderRetentionCheckResult{
+		results: []folderRetentionCheckResult{
 			{
 				Path:         "/",
 				Retention:    24,
@@ -177,7 +177,7 @@ func TestRetentionHookNotifications(t *testing.T) {
 	user.Permissions["/"] = []string{dataprovider.PermAny}
 	check := RetentionCheck{
 		Notifications: []RetentionCheckNotification{RetentionCheckNotificationHook},
-		results: []*folderRetentionCheckResult{
+		results: []folderRetentionCheckResult{
 			{
 				Path:         "/",
 				Retention:    24,

internal/common/eventmanager.go

@@ -17,6 +17,7 @@ package common
 import (
 	"bytes"
 	"context"
+	"encoding/csv"
 	"errors"
 	"fmt"
 	"io"
@@ -28,11 +29,13 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
+	"github.com/klauspost/compress/zip"
 	"github.com/robfig/cron/v3"
 	"github.com/rs/xid"
 	"github.com/sftpgo/sdk"
@@ -47,8 +50,8 @@ import (
 )
 
 const (
-	ipBlockedEventName      = "IP Blocked"
+	ipBlockedEventName = "IP Blocked"
-	emailAttachmentsMaxSize = int64(10 * 1024 * 1024)
+	maxAttachmentsSize = int64(10 * 1024 * 1024)
 )
 
 var (
@@ -412,6 +415,12 @@ func (r *eventRulesContainer) handleCertificateEvent(params EventParams) {
 	}
 }
 
+type executedRetentionCheck struct {
+	Username   string
+	ActionName string
+	Results    []folderRetentionCheckResult
+}
+
 // EventParams defines the supported event parameters
 type EventParams struct {
 	Name                  string
@@ -432,12 +441,25 @@ type EventParams struct {
 	sender                string
 	updateStatusFromError bool
 	errors                []string
+	retentionChecks       []executedRetentionCheck
 }
 
 func (p *EventParams) getACopy() *EventParams {
 	params := *p
 	params.errors = make([]string, len(p.errors))
 	copy(params.errors, p.errors)
+	retentionChecks := make([]executedRetentionCheck, 0, len(p.retentionChecks))
+	for _, c := range p.retentionChecks {
+		executedCheck := executedRetentionCheck{
+			Username:   c.Username,
+			ActionName: c.ActionName,
+		}
+		executedCheck.Results = make([]folderRetentionCheckResult, len(c.Results))
+		copy(executedCheck.Results, c.Results)
+		retentionChecks = append(retentionChecks, executedCheck)
+	}
+	params.retentionChecks = retentionChecks
+
 	return &params
 }
 
@@ -498,6 +520,52 @@ func (p *EventParams) getFolders() ([]vfs.BaseVirtualFolder, error) {
 	return []vfs.BaseVirtualFolder{folder}, nil
 }
 
+func (p *EventParams) getCompressedDataRetentionReport() ([]byte, error) {
+	if len(p.retentionChecks) == 0 {
+		return nil, errors.New("no data retention report available")
+	}
+	var b bytes.Buffer
+	wr := zip.NewWriter(&b)
+	for _, check := range p.retentionChecks {
+		if size := int64(len(b.Bytes())); size > maxAttachmentsSize {
+			eventManagerLog(logger.LevelError, "unable to get retention report, size too large: %s", util.ByteCountIEC(size))
+			return nil, fmt.Errorf("unable to get retention report, size too large: %s", util.ByteCountIEC(size))
+		}
+		data, err := getCSVRetentionReport(check.Results)
+		if err != nil {
+			return nil, fmt.Errorf("unable to get CSV report: %w", err)
+		}
+		fh := &zip.FileHeader{
+			Name:     fmt.Sprintf("%s-%s.csv", check.ActionName, check.Username),
+			Method:   zip.Deflate,
+			Modified: time.Now().UTC(),
+		}
+		f, err := wr.CreateHeader(fh)
+		if err != nil {
+			return nil, fmt.Errorf("unable to create zip header for file %q: %w", fh.Name, err)
+		}
+		_, err = io.Copy(f, bytes.NewBuffer(data))
+		if err != nil {
+			return nil, fmt.Errorf("unable to write content to zip file %q: %w", fh.Name, err)
+		}
+	}
+	if err := wr.Close(); err != nil {
+		return nil, fmt.Errorf("unable to close zip writer: %w", err)
+	}
+	return b.Bytes(), nil
+}
+
+func (p *EventParams) getRetentionReportsAsMailAttachment() (mail.File, error) {
+	var result mail.File
+	data, err := p.getCompressedDataRetentionReport()
+	if err != nil {
+		return result, err
+	}
+	result.Name = "retention-reports.zip"
+	result.Data = data
+	return result, nil
+}
+
 func (p *EventParams) getStringReplacements(addObjectData bool) []string {
 	replacements := []string{
 		"{{Name}}", p.Name,
@@ -530,6 +598,29 @@ func (p *EventParams) getStringReplacements(addObjectData bool) []string {
 	return replacements
 }
 
+func getCSVRetentionReport(results []folderRetentionCheckResult) ([]byte, error) {
+	var b bytes.Buffer
+	csvWriter := csv.NewWriter(&b)
+	err := csvWriter.Write([]string{"path", "retention (hours)", "deleted files", "deleted size (bytes)",
+		"elapsed (ms)", "info", "error"})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, result := range results {
+		err = csvWriter.Write([]string{result.Path, strconv.Itoa(result.Retention), strconv.Itoa(result.DeletedFiles),
+			strconv.FormatInt(result.DeletedSize, 10), strconv.FormatInt(result.Elapsed.Milliseconds(), 10),
+			result.Info, result.Error})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	csvWriter.Flush()
+	err = csvWriter.Error()
+	return b.Bytes(), err
+}
+
 func getFileReader(conn *BaseConnection, virtualPath string) (io.ReadCloser, func(), error) {
 	fs, fsPath, err := conn.GetFsAndResolvedPath(virtualPath)
 	if err != nil {
@@ -600,7 +691,7 @@ func getMailAttachments(user dataprovider.User, attachments []string, replacer *
 			return nil, fmt.Errorf("cannot attach non regular file %q", virtualPath)
 		}
 		totalSize += info.Size()
-		if totalSize > emailAttachmentsMaxSize {
+		if totalSize > maxAttachmentsSize {
 			return nil, fmt.Errorf("unable to send files as attachment, size too large: %s", util.ByteCountIEC(totalSize))
 		}
 		data, err := getFileContent(conn, virtualPath, int(info.Size()))
@@ -682,7 +773,7 @@ func getHTTPRuleActionEndpoint(c dataprovider.EventActionHTTPConfig, replacer *s
 }
 
 func writeHTTPPart(m *multipart.Writer, part dataprovider.HTTPPart, h textproto.MIMEHeader,
-	conn *BaseConnection, replacer *strings.Replacer,
+	conn *BaseConnection, replacer *strings.Replacer, params *EventParams,
 ) error {
 	partWriter, err := m.CreatePart(h)
 	if err != nil {
@@ -697,6 +788,18 @@ func writeHTTPPart(m *multipart.Writer, part dataprovider.HTTPPart, h textproto.
 		}
 		return nil
 	}
+	if part.Filepath == dataprovider.RetentionReportPlaceHolder {
+		data, err := params.getCompressedDataRetentionReport()
+		if err != nil {
+			return err
+		}
+		_, err = partWriter.Write(data)
+		if err != nil {
+			eventManagerLog(logger.LevelError, "unable to write part %q, err: %v", part.Name, err)
+			return err
+		}
+		return nil
+	}
 	err = writeFileContent(conn, util.CleanPath(replacer.Replace(part.Filepath)), partWriter)
 	if err != nil {
 		eventManagerLog(logger.LevelError, "unable to write file part %q, err: %v", part.Name, err)
@@ -706,13 +809,20 @@ func writeHTTPPart(m *multipart.Writer, part dataprovider.HTTPPart, h textproto.
 }
 
 func getHTTPRuleActionBody(c dataprovider.EventActionHTTPConfig, replacer *strings.Replacer,
-	cancel context.CancelFunc, user dataprovider.User,
+	cancel context.CancelFunc, user dataprovider.User, params *EventParams,
 ) (io.ReadCloser, string, error) {
 	var body io.ReadCloser
 	if c.Method == http.MethodGet {
 		return body, "", nil
 	}
 	if c.Body != "" {
+		if c.Body == dataprovider.RetentionReportPlaceHolder {
+			data, err := params.getCompressedDataRetentionReport()
+			if err != nil {
+				return body, "", err
+			}
+			return io.NopCloser(bytes.NewBuffer(data)), "", nil
+		}
 		return io.NopCloser(bytes.NewBufferString(replaceWithReplacer(c.Body, replacer))), "", nil
 	}
 	if len(c.Parts) > 0 {
@@ -745,11 +855,10 @@ func getHTTPRuleActionBody(c dataprovider.EventActionHTTPConfig, replacer *strin
 				if part.Body != "" {
 					h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"`, multipartQuoteEscaper.Replace(part.Name)))
 				} else {
-					filePath := util.CleanPath(replacer.Replace(part.Filepath))
 					h.Set("Content-Disposition",
 						fmt.Sprintf(`form-data; name="%s"; filename="%s"`,
-							multipartQuoteEscaper.Replace(part.Name), multipartQuoteEscaper.Replace(path.Base(filePath))))
+							multipartQuoteEscaper.Replace(part.Name), multipartQuoteEscaper.Replace(path.Base(part.Filepath))))
-					contentType := mime.TypeByExtension(path.Ext(filePath))
+					contentType := mime.TypeByExtension(path.Ext(part.Filepath))
 					if contentType == "" {
 						contentType = "application/octet-stream"
 					}
@@ -758,7 +867,7 @@ func getHTTPRuleActionBody(c dataprovider.EventActionHTTPConfig, replacer *strin
 				for _, keyVal := range part.Headers {
 					h.Set(keyVal.Key, replaceWithReplacer(keyVal.Value, replacer))
 				}
-				if err := writeHTTPPart(m, part, h, conn, replacer); err != nil {
+				if err := writeHTTPPart(m, part, h, conn, replacer, params); err != nil {
 					cancel()
 					return
 				}
@@ -791,13 +900,13 @@ func executeHTTPRuleAction(c dataprovider.EventActionHTTPConfig, params *EventPa
 	defer cancel()
 
 	var user dataprovider.User
-	if c.HasMultipartFile() {
+	if c.HasMultipartFiles() {
 		user, err = params.getUserFromSender()
 		if err != nil {
 			return err
 		}
 	}
-	body, contentType, err := getHTTPRuleActionBody(c, replacer, cancel, user)
+	body, contentType, err := getHTTPRuleActionBody(c, replacer, cancel, user, params)
 	if err != nil {
 		return err
 	}
@@ -888,15 +997,28 @@ func executeEmailRuleAction(c dataprovider.EventActionEmailConfig, params *Event
 	subject := replaceWithReplacer(c.Subject, replacer)
 	startTime := time.Now()
 	var files []mail.File
-	if len(c.Attachments) > 0 {
+	fileAttachments := make([]string, 0, len(c.Attachments))
+	for _, attachment := range c.Attachments {
+		if attachment == dataprovider.RetentionReportPlaceHolder {
+			f, err := params.getRetentionReportsAsMailAttachment()
+			if err != nil {
+				return err
+			}
+			files = append(files, f)
+			continue
+		}
+		fileAttachments = append(fileAttachments, attachment)
+	}
+	if len(fileAttachments) > 0 {
 		user, err := params.getUserFromSender()
 		if err != nil {
 			return err
 		}
-		files, err = getMailAttachments(user, c.Attachments, replacer)
+		res, err := getMailAttachments(user, fileAttachments, replacer)
 		if err != nil {
 			return err
 		}
+		files = append(files, res...)
 	}
 	err := smtp.SendEmail(c.Recipients, subject, body, smtp.EmailContentTypeTextPlain, files...)
 	eventManagerLog(logger.LevelDebug, "executed email notification action, elapsed: %s, error: %v",
@@ -1369,7 +1491,9 @@ func executeTransferQuotaResetRuleAction(conditions dataprovider.ConditionOption
 	return nil
 }
 
-func executeDataRetentionCheckForUser(user dataprovider.User, folders []dataprovider.FolderRetention) error {
+func executeDataRetentionCheckForUser(user dataprovider.User, folders []dataprovider.FolderRetention,
+	params *EventParams, actionName string,
+) error {
 	if err := user.LoadAndApplyGroupSettings(); err != nil {
 		eventManagerLog(logger.LevelDebug, "skipping scheduled retention check for user %s, cannot apply group settings: %v",
 			user.Username, err)
@@ -1383,6 +1507,13 @@ func executeDataRetentionCheckForUser(user dataprovider.User, folders []dataprov
 		eventManagerLog(logger.LevelError, "another retention check is already in progress for user %q", user.Username)
 		return fmt.Errorf("another retention check is in progress for user %q", user.Username)
 	}
+	defer func() {
+		params.retentionChecks = append(params.retentionChecks, executedRetentionCheck{
+			Username:   user.Username,
+			ActionName: actionName,
+			Results:    c.results,
+		})
+	}()
 	if err := c.Start(); err != nil {
 		eventManagerLog(logger.LevelError, "error checking retention for user %q: %v", user.Username, err)
 		return fmt.Errorf("error checking retention for user %q: %w", user.Username, err)
@@ -1391,7 +1522,7 @@ func executeDataRetentionCheckForUser(user dataprovider.User, folders []dataprov
 }
 
 func executeDataRetentionCheckRuleAction(config dataprovider.EventActionDataRetentionConfig,
-	conditions dataprovider.ConditionOptions, params *EventParams,
+	conditions dataprovider.ConditionOptions, params *EventParams, actionName string,
 ) error {
 	users, err := params.getUsers()
 	if err != nil {
@@ -1414,7 +1545,7 @@ func executeDataRetentionCheckRuleAction(config dataprovider.EventActionDataRete
 			}
 		}
 		executed++
-		if err = executeDataRetentionCheckForUser(user, config.Folders); err != nil {
+		if err = executeDataRetentionCheckForUser(user, config.Folders, params, actionName); err != nil {
 			failedChecks = append(failedChecks, user.Username)
 			params.AddError(err)
 			continue
@@ -1449,7 +1580,7 @@ func executeRuleAction(action dataprovider.BaseEventAction, params *EventParams,
 	case dataprovider.ActionTypeTransferQuotaReset:
 		err = executeTransferQuotaResetRuleAction(conditions, params)
 	case dataprovider.ActionTypeDataRetentionCheck:
-		err = executeDataRetentionCheckRuleAction(action.Options.RetentionConfig, conditions, params)
+		err = executeDataRetentionCheckRuleAction(action.Options.RetentionConfig, conditions, params, action.Name)
 	case dataprovider.ActionTypeFilesystem:
 		err = executeFsRuleAction(action.Options.FsConfig, conditions, params)
 	default:

internal/common/eventmanager_test.go

@@ -31,6 +31,7 @@ import (
 	"github.com/sftpgo/sdk"
 	sdkkms "github.com/sftpgo/sdk/kms"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/drakkan/sftpgo/v2/internal/dataprovider"
 	"github.com/drakkan/sftpgo/v2/internal/kms"
@@ -348,7 +349,7 @@ func TestEventManagerErrors(t *testing.T) {
 				Type: sdk.GroupTypePrimary,
 			},
 		},
-	}, nil)
+	}, nil, &EventParams{}, "")
 	assert.Error(t, err)
 	err = executeDeleteFsActionForUser(nil, nil, dataprovider.User{
 		Groups: []sdk.GroupMapping{
@@ -412,7 +413,7 @@ func TestEventManagerErrors(t *testing.T) {
 				Type: sdk.GroupTypePrimary,
 			},
 		},
-	})
+	}, &EventParams{})
 	assert.Error(t, err)
 
 	dataRetentionAction := dataprovider.BaseEventAction{
@@ -934,7 +935,7 @@ func TestEventRuleActions(t *testing.T) {
 
 	body, _, err := getHTTPRuleActionBody(dataprovider.EventActionHTTPConfig{
 		Method: http.MethodPost,
-	}, nil, nil, dataprovider.User{})
+	}, nil, nil, dataprovider.User{}, &EventParams{})
 	assert.NoError(t, err)
 	assert.Nil(t, body)
 
@@ -991,7 +992,7 @@ func TestEventRuleActionsNoGroupMatching(t *testing.T) {
 	if assert.Error(t, err) {
 		assert.Contains(t, err.Error(), "no transfer quota reset executed")
 	}
-	err = executeDataRetentionCheckRuleAction(dataprovider.EventActionDataRetentionConfig{}, conditions, &EventParams{})
+	err = executeDataRetentionCheckRuleAction(dataprovider.EventActionDataRetentionConfig{}, conditions, &EventParams{}, "")
 	if assert.Error(t, err) {
 		assert.Contains(t, err.Error(), "no retention check executed")
 	}
@@ -1033,7 +1034,7 @@ func TestGetFileContent(t *testing.T) {
 	_, err = getMailAttachments(user, []string{"/"}, replacer)
 	assert.Error(t, err)
 	// files too large
-	content := make([]byte, emailAttachmentsMaxSize/2+1)
+	content := make([]byte, maxAttachmentsSize/2+1)
 	_, err = rand.Read(content)
 	assert.NoError(t, err)
 	err = os.WriteFile(filepath.Join(user.GetHomeDir(), "file1.txt"), content, 0666)
@@ -1402,10 +1403,11 @@ func TestScheduledActions(t *testing.T) {
 
 func TestEventParamsCopy(t *testing.T) {
 	params := EventParams{
-		Name:   "name",
+		Name:            "name",
-		Event:  "event",
+		Event:           "event",
-		Status: 1,
+		Status:          1,
-		errors: []string{"error1"},
+		errors:          []string{"error1"},
+		retentionChecks: []executedRetentionCheck{},
 	}
 	paramsCopy := params.getACopy()
 	assert.Equal(t, params, *paramsCopy)
@@ -1422,6 +1424,35 @@ func TestEventParamsCopy(t *testing.T) {
 	assert.Equal(t, "event mod", paramsCopy.Event)
 	assert.Equal(t, 1, params.Status)
 	assert.Equal(t, 2, paramsCopy.Status)
+	params = EventParams{
+		retentionChecks: []executedRetentionCheck{
+			{
+				Username:   "u",
+				ActionName: "a",
+				Results: []folderRetentionCheckResult{
+					{
+						Path:      "p",
+						Retention: 1,
+					},
+				},
+			},
+		},
+	}
+	paramsCopy = params.getACopy()
+	require.Len(t, paramsCopy.retentionChecks, 1)
+	paramsCopy.retentionChecks[0].Username = "u_copy"
+	paramsCopy.retentionChecks[0].ActionName = "a_copy"
+	require.Len(t, paramsCopy.retentionChecks[0].Results, 1)
+	paramsCopy.retentionChecks[0].Results[0].Path = "p_copy"
+	paramsCopy.retentionChecks[0].Results[0].Retention = 2
+	assert.Equal(t, "u", params.retentionChecks[0].Username)
+	assert.Equal(t, "a", params.retentionChecks[0].ActionName)
+	assert.Equal(t, "p", params.retentionChecks[0].Results[0].Path)
+	assert.Equal(t, 1, params.retentionChecks[0].Results[0].Retention)
+	assert.Equal(t, "u_copy", paramsCopy.retentionChecks[0].Username)
+	assert.Equal(t, "a_copy", paramsCopy.retentionChecks[0].ActionName)
+	assert.Equal(t, "p_copy", paramsCopy.retentionChecks[0].Results[0].Path)
+	assert.Equal(t, 2, paramsCopy.retentionChecks[0].Results[0].Retention)
 }
 
 func TestEventParamsStatusFromError(t *testing.T) {
@@ -1454,13 +1485,13 @@ func TestWriteHTTPPartsError(t *testing.T) {
 		errTest: io.ErrShortWrite,
 	})
 
-	err := writeHTTPPart(m, dataprovider.HTTPPart{}, nil, nil, nil)
+	err := writeHTTPPart(m, dataprovider.HTTPPart{}, nil, nil, nil, &EventParams{})
 	assert.ErrorIs(t, err, io.ErrShortWrite)
 
 	body := "test body"
 	m = multipart.NewWriter(&testWriter{sentinel: body})
 	err = writeHTTPPart(m, dataprovider.HTTPPart{
 		Body: body,
-	}, nil, nil, nil)
+	}, nil, nil, nil, &EventParams{})
 	assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
 }

internal/common/protocol_test.go

@@ -4136,6 +4136,253 @@ func TestEventActionEmailAttachments(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestEventActionsRetentionReports(t *testing.T) {
+	smtpCfg := smtp.Config{
+		Host:          "127.0.0.1",
+		Port:          2525,
+		From:          "notify@example.com",
+		TemplatesPath: "templates",
+	}
+	err := smtpCfg.Initialize(configDir)
+	require.NoError(t, err)
+
+	testDir := "/d"
+	a1 := dataprovider.BaseEventAction{
+		Name: "action1",
+		Type: dataprovider.ActionTypeDataRetentionCheck,
+		Options: dataprovider.BaseEventActionOptions{
+			RetentionConfig: dataprovider.EventActionDataRetentionConfig{
+				Folders: []dataprovider.FolderRetention{
+					{
+						Path:                  testDir,
+						Retention:             1,
+						DeleteEmptyDirs:       true,
+						IgnoreUserPermissions: true,
+					},
+				},
+			},
+		},
+	}
+	action1, _, err := httpdtest.AddEventAction(a1, http.StatusCreated)
+	assert.NoError(t, err)
+	a2 := dataprovider.BaseEventAction{
+		Name: "action2",
+		Type: dataprovider.ActionTypeEmail,
+		Options: dataprovider.BaseEventActionOptions{
+			EmailConfig: dataprovider.EventActionEmailConfig{
+				Recipients:  []string{"test@example.com"},
+				Subject:     `"{{Event}}" from "{{Name}}"`,
+				Body:        "Fs path {{FsPath}}, size: {{FileSize}}, protocol: {{Protocol}}, IP: {{IP}}",
+				Attachments: []string{dataprovider.RetentionReportPlaceHolder},
+			},
+		},
+	}
+	action2, _, err := httpdtest.AddEventAction(a2, http.StatusCreated)
+	assert.NoError(t, err)
+	a3 := dataprovider.BaseEventAction{
+		Name: "action3",
+		Type: dataprovider.ActionTypeHTTP,
+		Options: dataprovider.BaseEventActionOptions{
+			HTTPConfig: dataprovider.EventActionHTTPConfig{
+				Endpoint: fmt.Sprintf("http://%s/", httpAddr),
+				Timeout:  20,
+				Method:   http.MethodPost,
+				Body:     dataprovider.RetentionReportPlaceHolder,
+			},
+		},
+	}
+	action3, _, err := httpdtest.AddEventAction(a3, http.StatusCreated)
+	assert.NoError(t, err)
+	a4 := dataprovider.BaseEventAction{
+		Name: "action4",
+		Type: dataprovider.ActionTypeHTTP,
+		Options: dataprovider.BaseEventActionOptions{
+			HTTPConfig: dataprovider.EventActionHTTPConfig{
+				Endpoint: fmt.Sprintf("http://%s/multipart", httpAddr),
+				Timeout:  20,
+				Method:   http.MethodPost,
+				Parts: []dataprovider.HTTPPart{
+					{
+						Name:     "reports.zip",
+						Filepath: dataprovider.RetentionReportPlaceHolder,
+					},
+				},
+			},
+		},
+	}
+	action4, _, err := httpdtest.AddEventAction(a4, http.StatusCreated)
+	assert.NoError(t, err)
+	r1 := dataprovider.EventRule{
+		Name:    "test rule1",
+		Trigger: dataprovider.EventTriggerFsEvent,
+		Conditions: dataprovider.EventConditions{
+			FsEvents: []string{"upload"},
+		},
+		Actions: []dataprovider.EventAction{
+			{
+				BaseEventAction: dataprovider.BaseEventAction{
+					Name: action1.Name,
+				},
+				Order: 1,
+				Options: dataprovider.EventActionOptions{
+					ExecuteSync:   true,
+					StopOnFailure: true,
+				},
+			},
+			{
+				BaseEventAction: dataprovider.BaseEventAction{
+					Name: action2.Name,
+				},
+				Order: 2,
+				Options: dataprovider.EventActionOptions{
+					ExecuteSync:   true,
+					StopOnFailure: true,
+				},
+			},
+			{
+				BaseEventAction: dataprovider.BaseEventAction{
+					Name: action3.Name,
+				},
+				Order: 3,
+				Options: dataprovider.EventActionOptions{
+					ExecuteSync:   true,
+					StopOnFailure: true,
+				},
+			},
+			{
+				BaseEventAction: dataprovider.BaseEventAction{
+					Name: action4.Name,
+				},
+				Order: 4,
+				Options: dataprovider.EventActionOptions{
+					ExecuteSync:   true,
+					StopOnFailure: true,
+				},
+			},
+		},
+	}
+	rule1, _, err := httpdtest.AddEventRule(r1, http.StatusCreated)
+	assert.NoError(t, err)
+
+	user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated)
+	assert.NoError(t, err)
+	conn, client, err := getSftpClient(user)
+	if assert.NoError(t, err) {
+		defer conn.Close()
+		defer client.Close()
+
+		subdir := path.Join(testDir, "sub")
+		err = client.MkdirAll(subdir)
+		assert.NoError(t, err)
+
+		lastReceivedEmail.reset()
+		f, err := client.Create(testFileName)
+		assert.NoError(t, err)
+		_, err = f.Write(testFileContent)
+		assert.NoError(t, err)
+		err = f.Close()
+		assert.NoError(t, err)
+
+		email := lastReceivedEmail.get()
+		assert.Len(t, email.To, 1)
+		assert.True(t, util.Contains(email.To, "test@example.com"))
+		assert.Contains(t, email.Data, fmt.Sprintf(`Subject: "upload" from "%s"`, user.Username))
+		assert.Contains(t, email.Data, "Content-Disposition: attachment")
+		_, err = client.Stat(testDir)
+		assert.NoError(t, err)
+		_, err = client.Stat(subdir)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+
+		err = client.Mkdir(subdir)
+		assert.NoError(t, err)
+		newName := path.Join(testDir, testFileName)
+		err = client.Rename(testFileName, newName)
+		assert.NoError(t, err)
+		err = client.Chtimes(newName, time.Now().Add(-24*time.Hour), time.Now().Add(-24*time.Hour))
+		assert.NoError(t, err)
+
+		lastReceivedEmail.reset()
+		f, err = client.Create(testFileName)
+		assert.NoError(t, err)
+		_, err = f.Write(testFileContent)
+		assert.NoError(t, err)
+		err = f.Close()
+		assert.NoError(t, err)
+		email = lastReceivedEmail.get()
+		assert.Len(t, email.To, 1)
+		_, err = client.Stat(subdir)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+		_, err = client.Stat(subdir)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+	}
+	// now remove the retention check to test errors
+	rule1.Actions = []dataprovider.EventAction{
+		{
+			BaseEventAction: dataprovider.BaseEventAction{
+				Name: action2.Name,
+			},
+			Order: 2,
+			Options: dataprovider.EventActionOptions{
+				ExecuteSync:   true,
+				StopOnFailure: false,
+			},
+		},
+		{
+			BaseEventAction: dataprovider.BaseEventAction{
+				Name: action3.Name,
+			},
+			Order: 3,
+			Options: dataprovider.EventActionOptions{
+				ExecuteSync:   true,
+				StopOnFailure: false,
+			},
+		},
+		{
+			BaseEventAction: dataprovider.BaseEventAction{
+				Name: action4.Name,
+			},
+			Order: 4,
+			Options: dataprovider.EventActionOptions{
+				ExecuteSync:   true,
+				StopOnFailure: false,
+			},
+		},
+	}
+	_, _, err = httpdtest.UpdateEventRule(rule1, http.StatusOK)
+	assert.NoError(t, err)
+	conn, client, err = getSftpClient(user)
+	if assert.NoError(t, err) {
+		defer conn.Close()
+		defer client.Close()
+
+		f, err := client.Create(testFileName)
+		assert.NoError(t, err)
+		_, err = f.Write(testFileContent)
+		assert.NoError(t, err)
+		err = f.Close()
+		assert.Error(t, err)
+	}
+
+	_, err = httpdtest.RemoveEventRule(rule1, http.StatusOK)
+	assert.NoError(t, err)
+	_, err = httpdtest.RemoveEventAction(action1, http.StatusOK)
+	assert.NoError(t, err)
+	_, err = httpdtest.RemoveEventAction(action2, http.StatusOK)
+	assert.NoError(t, err)
+	_, err = httpdtest.RemoveEventAction(action3, http.StatusOK)
+	assert.NoError(t, err)
+	_, err = httpdtest.RemoveEventAction(action4, http.StatusOK)
+	assert.NoError(t, err)
+	_, err = httpdtest.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+
+	smtpCfg = smtp.Config{}
+	err = smtpCfg.Initialize(configDir)
+	require.NoError(t, err)
+}
+
 func TestEventRuleFirstUploadDownloadActions(t *testing.T) {
 	smtpCfg := smtp.Config{
 		Host:          "127.0.0.1",

internal/dataprovider/eventrule.go

@@ -122,6 +122,11 @@ const (
 	FilesystemActionExist
 )
 
+const (
+	// RetentionReportPlaceHolder defines the placeholder for data retention reports
+	RetentionReportPlaceHolder = "{{RetentionReports}}"
+)
+
 var (
 	supportedFsActions = []int{FilesystemActionRename, FilesystemActionDelete, FilesystemActionMkdirs,
 		FilesystemActionExist}
@@ -229,7 +234,9 @@ func (p *HTTPPart) validate() error {
 		}
 	} else {
 		p.Body = ""
-		p.Filepath = util.CleanPath(p.Filepath)
+		if p.Filepath != RetentionReportPlaceHolder {
+			p.Filepath = util.CleanPath(p.Filepath)
+		}
 	}
 	return nil
 }
@@ -249,17 +256,24 @@ type EventActionHTTPConfig struct {
 }
 
 func (c *EventActionHTTPConfig) isTimeoutNotValid() bool {
-	if c.HasMultipartFile() {
+	if c.HasMultipartFiles() {
 		return false
 	}
-	return c.Timeout < 1 || c.Timeout > 120
+	return c.Timeout < 1 || c.Timeout > 180
 }
 
 func (c *EventActionHTTPConfig) validateMultiparts() error {
+	filePaths := make(map[string]bool)
 	for idx := range c.Parts {
 		if err := c.Parts[idx].validate(); err != nil {
 			return err
 		}
+		if filePath := c.Parts[idx].Filepath; filePath != "" {
+			if filePaths[filePath] {
+				return fmt.Errorf("filepath %q is duplicated", filePath)
+			}
+			filePaths[filePath] = true
+		}
 	}
 	if len(c.Parts) > 0 {
 		if c.Body != "" {
@@ -315,7 +329,7 @@ func (c *EventActionHTTPConfig) validate(additionalData string) error {
 
 // GetContext returns the context and the cancel func to use for the HTTP request
 func (c *EventActionHTTPConfig) GetContext() (context.Context, context.CancelFunc) {
-	if c.HasMultipartFile() {
+	if c.HasMultipartFiles() {
 		return context.WithCancel(context.Background())
 	}
 	return context.WithTimeout(context.Background(), time.Duration(c.Timeout)*time.Second)
@@ -334,10 +348,10 @@ func (c *EventActionHTTPConfig) HasObjectData() bool {
 	return false
 }
 
-// HasMultipartFile returns true if a file must be uploaded via a multipart request
+// HasMultipartFiles returns true if at least a file must be uploaded via a multipart request
-func (c *EventActionHTTPConfig) HasMultipartFile() bool {
+func (c *EventActionHTTPConfig) HasMultipartFiles() bool {
 	for _, part := range c.Parts {
-		if part.Filepath != "" {
+		if part.Filepath != "" && part.Filepath != RetentionReportPlaceHolder {
 			return true
 		}
 	}
@@ -427,6 +441,15 @@ func (c EventActionEmailConfig) GetAttachmentsAsString() string {
 	return strings.Join(c.Attachments, ",")
 }
 
+func (c *EventActionEmailConfig) hasFilesAttachments() bool {
+	for _, a := range c.Attachments {
+		if a != RetentionReportPlaceHolder {
+			return true
+		}
+	}
+	return false
+}
+
 func (c *EventActionEmailConfig) validate() error {
 	if len(c.Recipients) == 0 {
 		return util.NewValidationError("at least one email recipient is required")
@@ -448,7 +471,11 @@ func (c *EventActionEmailConfig) validate() error {
 		if val == "" {
 			return util.NewValidationError("invalid path to attach")
 		}
-		c.Attachments[idx] = util.CleanPath(val)
+		if val == RetentionReportPlaceHolder {
+			c.Attachments[idx] = val
+		} else {
+			c.Attachments[idx] = util.CleanPath(val)
+		}
 	}
 	c.Attachments = util.RemoveDuplicates(c.Attachments, false)
 	return nil
@@ -1290,12 +1317,12 @@ func (r *EventRule) CheckActionsConsistency(providerObjectType string) error {
 		}
 	}
 	for _, action := range r.Actions {
-		if action.Type == ActionTypeEmail && len(action.BaseEventAction.Options.EmailConfig.Attachments) > 0 {
+		if action.Type == ActionTypeEmail && action.BaseEventAction.Options.EmailConfig.hasFilesAttachments() {
 			if !r.hasUserAssociated(providerObjectType) {
 				return errors.New("cannot send an email with attachments for a rule with no user associated")
 			}
 		}
-		if action.Type == ActionTypeHTTP && action.BaseEventAction.Options.HTTPConfig.HasMultipartFile() {
+		if action.Type == ActionTypeHTTP && action.BaseEventAction.Options.HTTPConfig.HasMultipartFiles() {
 			if !r.hasUserAssociated(providerObjectType) {
 				return errors.New("cannot upload file/s for a rule with no user associated")
 			}

internal/smtp/smtp.go

@@ -120,7 +120,7 @@ func (c *Config) Initialize(configDir string) error {
 	smtpServer.Encryption = c.getEncryption()
 	smtpServer.KeepAlive = false
 	smtpServer.ConnectTimeout = 10 * time.Second
-	smtpServer.SendTimeout = 30 * time.Second
+	smtpServer.SendTimeout = 120 * time.Second
 	if c.Domain != "" {
 		smtpServer.Helo = c.Domain
 	}

openapi/openapi.yaml

@@ -6047,7 +6047,7 @@ components:
         timeout:
           type: integer
           minimum: 1
-          maximum: 120
+          maximum: 180
           description: 'Ignored for multipart requests with files as attachments'
         skip_tls_verify:
           type: boolean

templates/webadmin/eventaction.html

@@ -677,6 +677,9 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
                 <p>
                     <span class="shortcut"><b>{{`{{ObjectData}}`}}</b></span> => Provider object data serialized as JSON with sensitive fields removed.
                 </p>
+                <p>
+                    <span class="shortcut"><b>{{`{{RetentionReports}}`}}</b></span> => Data retention reports as zip compressed CSV files. Supported as email attachment, file path for multipart HTTP request and as single parameter for HTTP requests body.
+                </p>
             </div>
             <div class="modal-footer">
                 <button class="btn btn-primary" type="button" data-dismiss="modal">OK</button>