diff --git a/README.md b/README.md
index 9c7bbb8d..2d3bfa82 100644
--- a/README.md
+++ b/README.md
@@ -40,7 +40,7 @@ More [info](https://github.com/drakkan/sftpgo/issues/452).
SFTPGo is an Open Source project and you can of course use it for free but please don't ask for free support as well.
-We will check the reported issues to see if you are experiencing a bug and if so we'll will fix it, but will only provide support to project sponsors/donors.
+We will check the reported issues to see if you are experiencing a bug and if so we'll will fix it, but will only provide support to project [sponsors/donors](#sponsors).
If you report an invalid issue or ask for step-by-step support, your issue will remain open with no answer or will be closed as invalid without further explanation. Thanks for understanding.
diff --git a/docs/eventmanager.md b/docs/eventmanager.md
index 5cf5b4f0..7e8a3bb1 100644
--- a/docs/eventmanager.md
+++ b/docs/eventmanager.md
@@ -36,6 +36,7 @@ The following placeholders are supported:
- `{{IP}}`. Client IP address.
- `{{Timestamp}}`. Event timestamp as nanoseconds since epoch.
- `{{ObjectData}}`. Provider object data serialized as JSON with sensitive fields removed.
+- `{{RetentionReports}}`. Data retention reports as zip compressed CSV files. Supported as email attachment, file path for multipart HTTP request and as single parameter for HTTP requests body.
Event rules are based on the premise that an event occours. To each rule you can associate one or more actions.
The following trigger events are supported:
diff --git a/go.mod b/go.mod
index 775908c1..1d0aacac 100644
--- a/go.mod
+++ b/go.mod
@@ -55,10 +55,10 @@ require (
github.com/shirou/gopsutil/v3 v3.22.8
github.com/spf13/afero v1.9.2
github.com/spf13/cobra v1.5.0
- github.com/spf13/viper v1.12.0
+ github.com/spf13/viper v1.13.0
github.com/stretchr/testify v1.8.0
github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62
- github.com/unrolled/secure v1.12.0
+ github.com/unrolled/secure v1.13.0
github.com/wagslane/go-password-validator v0.3.0
github.com/xhit/go-simple-mail/v2 v2.11.0
github.com/yl2chen/cidranger v1.0.3-0.20210928021809-d1cb2c52f37a
@@ -68,7 +68,7 @@ require (
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b
golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094
- golang.org/x/sys v0.0.0-20220829200755-d48e67d00261
+ golang.org/x/sys v0.0.0-20220906135438-9e1f76180b77
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9
google.golang.org/api v0.94.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0
@@ -77,7 +77,7 @@ require (
require (
cloud.google.com/go v0.104.0 // indirect
cloud.google.com/go/compute v1.9.0 // indirect
- cloud.google.com/go/iam v0.3.0 // indirect
+ cloud.google.com/go/iam v0.4.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect
github.com/ajg/form v1.5.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.7 // indirect
diff --git a/go.sum b/go.sum
index 80e33a1b..5159f48e 100644
--- a/go.sum
+++ b/go.sum
@@ -54,8 +54,9 @@ cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1
cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY=
cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c=
cloud.google.com/go/iam v0.1.1/go.mod h1:CKqrcnI/suGpybEHxZ7BMehL0oA4LpdyJdUlTl9jVMw=
-cloud.google.com/go/iam v0.3.0 h1:exkAomrVUuzx9kWFI1wm3KI0uoDeUFPB4kKGzx6x+Gc=
cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY=
+cloud.google.com/go/iam v0.4.0 h1:YBYU00SCDzZJdHqVc4I5d6lsklcYIjQZa1YmEz4jlSE=
+cloud.google.com/go/iam v0.4.0/go.mod h1:cbaZxyScUhxl7ZAkNWiALgihfP75wS/fUsVNaa1r3vA=
cloud.google.com/go/kms v1.1.0/go.mod h1:WdbppnCDMDpOvoYBMn1+gNmOeEoZYqAv+HeuKARGCXI=
cloud.google.com/go/kms v1.4.0 h1:iElbfoE61VeLhnZcGOltqL8HIly8Nhbe5t6JlH9GXjo=
cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA=
@@ -736,8 +737,8 @@ github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmq
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ=
-github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI=
+github.com/spf13/viper v1.13.0 h1:BWSJ/M+f+3nmdz9bxB+bWX28kkALN2ok11D0rSo8EJU=
+github.com/spf13/viper v1.13.0/go.mod h1:Icm2xNL3/8uyh/wFuB1jI7TiTNKp8632Nwegu+zgdYw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
@@ -765,8 +766,8 @@ github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 h1:PM5hJF7HVfNWmCjM
github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208/go.mod h1:BzWtXXrXzZUvMacR0oF/fbDDgUPO8L36tDMmRAf14ns=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/unrolled/secure v1.12.0 h1:7k3jcgLwfjiKkhQde6VbQ3D4KDLtDBqDd/hs3PPANDY=
-github.com/unrolled/secure v1.12.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
+github.com/unrolled/secure v1.13.0 h1:sdr3Phw2+f8Px8HE5sd1EHdj1aV3yUwed/uZXChLFsk=
+github.com/unrolled/secure v1.13.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
github.com/wagslane/go-password-validator v0.3.0 h1:vfxOPzGHkz5S146HDpavl0cw1DSVP061Ry2PX0/ON6I=
github.com/wagslane/go-password-validator v0.3.0/go.mod h1:TI1XJ6T5fRdRnHqHt14pvy1tNVnrwe7m3/f1f2fDphQ=
github.com/xhit/go-simple-mail/v2 v2.11.0 h1:o/056V50zfkO3Mm5tVdo9rG3ryg4ZmJ2XW5GMinHfVs=
@@ -976,8 +977,8 @@ golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY=
-golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220906135438-9e1f76180b77 h1:C1tElbkWrsSkn3IRl1GCW/gETw1TywWIPgwZtXTZbYg=
+golang.org/x/sys v0.0.0-20220906135438-9e1f76180b77/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
diff --git a/internal/common/dataretention.go b/internal/common/dataretention.go
index 0f806fde..7b2fe17d 100644
--- a/internal/common/dataretention.go
+++ b/internal/common/dataretention.go
@@ -147,7 +147,7 @@ type RetentionCheck struct {
// email to use if the notification method is set to email
Email string `json:"email,omitempty"`
// Cleanup results
- results []*folderRetentionCheckResult `json:"-"`
+ results []folderRetentionCheckResult `json:"-"`
conn *BaseConnection
}
@@ -223,10 +223,12 @@ func (c *RetentionCheck) removeFile(virtualPath string, info os.FileInfo) error
func (c *RetentionCheck) cleanupFolder(folderPath string) error {
deleteFilesPerms := []string{dataprovider.PermDelete, dataprovider.PermDeleteFiles}
startTime := time.Now()
- result := &folderRetentionCheckResult{
+ result := folderRetentionCheckResult{
Path: folderPath,
}
- c.results = append(c.results, result)
+ defer func() {
+ c.results = append(c.results, result)
+ }()
if !c.conn.User.HasPerm(dataprovider.PermListItems, folderPath) || !c.conn.User.HasAnyPerm(deleteFilesPerms, folderPath) {
result.Elapsed = time.Since(startTime)
result.Info = "data retention check skipped: no permissions"
@@ -301,7 +303,15 @@ func (c *RetentionCheck) cleanupFolder(folderPath string) error {
}
func (c *RetentionCheck) checkEmptyDirRemoval(folderPath string) {
- if folderPath != "/" && c.conn.User.HasAnyPerm([]string{
+ if folderPath == "/" {
+ return
+ }
+ for _, folder := range c.Folders {
+ if folderPath == folder.Path {
+ return
+ }
+ }
+ if c.conn.User.HasAnyPerm([]string{
dataprovider.PermDelete,
dataprovider.PermDeleteDirs,
}, path.Dir(folderPath),
diff --git a/internal/common/dataretention_test.go b/internal/common/dataretention_test.go
index 3a566c54..8ee10b04 100644
--- a/internal/common/dataretention_test.go
+++ b/internal/common/dataretention_test.go
@@ -131,7 +131,7 @@ func TestRetentionEmailNotifications(t *testing.T) {
check := RetentionCheck{
Notifications: []RetentionCheckNotification{RetentionCheckNotificationEmail},
Email: "notification@example.com",
- results: []*folderRetentionCheckResult{
+ results: []folderRetentionCheckResult{
{
Path: "/",
Retention: 24,
@@ -177,7 +177,7 @@ func TestRetentionHookNotifications(t *testing.T) {
user.Permissions["/"] = []string{dataprovider.PermAny}
check := RetentionCheck{
Notifications: []RetentionCheckNotification{RetentionCheckNotificationHook},
- results: []*folderRetentionCheckResult{
+ results: []folderRetentionCheckResult{
{
Path: "/",
Retention: 24,
diff --git a/internal/common/eventmanager.go b/internal/common/eventmanager.go
index c21ffef6..00e42250 100644
--- a/internal/common/eventmanager.go
+++ b/internal/common/eventmanager.go
@@ -17,6 +17,7 @@ package common
import (
"bytes"
"context"
+ "encoding/csv"
"errors"
"fmt"
"io"
@@ -28,11 +29,13 @@ import (
"os"
"os/exec"
"path"
+ "strconv"
"strings"
"sync"
"sync/atomic"
"time"
+ "github.com/klauspost/compress/zip"
"github.com/robfig/cron/v3"
"github.com/rs/xid"
"github.com/sftpgo/sdk"
@@ -47,8 +50,8 @@ import (
)
const (
- ipBlockedEventName = "IP Blocked"
- emailAttachmentsMaxSize = int64(10 * 1024 * 1024)
+ ipBlockedEventName = "IP Blocked"
+ maxAttachmentsSize = int64(10 * 1024 * 1024)
)
var (
@@ -412,6 +415,12 @@ func (r *eventRulesContainer) handleCertificateEvent(params EventParams) {
}
}
+type executedRetentionCheck struct {
+ Username string
+ ActionName string
+ Results []folderRetentionCheckResult
+}
+
// EventParams defines the supported event parameters
type EventParams struct {
Name string
@@ -432,12 +441,25 @@ type EventParams struct {
sender string
updateStatusFromError bool
errors []string
+ retentionChecks []executedRetentionCheck
}
func (p *EventParams) getACopy() *EventParams {
params := *p
params.errors = make([]string, len(p.errors))
copy(params.errors, p.errors)
+ retentionChecks := make([]executedRetentionCheck, 0, len(p.retentionChecks))
+ for _, c := range p.retentionChecks {
+ executedCheck := executedRetentionCheck{
+ Username: c.Username,
+ ActionName: c.ActionName,
+ }
+ executedCheck.Results = make([]folderRetentionCheckResult, len(c.Results))
+ copy(executedCheck.Results, c.Results)
+ retentionChecks = append(retentionChecks, executedCheck)
+ }
+ params.retentionChecks = retentionChecks
+
return ¶ms
}
@@ -498,6 +520,52 @@ func (p *EventParams) getFolders() ([]vfs.BaseVirtualFolder, error) {
return []vfs.BaseVirtualFolder{folder}, nil
}
+func (p *EventParams) getCompressedDataRetentionReport() ([]byte, error) {
+ if len(p.retentionChecks) == 0 {
+ return nil, errors.New("no data retention report available")
+ }
+ var b bytes.Buffer
+ wr := zip.NewWriter(&b)
+ for _, check := range p.retentionChecks {
+ if size := int64(len(b.Bytes())); size > maxAttachmentsSize {
+ eventManagerLog(logger.LevelError, "unable to get retention report, size too large: %s", util.ByteCountIEC(size))
+ return nil, fmt.Errorf("unable to get retention report, size too large: %s", util.ByteCountIEC(size))
+ }
+ data, err := getCSVRetentionReport(check.Results)
+ if err != nil {
+ return nil, fmt.Errorf("unable to get CSV report: %w", err)
+ }
+ fh := &zip.FileHeader{
+ Name: fmt.Sprintf("%s-%s.csv", check.ActionName, check.Username),
+ Method: zip.Deflate,
+ Modified: time.Now().UTC(),
+ }
+ f, err := wr.CreateHeader(fh)
+ if err != nil {
+ return nil, fmt.Errorf("unable to create zip header for file %q: %w", fh.Name, err)
+ }
+ _, err = io.Copy(f, bytes.NewBuffer(data))
+ if err != nil {
+ return nil, fmt.Errorf("unable to write content to zip file %q: %w", fh.Name, err)
+ }
+ }
+ if err := wr.Close(); err != nil {
+ return nil, fmt.Errorf("unable to close zip writer: %w", err)
+ }
+ return b.Bytes(), nil
+}
+
+func (p *EventParams) getRetentionReportsAsMailAttachment() (mail.File, error) {
+ var result mail.File
+ data, err := p.getCompressedDataRetentionReport()
+ if err != nil {
+ return result, err
+ }
+ result.Name = "retention-reports.zip"
+ result.Data = data
+ return result, nil
+}
+
func (p *EventParams) getStringReplacements(addObjectData bool) []string {
replacements := []string{
"{{Name}}", p.Name,
@@ -530,6 +598,29 @@ func (p *EventParams) getStringReplacements(addObjectData bool) []string {
return replacements
}
+func getCSVRetentionReport(results []folderRetentionCheckResult) ([]byte, error) {
+ var b bytes.Buffer
+ csvWriter := csv.NewWriter(&b)
+ err := csvWriter.Write([]string{"path", "retention (hours)", "deleted files", "deleted size (bytes)",
+ "elapsed (ms)", "info", "error"})
+ if err != nil {
+ return nil, err
+ }
+
+ for _, result := range results {
+ err = csvWriter.Write([]string{result.Path, strconv.Itoa(result.Retention), strconv.Itoa(result.DeletedFiles),
+ strconv.FormatInt(result.DeletedSize, 10), strconv.FormatInt(result.Elapsed.Milliseconds(), 10),
+ result.Info, result.Error})
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ csvWriter.Flush()
+ err = csvWriter.Error()
+ return b.Bytes(), err
+}
+
func getFileReader(conn *BaseConnection, virtualPath string) (io.ReadCloser, func(), error) {
fs, fsPath, err := conn.GetFsAndResolvedPath(virtualPath)
if err != nil {
@@ -600,7 +691,7 @@ func getMailAttachments(user dataprovider.User, attachments []string, replacer *
return nil, fmt.Errorf("cannot attach non regular file %q", virtualPath)
}
totalSize += info.Size()
- if totalSize > emailAttachmentsMaxSize {
+ if totalSize > maxAttachmentsSize {
return nil, fmt.Errorf("unable to send files as attachment, size too large: %s", util.ByteCountIEC(totalSize))
}
data, err := getFileContent(conn, virtualPath, int(info.Size()))
@@ -682,7 +773,7 @@ func getHTTPRuleActionEndpoint(c dataprovider.EventActionHTTPConfig, replacer *s
}
func writeHTTPPart(m *multipart.Writer, part dataprovider.HTTPPart, h textproto.MIMEHeader,
- conn *BaseConnection, replacer *strings.Replacer,
+ conn *BaseConnection, replacer *strings.Replacer, params *EventParams,
) error {
partWriter, err := m.CreatePart(h)
if err != nil {
@@ -697,6 +788,18 @@ func writeHTTPPart(m *multipart.Writer, part dataprovider.HTTPPart, h textproto.
}
return nil
}
+ if part.Filepath == dataprovider.RetentionReportPlaceHolder {
+ data, err := params.getCompressedDataRetentionReport()
+ if err != nil {
+ return err
+ }
+ _, err = partWriter.Write(data)
+ if err != nil {
+ eventManagerLog(logger.LevelError, "unable to write part %q, err: %v", part.Name, err)
+ return err
+ }
+ return nil
+ }
err = writeFileContent(conn, util.CleanPath(replacer.Replace(part.Filepath)), partWriter)
if err != nil {
eventManagerLog(logger.LevelError, "unable to write file part %q, err: %v", part.Name, err)
@@ -706,13 +809,20 @@ func writeHTTPPart(m *multipart.Writer, part dataprovider.HTTPPart, h textproto.
}
func getHTTPRuleActionBody(c dataprovider.EventActionHTTPConfig, replacer *strings.Replacer,
- cancel context.CancelFunc, user dataprovider.User,
+ cancel context.CancelFunc, user dataprovider.User, params *EventParams,
) (io.ReadCloser, string, error) {
var body io.ReadCloser
if c.Method == http.MethodGet {
return body, "", nil
}
if c.Body != "" {
+ if c.Body == dataprovider.RetentionReportPlaceHolder {
+ data, err := params.getCompressedDataRetentionReport()
+ if err != nil {
+ return body, "", err
+ }
+ return io.NopCloser(bytes.NewBuffer(data)), "", nil
+ }
return io.NopCloser(bytes.NewBufferString(replaceWithReplacer(c.Body, replacer))), "", nil
}
if len(c.Parts) > 0 {
@@ -745,11 +855,10 @@ func getHTTPRuleActionBody(c dataprovider.EventActionHTTPConfig, replacer *strin
if part.Body != "" {
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"`, multipartQuoteEscaper.Replace(part.Name)))
} else {
- filePath := util.CleanPath(replacer.Replace(part.Filepath))
h.Set("Content-Disposition",
fmt.Sprintf(`form-data; name="%s"; filename="%s"`,
- multipartQuoteEscaper.Replace(part.Name), multipartQuoteEscaper.Replace(path.Base(filePath))))
- contentType := mime.TypeByExtension(path.Ext(filePath))
+ multipartQuoteEscaper.Replace(part.Name), multipartQuoteEscaper.Replace(path.Base(part.Filepath))))
+ contentType := mime.TypeByExtension(path.Ext(part.Filepath))
if contentType == "" {
contentType = "application/octet-stream"
}
@@ -758,7 +867,7 @@ func getHTTPRuleActionBody(c dataprovider.EventActionHTTPConfig, replacer *strin
for _, keyVal := range part.Headers {
h.Set(keyVal.Key, replaceWithReplacer(keyVal.Value, replacer))
}
- if err := writeHTTPPart(m, part, h, conn, replacer); err != nil {
+ if err := writeHTTPPart(m, part, h, conn, replacer, params); err != nil {
cancel()
return
}
@@ -791,13 +900,13 @@ func executeHTTPRuleAction(c dataprovider.EventActionHTTPConfig, params *EventPa
defer cancel()
var user dataprovider.User
- if c.HasMultipartFile() {
+ if c.HasMultipartFiles() {
user, err = params.getUserFromSender()
if err != nil {
return err
}
}
- body, contentType, err := getHTTPRuleActionBody(c, replacer, cancel, user)
+ body, contentType, err := getHTTPRuleActionBody(c, replacer, cancel, user, params)
if err != nil {
return err
}
@@ -888,15 +997,28 @@ func executeEmailRuleAction(c dataprovider.EventActionEmailConfig, params *Event
subject := replaceWithReplacer(c.Subject, replacer)
startTime := time.Now()
var files []mail.File
- if len(c.Attachments) > 0 {
+ fileAttachments := make([]string, 0, len(c.Attachments))
+ for _, attachment := range c.Attachments {
+ if attachment == dataprovider.RetentionReportPlaceHolder {
+ f, err := params.getRetentionReportsAsMailAttachment()
+ if err != nil {
+ return err
+ }
+ files = append(files, f)
+ continue
+ }
+ fileAttachments = append(fileAttachments, attachment)
+ }
+ if len(fileAttachments) > 0 {
user, err := params.getUserFromSender()
if err != nil {
return err
}
- files, err = getMailAttachments(user, c.Attachments, replacer)
+ res, err := getMailAttachments(user, fileAttachments, replacer)
if err != nil {
return err
}
+ files = append(files, res...)
}
err := smtp.SendEmail(c.Recipients, subject, body, smtp.EmailContentTypeTextPlain, files...)
eventManagerLog(logger.LevelDebug, "executed email notification action, elapsed: %s, error: %v",
@@ -1369,7 +1491,9 @@ func executeTransferQuotaResetRuleAction(conditions dataprovider.ConditionOption
return nil
}
-func executeDataRetentionCheckForUser(user dataprovider.User, folders []dataprovider.FolderRetention) error {
+func executeDataRetentionCheckForUser(user dataprovider.User, folders []dataprovider.FolderRetention,
+ params *EventParams, actionName string,
+) error {
if err := user.LoadAndApplyGroupSettings(); err != nil {
eventManagerLog(logger.LevelDebug, "skipping scheduled retention check for user %s, cannot apply group settings: %v",
user.Username, err)
@@ -1383,6 +1507,13 @@ func executeDataRetentionCheckForUser(user dataprovider.User, folders []dataprov
eventManagerLog(logger.LevelError, "another retention check is already in progress for user %q", user.Username)
return fmt.Errorf("another retention check is in progress for user %q", user.Username)
}
+ defer func() {
+ params.retentionChecks = append(params.retentionChecks, executedRetentionCheck{
+ Username: user.Username,
+ ActionName: actionName,
+ Results: c.results,
+ })
+ }()
if err := c.Start(); err != nil {
eventManagerLog(logger.LevelError, "error checking retention for user %q: %v", user.Username, err)
return fmt.Errorf("error checking retention for user %q: %w", user.Username, err)
@@ -1391,7 +1522,7 @@ func executeDataRetentionCheckForUser(user dataprovider.User, folders []dataprov
}
func executeDataRetentionCheckRuleAction(config dataprovider.EventActionDataRetentionConfig,
- conditions dataprovider.ConditionOptions, params *EventParams,
+ conditions dataprovider.ConditionOptions, params *EventParams, actionName string,
) error {
users, err := params.getUsers()
if err != nil {
@@ -1414,7 +1545,7 @@ func executeDataRetentionCheckRuleAction(config dataprovider.EventActionDataRete
}
}
executed++
- if err = executeDataRetentionCheckForUser(user, config.Folders); err != nil {
+ if err = executeDataRetentionCheckForUser(user, config.Folders, params, actionName); err != nil {
failedChecks = append(failedChecks, user.Username)
params.AddError(err)
continue
@@ -1449,7 +1580,7 @@ func executeRuleAction(action dataprovider.BaseEventAction, params *EventParams,
case dataprovider.ActionTypeTransferQuotaReset:
err = executeTransferQuotaResetRuleAction(conditions, params)
case dataprovider.ActionTypeDataRetentionCheck:
- err = executeDataRetentionCheckRuleAction(action.Options.RetentionConfig, conditions, params)
+ err = executeDataRetentionCheckRuleAction(action.Options.RetentionConfig, conditions, params, action.Name)
case dataprovider.ActionTypeFilesystem:
err = executeFsRuleAction(action.Options.FsConfig, conditions, params)
default:
diff --git a/internal/common/eventmanager_test.go b/internal/common/eventmanager_test.go
index 304cef30..6f34a764 100644
--- a/internal/common/eventmanager_test.go
+++ b/internal/common/eventmanager_test.go
@@ -31,6 +31,7 @@ import (
"github.com/sftpgo/sdk"
sdkkms "github.com/sftpgo/sdk/kms"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
"github.com/drakkan/sftpgo/v2/internal/dataprovider"
"github.com/drakkan/sftpgo/v2/internal/kms"
@@ -348,7 +349,7 @@ func TestEventManagerErrors(t *testing.T) {
Type: sdk.GroupTypePrimary,
},
},
- }, nil)
+ }, nil, &EventParams{}, "")
assert.Error(t, err)
err = executeDeleteFsActionForUser(nil, nil, dataprovider.User{
Groups: []sdk.GroupMapping{
@@ -412,7 +413,7 @@ func TestEventManagerErrors(t *testing.T) {
Type: sdk.GroupTypePrimary,
},
},
- })
+ }, &EventParams{})
assert.Error(t, err)
dataRetentionAction := dataprovider.BaseEventAction{
@@ -934,7 +935,7 @@ func TestEventRuleActions(t *testing.T) {
body, _, err := getHTTPRuleActionBody(dataprovider.EventActionHTTPConfig{
Method: http.MethodPost,
- }, nil, nil, dataprovider.User{})
+ }, nil, nil, dataprovider.User{}, &EventParams{})
assert.NoError(t, err)
assert.Nil(t, body)
@@ -991,7 +992,7 @@ func TestEventRuleActionsNoGroupMatching(t *testing.T) {
if assert.Error(t, err) {
assert.Contains(t, err.Error(), "no transfer quota reset executed")
}
- err = executeDataRetentionCheckRuleAction(dataprovider.EventActionDataRetentionConfig{}, conditions, &EventParams{})
+ err = executeDataRetentionCheckRuleAction(dataprovider.EventActionDataRetentionConfig{}, conditions, &EventParams{}, "")
if assert.Error(t, err) {
assert.Contains(t, err.Error(), "no retention check executed")
}
@@ -1033,7 +1034,7 @@ func TestGetFileContent(t *testing.T) {
_, err = getMailAttachments(user, []string{"/"}, replacer)
assert.Error(t, err)
// files too large
- content := make([]byte, emailAttachmentsMaxSize/2+1)
+ content := make([]byte, maxAttachmentsSize/2+1)
_, err = rand.Read(content)
assert.NoError(t, err)
err = os.WriteFile(filepath.Join(user.GetHomeDir(), "file1.txt"), content, 0666)
@@ -1402,10 +1403,11 @@ func TestScheduledActions(t *testing.T) {
func TestEventParamsCopy(t *testing.T) {
params := EventParams{
- Name: "name",
- Event: "event",
- Status: 1,
- errors: []string{"error1"},
+ Name: "name",
+ Event: "event",
+ Status: 1,
+ errors: []string{"error1"},
+ retentionChecks: []executedRetentionCheck{},
}
paramsCopy := params.getACopy()
assert.Equal(t, params, *paramsCopy)
@@ -1422,6 +1424,35 @@ func TestEventParamsCopy(t *testing.T) {
assert.Equal(t, "event mod", paramsCopy.Event)
assert.Equal(t, 1, params.Status)
assert.Equal(t, 2, paramsCopy.Status)
+ params = EventParams{
+ retentionChecks: []executedRetentionCheck{
+ {
+ Username: "u",
+ ActionName: "a",
+ Results: []folderRetentionCheckResult{
+ {
+ Path: "p",
+ Retention: 1,
+ },
+ },
+ },
+ },
+ }
+ paramsCopy = params.getACopy()
+ require.Len(t, paramsCopy.retentionChecks, 1)
+ paramsCopy.retentionChecks[0].Username = "u_copy"
+ paramsCopy.retentionChecks[0].ActionName = "a_copy"
+ require.Len(t, paramsCopy.retentionChecks[0].Results, 1)
+ paramsCopy.retentionChecks[0].Results[0].Path = "p_copy"
+ paramsCopy.retentionChecks[0].Results[0].Retention = 2
+ assert.Equal(t, "u", params.retentionChecks[0].Username)
+ assert.Equal(t, "a", params.retentionChecks[0].ActionName)
+ assert.Equal(t, "p", params.retentionChecks[0].Results[0].Path)
+ assert.Equal(t, 1, params.retentionChecks[0].Results[0].Retention)
+ assert.Equal(t, "u_copy", paramsCopy.retentionChecks[0].Username)
+ assert.Equal(t, "a_copy", paramsCopy.retentionChecks[0].ActionName)
+ assert.Equal(t, "p_copy", paramsCopy.retentionChecks[0].Results[0].Path)
+ assert.Equal(t, 2, paramsCopy.retentionChecks[0].Results[0].Retention)
}
func TestEventParamsStatusFromError(t *testing.T) {
@@ -1454,13 +1485,13 @@ func TestWriteHTTPPartsError(t *testing.T) {
errTest: io.ErrShortWrite,
})
- err := writeHTTPPart(m, dataprovider.HTTPPart{}, nil, nil, nil)
+ err := writeHTTPPart(m, dataprovider.HTTPPart{}, nil, nil, nil, &EventParams{})
assert.ErrorIs(t, err, io.ErrShortWrite)
body := "test body"
m = multipart.NewWriter(&testWriter{sentinel: body})
err = writeHTTPPart(m, dataprovider.HTTPPart{
Body: body,
- }, nil, nil, nil)
+ }, nil, nil, nil, &EventParams{})
assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
}
diff --git a/internal/common/protocol_test.go b/internal/common/protocol_test.go
index 6bff5980..56b624c4 100644
--- a/internal/common/protocol_test.go
+++ b/internal/common/protocol_test.go
@@ -4136,6 +4136,253 @@ func TestEventActionEmailAttachments(t *testing.T) {
require.NoError(t, err)
}
+func TestEventActionsRetentionReports(t *testing.T) {
+ smtpCfg := smtp.Config{
+ Host: "127.0.0.1",
+ Port: 2525,
+ From: "notify@example.com",
+ TemplatesPath: "templates",
+ }
+ err := smtpCfg.Initialize(configDir)
+ require.NoError(t, err)
+
+ testDir := "/d"
+ a1 := dataprovider.BaseEventAction{
+ Name: "action1",
+ Type: dataprovider.ActionTypeDataRetentionCheck,
+ Options: dataprovider.BaseEventActionOptions{
+ RetentionConfig: dataprovider.EventActionDataRetentionConfig{
+ Folders: []dataprovider.FolderRetention{
+ {
+ Path: testDir,
+ Retention: 1,
+ DeleteEmptyDirs: true,
+ IgnoreUserPermissions: true,
+ },
+ },
+ },
+ },
+ }
+ action1, _, err := httpdtest.AddEventAction(a1, http.StatusCreated)
+ assert.NoError(t, err)
+ a2 := dataprovider.BaseEventAction{
+ Name: "action2",
+ Type: dataprovider.ActionTypeEmail,
+ Options: dataprovider.BaseEventActionOptions{
+ EmailConfig: dataprovider.EventActionEmailConfig{
+ Recipients: []string{"test@example.com"},
+ Subject: `"{{Event}}" from "{{Name}}"`,
+ Body: "Fs path {{FsPath}}, size: {{FileSize}}, protocol: {{Protocol}}, IP: {{IP}}",
+ Attachments: []string{dataprovider.RetentionReportPlaceHolder},
+ },
+ },
+ }
+ action2, _, err := httpdtest.AddEventAction(a2, http.StatusCreated)
+ assert.NoError(t, err)
+ a3 := dataprovider.BaseEventAction{
+ Name: "action3",
+ Type: dataprovider.ActionTypeHTTP,
+ Options: dataprovider.BaseEventActionOptions{
+ HTTPConfig: dataprovider.EventActionHTTPConfig{
+ Endpoint: fmt.Sprintf("http://%s/", httpAddr),
+ Timeout: 20,
+ Method: http.MethodPost,
+ Body: dataprovider.RetentionReportPlaceHolder,
+ },
+ },
+ }
+ action3, _, err := httpdtest.AddEventAction(a3, http.StatusCreated)
+ assert.NoError(t, err)
+ a4 := dataprovider.BaseEventAction{
+ Name: "action4",
+ Type: dataprovider.ActionTypeHTTP,
+ Options: dataprovider.BaseEventActionOptions{
+ HTTPConfig: dataprovider.EventActionHTTPConfig{
+ Endpoint: fmt.Sprintf("http://%s/multipart", httpAddr),
+ Timeout: 20,
+ Method: http.MethodPost,
+ Parts: []dataprovider.HTTPPart{
+ {
+ Name: "reports.zip",
+ Filepath: dataprovider.RetentionReportPlaceHolder,
+ },
+ },
+ },
+ },
+ }
+ action4, _, err := httpdtest.AddEventAction(a4, http.StatusCreated)
+ assert.NoError(t, err)
+ r1 := dataprovider.EventRule{
+ Name: "test rule1",
+ Trigger: dataprovider.EventTriggerFsEvent,
+ Conditions: dataprovider.EventConditions{
+ FsEvents: []string{"upload"},
+ },
+ Actions: []dataprovider.EventAction{
+ {
+ BaseEventAction: dataprovider.BaseEventAction{
+ Name: action1.Name,
+ },
+ Order: 1,
+ Options: dataprovider.EventActionOptions{
+ ExecuteSync: true,
+ StopOnFailure: true,
+ },
+ },
+ {
+ BaseEventAction: dataprovider.BaseEventAction{
+ Name: action2.Name,
+ },
+ Order: 2,
+ Options: dataprovider.EventActionOptions{
+ ExecuteSync: true,
+ StopOnFailure: true,
+ },
+ },
+ {
+ BaseEventAction: dataprovider.BaseEventAction{
+ Name: action3.Name,
+ },
+ Order: 3,
+ Options: dataprovider.EventActionOptions{
+ ExecuteSync: true,
+ StopOnFailure: true,
+ },
+ },
+ {
+ BaseEventAction: dataprovider.BaseEventAction{
+ Name: action4.Name,
+ },
+ Order: 4,
+ Options: dataprovider.EventActionOptions{
+ ExecuteSync: true,
+ StopOnFailure: true,
+ },
+ },
+ },
+ }
+ rule1, _, err := httpdtest.AddEventRule(r1, http.StatusCreated)
+ assert.NoError(t, err)
+
+ user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated)
+ assert.NoError(t, err)
+ conn, client, err := getSftpClient(user)
+ if assert.NoError(t, err) {
+ defer conn.Close()
+ defer client.Close()
+
+ subdir := path.Join(testDir, "sub")
+ err = client.MkdirAll(subdir)
+ assert.NoError(t, err)
+
+ lastReceivedEmail.reset()
+ f, err := client.Create(testFileName)
+ assert.NoError(t, err)
+ _, err = f.Write(testFileContent)
+ assert.NoError(t, err)
+ err = f.Close()
+ assert.NoError(t, err)
+
+ email := lastReceivedEmail.get()
+ assert.Len(t, email.To, 1)
+ assert.True(t, util.Contains(email.To, "test@example.com"))
+ assert.Contains(t, email.Data, fmt.Sprintf(`Subject: "upload" from "%s"`, user.Username))
+ assert.Contains(t, email.Data, "Content-Disposition: attachment")
+ _, err = client.Stat(testDir)
+ assert.NoError(t, err)
+ _, err = client.Stat(subdir)
+ assert.ErrorIs(t, err, os.ErrNotExist)
+
+ err = client.Mkdir(subdir)
+ assert.NoError(t, err)
+ newName := path.Join(testDir, testFileName)
+ err = client.Rename(testFileName, newName)
+ assert.NoError(t, err)
+ err = client.Chtimes(newName, time.Now().Add(-24*time.Hour), time.Now().Add(-24*time.Hour))
+ assert.NoError(t, err)
+
+ lastReceivedEmail.reset()
+ f, err = client.Create(testFileName)
+ assert.NoError(t, err)
+ _, err = f.Write(testFileContent)
+ assert.NoError(t, err)
+ err = f.Close()
+ assert.NoError(t, err)
+ email = lastReceivedEmail.get()
+ assert.Len(t, email.To, 1)
+ _, err = client.Stat(subdir)
+ assert.ErrorIs(t, err, os.ErrNotExist)
+ _, err = client.Stat(subdir)
+ assert.ErrorIs(t, err, os.ErrNotExist)
+ }
+ // now remove the retention check to test errors
+ rule1.Actions = []dataprovider.EventAction{
+ {
+ BaseEventAction: dataprovider.BaseEventAction{
+ Name: action2.Name,
+ },
+ Order: 2,
+ Options: dataprovider.EventActionOptions{
+ ExecuteSync: true,
+ StopOnFailure: false,
+ },
+ },
+ {
+ BaseEventAction: dataprovider.BaseEventAction{
+ Name: action3.Name,
+ },
+ Order: 3,
+ Options: dataprovider.EventActionOptions{
+ ExecuteSync: true,
+ StopOnFailure: false,
+ },
+ },
+ {
+ BaseEventAction: dataprovider.BaseEventAction{
+ Name: action4.Name,
+ },
+ Order: 4,
+ Options: dataprovider.EventActionOptions{
+ ExecuteSync: true,
+ StopOnFailure: false,
+ },
+ },
+ }
+ _, _, err = httpdtest.UpdateEventRule(rule1, http.StatusOK)
+ assert.NoError(t, err)
+ conn, client, err = getSftpClient(user)
+ if assert.NoError(t, err) {
+ defer conn.Close()
+ defer client.Close()
+
+ f, err := client.Create(testFileName)
+ assert.NoError(t, err)
+ _, err = f.Write(testFileContent)
+ assert.NoError(t, err)
+ err = f.Close()
+ assert.Error(t, err)
+ }
+
+ _, err = httpdtest.RemoveEventRule(rule1, http.StatusOK)
+ assert.NoError(t, err)
+ _, err = httpdtest.RemoveEventAction(action1, http.StatusOK)
+ assert.NoError(t, err)
+ _, err = httpdtest.RemoveEventAction(action2, http.StatusOK)
+ assert.NoError(t, err)
+ _, err = httpdtest.RemoveEventAction(action3, http.StatusOK)
+ assert.NoError(t, err)
+ _, err = httpdtest.RemoveEventAction(action4, http.StatusOK)
+ assert.NoError(t, err)
+ _, err = httpdtest.RemoveUser(user, http.StatusOK)
+ assert.NoError(t, err)
+ err = os.RemoveAll(user.GetHomeDir())
+ assert.NoError(t, err)
+
+ smtpCfg = smtp.Config{}
+ err = smtpCfg.Initialize(configDir)
+ require.NoError(t, err)
+}
+
func TestEventRuleFirstUploadDownloadActions(t *testing.T) {
smtpCfg := smtp.Config{
Host: "127.0.0.1",
diff --git a/internal/dataprovider/eventrule.go b/internal/dataprovider/eventrule.go
index bb940606..5afcc293 100644
--- a/internal/dataprovider/eventrule.go
+++ b/internal/dataprovider/eventrule.go
@@ -122,6 +122,11 @@ const (
FilesystemActionExist
)
+const (
+ // RetentionReportPlaceHolder defines the placeholder for data retention reports
+ RetentionReportPlaceHolder = "{{RetentionReports}}"
+)
+
var (
supportedFsActions = []int{FilesystemActionRename, FilesystemActionDelete, FilesystemActionMkdirs,
FilesystemActionExist}
@@ -229,7 +234,9 @@ func (p *HTTPPart) validate() error {
}
} else {
p.Body = ""
- p.Filepath = util.CleanPath(p.Filepath)
+ if p.Filepath != RetentionReportPlaceHolder {
+ p.Filepath = util.CleanPath(p.Filepath)
+ }
}
return nil
}
@@ -249,17 +256,24 @@ type EventActionHTTPConfig struct {
}
func (c *EventActionHTTPConfig) isTimeoutNotValid() bool {
- if c.HasMultipartFile() {
+ if c.HasMultipartFiles() {
return false
}
- return c.Timeout < 1 || c.Timeout > 120
+ return c.Timeout < 1 || c.Timeout > 180
}
func (c *EventActionHTTPConfig) validateMultiparts() error {
+ filePaths := make(map[string]bool)
for idx := range c.Parts {
if err := c.Parts[idx].validate(); err != nil {
return err
}
+ if filePath := c.Parts[idx].Filepath; filePath != "" {
+ if filePaths[filePath] {
+ return fmt.Errorf("filepath %q is duplicated", filePath)
+ }
+ filePaths[filePath] = true
+ }
}
if len(c.Parts) > 0 {
if c.Body != "" {
@@ -315,7 +329,7 @@ func (c *EventActionHTTPConfig) validate(additionalData string) error {
// GetContext returns the context and the cancel func to use for the HTTP request
func (c *EventActionHTTPConfig) GetContext() (context.Context, context.CancelFunc) {
- if c.HasMultipartFile() {
+ if c.HasMultipartFiles() {
return context.WithCancel(context.Background())
}
return context.WithTimeout(context.Background(), time.Duration(c.Timeout)*time.Second)
@@ -334,10 +348,10 @@ func (c *EventActionHTTPConfig) HasObjectData() bool {
return false
}
-// HasMultipartFile returns true if a file must be uploaded via a multipart request
-func (c *EventActionHTTPConfig) HasMultipartFile() bool {
+// HasMultipartFiles returns true if at least a file must be uploaded via a multipart request
+func (c *EventActionHTTPConfig) HasMultipartFiles() bool {
for _, part := range c.Parts {
- if part.Filepath != "" {
+ if part.Filepath != "" && part.Filepath != RetentionReportPlaceHolder {
return true
}
}
@@ -427,6 +441,15 @@ func (c EventActionEmailConfig) GetAttachmentsAsString() string {
return strings.Join(c.Attachments, ",")
}
+func (c *EventActionEmailConfig) hasFilesAttachments() bool {
+ for _, a := range c.Attachments {
+ if a != RetentionReportPlaceHolder {
+ return true
+ }
+ }
+ return false
+}
+
func (c *EventActionEmailConfig) validate() error {
if len(c.Recipients) == 0 {
return util.NewValidationError("at least one email recipient is required")
@@ -448,7 +471,11 @@ func (c *EventActionEmailConfig) validate() error {
if val == "" {
return util.NewValidationError("invalid path to attach")
}
- c.Attachments[idx] = util.CleanPath(val)
+ if val == RetentionReportPlaceHolder {
+ c.Attachments[idx] = val
+ } else {
+ c.Attachments[idx] = util.CleanPath(val)
+ }
}
c.Attachments = util.RemoveDuplicates(c.Attachments, false)
return nil
@@ -1290,12 +1317,12 @@ func (r *EventRule) CheckActionsConsistency(providerObjectType string) error {
}
}
for _, action := range r.Actions {
- if action.Type == ActionTypeEmail && len(action.BaseEventAction.Options.EmailConfig.Attachments) > 0 {
+ if action.Type == ActionTypeEmail && action.BaseEventAction.Options.EmailConfig.hasFilesAttachments() {
if !r.hasUserAssociated(providerObjectType) {
return errors.New("cannot send an email with attachments for a rule with no user associated")
}
}
- if action.Type == ActionTypeHTTP && action.BaseEventAction.Options.HTTPConfig.HasMultipartFile() {
+ if action.Type == ActionTypeHTTP && action.BaseEventAction.Options.HTTPConfig.HasMultipartFiles() {
if !r.hasUserAssociated(providerObjectType) {
return errors.New("cannot upload file/s for a rule with no user associated")
}
diff --git a/internal/smtp/smtp.go b/internal/smtp/smtp.go
index ff16746b..617a102f 100644
--- a/internal/smtp/smtp.go
+++ b/internal/smtp/smtp.go
@@ -120,7 +120,7 @@ func (c *Config) Initialize(configDir string) error {
smtpServer.Encryption = c.getEncryption()
smtpServer.KeepAlive = false
smtpServer.ConnectTimeout = 10 * time.Second
- smtpServer.SendTimeout = 30 * time.Second
+ smtpServer.SendTimeout = 120 * time.Second
if c.Domain != "" {
smtpServer.Helo = c.Domain
}
diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml
index 46410176..69655783 100644
--- a/openapi/openapi.yaml
+++ b/openapi/openapi.yaml
@@ -6047,7 +6047,7 @@ components:
timeout:
type: integer
minimum: 1
- maximum: 120
+ maximum: 180
description: 'Ignored for multipart requests with files as attachments'
skip_tls_verify:
type: boolean
diff --git a/templates/webadmin/eventaction.html b/templates/webadmin/eventaction.html
index 6f6a6519..baa506d1 100644
--- a/templates/webadmin/eventaction.html
+++ b/templates/webadmin/eventaction.html
@@ -677,6 +677,9 @@ along with this program. If not, see .
{{`{{ObjectData}}`}} => Provider object data serialized as JSON with sensitive fields removed.
+
+ {{`{{RetentionReports}}`}} => Data retention reports as zip compressed CSV files. Supported as email attachment, file path for multipart HTTP request and as single parameter for HTTP requests body.
+