diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml
index a04c2c43..5b75cfb9 100644
--- a/.github/workflows/development.yml
+++ b/.github/workflows/development.yml
@@ -11,11 +11,11 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
- go: [1.18]
+ go: [1.19]
os: [ubuntu-latest, macos-latest]
upload-coverage: [true]
include:
- - go: 1.18
+ - go: 1.19
os: windows-latest
upload-coverage: false
@@ -232,7 +232,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v3
with:
- go-version: 1.18
+ go-version: 1.19
- name: Build
run: |
@@ -306,7 +306,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v3
with:
- go-version: 1.18
+ go-version: 1.19
- name: Build
run: |
@@ -383,7 +383,7 @@ jobs:
matrix:
include:
- arch: amd64
- go: 1.18
+ go: 1.19
go-arch: amd64
- arch: aarch64
distro: ubuntu18.04
@@ -504,7 +504,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v3
with:
- go-version: 1.18
+ go-version: 1.19
- uses: actions/checkout@v3
- name: Run golangci-lint
uses: golangci/golangci-lint-action@v3
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 062f422b..04ae2ccc 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -5,7 +5,7 @@ on:
tags: 'v*'
env:
- GO_VERSION: 1.18.3
+ GO_VERSION: 1.19
jobs:
prepare-sources-with-deps:
diff --git a/Dockerfile b/Dockerfile
index 18b06bfa..028850de 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.18-bullseye as builder
+FROM golang:1.19-bullseye as builder
ENV GOFLAGS="-mod=readonly"
diff --git a/Dockerfile.alpine b/Dockerfile.alpine
index 4ac4dfc0..96b87540 100644
--- a/Dockerfile.alpine
+++ b/Dockerfile.alpine
@@ -1,4 +1,4 @@
-FROM golang:1.18-alpine3.16 AS builder
+FROM golang:1.19-alpine3.16 AS builder
ENV GOFLAGS="-mod=readonly"
diff --git a/Dockerfile.distroless b/Dockerfile.distroless
index 9dd1d4a2..f144bb22 100644
--- a/Dockerfile.distroless
+++ b/Dockerfile.distroless
@@ -1,4 +1,4 @@
-FROM golang:1.18-bullseye as builder
+FROM golang:1.19-bullseye as builder
ENV CGO_ENABLED=0 GOFLAGS="-mod=readonly"
diff --git a/docs/eventmanager.md b/docs/eventmanager.md
index 3ab1c683..ac5e4a35 100644
--- a/docs/eventmanager.md
+++ b/docs/eventmanager.md
@@ -11,6 +11,7 @@ The following actions are supported:
- `User quota reset`. The quota used by users will be updated based on current usage.
- `Folder quota reset`. The quota used by virtual folders will be updated based on current usage.
- `Transfer quota reset`. The transfer quota values will be reset to `0`.
+- `Data retention check`. You can define per-folder retention policies.
The following placeholders are supported:
diff --git a/go.mod b/go.mod
index f2880856..7a05cd1b 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
module github.com/drakkan/sftpgo/v2
-go 1.18
+go 1.19
require (
cloud.google.com/go/storage v1.24.0
@@ -64,13 +64,13 @@ require (
github.com/yl2chen/cidranger v1.0.3-0.20210928021809-d1cb2c52f37a
go.etcd.io/bbolt v1.3.6
go.uber.org/automaxprocs v1.5.1
- gocloud.dev v0.25.0
+ gocloud.dev v0.26.0
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
golang.org/x/net v0.0.0-20220728211354-c7608f3a8462
golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c
- golang.org/x/sys v0.0.0-20220731174439-a90be440212d
+ golang.org/x/sys v0.0.0-20220803195053-6e608f9ce704
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9
- google.golang.org/api v0.90.0
+ google.golang.org/api v0.91.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0
)
@@ -155,7 +155,7 @@ require (
golang.org/x/tools v0.1.12 // indirect
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
google.golang.org/appengine v1.6.7 // indirect
- google.golang.org/genproto v0.0.0-20220802133213-ce4fa296bf78 // indirect
+ google.golang.org/genproto v0.0.0-20220804142021-4e6b2dfa6612 // indirect
google.golang.org/grpc v1.48.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/ini.v1 v1.66.6 // indirect
diff --git a/go.sum b/go.sum
index b2c6fa84..0a2f485b 100644
--- a/go.sum
+++ b/go.sum
@@ -602,7 +602,6 @@ github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope
github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E=
-github.com/mattn/go-ieproxy v0.0.3/go.mod h1:6ZpRmhBaYuBX1U2za+9rC9iCGLsSp2tftelZne7CPko=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
@@ -808,8 +807,8 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
-gocloud.dev v0.25.0 h1:Y7vDq8xj7SyM848KXf32Krda2e6jQ4CLh/mTeCSqXtk=
-gocloud.dev v0.25.0/go.mod h1:7HegHVCYZrMiU3IE1qtnzf/vRrDwLYnRNR3EhWX8x9Y=
+gocloud.dev v0.26.0 h1:4rM/SVL0lLs+rhC0Gmc+gt/82DBpb7nbpIZKXXnfMXg=
+gocloud.dev v0.26.0/go.mod h1:mkUgejbnbLotorqDyvedJO20XcZNTynmSeVSQS9btVg=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -955,7 +954,6 @@ golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220110181412-a018aaa089fe/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -973,8 +971,8 @@ golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80=
-golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220803195053-6e608f9ce704 h1:Y7NOhdqIOU8kYI7BxsgL38d0ot0raxvcW+EMQU2QrT4=
+golang.org/x/sys v0.0.0-20220803195053-6e608f9ce704/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1117,8 +1115,8 @@ google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3
google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g=
google.golang.org/api v0.86.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
-google.golang.org/api v0.90.0 h1:WMnUWAvihIClUYFNeFA69VTuR3duKS3IalMGDQcLvq8=
-google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
+google.golang.org/api v0.91.0 h1:731+JzuwaJoZXRQGmPoBiV+SrsAfUaIkdMCWTcQNPyA=
+google.golang.org/api v0.91.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1225,8 +1223,8 @@ google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljW
google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220802133213-ce4fa296bf78 h1:QntLWYqZeuBtJkth3m/6DLznnI0AHJr+AgJXvVh/izw=
-google.golang.org/genproto v0.0.0-20220802133213-ce4fa296bf78/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc=
+google.golang.org/genproto v0.0.0-20220804142021-4e6b2dfa6612 h1:NX3L5YesD5qgxxrPHdKqHH38Ao0AG6poRXG+JljPsGU=
+google.golang.org/genproto v0.0.0-20220804142021-4e6b2dfa6612/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
diff --git a/internal/common/connection.go b/internal/common/connection.go
index 06a6e8ca..a70e5a5f 100644
--- a/internal/common/connection.go
+++ b/internal/common/connection.go
@@ -622,7 +622,7 @@ func (c *BaseConnection) DoStat(virtualPath string, mode int, checkFilePatterns
info, err = fs.Stat(c.getRealFsPath(fsPath))
}
if err != nil {
- c.Log(logger.LevelError, "stat error for path %#v: %+v", virtualPath, err)
+ c.Log(logger.LevelWarn, "stat error for path %#v: %+v", virtualPath, err)
return info, c.GetFsError(fs, err)
}
if vfs.IsCryptOsFs(fs) {
diff --git a/internal/common/dataretention.go b/internal/common/dataretention.go
index 0f73ddbb..214832c2 100644
--- a/internal/common/dataretention.go
+++ b/internal/common/dataretention.go
@@ -66,7 +66,7 @@ func (c *ActiveRetentionChecks) Get() []RetentionCheck {
checks := make([]RetentionCheck, 0, len(c.Checks))
for _, check := range c.Checks {
- foldersCopy := make([]FolderRetention, len(check.Folders))
+ foldersCopy := make([]dataprovider.FolderRetention, len(check.Folders))
copy(foldersCopy, check.Folders)
notificationsCopy := make([]string, len(check.Notifications))
copy(notificationsCopy, check.Notifications)
@@ -124,37 +124,6 @@ func (c *ActiveRetentionChecks) remove(username string) bool {
return false
}
-// FolderRetention defines the retention policy for the specified directory path
-type FolderRetention struct {
- // Path is the exposed virtual directory path, if no other specific retention is defined,
- // the retention applies for sub directories too. For example if retention is defined
- // for the paths "/" and "/sub" then the retention for "/" is applied for any file outside
- // the "/sub" directory
- Path string `json:"path"`
- // Retention time in hours. 0 means exclude this path
- Retention int `json:"retention"`
- // DeleteEmptyDirs defines if empty directories will be deleted.
- // The user need the delete permission
- DeleteEmptyDirs bool `json:"delete_empty_dirs,omitempty"`
- // IgnoreUserPermissions defines if delete files even if the user does not have the delete permission.
- // The default is "false" which means that files will be skipped if the user does not have the permission
- // to delete them. This applies to sub directories too.
- IgnoreUserPermissions bool `json:"ignore_user_permissions,omitempty"`
-}
-
-func (f *FolderRetention) isValid() error {
- f.Path = path.Clean(f.Path)
- if !path.IsAbs(f.Path) {
- return util.NewValidationError(fmt.Sprintf("folder retention: invalid path %#v, please specify an absolute POSIX path",
- f.Path))
- }
- if f.Retention < 0 {
- return util.NewValidationError(fmt.Sprintf("invalid folder retention %v, it must be greater or equal to zero",
- f.Retention))
- }
- return nil
-}
-
type folderRetentionCheckResult struct {
Path string `json:"path"`
Retention int `json:"retention"`
@@ -172,7 +141,7 @@ type RetentionCheck struct {
// retention check start time as unix timestamp in milliseconds
StartTime int64 `json:"start_time"`
// affected folders
- Folders []FolderRetention `json:"folders"`
+ Folders []dataprovider.FolderRetention `json:"folders"`
// how cleanup results will be notified
Notifications []RetentionCheckNotification `json:"notifications,omitempty"`
// email to use if the notification method is set to email
@@ -188,7 +157,7 @@ func (c *RetentionCheck) Validate() error {
nothingToDo := true
for idx := range c.Folders {
f := &c.Folders[idx]
- if err := f.isValid(); err != nil {
+ if err := f.Validate(); err != nil {
return err
}
if f.Retention > 0 {
@@ -230,7 +199,7 @@ func (c *RetentionCheck) updateUserPermissions() {
}
}
-func (c *RetentionCheck) getFolderRetention(folderPath string) (FolderRetention, error) {
+func (c *RetentionCheck) getFolderRetention(folderPath string) (dataprovider.FolderRetention, error) {
dirsForPath := util.GetDirsForVirtualPath(folderPath)
for _, dirPath := range dirsForPath {
for _, folder := range c.Folders {
@@ -240,7 +209,7 @@ func (c *RetentionCheck) getFolderRetention(folderPath string) (FolderRetention,
}
}
- return FolderRetention{}, fmt.Errorf("unable to find folder retention for %#v", folderPath)
+ return dataprovider.FolderRetention{}, fmt.Errorf("unable to find folder retention for %#v", folderPath)
}
func (c *RetentionCheck) removeFile(virtualPath string, info os.FileInfo) error {
@@ -346,7 +315,7 @@ func (c *RetentionCheck) checkEmptyDirRemoval(folderPath string) {
}
// Start starts the retention check
-func (c *RetentionCheck) Start() {
+func (c *RetentionCheck) Start() error {
c.conn.Log(logger.LevelInfo, "retention check started")
defer RetentionChecks.remove(c.conn.User.Username)
defer c.conn.CloseFS() //nolint:errcheck
@@ -357,13 +326,14 @@ func (c *RetentionCheck) Start() {
if err := c.cleanupFolder(folder.Path); err != nil {
c.conn.Log(logger.LevelError, "retention check failed, unable to cleanup folder %#v", folder.Path)
c.sendNotifications(time.Since(startTime), err)
- return
+ return err
}
}
}
c.conn.Log(logger.LevelInfo, "retention check completed")
c.sendNotifications(time.Since(startTime), nil)
+ return nil
}
func (c *RetentionCheck) sendNotifications(elapsed time.Duration, err error) {
diff --git a/internal/common/dataretention_test.go b/internal/common/dataretention_test.go
index 1c66b20d..3a566c54 100644
--- a/internal/common/dataretention_test.go
+++ b/internal/common/dataretention_test.go
@@ -32,25 +32,17 @@ import (
func TestRetentionValidation(t *testing.T) {
check := RetentionCheck{}
- check.Folders = append(check.Folders, FolderRetention{
- Path: "relative",
- Retention: 10,
- })
- err := check.Validate()
- require.Error(t, err)
- assert.Contains(t, err.Error(), "please specify an absolute POSIX path")
-
- check.Folders = []FolderRetention{
+ check.Folders = []dataprovider.FolderRetention{
{
Path: "/",
Retention: -1,
},
}
- err = check.Validate()
+ err := check.Validate()
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid folder retention")
- check.Folders = []FolderRetention{
+ check.Folders = []dataprovider.FolderRetention{
{
Path: "/ab/..",
Retention: 0,
@@ -61,7 +53,7 @@ func TestRetentionValidation(t *testing.T) {
assert.Contains(t, err.Error(), "nothing to delete")
assert.Equal(t, "/", check.Folders[0].Path)
- check.Folders = append(check.Folders, FolderRetention{
+ check.Folders = append(check.Folders, dataprovider.FolderRetention{
Path: "/../..",
Retention: 24,
})
@@ -69,7 +61,7 @@ func TestRetentionValidation(t *testing.T) {
require.Error(t, err)
assert.Contains(t, err.Error(), `duplicated folder path "/"`)
- check.Folders = []FolderRetention{
+ check.Folders = []dataprovider.FolderRetention{
{
Path: "/dir1",
Retention: 48,
@@ -240,7 +232,7 @@ func TestRetentionPermissionsAndGetFolder(t *testing.T) {
user.Permissions["/dir2/sub2"] = []string{dataprovider.PermDelete}
check := RetentionCheck{
- Folders: []FolderRetention{
+ Folders: []dataprovider.FolderRetention{
{
Path: "/dir2",
Retention: 24 * 7,
@@ -300,7 +292,7 @@ func TestRetentionCheckAddRemove(t *testing.T) {
user.Permissions = make(map[string][]string)
user.Permissions["/"] = []string{dataprovider.PermAny}
check := RetentionCheck{
- Folders: []FolderRetention{
+ Folders: []dataprovider.FolderRetention{
{
Path: "/",
Retention: 48,
@@ -334,7 +326,7 @@ func TestCleanupErrors(t *testing.T) {
user.Permissions = make(map[string][]string)
user.Permissions["/"] = []string{dataprovider.PermAny}
check := &RetentionCheck{
- Folders: []FolderRetention{
+ Folders: []dataprovider.FolderRetention{
{
Path: "/path",
Retention: 48,
diff --git a/internal/common/eventmanager.go b/internal/common/eventmanager.go
index 6e3aec41..90a1dd13 100644
--- a/internal/common/eventmanager.go
+++ b/internal/common/eventmanager.go
@@ -495,6 +495,31 @@ func executeEmailRuleAction(c dataprovider.EventActionEmailConfig, params EventP
return err
}
+func executeQuotaResetForUser(user dataprovider.User) error {
+ if err := user.LoadAndApplyGroupSettings(); err != nil {
+ eventManagerLog(logger.LevelDebug, "skipping scheduled quota reset for user %s, cannot apply group settings: %v",
+ user.Username, err)
+ return err
+ }
+ if !QuotaScans.AddUserQuotaScan(user.Username) {
+ eventManagerLog(logger.LevelError, "another quota scan is already in progress for user %s", user.Username)
+ return fmt.Errorf("another quota scan is in progress for user %s", user.Username)
+ }
+ defer QuotaScans.RemoveUserQuotaScan(user.Username)
+
+ numFiles, size, err := user.ScanQuota()
+ if err != nil {
+ eventManagerLog(logger.LevelError, "error scanning quota for user %s: %v", user.Username, err)
+ return err
+ }
+ err = dataprovider.UpdateUserQuota(&user, numFiles, size, true)
+ if err != nil {
+ eventManagerLog(logger.LevelError, "error updating quota for user %s: %v", user.Username, err)
+ return err
+ }
+ return nil
+}
+
func executeUsersQuotaResetRuleAction(conditions dataprovider.ConditionOptions) error {
users, err := dataprovider.DumpUsers()
if err != nil {
@@ -507,21 +532,7 @@ func executeUsersQuotaResetRuleAction(conditions dataprovider.ConditionOptions)
user.Username)
continue
}
- if !QuotaScans.AddUserQuotaScan(user.Username) {
- eventManagerLog(logger.LevelError, "another quota scan is already in progress for user %s", user.Username)
- failedResets = append(failedResets, user.Username)
- continue
- }
- numFiles, size, err := user.ScanQuota()
- QuotaScans.RemoveUserQuotaScan(user.Username)
- if err != nil {
- eventManagerLog(logger.LevelError, "error scanning quota for user %s: %v", user.Username, err)
- failedResets = append(failedResets, user.Username)
- continue
- }
- err = dataprovider.UpdateUserQuota(&user, numFiles, size, true)
- if err != nil {
- eventManagerLog(logger.LevelError, "error updating quota for user %s: %v", user.Username, err)
+ if err = executeQuotaResetForUser(user); err != nil {
failedResets = append(failedResets, user.Username)
continue
}
@@ -564,7 +575,6 @@ func executeFoldersQuotaResetRuleAction(conditions dataprovider.ConditionOptions
if err != nil {
eventManagerLog(logger.LevelError, "error updating quota for folder %s: %v", folder.Name, err)
failedResets = append(failedResets, folder.Name)
- continue
}
}
if len(failedResets) > 0 {
@@ -589,7 +599,6 @@ func executeTransferQuotaResetRuleAction(conditions dataprovider.ConditionOption
if err != nil {
eventManagerLog(logger.LevelError, "error updating transfer quota for user %s: %v", user.Username, err)
failedResets = append(failedResets, user.Username)
- continue
}
}
if len(failedResets) > 0 {
@@ -598,6 +607,52 @@ func executeTransferQuotaResetRuleAction(conditions dataprovider.ConditionOption
return nil
}
+func executeDataRetentionCheckForUser(user dataprovider.User, folders []dataprovider.FolderRetention) error {
+ if err := user.LoadAndApplyGroupSettings(); err != nil {
+ eventManagerLog(logger.LevelDebug, "skipping scheduled retention check for user %s, cannot apply group settings: %v",
+ user.Username, err)
+ return err
+ }
+ check := RetentionCheck{
+ Folders: folders,
+ }
+ c := RetentionChecks.Add(check, &user)
+ if c == nil {
+ eventManagerLog(logger.LevelError, "another retention check is already in progress for user %s", user.Username)
+ return fmt.Errorf("another retention check is in progress for user %s", user.Username)
+ }
+ if err := c.Start(); err != nil {
+ eventManagerLog(logger.LevelError, "error checking retention for user %s: %v", user.Username, err)
+ return err
+ }
+ return nil
+}
+
+func executeDataRetentionCheckRuleAction(config dataprovider.EventActionDataRetentionConfig,
+ conditions dataprovider.ConditionOptions,
+) error {
+ users, err := dataprovider.DumpUsers()
+ if err != nil {
+ return fmt.Errorf("unable to get users: %w", err)
+ }
+ var failedChecks []string
+ for _, user := range users {
+ if !checkEventConditionPatterns(user.Username, conditions.Names) {
+ eventManagerLog(logger.LevelDebug, "skipping scheduled retention check for user %s, name conditions don't match",
+ user.Username)
+ continue
+ }
+ if err = executeDataRetentionCheckForUser(user, config.Folders); err != nil {
+ failedChecks = append(failedChecks, user.Username)
+ continue
+ }
+ }
+ if len(failedChecks) > 0 {
+ return fmt.Errorf("retention check failed for users: %+v", failedChecks)
+ }
+ return nil
+}
+
func executeRuleAction(action dataprovider.BaseEventAction, params EventParams, conditions dataprovider.ConditionOptions) error {
switch action.Type {
case dataprovider.ActionTypeHTTP:
@@ -614,6 +669,8 @@ func executeRuleAction(action dataprovider.BaseEventAction, params EventParams,
return executeFoldersQuotaResetRuleAction(conditions)
case dataprovider.ActionTypeTransferQuotaReset:
return executeTransferQuotaResetRuleAction(conditions)
+ case dataprovider.ActionTypeDataRetentionCheck:
+ return executeDataRetentionCheckRuleAction(action.Options.RetentionConfig, conditions)
default:
return fmt.Errorf("unsupported action type: %d", action.Type)
}
diff --git a/internal/common/eventmanager_test.go b/internal/common/eventmanager_test.go
index 7cff17d0..cbdb8de9 100644
--- a/internal/common/eventmanager_test.go
+++ b/internal/common/eventmanager_test.go
@@ -265,6 +265,48 @@ func TestEventManagerErrors(t *testing.T) {
assert.Error(t, err)
err = executeTransferQuotaResetRuleAction(dataprovider.ConditionOptions{})
assert.Error(t, err)
+ err = executeQuotaResetForUser(dataprovider.User{
+ Groups: []sdk.GroupMapping{
+ {
+ Name: "agroup",
+ Type: sdk.GroupTypePrimary,
+ },
+ },
+ })
+ assert.Error(t, err)
+ err = executeDataRetentionCheckForUser(dataprovider.User{
+ Groups: []sdk.GroupMapping{
+ {
+ Name: "agroup",
+ Type: sdk.GroupTypePrimary,
+ },
+ },
+ }, nil)
+ assert.Error(t, err)
+
+ dataRetentionAction := dataprovider.BaseEventAction{
+ Type: dataprovider.ActionTypeDataRetentionCheck,
+ Options: dataprovider.BaseEventActionOptions{
+ RetentionConfig: dataprovider.EventActionDataRetentionConfig{
+ Folders: []dataprovider.FolderRetention{
+ {
+ Path: "/",
+ Retention: 24,
+ },
+ },
+ },
+ },
+ }
+ err = executeRuleAction(dataRetentionAction, EventParams{}, dataprovider.ConditionOptions{
+ Names: []dataprovider.ConditionPattern{
+ {
+ Pattern: "username1",
+ },
+ },
+ })
+ if assert.Error(t, err) {
+ assert.Contains(t, err.Error(), "unable to get users")
+ }
eventManager.loadRules()
@@ -447,6 +489,88 @@ func TestEventRuleActions(t *testing.T) {
assert.Error(t, err)
assert.True(t, QuotaScans.RemoveUserQuotaScan(username1))
+ dataRetentionAction := dataprovider.BaseEventAction{
+ Type: dataprovider.ActionTypeDataRetentionCheck,
+ Options: dataprovider.BaseEventActionOptions{
+ RetentionConfig: dataprovider.EventActionDataRetentionConfig{
+ Folders: []dataprovider.FolderRetention{
+ {
+ Path: "",
+ Retention: 24,
+ },
+ },
+ },
+ },
+ }
+ err = executeRuleAction(dataRetentionAction, EventParams{}, dataprovider.ConditionOptions{
+ Names: []dataprovider.ConditionPattern{
+ {
+ Pattern: username1,
+ },
+ },
+ })
+ assert.Error(t, err) // invalid config, no folder path specified
+ retentionDir := "testretention"
+ dataRetentionAction = dataprovider.BaseEventAction{
+ Type: dataprovider.ActionTypeDataRetentionCheck,
+ Options: dataprovider.BaseEventActionOptions{
+ RetentionConfig: dataprovider.EventActionDataRetentionConfig{
+ Folders: []dataprovider.FolderRetention{
+ {
+ Path: path.Join("/", retentionDir),
+ Retention: 24,
+ DeleteEmptyDirs: true,
+ },
+ },
+ },
+ },
+ }
+ // create some test files
+ file1 := filepath.Join(user1.GetHomeDir(), "file1.txt")
+ file2 := filepath.Join(user1.GetHomeDir(), retentionDir, "file2.txt")
+ file3 := filepath.Join(user1.GetHomeDir(), retentionDir, "file3.txt")
+ file4 := filepath.Join(user1.GetHomeDir(), retentionDir, "sub", "file4.txt")
+
+ err = os.MkdirAll(filepath.Dir(file4), os.ModePerm)
+ assert.NoError(t, err)
+
+ for _, f := range []string{file1, file2, file3, file4} {
+ err = os.WriteFile(f, []byte(""), 0666)
+ assert.NoError(t, err)
+ }
+ timeBeforeRetention := time.Now().Add(-48 * time.Hour)
+ err = os.Chtimes(file1, timeBeforeRetention, timeBeforeRetention)
+ assert.NoError(t, err)
+ err = os.Chtimes(file2, timeBeforeRetention, timeBeforeRetention)
+ assert.NoError(t, err)
+ err = os.Chtimes(file4, timeBeforeRetention, timeBeforeRetention)
+ assert.NoError(t, err)
+
+ err = executeRuleAction(dataRetentionAction, EventParams{}, dataprovider.ConditionOptions{
+ Names: []dataprovider.ConditionPattern{
+ {
+ Pattern: username1,
+ },
+ },
+ })
+ assert.NoError(t, err)
+ assert.FileExists(t, file1)
+ assert.NoFileExists(t, file2)
+ assert.FileExists(t, file3)
+ assert.NoDirExists(t, filepath.Dir(file4))
+ // simulate another check in progress
+ c := RetentionChecks.Add(RetentionCheck{}, &user1)
+ assert.NotNil(t, c)
+ err = executeRuleAction(dataRetentionAction, EventParams{}, dataprovider.ConditionOptions{
+ Names: []dataprovider.ConditionPattern{
+ {
+ Pattern: username1,
+ },
+ },
+ })
+ assert.Error(t, err)
+ RetentionChecks.remove(user1.Username)
+
err = os.RemoveAll(user1.GetHomeDir())
assert.NoError(t, err)
diff --git a/internal/common/protocol_test.go b/internal/common/protocol_test.go
index ee40a988..99080e8e 100644
--- a/internal/common/protocol_test.go
+++ b/internal/common/protocol_test.go
@@ -3454,7 +3454,7 @@ func TestRetentionAPI(t *testing.T) {
err = writeSFTPFile(uploadPath, 32, client)
assert.NoError(t, err)
- folderRetention := []common.FolderRetention{
+ folderRetention := []dataprovider.FolderRetention{
{
Path: "/",
Retention: 24,
@@ -3535,7 +3535,7 @@ func TestRetentionAPI(t *testing.T) {
err = client.Chtimes(innerUploadFilePath, time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour))
assert.NoError(t, err)
- folderRetention := []common.FolderRetention{
+ folderRetention := []dataprovider.FolderRetention{
{
Path: "/missing",
Retention: 24,
@@ -3576,7 +3576,7 @@ func TestRetentionAPI(t *testing.T) {
_, err = client.Stat(innerUploadFilePath)
assert.NoError(t, err)
- folderRetention = []common.FolderRetention{
+ folderRetention = []dataprovider.FolderRetention{
{
Path: "/" + testDir,
@@ -3611,7 +3611,7 @@ func TestRetentionAPI(t *testing.T) {
err = os.Chmod(dirPath, 0001)
assert.NoError(t, err)
- folderRetention := []common.FolderRetention{
+ folderRetention := []dataprovider.FolderRetention{
{
Path: "/adir",
diff --git a/internal/dataprovider/eventrule.go b/internal/dataprovider/eventrule.go
index e210e167..11a36a01 100644
--- a/internal/dataprovider/eventrule.go
+++ b/internal/dataprovider/eventrule.go
@@ -40,11 +40,13 @@ const (
ActionTypeUserQuotaReset
ActionTypeFolderQuotaReset
ActionTypeTransferQuotaReset
+ ActionTypeDataRetentionCheck
)
var (
supportedEventActions = []int{ActionTypeHTTP, ActionTypeCommand, ActionTypeEmail, ActionTypeBackup,
- ActionTypeUserQuotaReset, ActionTypeFolderQuotaReset, ActionTypeTransferQuotaReset}
+ ActionTypeUserQuotaReset, ActionTypeFolderQuotaReset, ActionTypeTransferQuotaReset,
+ ActionTypeDataRetentionCheck}
)
func isActionTypeValid(action int) bool {
@@ -65,6 +67,8 @@ func getActionTypeAsString(action int) string {
return "Folder quota reset"
case ActionTypeTransferQuotaReset:
return "Transfer quota reset"
+ case ActionTypeDataRetentionCheck:
+ return "Data retention check"
default:
return "Command"
}
@@ -149,13 +153,13 @@ type KeyValue struct {
// EventActionHTTPConfig defines the configuration for an HTTP event target
type EventActionHTTPConfig struct {
- Endpoint string `json:"endpoint"`
+ Endpoint string `json:"endpoint,omitempty"`
Username string `json:"username,omitempty"`
Password *kms.Secret `json:"password,omitempty"`
Headers []KeyValue `json:"headers,omitempty"`
- Timeout int `json:"timeout"`
+ Timeout int `json:"timeout,omitempty"`
SkipTLSVerify bool `json:"skip_tls_verify,omitempty"`
- Method string `json:"method"`
+ Method string `json:"method,omitempty"`
QueryParameters []KeyValue `json:"query_parameters,omitempty"`
Body string `json:"post_body,omitempty"`
}
@@ -218,9 +222,9 @@ func (c *EventActionHTTPConfig) GetHTTPClient() *http.Client {
// EventActionCommandConfig defines the configuration for a command event target
type EventActionCommandConfig struct {
- Cmd string `json:"cmd"`
- Timeout int `json:"timeout"`
- EnvVars []KeyValue `json:"env_vars"`
+ Cmd string `json:"cmd,omitempty"`
+ Timeout int `json:"timeout,omitempty"`
+ EnvVars []KeyValue `json:"env_vars,omitempty"`
}
func (c *EventActionCommandConfig) validate() error {
@@ -243,46 +247,111 @@ func (c *EventActionCommandConfig) validate() error {
// EventActionEmailConfig defines the configuration options for SMTP event actions
type EventActionEmailConfig struct {
- Recipients []string `json:"recipients"`
- Subject string `json:"subject"`
- Body string `json:"body"`
+ Recipients []string `json:"recipients,omitempty"`
+ Subject string `json:"subject,omitempty"`
+ Body string `json:"body,omitempty"`
}
// GetRecipientsAsString returns the list of recipients as comma separated string
-func (o EventActionEmailConfig) GetRecipientsAsString() string {
- return strings.Join(o.Recipients, ",")
+func (c EventActionEmailConfig) GetRecipientsAsString() string {
+ return strings.Join(c.Recipients, ",")
}
-func (o *EventActionEmailConfig) validate() error {
- if len(o.Recipients) == 0 {
+func (c *EventActionEmailConfig) validate() error {
+ if len(c.Recipients) == 0 {
return util.NewValidationError("at least one email recipient is required")
}
- o.Recipients = util.RemoveDuplicates(o.Recipients, false)
- for _, r := range o.Recipients {
+ c.Recipients = util.RemoveDuplicates(c.Recipients, false)
+ for _, r := range c.Recipients {
if r == "" {
return util.NewValidationError("invalid email recipients")
}
}
- if o.Subject == "" {
+ if c.Subject == "" {
return util.NewValidationError("email subject is required")
}
- if o.Body == "" {
+ if c.Body == "" {
return util.NewValidationError("email body is required")
}
return nil
}
+// FolderRetention defines a folder retention configuration
+type FolderRetention struct {
+ // Path is the exposed virtual directory path, if no other specific retention is defined,
+ // the retention applies for sub directories too. For example if retention is defined
+ // for the paths "/" and "/sub" then the retention for "/" is applied for any file outside
+ // the "/sub" directory
+ Path string `json:"path"`
+ // Retention time in hours. 0 means exclude this path
+ Retention int `json:"retention"`
+ // DeleteEmptyDirs defines if empty directories will be deleted.
+ // The user need the delete permission
+ DeleteEmptyDirs bool `json:"delete_empty_dirs,omitempty"`
+ // IgnoreUserPermissions defines whether to delete files even if the user does not have the delete permission.
+ // The default is "false" which means that files will be skipped if the user does not have the permission
+ // to delete them. This applies to sub directories too.
+ IgnoreUserPermissions bool `json:"ignore_user_permissions,omitempty"`
+}
+
+// Validate returns an error if the configuration is not valid
+func (f *FolderRetention) Validate() error {
+ f.Path = util.CleanPath(f.Path)
+ if f.Retention < 0 {
+ return util.NewValidationError(fmt.Sprintf("invalid folder retention %v, it must be greater or equal to zero",
+ f.Retention))
+ }
+ return nil
+}
+
+// EventActionDataRetentionConfig defines the configuration for a data retention check
+type EventActionDataRetentionConfig struct {
+ Folders []FolderRetention `json:"folders,omitempty"`
+}
+
+func (c *EventActionDataRetentionConfig) validate() error {
+ folderPaths := make(map[string]bool)
+ nothingToDo := true
+ for idx := range c.Folders {
+ f := &c.Folders[idx]
+ if err := f.Validate(); err != nil {
+ return err
+ }
+ if f.Retention > 0 {
+ nothingToDo = false
+ }
+ if _, ok := folderPaths[f.Path]; ok {
+ return util.NewValidationError(fmt.Sprintf("duplicated folder path %#v", f.Path))
+ }
+ folderPaths[f.Path] = true
+ }
+ if nothingToDo {
+ return util.NewValidationError("nothing to delete!")
+ }
+ return nil
+}
+
// BaseEventActionOptions defines the supported configuration options for a base event actions
type BaseEventActionOptions struct {
- HTTPConfig EventActionHTTPConfig `json:"http_config"`
- CmdConfig EventActionCommandConfig `json:"cmd_config"`
- EmailConfig EventActionEmailConfig `json:"email_config"`
+ HTTPConfig EventActionHTTPConfig `json:"http_config"`
+ CmdConfig EventActionCommandConfig `json:"cmd_config"`
+ EmailConfig EventActionEmailConfig `json:"email_config"`
+ RetentionConfig EventActionDataRetentionConfig `json:"retention_config"`
}
func (o *BaseEventActionOptions) getACopy() BaseEventActionOptions {
o.SetEmptySecretsIfNil()
emailRecipients := make([]string, len(o.EmailConfig.Recipients))
copy(emailRecipients, o.EmailConfig.Recipients)
+ folders := make([]FolderRetention, 0, len(o.RetentionConfig.Folders))
+ for _, folder := range o.RetentionConfig.Folders {
+ folders = append(folders, FolderRetention{
+ Path: folder.Path,
+ Retention: folder.Retention,
+ DeleteEmptyDirs: folder.DeleteEmptyDirs,
+ IgnoreUserPermissions: folder.IgnoreUserPermissions,
+ })
+ }
return BaseEventActionOptions{
HTTPConfig: EventActionHTTPConfig{
@@ -306,6 +375,9 @@ func (o *BaseEventActionOptions) getACopy() BaseEventActionOptions {
Subject: o.EmailConfig.Subject,
Body: o.EmailConfig.Body,
},
+ RetentionConfig: EventActionDataRetentionConfig{
+ Folders: folders,
+ },
}
}
@@ -334,19 +406,28 @@ func (o *BaseEventActionOptions) validate(action int, name string) error {
case ActionTypeHTTP:
o.CmdConfig = EventActionCommandConfig{}
o.EmailConfig = EventActionEmailConfig{}
+ o.RetentionConfig = EventActionDataRetentionConfig{}
return o.HTTPConfig.validate(name)
case ActionTypeCommand:
o.HTTPConfig = EventActionHTTPConfig{}
o.EmailConfig = EventActionEmailConfig{}
+ o.RetentionConfig = EventActionDataRetentionConfig{}
return o.CmdConfig.validate()
case ActionTypeEmail:
o.HTTPConfig = EventActionHTTPConfig{}
o.CmdConfig = EventActionCommandConfig{}
+ o.RetentionConfig = EventActionDataRetentionConfig{}
return o.EmailConfig.validate()
+ case ActionTypeDataRetentionCheck:
+ o.HTTPConfig = EventActionHTTPConfig{}
+ o.CmdConfig = EventActionCommandConfig{}
+ o.EmailConfig = EventActionEmailConfig{}
+ return o.RetentionConfig.validate()
default:
o.HTTPConfig = EventActionHTTPConfig{}
o.CmdConfig = EventActionCommandConfig{}
o.EmailConfig = EventActionEmailConfig{}
+ o.RetentionConfig = EventActionDataRetentionConfig{}
}
return nil
}
diff --git a/internal/httpd/api_retention.go b/internal/httpd/api_retention.go
index ff98cbff..a3ba39f5 100644
--- a/internal/httpd/api_retention.go
+++ b/internal/httpd/api_retention.go
@@ -71,6 +71,6 @@ func startRetentionCheck(w http.ResponseWriter, r *http.Request) {
http.StatusConflict)
return
}
- go c.Start()
+ go c.Start() //nolint:errcheck
sendAPIResponse(w, r, err, "Check started", http.StatusAccepted)
}
diff --git a/internal/httpd/httpd_test.go b/internal/httpd/httpd_test.go
index 54ff1df1..11c63f36 100644
--- a/internal/httpd/httpd_test.go
+++ b/internal/httpd/httpd_test.go
@@ -1100,6 +1100,27 @@ func TestBasicActionRulesHandling(t *testing.T) {
}
assert.True(t, found)
a.Description = "new description"
+ a.Type = dataprovider.ActionTypeDataRetentionCheck
+ a.Options = dataprovider.BaseEventActionOptions{
+ RetentionConfig: dataprovider.EventActionDataRetentionConfig{
+ Folders: []dataprovider.FolderRetention{
+ {
+ Path: "/",
+ Retention: 144,
+ },
+ {
+ Path: "/p1",
+ Retention: 0,
+ },
+ {
+ Path: "/p2",
+ Retention: 12,
+ },
+ },
+ },
+ }
+ _, _, err = httpdtest.UpdateEventAction(a, http.StatusOK)
+ assert.NoError(t, err)
a.Type = dataprovider.ActionTypeCommand
a.Options = dataprovider.BaseEventActionOptions{
CmdConfig: dataprovider.EventActionCommandConfig{
@@ -1526,6 +1547,51 @@ func TestEventActionValidation(t *testing.T) {
_, resp, err = httpdtest.AddEventAction(action, http.StatusBadRequest)
assert.NoError(t, err)
assert.Contains(t, string(resp), "email body is required")
+
+ action.Type = dataprovider.ActionTypeDataRetentionCheck
+ action.Options.RetentionConfig = dataprovider.EventActionDataRetentionConfig{
+ Folders: nil,
+ }
+ _, resp, err = httpdtest.AddEventAction(action, http.StatusBadRequest)
+ assert.NoError(t, err)
+ assert.Contains(t, string(resp), "nothing to delete")
+ action.Options.RetentionConfig = dataprovider.EventActionDataRetentionConfig{
+ Folders: []dataprovider.FolderRetention{
+ {
+ Path: "/",
+ Retention: 0,
+ },
+ },
+ }
+ _, resp, err = httpdtest.AddEventAction(action, http.StatusBadRequest)
+ assert.NoError(t, err)
+ assert.Contains(t, string(resp), "nothing to delete")
+ action.Options.RetentionConfig = dataprovider.EventActionDataRetentionConfig{
+ Folders: []dataprovider.FolderRetention{
+ {
+ Path: "../path",
+ Retention: 1,
+ },
+ {
+ Path: "/path",
+ Retention: 10,
+ },
+ },
+ }
+ _, resp, err = httpdtest.AddEventAction(action, http.StatusBadRequest)
+ assert.NoError(t, err)
+ assert.Contains(t, string(resp), "duplicated folder path")
+ action.Options.RetentionConfig = dataprovider.EventActionDataRetentionConfig{
+ Folders: []dataprovider.FolderRetention{
+ {
+ Path: "p",
+ Retention: -1,
+ },
+ },
+ }
+ _, resp, err = httpdtest.AddEventAction(action, http.StatusBadRequest)
+ assert.NoError(t, err)
+ assert.Contains(t, string(resp), "invalid folder retention")
}
func TestEventRuleValidation(t *testing.T) {
@@ -3235,7 +3301,7 @@ func TestRetentionAPI(t *testing.T) {
err = os.WriteFile(localFilePath, []byte("test data"), os.ModePerm)
assert.NoError(t, err)
- folderRetention := []common.FolderRetention{
+ folderRetention := []dataprovider.FolderRetention{
{
Path: "/",
Retention: 0,
@@ -3282,7 +3348,8 @@ func TestRetentionAPI(t *testing.T) {
_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusConflict)
assert.NoError(t, err)
- c.Start()
+ err = c.Start()
+ assert.NoError(t, err)
assert.Len(t, common.RetentionChecks.Get(), 0)
admin := getTestAdmin()
@@ -18692,6 +18759,53 @@ func TestWebEventAction(t *testing.T) {
assert.Empty(t, actionGet.Options.CmdConfig.Cmd)
assert.Equal(t, 0, actionGet.Options.CmdConfig.Timeout)
assert.Len(t, actionGet.Options.CmdConfig.EnvVars, 0)
+ // change action type to data retention check
+ action.Type = dataprovider.ActionTypeDataRetentionCheck
+ form.Set("type", fmt.Sprintf("%d", action.Type))
+ form.Set("folder_retention_path10", "p1")
+ form.Set("folder_retention_val10", "a")
+ req, err = http.NewRequest(http.MethodPost, path.Join(webAdminEventActionPath, action.Name),
+ bytes.NewBuffer([]byte(form.Encode())))
+ assert.NoError(t, err)
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ setJWTCookieForReq(req, webToken)
+ rr = executeRequest(req)
+ checkResponseCode(t, http.StatusOK, rr)
+ assert.Contains(t, rr.Body.String(), "invalid retention for path")
+ form.Set("folder_retention_val10", "24")
+ form.Set("folder_retention_options10", "1")
+ form.Add("folder_retention_options10", "2")
+ form.Set("folder_retention_path11", "../p2")
+ form.Set("folder_retention_val11", "48")
+ form.Set("folder_retention_options11", "1")
+ form.Add("folder_retention_options12", "2") // ignored
+ req, err = http.NewRequest(http.MethodPost, path.Join(webAdminEventActionPath, action.Name),
+ bytes.NewBuffer([]byte(form.Encode())))
+ assert.NoError(t, err)
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ setJWTCookieForReq(req, webToken)
+ rr = executeRequest(req)
+ checkResponseCode(t, http.StatusSeeOther, rr)
+ // check the update
+ actionGet, _, err = httpdtest.GetEventActionByName(action.Name, http.StatusOK)
+ assert.NoError(t, err)
+ assert.Equal(t, action.Type, actionGet.Type)
+ if assert.Len(t, actionGet.Options.RetentionConfig.Folders, 2) {
+ for _, folder := range actionGet.Options.RetentionConfig.Folders {
+ switch folder.Path {
+ case "/p1":
+ assert.Equal(t, 24, folder.Retention)
+ assert.True(t, folder.DeleteEmptyDirs)
+ assert.True(t, folder.IgnoreUserPermissions)
+ case "/p2":
+ assert.Equal(t, 48, folder.Retention)
+ assert.True(t, folder.DeleteEmptyDirs)
+ assert.False(t, folder.IgnoreUserPermissions)
+ default:
+ t.Errorf("unexpected folder path %v", folder.Path)
+ }
+ }
+ }
req, err = http.NewRequest(http.MethodDelete, path.Join(webAdminEventActionPath, action.Name), nil)
assert.NoError(t, err)
diff --git a/internal/httpd/internal_test.go b/internal/httpd/internal_test.go
index 5868aa9b..477de947 100644
--- a/internal/httpd/internal_test.go
+++ b/internal/httpd/internal_test.go
@@ -767,7 +767,7 @@ func TestRetentionInvalidTokenClaims(t *testing.T) {
user.Filters.AllowAPIKeyAuth = true
err := dataprovider.AddUser(&user, "", "")
assert.NoError(t, err)
- folderRetention := []common.FolderRetention{
+ folderRetention := []dataprovider.FolderRetention{
{
Path: "/",
Retention: 0,
diff --git a/internal/httpd/webadmin.go b/internal/httpd/webadmin.go
index 38a65fda..8b07f398 100644
--- a/internal/httpd/webadmin.go
+++ b/internal/httpd/webadmin.go
@@ -1836,6 +1836,30 @@ func getKeyValsFromPostFields(r *http.Request, key, val string) []dataprovider.K
return res
}
+func getFoldersRetentionFromPostFields(r *http.Request) ([]dataprovider.FolderRetention, error) {
+ var res []dataprovider.FolderRetention
+ for k := range r.Form {
+ if strings.HasPrefix(k, "folder_retention_path") {
+ folderPath := r.Form.Get(k)
+ if folderPath != "" {
+ idx := strings.TrimPrefix(k, "folder_retention_path")
+ retention, err := strconv.Atoi(r.Form.Get(fmt.Sprintf("folder_retention_val%s", idx)))
+ if err != nil {
+ return nil, fmt.Errorf("invalid retention for path %q: %w", folderPath, err)
+ }
+ options := r.Form[fmt.Sprintf("folder_retention_options%s", idx)]
+ res = append(res, dataprovider.FolderRetention{
+ Path: folderPath,
+ Retention: retention,
+ DeleteEmptyDirs: util.Contains(options, "1"),
+ IgnoreUserPermissions: util.Contains(options, "2"),
+ })
+ }
+ }
+ }
+ return res, nil
+}
+
func getEventActionOptionsFromPostFields(r *http.Request) (dataprovider.BaseEventActionOptions, error) {
httpTimeout, err := strconv.Atoi(r.Form.Get("http_timeout"))
if err != nil {
@@ -1845,6 +1869,10 @@ func getEventActionOptionsFromPostFields(r *http.Request) (dataprovider.BaseEven
if err != nil {
return dataprovider.BaseEventActionOptions{}, fmt.Errorf("invalid command timeout: %w", err)
}
+ foldersRetention, err := getFoldersRetentionFromPostFields(r)
+ if err != nil {
+ return dataprovider.BaseEventActionOptions{}, err
+ }
options := dataprovider.BaseEventActionOptions{
HTTPConfig: dataprovider.EventActionHTTPConfig{
Endpoint: r.Form.Get("http_endpoint"),
@@ -1867,6 +1895,9 @@ func getEventActionOptionsFromPostFields(r *http.Request) (dataprovider.BaseEven
Subject: r.Form.Get("email_subject"),
Body: r.Form.Get("email_body"),
},
+ RetentionConfig: dataprovider.EventActionDataRetentionConfig{
+ Folders: foldersRetention,
+ },
}
return options, nil
}
diff --git a/internal/httpdtest/httpdtest.go b/internal/httpdtest/httpdtest.go
index b1ca67ab..fa5182ba 100644
--- a/internal/httpdtest/httpdtest.go
+++ b/internal/httpdtest/httpdtest.go
@@ -920,7 +920,7 @@ func GetRetentionChecks(expectedStatusCode int) ([]common.ActiveRetentionChecks,
}
// StartRetentionCheck starts a new retention check
-func StartRetentionCheck(username string, retention []common.FolderRetention, expectedStatusCode int) ([]byte, error) {
+func StartRetentionCheck(username string, retention []dataprovider.FolderRetention, expectedStatusCode int) ([]byte, error) {
var body []byte
asJSON, _ := json.Marshal(retention)
resp, err := sendHTTPRequest(http.MethodPost, buildURLRelativeToBase(retentionBasePath, username, "check"),
@@ -1346,6 +1346,9 @@ func checkEventAction(expected, actual dataprovider.BaseEventAction) error {
if err := compareEventActionEmailConfigFields(expected.Options.EmailConfig, actual.Options.EmailConfig); err != nil {
return err
}
+ if err := compareEventActionDataRetentionFields(expected.Options.RetentionConfig, actual.Options.RetentionConfig); err != nil {
+ return err
+ }
return compareEventActionHTTPConfigFields(expected.Options.HTTPConfig, actual.Options.HTTPConfig)
}
@@ -2248,6 +2251,34 @@ func compareEventActionCmdConfigFields(expected, actual dataprovider.EventAction
return nil
}
+func compareEventActionDataRetentionFields(expected, actual dataprovider.EventActionDataRetentionConfig) error {
+ if len(expected.Folders) != len(actual.Folders) {
+ return errors.New("retention folders mismatch")
+ }
+ for _, f1 := range expected.Folders {
+ found := false
+ for _, f2 := range actual.Folders {
+ if f1.Path == f2.Path {
+ found = true
+ if f1.Retention != f2.Retention {
+ return fmt.Errorf("retention mismatch for folder %s", f1.Path)
+ }
+ if f1.DeleteEmptyDirs != f2.DeleteEmptyDirs {
+ return fmt.Errorf("delete_empty_dirs mismatch for folder %s", f1.Path)
+ }
+ if f1.IgnoreUserPermissions != f2.IgnoreUserPermissions {
+ return fmt.Errorf("ignore_user_permissions mismatch for folder %s", f1.Path)
+ }
+ break
+ }
+ }
+ if !found {
+ return errors.New("retention folders mismatch")
+ }
+ }
+ return nil
+}
+
func compareEqualGroupSettingsFields(expected sdk.BaseGroupUserSettings, actual sdk.BaseGroupUserSettings) error {
if expected.HomeDir != actual.HomeDir {
return errors.New("home dir mismatch")
diff --git a/internal/sftpd/scp.go b/internal/sftpd/scp.go
index e7dc1093..7006a6ab 100644
--- a/internal/sftpd/scp.go
+++ b/internal/sftpd/scp.go
@@ -635,8 +635,10 @@ func (c *scpCommand) readProtocolMessage() (string, error) {
return command.String(), err
}
-// send an error message and close the channel
-//nolint:errcheck // we don't check write errors here, we have to close the channel anyway
+// sendErrorMessage sends an error message and close the channel
+// we don't check write errors here, we have to close the channel anyway
+//
+//nolint:errcheck
func (c *scpCommand) sendErrorMessage(fs vfs.Fs, err error) {
c.connection.channel.Write(errMsg)
if fs != nil {
diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go
index 76adf40b..2b9377be 100644
--- a/internal/telemetry/telemetry.go
+++ b/internal/telemetry/telemetry.go
@@ -13,9 +13,9 @@
// along with this program. If not, see .
// Package telemetry provides telemetry information for SFTPGo, such as:
-// - health information (for health checks)
-// - metrics
-// - profiling information
+// - health information (for health checks)
+// - metrics
+// - profiling information
package telemetry
import (
diff --git a/internal/vfs/folder.go b/internal/vfs/folder.go
index db6eb5cb..a6d055d9 100644
--- a/internal/vfs/folder.go
+++ b/internal/vfs/folder.go
@@ -15,6 +15,7 @@
package vfs
import (
+ "errors"
"fmt"
"strconv"
"strings"
@@ -156,6 +157,24 @@ func (v *BaseVirtualFolder) HasRedactedSecret() bool {
return v.FsConfig.HasRedactedSecret()
}
+// hasPathPlaceholder returns true if the folder has a path placeholder
+func (v *BaseVirtualFolder) hasPathPlaceholder() bool {
+ placeholder := "%username%"
+ switch v.FsConfig.Provider {
+ case sdk.S3FilesystemProvider:
+ return strings.Contains(v.FsConfig.S3Config.KeyPrefix, placeholder)
+ case sdk.GCSFilesystemProvider:
+ return strings.Contains(v.FsConfig.GCSConfig.KeyPrefix, placeholder)
+ case sdk.AzureBlobFilesystemProvider:
+ return strings.Contains(v.FsConfig.AzBlobConfig.KeyPrefix, placeholder)
+ case sdk.SFTPFilesystemProvider:
+ return strings.Contains(v.FsConfig.SFTPConfig.Prefix, placeholder)
+ case sdk.LocalFilesystemProvider, sdk.CryptedFilesystemProvider:
+ return strings.Contains(v.MappedPath, placeholder)
+ }
+ return false
+}
+
// VirtualFolder defines a mapping between an SFTPGo exposed virtual path and a
// filesystem path outside the user home directory.
// The specified paths must be absolute and the virtual path cannot be "/",
@@ -205,6 +224,9 @@ func (v *VirtualFolder) CheckMetadataConsistency() error {
// ScanQuota scans the folder and returns the number of files and their size
func (v *VirtualFolder) ScanQuota() (int, int64, error) {
+ if v.hasPathPlaceholder() {
+ return 0, 0, errors.New("cannot scan quota: this folder has a path placeholder")
+ }
fs, err := v.GetFilesystem("", nil)
if err != nil {
return 0, 0, err
diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml
index 8242c9e5..f9390858 100644
--- a/openapi/openapi.yaml
+++ b/openapi/openapi.yaml
@@ -6024,6 +6024,13 @@ components:
type: string
body:
type: string
+ EventActionDataRetentionConfig:
+ type: object
+ properties:
+ folders:
+ type: array
+ items:
+ $ref: '#/components/schemas/FolderRetention'
BaseEventActionOptions:
type: object
properties:
@@ -6033,6 +6040,8 @@ components:
$ref: '#/components/schemas/EventActionCommandConfig'
email_config:
$ref: '#/components/schemas/EventActionEmailConfig'
+ retention_config:
+ $ref: '#/components/schemas/EventActionDataRetentionConfig'
BaseEventAction:
type: object
properties:
diff --git a/templates/webadmin/eventaction.html b/templates/webadmin/eventaction.html
index 560a1934..899552cb 100644
--- a/templates/webadmin/eventaction.html
+++ b/templates/webadmin/eventaction.html
@@ -342,6 +342,67 @@ along with this program. If not, see .
+
+
+ Data retention
+
+
+
Set the data retention, as hours, per path. Retention applies recursively. Setting 0 as retention means excluding the specified path. "Ignore user permissions" defines whether to delete files even if the user does not have the "delete" permission, by default files will be skipped if the user does not have the "delete" permission.
@@ -501,6 +562,40 @@ along with this program. If not, see .
$(this).closest(".form_field_cmd_env_outer_row").remove();
});
+ $("body").on("click", ".add_new_data_retention_field_btn", function () {
+ var index = $(".form_field_data_retention_outer").find(".form_field_data_retention_outer_row").length;
+ while (document.getElementById("idFolderRetentionPath"+index) != null){
+ index++;
+ }
+ $(".form_field_data_retention_outer").append(`
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ `);
+ $("#idFolderRetentionOptions"+index).selectpicker();
+ });
+
+ $("body").on("click", ".remove_data_retention_btn_frm_field", function () {
+ $(this).closest(".form_field_data_retention_outer_row").remove();
+ });
+
function onTypeChanged(val){
$('.action-type').hide();
switch (val) {
@@ -516,6 +611,10 @@ along with this program. If not, see .
case 3:
$('.action-smtp').show();
break;
+ case '8':
+ case 8:
+ $('.action-dataretention').show();
+ break;
}
}
diff --git a/templates/webadmin/eventrule.html b/templates/webadmin/eventrule.html
index 8a614c36..6b2d1729 100644
--- a/templates/webadmin/eventrule.html
+++ b/templates/webadmin/eventrule.html
@@ -520,7 +520,6 @@ along with this program. If not, see .
{{- range .Actions}}
$("#idActionName"+index).append($('