add basic REST APIs for data retention

Fixes #495

README.md

@@ -18,7 +18,7 @@ Several storage backends are supported: local filesystem, encrypted local filesy
 - SQLite, MySQL, PostgreSQL, CockroachDB, Bolt (key/value store in pure Go) and in-memory data providers are supported.
 - Chroot isolation for local accounts. Cloud-based accounts can be restricted to a certain base path.
 - Per user and per directory virtual permissions, for each exposed path you can allow or deny: directory listing, upload, overwrite, download, delete, rename, create directories, create symlinks, change owner/group/file mode.
-- [REST API](./docs/rest-api.md) for users and folders management, backup, restore and real time reports of the active connections with possibility of forcibly closing a connection.
+- [REST API](./docs/rest-api.md) for users and folders management, data retention, backup, restore and real time reports of the active connections with possibility of forcibly closing a connection.
 - [Web based administration interface](./docs/web-admin.md) to easily manage users, folders and connections.
 - [Web client interface](./docs/web-client.md) so that end users can change their credentials and browse their files.
 - Public key and password authentication. Multiple public keys per user are supported.

common/common.go

@@ -887,8 +887,8 @@ type ActiveVirtualFolderQuotaScan struct {
 // ActiveScans holds the active quota scans
 type ActiveScans struct {
 	sync.RWMutex
-	UserHomeScans []ActiveQuotaScan
+	UserScans   []ActiveQuotaScan
-	FolderScans   []ActiveVirtualFolderQuotaScan
+	FolderScans []ActiveVirtualFolderQuotaScan
 }
 
 // GetUsersQuotaScans returns the active quota scans for users home directories
@@ -896,8 +896,8 @@ func (s *ActiveScans) GetUsersQuotaScans() []ActiveQuotaScan {
 	s.RLock()
 	defer s.RUnlock()
 
-	scans := make([]ActiveQuotaScan, len(s.UserHomeScans))
+	scans := make([]ActiveQuotaScan, len(s.UserScans))
-	copy(scans, s.UserHomeScans)
+	copy(scans, s.UserScans)
 	return scans
 }
 
@@ -907,12 +907,12 @@ func (s *ActiveScans) AddUserQuotaScan(username string) bool {
 	s.Lock()
 	defer s.Unlock()
 
-	for _, scan := range s.UserHomeScans {
+	for _, scan := range s.UserScans {
 		if scan.Username == username {
 			return false
 		}
 	}
-	s.UserHomeScans = append(s.UserHomeScans, ActiveQuotaScan{
+	s.UserScans = append(s.UserScans, ActiveQuotaScan{
 		Username:  username,
 		StartTime: util.GetTimeAsMsSinceEpoch(time.Now()),
 	})
@@ -925,18 +925,15 @@ func (s *ActiveScans) RemoveUserQuotaScan(username string) bool {
 	s.Lock()
 	defer s.Unlock()
 
-	indexToRemove := -1
+	for idx, scan := range s.UserScans {
-	for i, scan := range s.UserHomeScans {
 		if scan.Username == username {
-			indexToRemove = i
+			lastIdx := len(s.UserScans) - 1
-			break
+			s.UserScans[idx] = s.UserScans[lastIdx]
+			s.UserScans = s.UserScans[:lastIdx]
+			return true
 		}
 	}
-	if indexToRemove >= 0 {
+
-		s.UserHomeScans[indexToRemove] = s.UserHomeScans[len(s.UserHomeScans)-1]
-		s.UserHomeScans = s.UserHomeScans[:len(s.UserHomeScans)-1]
-		return true
-	}
 	return false
 }
 
@@ -973,17 +970,14 @@ func (s *ActiveScans) RemoveVFolderQuotaScan(folderName string) bool {
 	s.Lock()
 	defer s.Unlock()
 
-	indexToRemove := -1
+	for idx, scan := range s.FolderScans {
-	for i, scan := range s.FolderScans {
 		if scan.Name == folderName {
-			indexToRemove = i
+			lastIdx := len(s.FolderScans) - 1
-			break
+			s.FolderScans[idx] = s.FolderScans[lastIdx]
+			s.FolderScans = s.FolderScans[:lastIdx]
+			return true
 		}
 	}
-	if indexToRemove >= 0 {
+
-		s.FolderScans[indexToRemove] = s.FolderScans[len(s.FolderScans)-1]
-		s.FolderScans = s.FolderScans[:len(s.FolderScans)-1]
-		return true
-	}
 	return false
 }

common/common_test.go

@@ -535,13 +535,18 @@ func TestQuotaScans(t *testing.T) {
 	username := "username"
 	assert.True(t, QuotaScans.AddUserQuotaScan(username))
 	assert.False(t, QuotaScans.AddUserQuotaScan(username))
-	if assert.Len(t, QuotaScans.GetUsersQuotaScans(), 1) {
+	usersScans := QuotaScans.GetUsersQuotaScans()
-		assert.Equal(t, QuotaScans.GetUsersQuotaScans()[0].Username, username)
+	if assert.Len(t, usersScans, 1) {
+		assert.Equal(t, usersScans[0].Username, username)
+		assert.Equal(t, QuotaScans.UserScans[0].StartTime, usersScans[0].StartTime)
+		QuotaScans.UserScans[0].StartTime = 0
+		assert.NotEqual(t, QuotaScans.UserScans[0].StartTime, usersScans[0].StartTime)
 	}
 
 	assert.True(t, QuotaScans.RemoveUserQuotaScan(username))
 	assert.False(t, QuotaScans.RemoveUserQuotaScan(username))
 	assert.Len(t, QuotaScans.GetUsersQuotaScans(), 0)
+	assert.Len(t, usersScans, 1)
 
 	folderName := "folder"
 	assert.True(t, QuotaScans.AddVFolderQuotaScan(folderName))

common/dataretention.go

@@ -0,0 +1,269 @@
+package common
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"sync"
+	"time"
+
+	"github.com/drakkan/sftpgo/v2/dataprovider"
+	"github.com/drakkan/sftpgo/v2/logger"
+	"github.com/drakkan/sftpgo/v2/util"
+)
+
+var (
+	// RetentionChecks is the list of active quota scans
+	RetentionChecks ActiveRetentionChecks
+)
+
+// ActiveRetentionChecks holds the active quota scans
+type ActiveRetentionChecks struct {
+	sync.RWMutex
+	Checks []RetentionCheck
+}
+
+// Get returns the active retention checks
+func (c *ActiveRetentionChecks) Get() []RetentionCheck {
+	c.RLock()
+	defer c.RUnlock()
+
+	checks := make([]RetentionCheck, 0, len(c.Checks))
+	for _, check := range c.Checks {
+		foldersCopy := make([]FolderRetention, len(check.Folders))
+		copy(foldersCopy, check.Folders)
+		checks = append(checks, RetentionCheck{
+			Username:  check.Username,
+			StartTime: check.StartTime,
+			Folders:   foldersCopy,
+		})
+	}
+	return checks
+}
+
+// Add a new retention check, returns nil if a retention check for the given
+// username is already active. The returned result can be used to start the check
+func (c *ActiveRetentionChecks) Add(check RetentionCheck, user *dataprovider.User) *RetentionCheck {
+	c.Lock()
+	defer c.Unlock()
+
+	for _, val := range c.Checks {
+		if val.Username == user.Username {
+			return nil
+		}
+	}
+	// we silently ignore file patterns
+	user.Filters.FilePatterns = nil
+	conn := NewBaseConnection("", "", "", "", *user)
+	conn.ID = fmt.Sprintf("retention_check_%v", user.Username)
+	check.Username = user.Username
+	check.StartTime = util.GetTimeAsMsSinceEpoch(time.Now())
+	check.conn = conn
+	check.updateUserPermissions()
+	c.Checks = append(c.Checks, check)
+
+	return &check
+}
+
+// remove a user from the ones with active retention checks
+// and returns true if the user is removed
+func (c *ActiveRetentionChecks) remove(username string) bool {
+	c.Lock()
+	defer c.Unlock()
+
+	for idx, check := range c.Checks {
+		if check.Username == username {
+			lastIdx := len(c.Checks) - 1
+			c.Checks[idx] = c.Checks[lastIdx]
+			c.Checks = c.Checks[:lastIdx]
+			return true
+		}
+	}
+
+	return false
+}
+
+// FolderRetention defines the retention policy for the specified directory path
+type FolderRetention struct {
+	// Path is the exposed virtual directory path, if no other specific retention is defined,
+	// the retention applies for sub directories too. For example if retention is defined
+	// for the paths "/" and "/sub" then the retention for "/" is applied for any file outside
+	// the "/sub" directory
+	Path string `json:"path"`
+	// Retention time in hours. 0 means exclude this path
+	Retention int `json:"retention"`
+	// DeleteEmptyDirs defines if empty directories will be deleted.
+	// The user need the delete permission
+	DeleteEmptyDirs bool `json:"delete_empty_dirs,omitempty"`
+	// IgnoreUserPermissions defines if delete files even if the user does not have the delete permission.
+	// The default is "false" which means that files will be skipped if the user does not have the permission
+	// to delete them. This applies to sub directories too.
+	IgnoreUserPermissions bool `json:"ignore_user_permissions,omitempty"`
+}
+
+func (f *FolderRetention) isValid() error {
+	f.Path = path.Clean(f.Path)
+	if !path.IsAbs(f.Path) {
+		return util.NewValidationError(fmt.Sprintf("folder retention: invalid path %#v, please specify an absolute POSIX path",
+			f.Path))
+	}
+	if f.Retention < 0 {
+		return util.NewValidationError(fmt.Sprintf("invalid folder retention %v, it must be greater or equal to zero",
+			f.Retention))
+	}
+	return nil
+}
+
+// RetentionCheck defines an active retention check
+type RetentionCheck struct {
+	// Username to which the retention check refers
+	Username string `json:"username"`
+	// retention check start time as unix timestamp in milliseconds
+	StartTime int64 `json:"start_time"`
+	// affected folders
+	Folders []FolderRetention `json:"folders"`
+	// Cleanup results
+	conn *BaseConnection
+}
+
+// Validate returns an error if the specified folders are not valid
+func (c *RetentionCheck) Validate() error {
+	folderPaths := make(map[string]bool)
+	nothingToDo := true
+	for idx := range c.Folders {
+		f := &c.Folders[idx]
+		if err := f.isValid(); err != nil {
+			return err
+		}
+		if f.Retention > 0 {
+			nothingToDo = false
+		}
+		if _, ok := folderPaths[f.Path]; ok {
+			return util.NewValidationError(fmt.Sprintf("duplicated folder path %#v", f.Path))
+		}
+		folderPaths[f.Path] = true
+	}
+	if nothingToDo {
+		return util.NewValidationError("nothing to delete!")
+	}
+	return nil
+}
+
+func (c *RetentionCheck) updateUserPermissions() {
+	for _, folder := range c.Folders {
+		if folder.IgnoreUserPermissions {
+			c.conn.User.Permissions[folder.Path] = []string{dataprovider.PermAny}
+		}
+	}
+}
+
+func (c *RetentionCheck) getFolderRetention(folderPath string) (FolderRetention, error) {
+	dirsForPath := util.GetDirsForVirtualPath(folderPath)
+	for _, dirPath := range dirsForPath {
+		for _, folder := range c.Folders {
+			if folder.Path == dirPath {
+				return folder, nil
+			}
+		}
+	}
+
+	return FolderRetention{}, fmt.Errorf("unable to find folder retention for %#v", folderPath)
+}
+
+func (c *RetentionCheck) removeFile(virtualPath string, info os.FileInfo) error {
+	fs, fsPath, err := c.conn.GetFsAndResolvedPath(virtualPath)
+	if err != nil {
+		return err
+	}
+	return c.conn.RemoveFile(fs, fsPath, virtualPath, info)
+}
+
+func (c *RetentionCheck) cleanupFolder(folderPath string) error {
+	cleanupPerms := []string{dataprovider.PermListItems, dataprovider.PermDelete}
+	if !c.conn.User.HasPerms(cleanupPerms, folderPath) {
+		c.conn.Log(logger.LevelInfo, "user %#v does not have permissions to check retention on %#v, retention check skipped",
+			c.conn.User, folderPath)
+		return nil
+	}
+
+	folderRetention, err := c.getFolderRetention(folderPath)
+	if err != nil {
+		c.conn.Log(logger.LevelError, "unable to get folder retention for path %#v", folderPath)
+		return err
+	}
+	if folderRetention.Retention == 0 {
+		c.conn.Log(logger.LevelDebug, "retention check skipped for folder %#v, retention is set to 0", folderPath)
+		return nil
+	}
+	c.conn.Log(logger.LevelDebug, "start retention check for folder %#v, retention: %v hours, delete empty dirs? %v, ignore user perms? %v",
+		folderPath, folderRetention.Retention, folderRetention.DeleteEmptyDirs, folderRetention.IgnoreUserPermissions)
+	files, err := c.conn.ListDir(folderPath)
+	if err != nil {
+		if err == c.conn.GetNotExistError() {
+			c.conn.Log(logger.LevelDebug, "folder %#v does not exist, retention check skipped", folderPath)
+			return nil
+		}
+		c.conn.Log(logger.LevelWarn, "unable to list directory %#v", folderPath)
+		return err
+	}
+	deletedFiles := 0
+	deletedSize := int64(0)
+	for _, info := range files {
+		virtualPath := path.Join(folderPath, info.Name())
+		if info.IsDir() {
+			if err := c.cleanupFolder(virtualPath); err != nil {
+				c.conn.Log(logger.LevelWarn, "unable to cleanup folder %#v: %v", virtualPath, err)
+				return err
+			}
+		} else {
+			retentionTime := info.ModTime().Add(time.Duration(folderRetention.Retention) * time.Hour)
+			if retentionTime.Before(time.Now()) {
+				if err := c.removeFile(virtualPath, info); err != nil {
+					c.conn.Log(logger.LevelWarn, "unable to remove file %#v, retention %v: %v",
+						virtualPath, retentionTime, err)
+					return err
+				}
+				c.conn.Log(logger.LevelDebug, "removed file %#v, modification time: %v, retention: %v hours, retention time: %v",
+					virtualPath, info.ModTime(), folderRetention.Retention, retentionTime)
+				deletedFiles++
+				deletedSize += info.Size()
+			}
+		}
+	}
+
+	if folderRetention.DeleteEmptyDirs {
+		c.checkEmptyDirRemoval(folderPath)
+	}
+	c.conn.Log(logger.LevelDebug, "retention check completed for folder %#v, deleted files: %v, deleted size: %v bytes",
+		folderPath, deletedFiles, deletedSize)
+
+	return nil
+}
+
+func (c *RetentionCheck) checkEmptyDirRemoval(folderPath string) {
+	if folderPath != "/" && c.conn.User.HasPerm(dataprovider.PermDelete, path.Dir(folderPath)) {
+		files, err := c.conn.ListDir(folderPath)
+		if err == nil && len(files) == 0 {
+			err = c.conn.RemoveDir(folderPath)
+			c.conn.Log(logger.LevelDebug, "tryed to remove empty dir %#v, error: %v", folderPath, err)
+		}
+	}
+}
+
+// Start starts the retention check
+func (c *RetentionCheck) Start() {
+	c.conn.Log(logger.LevelInfo, "retention check started")
+	defer RetentionChecks.remove(c.conn.User.Username)
+	defer c.conn.CloseFS() //nolint:errcheck
+
+	for _, folder := range c.Folders {
+		if folder.Retention > 0 {
+			if err := c.cleanupFolder(folder.Path); err != nil {
+				c.conn.Log(logger.LevelWarn, "retention check failed, unable to cleanup folder %#v", folder.Path)
+				return
+			}
+		}
+	}
+
+	c.conn.Log(logger.LevelInfo, "retention check completed")
+}

common/dataretention_test.go

@@ -0,0 +1,187 @@
+package common
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/drakkan/sftpgo/v2/dataprovider"
+	"github.com/drakkan/sftpgo/v2/sdk"
+)
+
+func TestRetentionValidation(t *testing.T) {
+	check := RetentionCheck{}
+	check.Folders = append(check.Folders, FolderRetention{
+		Path:      "relative",
+		Retention: 10,
+	})
+	err := check.Validate()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "please specify an absolute POSIX path")
+
+	check.Folders = []FolderRetention{
+		{
+			Path:      "/",
+			Retention: -1,
+		},
+	}
+	err = check.Validate()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "invalid folder retention")
+
+	check.Folders = []FolderRetention{
+		{
+			Path:      "/ab/..",
+			Retention: 0,
+		},
+	}
+	err = check.Validate()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "nothing to delete")
+	assert.Equal(t, "/", check.Folders[0].Path)
+
+	check.Folders = append(check.Folders, FolderRetention{
+		Path:      "/../..",
+		Retention: 24,
+	})
+	err = check.Validate()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), `duplicated folder path "/"`)
+
+	check.Folders = []FolderRetention{
+		{
+			Path:      "/dir1",
+			Retention: 48,
+		},
+		{
+			Path:      "/dir2",
+			Retention: 96,
+		},
+	}
+	err = check.Validate()
+	assert.NoError(t, err)
+}
+
+func TestRetentionPermissionsAndGetFolder(t *testing.T) {
+	user := dataprovider.User{
+		BaseUser: sdk.BaseUser{
+			Username: "user1",
+		},
+	}
+	user.Permissions = make(map[string][]string)
+	user.Permissions["/"] = []string{dataprovider.PermListItems, dataprovider.PermDelete}
+	user.Permissions["/dir1"] = []string{dataprovider.PermListItems}
+	user.Permissions["/dir2/sub1"] = []string{dataprovider.PermCreateDirs}
+	user.Permissions["/dir2/sub2"] = []string{dataprovider.PermDelete}
+
+	check := RetentionCheck{
+		Folders: []FolderRetention{
+			{
+				Path:                  "/dir2",
+				Retention:             24 * 7,
+				IgnoreUserPermissions: true,
+			},
+			{
+				Path:                  "/dir3",
+				Retention:             24 * 7,
+				IgnoreUserPermissions: false,
+			},
+			{
+				Path:                  "/dir2/sub1/sub",
+				Retention:             24,
+				IgnoreUserPermissions: true,
+			},
+		},
+	}
+
+	conn := NewBaseConnection("", "", "", "", user)
+	conn.ID = fmt.Sprintf("retention_check_%v", user.Username)
+	check.conn = conn
+	check.updateUserPermissions()
+	assert.Equal(t, []string{dataprovider.PermListItems, dataprovider.PermDelete}, conn.User.Permissions["/"])
+	assert.Equal(t, []string{dataprovider.PermListItems}, conn.User.Permissions["/dir1"])
+	assert.Equal(t, []string{dataprovider.PermAny}, conn.User.Permissions["/dir2"])
+	assert.Equal(t, []string{dataprovider.PermAny}, conn.User.Permissions["/dir2/sub1/sub"])
+	assert.Equal(t, []string{dataprovider.PermCreateDirs}, conn.User.Permissions["/dir2/sub1"])
+	assert.Equal(t, []string{dataprovider.PermDelete}, conn.User.Permissions["/dir2/sub2"])
+
+	_, err := check.getFolderRetention("/")
+	assert.Error(t, err)
+	folder, err := check.getFolderRetention("/dir3")
+	assert.NoError(t, err)
+	assert.Equal(t, "/dir3", folder.Path)
+	folder, err = check.getFolderRetention("/dir2/sub3")
+	assert.NoError(t, err)
+	assert.Equal(t, "/dir2", folder.Path)
+	folder, err = check.getFolderRetention("/dir2/sub2")
+	assert.NoError(t, err)
+	assert.Equal(t, "/dir2", folder.Path)
+	folder, err = check.getFolderRetention("/dir2/sub1")
+	assert.NoError(t, err)
+	assert.Equal(t, "/dir2", folder.Path)
+	folder, err = check.getFolderRetention("/dir2/sub1/sub/sub")
+	assert.NoError(t, err)
+	assert.Equal(t, "/dir2/sub1/sub", folder.Path)
+}
+
+func TestRetentionCheckAddRemove(t *testing.T) {
+	username := "username"
+	user := dataprovider.User{
+		BaseUser: sdk.BaseUser{
+			Username: username,
+		},
+	}
+	user.Permissions = make(map[string][]string)
+	user.Permissions["/"] = []string{dataprovider.PermAny}
+	check := RetentionCheck{
+		Folders: []FolderRetention{
+			{
+				Path:      "/",
+				Retention: 48,
+			},
+		},
+	}
+	assert.NotNil(t, RetentionChecks.Add(check, &user))
+	checks := RetentionChecks.Get()
+	require.Len(t, checks, 1)
+	assert.Equal(t, username, checks[0].Username)
+	assert.Greater(t, checks[0].StartTime, int64(0))
+	require.Len(t, checks[0].Folders, 1)
+	assert.Equal(t, check.Folders[0].Path, checks[0].Folders[0].Path)
+	assert.Equal(t, check.Folders[0].Retention, checks[0].Folders[0].Retention)
+
+	assert.Nil(t, RetentionChecks.Add(check, &user))
+	assert.True(t, RetentionChecks.remove(username))
+	require.Len(t, RetentionChecks.Get(), 0)
+	assert.False(t, RetentionChecks.remove(username))
+}
+
+func TestCleanupErrors(t *testing.T) {
+	user := dataprovider.User{
+		BaseUser: sdk.BaseUser{
+			Username: "u",
+		},
+	}
+	user.Permissions = make(map[string][]string)
+	user.Permissions["/"] = []string{dataprovider.PermAny}
+	check := &RetentionCheck{
+		Folders: []FolderRetention{
+			{
+				Path:      "/path",
+				Retention: 48,
+			},
+		},
+	}
+	check = RetentionChecks.Add(*check, &user)
+	require.NotNil(t, check)
+
+	err := check.removeFile("missing file", nil)
+	assert.Error(t, err)
+
+	err = check.cleanupFolder("/")
+	assert.Error(t, err)
+
+	assert.True(t, RetentionChecks.remove(user.Username))
+}

common/protocol_test.go

@@ -2325,6 +2325,225 @@ func TestGetQuotaError(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestRetentionAPI(t *testing.T) {
+	u := getTestUser()
+	user, _, err := httpdtest.AddUser(u, http.StatusCreated)
+	assert.NoError(t, err)
+	uploadPath := path.Join(testDir, testFileName)
+
+	conn, client, err := getSftpClient(user)
+	if assert.NoError(t, err) {
+		defer conn.Close()
+		defer client.Close()
+
+		err = client.Mkdir(testDir)
+		assert.NoError(t, err)
+		err = writeSFTPFile(uploadPath, 32, client)
+		assert.NoError(t, err)
+
+		folderRetention := []common.FolderRetention{
+			{
+				Path:            "/",
+				Retention:       24,
+				DeleteEmptyDirs: true,
+			},
+		}
+		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+		assert.NoError(t, err)
+
+		assert.Eventually(t, func() bool {
+			return len(common.RetentionChecks.Get()) == 0
+		}, 1000*time.Millisecond, 50*time.Millisecond)
+
+		_, err = client.Stat(uploadPath)
+		assert.NoError(t, err)
+
+		err = client.Chtimes(uploadPath, time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour))
+		assert.NoError(t, err)
+
+		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+		assert.NoError(t, err)
+
+		assert.Eventually(t, func() bool {
+			return len(common.RetentionChecks.Get()) == 0
+		}, 1000*time.Millisecond, 50*time.Millisecond)
+
+		_, err = client.Stat(uploadPath)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+
+		_, err = client.Stat(testDir)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+
+		err = client.Mkdir(testDir)
+		assert.NoError(t, err)
+		err = writeSFTPFile(uploadPath, 32, client)
+		assert.NoError(t, err)
+
+		folderRetention[0].DeleteEmptyDirs = false
+		err = client.Chtimes(uploadPath, time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour))
+		assert.NoError(t, err)
+
+		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+		assert.NoError(t, err)
+
+		assert.Eventually(t, func() bool {
+			return len(common.RetentionChecks.Get()) == 0
+		}, 1000*time.Millisecond, 50*time.Millisecond)
+
+		_, err = client.Stat(uploadPath)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+
+		_, err = client.Stat(testDir)
+		assert.NoError(t, err)
+
+		err = writeSFTPFile(uploadPath, 32, client)
+		assert.NoError(t, err)
+		err = client.Chtimes(uploadPath, time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour))
+		assert.NoError(t, err)
+	}
+
+	// remove delete permissions to the user
+	user.Permissions["/"+testDir] = []string{dataprovider.PermListItems, dataprovider.PermUpload,
+		dataprovider.PermCreateDirs, dataprovider.PermChtimes}
+	user, _, err = httpdtest.UpdateUser(user, http.StatusOK, "")
+	assert.NoError(t, err)
+
+	conn, client, err = getSftpClient(user)
+	if assert.NoError(t, err) {
+		defer conn.Close()
+		defer client.Close()
+
+		innerUploadFilePath := path.Join("/"+testDir, testDir, testFileName)
+		err = client.Mkdir(path.Join(testDir, testDir))
+		assert.NoError(t, err)
+
+		err = writeSFTPFile(innerUploadFilePath, 32, client)
+		assert.NoError(t, err)
+		err = client.Chtimes(innerUploadFilePath, time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour))
+		assert.NoError(t, err)
+
+		folderRetention := []common.FolderRetention{
+			{
+				Path:      "/missing",
+				Retention: 24,
+			},
+			{
+				Path:            "/" + testDir,
+				Retention:       24,
+				DeleteEmptyDirs: true,
+			},
+			{
+				Path:                  path.Dir(innerUploadFilePath),
+				Retention:             0,
+				IgnoreUserPermissions: true,
+			},
+		}
+		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+		assert.NoError(t, err)
+
+		assert.Eventually(t, func() bool {
+			return len(common.RetentionChecks.Get()) == 0
+		}, 1000*time.Millisecond, 50*time.Millisecond)
+
+		_, err = client.Stat(uploadPath)
+		assert.NoError(t, err)
+		_, err = client.Stat(innerUploadFilePath)
+		assert.NoError(t, err)
+
+		folderRetention[1].IgnoreUserPermissions = true
+		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+		assert.NoError(t, err)
+
+		assert.Eventually(t, func() bool {
+			return len(common.RetentionChecks.Get()) == 0
+		}, 1000*time.Millisecond, 50*time.Millisecond)
+
+		_, err = client.Stat(uploadPath)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+		_, err = client.Stat(innerUploadFilePath)
+		assert.NoError(t, err)
+
+		folderRetention = []common.FolderRetention{
+
+			{
+				Path:                  "/" + testDir,
+				Retention:             24,
+				DeleteEmptyDirs:       true,
+				IgnoreUserPermissions: true,
+			},
+		}
+
+		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+		assert.NoError(t, err)
+
+		assert.Eventually(t, func() bool {
+			return len(common.RetentionChecks.Get()) == 0
+		}, 1000*time.Millisecond, 50*time.Millisecond)
+
+		_, err = client.Stat(innerUploadFilePath)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+	}
+	// finally test some errors removing files or folders
+	if runtime.GOOS != osWindows {
+		dirPath := filepath.Join(user.HomeDir, "adir", "sub")
+		err := os.MkdirAll(dirPath, os.ModePerm)
+		assert.NoError(t, err)
+		filePath := filepath.Join(dirPath, "f.dat")
+		err = os.WriteFile(filePath, nil, os.ModePerm)
+		assert.NoError(t, err)
+
+		err = os.Chtimes(filePath, time.Now().Add(-72*time.Hour), time.Now().Add(-72*time.Hour))
+		assert.NoError(t, err)
+
+		err = os.Chmod(dirPath, 0001)
+		assert.NoError(t, err)
+
+		folderRetention := []common.FolderRetention{
+
+			{
+				Path:                  "/adir",
+				Retention:             24,
+				DeleteEmptyDirs:       true,
+				IgnoreUserPermissions: true,
+			},
+		}
+
+		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+		assert.NoError(t, err)
+
+		assert.Eventually(t, func() bool {
+			return len(common.RetentionChecks.Get()) == 0
+		}, 1000*time.Millisecond, 50*time.Millisecond)
+
+		err = os.Chmod(dirPath, 0555)
+		assert.NoError(t, err)
+
+		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+		assert.NoError(t, err)
+
+		assert.Eventually(t, func() bool {
+			return len(common.RetentionChecks.Get()) == 0
+		}, 1000*time.Millisecond, 50*time.Millisecond)
+
+		err = os.Chmod(dirPath, os.ModePerm)
+		assert.NoError(t, err)
+
+		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+		assert.NoError(t, err)
+
+		assert.Eventually(t, func() bool {
+			return len(common.RetentionChecks.Get()) == 0
+		}, 1000*time.Millisecond, 50*time.Millisecond)
+
+		assert.NoDirExists(t, dirPath)
+	}
+
+	_, err = httpdtest.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
 func TestRenameDir(t *testing.T) {
 	u := getTestUser()
 	testDir := "/dir-to-rename"

dataprovider/admin.go

@@ -36,6 +36,7 @@ const (
 	PermAdminManageSystem     = "manage_system"
 	PermAdminManageDefender   = "manage_defender"
 	PermAdminViewDefender     = "view_defender"
+	PermAdminRetentionChecks  = "retention_checks"
 )
 
 var (
@@ -43,7 +44,7 @@ var (
 	validAdminPerms = []string{PermAdminAny, PermAdminAddUsers, PermAdminChangeUsers, PermAdminDeleteUsers,
 		PermAdminViewUsers, PermAdminViewConnections, PermAdminCloseConnections, PermAdminViewServerStatus,
 		PermAdminManageAdmins, PermAdminManageAPIKeys, PermAdminQuotaScans, PermAdminManageSystem,
-		PermAdminManageDefender, PermAdminViewDefender}
+		PermAdminManageDefender, PermAdminViewDefender, PermAdminRetentionChecks}
 )
 
 // TOTPConfig defines the time-based one time password configuration

docs/kms.md

@@ -8,7 +8,7 @@ The `secrets` section of the `kms` configuration allows to configure how to encr
 
 - `url` defines the URI to the KMS service
 - `master_key`, defines the master encryption key as string. If not empty, it takes precedence over `master_key_path`.
-- `master_key_path` defines the absolute path to a file containing the master encryption key. This could be, for example, a docker secrets or a file protected with filesystem level permissions.
+- `master_key_path` defines the absolute path to a file containing the master encryption key. This could be, for example, a docker secret or a file protected with filesystem level permissions.
 
 ### Local provider
 

docs/rest-api.md

@@ -1,6 +1,6 @@
 # REST API
 
-SFTPGo exposes REST API to manage, backup, and restore users and folders, and to get real time reports of the active connections with the ability to forcibly close a connection.
+SFTPGo exposes REST API to manage, backup, and restore users and folders, data retention, and to get real time reports of the active connections with the ability to forcibly close a connection.
 
 If quota tracking is enabled in the configuration file, then the used size and number of files are updated each time a file is added/removed. If files are added/removed not using SFTP/SCP, or if you change `track_quota` from `2` to `1`, you can rescan the users home dir and update the used quota using the REST API.
 
@@ -35,6 +35,7 @@ You can create other administrator and assign them the following permissions:
 - manage API keys
 - manage system
 - manage admins
+- manage data retention
 
 You can also restrict administrator access based on the source IP address. If you are running SFTPGo behind a reverse proxy you need to allow both the proxy IP address and the real client IP.
 
@@ -62,6 +63,35 @@ API keys are not allowed for the following REST APIs:
 
 Please keep in mind that using an API key not associated with any administrator it is still possible to create a new administrator, with full permissions, and then impersonate it: be careful if you share unassociated API keys with third parties and with the `manage adminis` permission granted, they will basically allow full access, the only restriction is that the impersonated admin cannot be modified.
 
+The data retention APIs allow you to define per-folder retention policies for each user. To clarify this concept let's show an example, a data retention check accepts a POST body like this one:
+
+```json
+[
+  {
+    "path": "/folder1",
+    "retention": 72
+  },
+  {
+    "path": "/folder1/subfolder",
+    "retention": 0
+  },
+  {
+    "path": "/folder2",
+    "retention": 24
+  }
+]
+```
+
+In the above example we asked to SFTPGo:
+
+- to delete all the files with modification time older than 72 hours in `/folder1`
+- to exclude `/folder1/subfolder`, no files will be deleted here
+- to delete all the files with modification time older than 24 hours in `/folder2`
+
+You can find an example script that shows how to manage data retention [here](../examples/data-retention). Checks the REST API schema for full details.
+
+:warning: Deleting files is an irreversible action, please make sure you fully understand what you are doing before using this feature, you may have users with overlapping home directories or virtual folders shared between multiple users, it is relatively easy to inadvertently delete files you need.
+
 The OpenAPI 3 schema for the exposed API can be found inside the source tree: [openapi.yaml](../httpd/schema/openapi.yaml "OpenAPI 3 specs"). If you want to render the schema without importing it manually, you can explore it on [Stoplight](https://sftpgo.stoplight.io/docs/sftpgo/openapi.yaml).
 
 You can generate your own REST client in your preferred programming language, or even bash scripts, using an OpenAPI generator such as [swagger-codegen](https://github.com/swagger-api/swagger-codegen) or [OpenAPI Generator](https://openapi-generator.tech/).

examples/data-retention/README.md

@@ -0,0 +1,34 @@
+# File retention policies
+
+The `checkretention` example script shows how to use the SFTPGo REST API to manage data retention.
+
+:warning: Deleting files is an irreversible action, please make sure you fully understand what you are doing before using this feature, you may have users with overlapping home directories or virtual folders shared between multiple users, it is relatively easy to inadvertently delete files you need.
+
+The example shows how to setup a really simple retention policy, for each user it sends this request:
+
+```json
+[
+  {
+    "path": "/",
+    "retention": 168,
+    "delete_empty_dirs": true,
+    "ignore_user_permissions": false
+  }
+]
+```
+
+so alls files with modification time older than 168 hours (7 days) will be deleted. Empty directories will be removed and the check will respect user's permissions, so if the user cannot delete a file/folder it will be skipped.
+
+You can define different retention policies per-user and per-folder and you can exclude a folder setting the retention to `0`.
+
+You can use this script as a starting point, please edit it according to your needs.
+
+The script is written in Python and has the following requirements:
+
+- python3 or python2
+- python [Requests](https://requests.readthedocs.io/en/master/) module
+
+The provided example tries to connect to an SFTPGo instance running on `127.0.0.1:8080` using the following credentials:
+
+- username: `admin`
+- password: `password`

examples/data-retention/checkretention

@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+
+from datetime import datetime
+import sys
+import time
+
+import pytz
+import requests
+
+try:
+	import urllib.parse as urlparse
+except ImportError:
+	import urlparse
+
+# change base_url to point to your SFTPGo installation
+base_url = "http://127.0.0.1:8080"
+# set to False if you want to skip TLS certificate validation
+verify_tls_cert = True
+# set the credentials for a valid admin here
+admin_user = "admin"
+admin_password = "password"
+
+
+class CheckRetention:
+
+	def __init__(self):
+		self.limit = 100
+		self.offset = 0
+		self.access_token = ""
+		self.access_token_expiration = None
+
+	def printLog(self, message):
+		print("{} - {}".format(datetime.now(), message))
+
+	def checkAccessToken(self):
+		if self.access_token != "" and self.access_token_expiration:
+			expire_diff = self.access_token_expiration - datetime.now(tz=pytz.UTC)
+			# we don't use total_seconds to be python 2 compatible
+			seconds_to_expire = expire_diff.days * 86400 + expire_diff.seconds
+			if seconds_to_expire > 180:
+				return
+
+		auth = requests.auth.HTTPBasicAuth(admin_user, admin_password)
+		r = requests.get(urlparse.urljoin(base_url, "api/v2/token"), auth=auth, verify=verify_tls_cert, timeout=10)
+		if r.status_code != 200:
+			self.printLog("error getting access token: {}".format(r.text))
+			sys.exit(1)
+		self.access_token = r.json()["access_token"]
+		self.access_token_expiration = pytz.timezone("UTC").localize(datetime.strptime(r.json()["expires_at"],
+																					"%Y-%m-%dT%H:%M:%SZ"))
+
+	def getAuthHeader(self):
+		self.checkAccessToken()
+		return {"Authorization": "Bearer " + self.access_token}
+
+	def waitForRentionCheck(self, username):
+		while True:
+			auth_header = self.getAuthHeader()
+			r = requests.get(urlparse.urljoin(base_url, "api/v2/retention/users/checks"), headers=auth_header, verify=verify_tls_cert,
+							timeout=10)
+			if r.status_code != 200:
+				self.printLog("error getting retention checks while waiting for {}: {}".format(username, r.text))
+				sys.exit(1)
+
+			checking = False
+			for check in r.json():
+				if check["username"] == username:
+					checking = True
+			if not checking:
+				break
+			self.printLog("waiting for the retention check to complete for user {}".format(username))
+			time.sleep(2)
+
+		self.printLog("retention check for user {} finished".format(username))
+
+	def checkUserRetention(self, username):
+		self.printLog("starting retention check for user {}".format(username))
+		auth_header = self.getAuthHeader()
+		retention = [
+						{
+							"path": "/",
+							"retention": 168,
+							"delete_empty_dirs": True,
+							"ignore_user_permissions": False
+						}
+					]
+		r = requests.post(urlparse.urljoin(base_url, "api/v2/retention/users/" + username + "/check"), headers=auth_header,
+						 json=retention, verify=verify_tls_cert, timeout=10)
+		if r.status_code != 202:
+			self.printLog("error starting retention check for user {}: {}".format(username, r.text))
+			sys.exit(1)
+		self.waitForRentionCheck(username)
+
+	def checkUsersRetention(self):
+		while True:
+			self.printLog("get users, limit {} offset {}".format(self.limit, self.offset))
+			auth_header = self.getAuthHeader()
+			payload = {"limit":self.limit, "offset":self.offset}
+			r = requests.get(urlparse.urljoin(base_url, "api/v2/users"), headers=auth_header, params=payload,
+						verify=verify_tls_cert, timeout=10)
+			if r.status_code != 200:
+				self.printLog("error getting users: {}".format(r.text))
+				sys.exit(1)
+			users = r.json()
+			for user in users:
+				self.checkUserRetention(user["username"])
+
+			self.offset += len(users)
+			if len(users) < self.limit:
+				break
+
+
+if __name__ == '__main__':
+	c = CheckRetention()
+	c.checkUsersRetention()

examples/quotascan/README.md

@@ -6,7 +6,7 @@ The stored quota may be incorrect for several reasons, such as an unexpected shu
 
 A quota scan updates the number of files and their total size for the specified user and the virtual folders, if any, included in his quota.
 
-If you want to track quotas, a scheduled quota scan is recommended. You could use this example as a starting point.
+If you want to track quotas, a scheduled quota scan is recommended. You can use this example as a starting point.
 
 The script is written in Python and has the following requirements:
 

examples/quotascan/scanuserquota

@@ -65,7 +65,7 @@ class UpdateQuota:
 	def waitForQuotaUpdate(self, username):
 		while True:
 			auth_header = self.getAuthHeader()
-			r = requests.get(urlparse.urljoin(base_url, "api/v2/quota-scans"), headers=auth_header, verify=verify_tls_cert,
+			r = requests.get(urlparse.urljoin(base_url, "api/v2/quotas/users/scans"), headers=auth_header, verify=verify_tls_cert,
 							timeout=10)
 			if r.status_code != 200:
 				self.printLog("error getting quota scans while waiting for {}: {}".format(username, r.text))
@@ -85,8 +85,8 @@ class UpdateQuota:
 	def updateUserQuota(self, username):
 		self.printLog("starting quota update for user {}".format(username))
 		auth_header = self.getAuthHeader()
-		r = requests.post(urlparse.urljoin(base_url, "api/v2/quota-scans"), headers=auth_header,
+		r = requests.post(urlparse.urljoin(base_url, "api/v2/quotas/users/" + username + "/scan"), headers=auth_header,
-						json={"username":username}, verify=verify_tls_cert, timeout=10)
+						 verify=verify_tls_cert, timeout=10)
 		if r.status_code != 202:
 			self.printLog("error starting quota scan for user {}: {}".format(username, r.text))
 			sys.exit(1)

go.mod

@@ -7,14 +7,14 @@ require (
 	github.com/Azure/azure-storage-blob-go v0.14.0
 	github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
 	github.com/alexedwards/argon2id v0.0.0-20210511081203-7d35d68092b8
-	github.com/aws/aws-sdk-go v1.40.45
+	github.com/aws/aws-sdk-go v1.40.49
 	github.com/cockroachdb/cockroach-go/v2 v2.1.1
 	github.com/eikenb/pipeat v0.0.0-20210603033007-44fc3ffce52b
-	github.com/fatih/color v1.12.0 // indirect
+	github.com/fatih/color v1.13.0 // indirect
 	github.com/fclairamb/ftpserverlib v0.16.0
 	github.com/fclairamb/go-log v0.1.0
 	github.com/go-chi/chi/v5 v5.0.4
-	github.com/go-chi/jwtauth/v5 v5.0.1
+	github.com/go-chi/jwtauth/v5 v5.0.2
 	github.com/go-chi/render v1.0.1
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/golang/mock v1.6.0
@@ -51,7 +51,7 @@ require (
 	github.com/shirou/gopsutil/v3 v3.21.8
 	github.com/spf13/afero v1.6.0
 	github.com/spf13/cobra v1.2.1
-	github.com/spf13/viper v1.8.1
+	github.com/spf13/viper v1.9.0
 	github.com/stretchr/testify v1.7.0
 	github.com/studio-b12/gowebdav v0.0.0-20210917133250-a3a86976a1df
 	github.com/wagslane/go-password-validator v0.3.0
@@ -60,18 +60,18 @@ require (
 	go.uber.org/automaxprocs v1.4.0
 	gocloud.dev v0.24.0
 	golang.org/x/crypto v0.0.0-20210915214749-c084706c2272
-	golang.org/x/net v0.0.0-20210917221730-978cfadd31cf
+	golang.org/x/net v0.0.0-20210924151903-3ad01bbaa167
-	golang.org/x/sys v0.0.0-20210917161153-d61c044b1678
+	golang.org/x/sys v0.0.0-20210925032602-92d5a993a665
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac
 	google.golang.org/api v0.57.0
-	google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 // indirect
+	google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0 // indirect
-	google.golang.org/grpc v1.40.0
+	google.golang.org/grpc v1.41.0
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
 require (
-	cloud.google.com/go v0.94.1 // indirect
+	cloud.google.com/go v0.95.0 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.3 // indirect
 	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -100,7 +100,7 @@ require (
 	github.com/lestrrat-go/iter v1.0.1 // indirect
 	github.com/lestrrat-go/option v1.0.0 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
-	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-colorable v0.1.10 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
@@ -130,5 +130,5 @@ replace (
 	github.com/eikenb/pipeat => github.com/drakkan/pipeat v0.0.0-20210805162858-70e57fa8a639
 	github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9
 	golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20210918082254-e7eb8487714b
-	golang.org/x/net => github.com/drakkan/net v0.0.0-20210918081947-af83c6eab079
+	golang.org/x/net => github.com/drakkan/net v0.0.0-20210925100637-0aaabc6c2a0b
 )

go.sum

@@ -31,8 +31,9 @@ cloud.google.com/go v0.92.2/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y
 cloud.google.com/go v0.92.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
 cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
 cloud.google.com/go v0.94.0/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4=
-cloud.google.com/go v0.94.1 h1:DwuSvDZ1pTYGbXo8yOJevCTr3BoBlE+OVkHAKiYQUXc=
 cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4=
+cloud.google.com/go v0.95.0 h1:JVWssQIj9cLwHmLjqWLptFa83o7HgqUictM6eyvGWJE=
+cloud.google.com/go v0.95.0/go.mod h1:MzZUAH870Y7E+c14j23Ir66FC1+PK8WLG7OG4SjP+0k=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
@@ -43,6 +44,7 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
 cloud.google.com/go/firestore v1.5.0/go.mod h1:c4nNYR1qdq7eaZ+jSc5fonrQN2k3M7sWATcYTiakjEo=
+cloud.google.com/go/firestore v1.6.0/go.mod h1:afJwI0vaXwAG54kI7A//lP/lSPDkQORQuMkv56TxEPU=
 cloud.google.com/go/kms v0.1.0 h1:VXAb5OzejDcyhFzIDeZ5n5AUdlsFnCyexuascIwWMj0=
 cloud.google.com/go/kms v0.1.0/go.mod h1:8Qp8PCAypHg4FdmlyW1QRAv09BGQ9Uzh7JnmIZxPk+c=
 cloud.google.com/go/monitoring v0.1.0/go.mod h1:Hpm3XfzJv+UTiXzCG5Ffp0wijzHTC7Cv4eR7o3x/fEE=
@@ -134,8 +136,8 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo
 github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go v1.38.68/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go v1.40.34/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
-github.com/aws/aws-sdk-go v1.40.45 h1:QN1nsY27ssD/JmW4s83qmSb+uL6DG4GmCDzjmJB4xUI=
+github.com/aws/aws-sdk-go v1.40.49 h1:kIbJYc4FZA2r4yxNU5giIR4HHLRkG9roFReWAsk0ZVQ=
-github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
+github.com/aws/aws-sdk-go v1.40.49/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
 github.com/aws/aws-sdk-go-v2 v1.7.0/go.mod h1:tb9wi5s61kTDA5qCkcDbt3KRVV74GGslQkl/DRdX/P4=
 github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
 github.com/aws/aws-sdk-go-v2/config v1.7.0/go.mod h1:w9+nMZ7soXCe5nT46Ri354SNhXDQ6v+V5wqDjnZE+GY=
@@ -180,6 +182,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/cockroachdb/cockroach-go/v2 v2.1.1 h1:3XzfSMuUT0wBe1a3o5C0eOTcArhmmFAg2Jzh/7hhKqo=
 github.com/cockroachdb/cockroach-go/v2 v2.1.1/go.mod h1:7NtUnP6eK+l6k483WSYNrq3Kb23bWV10IRV1TyeSpwM=
@@ -198,9 +201,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/decred/dcrd/chaincfg/chainhash v1.0.2/go.mod h1:BpbrGgrPTr3YJYRN3Bm+D9NuaFd+zGyNeIKgrhCXK60=
 github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
-github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0/go.mod h1:J70FGZSbzsjecRTiTzER+3f1KZLNaXkuv+yeFTKoxM8=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0 h1:Fe5DW39aaoS/fqZiYlylEqQWIKznnbatWSHpWdFA3oQ=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs=
@@ -213,8 +214,8 @@ github.com/drakkan/crypto v0.0.0-20210918082254-e7eb8487714b h1:MZY6RAQFVhJous68
 github.com/drakkan/crypto v0.0.0-20210918082254-e7eb8487714b/go.mod h1:0hNoheD1tVu/m8WMkw/chBXf5VpwzL5fHQU25k79NKo=
 github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 h1:LPH1dEblAOO/LoG7yHPMtBLXhQmjaga91/DDjWk9jWA=
 github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
-github.com/drakkan/net v0.0.0-20210918081947-af83c6eab079 h1:tUP5m4c14gFrz/N4P6Z6q/59viuytG1A0q7BWh3VPFo=
+github.com/drakkan/net v0.0.0-20210925100637-0aaabc6c2a0b h1:jIHHXQ4e5CGytFZPkDuCOM7YPL04z6jvnG/Iqm94PAY=
-github.com/drakkan/net v0.0.0-20210918081947-af83c6eab079/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+github.com/drakkan/net v0.0.0-20210925100637-0aaabc6c2a0b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 github.com/drakkan/pipeat v0.0.0-20210805162858-70e57fa8a639 h1:8tfGdb4kg/YCvAbIrsMazgoNtnqdOqQVDKW12uUCuuU=
 github.com/drakkan/pipeat v0.0.0-20210805162858-70e57fa8a639/go.mod h1:kltMsfRMTHSFdMbK66XdS8mfMW77+FZA1fGY1xYMF84=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@@ -229,11 +230,12 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
+github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
-github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc=
+github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
-github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
+github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/fclairamb/ftpserverlib v0.16.0 h1:uATwg14csmYcYaf3n7G16FCzuqYYI28PjBL4Jsk+yXo=
 github.com/fclairamb/ftpserverlib v0.16.0/go.mod h1:+Doq95UijHTIaJcWREhyu9dyQOqyoULbVU3OXgs8wEI=
 github.com/fclairamb/go-log v0.1.0 h1:fNoqk8w62i4EDEuRzDgHdDVTqMYSyr3DS981R7F2x/Y=
@@ -250,11 +252,10 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
-github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs=
 github.com/go-chi/chi/v5 v5.0.4 h1:5e494iHzsYBiyXQAHHuI4tyJS9M3V84OuX3ufIIGHFo=
 github.com/go-chi/chi/v5 v5.0.4/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
-github.com/go-chi/jwtauth/v5 v5.0.1 h1:eyJ6Yx5VphEfjkqpZ7+LJEWThzyIcF5aN2QVpgqSIu0=
+github.com/go-chi/jwtauth/v5 v5.0.2 h1:CSKtr+b6Jnfy5T27sMaiBPxaVE/bjnjS3ramFQ0526w=
-github.com/go-chi/jwtauth/v5 v5.0.1/go.mod h1:+JtcRYGZsnA4+ur1LFlb4Bei3O9WeUzoMfDZWfUJuoY=
+github.com/go-chi/jwtauth/v5 v5.0.2/go.mod h1:TeA7vmPe3uYThvHw8O8W13HOOpOd4MTgToxL41gZyjs=
 github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
 github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -286,7 +287,6 @@ github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
-github.com/goccy/go-json v0.4.8/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.7.6/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.7.8 h1:CvMH7LotYymYuLGEohBM1lTZWX4g6jzWUUl2aLFuBoE=
 github.com/goccy/go-json v0.7.8/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
@@ -400,8 +400,10 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/api v1.8.1/go.mod h1:sDjTOq0yUyv5G4h+BqSea7Fn6BU+XbolEz1952UB+mk=
+github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/consul/sdk v0.7.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM=
+github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
@@ -537,24 +539,19 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
-github.com/lestrrat-go/backoff/v2 v2.0.7/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
 github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A=
 github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
 github.com/lestrrat-go/blackmagic v1.0.0 h1:XzdxDbuQTz0RZZEmdU7cnQxUtFUzgCSPq8RCz4BxIi4=
 github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ=
-github.com/lestrrat-go/codegen v1.0.0/go.mod h1:JhJw6OQAuPEfVKUCLItpaVLumDGWQznd1VaXrBk9TdM=
 github.com/lestrrat-go/codegen v1.0.1/go.mod h1:JhJw6OQAuPEfVKUCLItpaVLumDGWQznd1VaXrBk9TdM=
 github.com/lestrrat-go/httpcc v1.0.0 h1:FszVC6cKfDvBKcJv646+lkh4GydQg2Z29scgUfkOpYc=
 github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++0Gf8MBnAvE=
 github.com/lestrrat-go/iter v1.0.1 h1:q8faalr2dY6o8bV45uwrxq12bRa1ezKrB6oM9FUgN4A=
 github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc=
-github.com/lestrrat-go/jwx v1.1.6/go.mod h1:c+R8G7qsaFNmTzYjU98A+sMh8Bo/MJqO9GnpqR+X024=
 github.com/lestrrat-go/jwx v1.2.6 h1:XAgfuHaOB7fDZ/6WhVgl8K89af768dU+3Nx4DlTbLIk=
 github.com/lestrrat-go/jwx v1.2.6/go.mod h1:tJuGuAI3LC71IicTx82Mz1n3w9woAs2bYJZpkjJQ5aU=
-github.com/lestrrat-go/option v0.0.0-20210103042652-6f1ecfceda35/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
 github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFeEO4=
 github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
-github.com/lestrrat-go/pdebug/v3 v3.0.1/go.mod h1:za+m+Ve24yCxTEhR59N7UlnJomWwCiIqbJRmKeiADU4=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@@ -572,8 +569,9 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.10 h1:KWqbp83oZ6YOEgIbNW3BM1Jbe2tz4jgmWA9FOuAF8bw=
+github.com/mattn/go-colorable v0.1.10/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI=
 github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -705,6 +703,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo=
@@ -735,8 +734,9 @@ github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmq
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44=
 github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
+github.com/spf13/viper v1.9.0 h1:yR6EXjTp0y0cLN8OZg1CRZmOBdI88UcGkhgyJhu6nZk=
+github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4=
 github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/handy v0.0.0-20200128134331-0f66f006fb2e/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
@@ -950,8 +950,8 @@ golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210917161153-d61c044b1678 h1:J27LZFQBFoihqXoegpscI10HpjZ7B5WQLLKL2FZXQKw=
+golang.org/x/sys v0.0.0-20210925032602-92d5a993a665 h1:QOQNt6vCjMpXE7JSK5VvAzJC1byuN3FgTNSBwf+CJgI=
-golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210925032602-92d5a993a665/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1148,8 +1148,9 @@ google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEc
 google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 h1:ysnBoUyeL/H6RCvNRhWHjKoDEmguI+mPU+qHgK8qv/w=
+google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
+google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0 h1:5Tbluzus3QxoAJx4IefGt1W0HQZW4nuMrVk684jI74Q=
+google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@@ -1175,8 +1176,9 @@ google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
 google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
-google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
+google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E=
+google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

httpd/api_quota.go

@@ -2,6 +2,7 @@ package httpd
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 
 	"github.com/go-chi/render"
@@ -191,7 +192,8 @@ func doStartUserQuotaScan(w http.ResponseWriter, r *http.Request, username strin
 		return
 	}
 	if !common.QuotaScans.AddUserQuotaScan(user.Username) {
-		sendAPIResponse(w, r, err, "Another scan is already in progress", http.StatusConflict)
+		sendAPIResponse(w, r, err, fmt.Sprintf("Another scan is already in progress for user %#v", username),
+			http.StatusConflict)
 		return
 	}
 	go doUserQuotaScan(user) //nolint:errcheck
@@ -209,7 +211,8 @@ func doStartFolderQuotaScan(w http.ResponseWriter, r *http.Request, name string)
 		return
 	}
 	if !common.QuotaScans.AddVFolderQuotaScan(folder.Name) {
-		sendAPIResponse(w, r, err, "Another scan is already in progress", http.StatusConflict)
+		sendAPIResponse(w, r, err, fmt.Sprintf("Another scan is already in progress for folder %#v", name),
+			http.StatusConflict)
 		return
 	}
 	go doFolderQuotaScan(folder) //nolint:errcheck

httpd/api_retention.go

@@ -0,0 +1,44 @@
+package httpd
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/go-chi/render"
+
+	"github.com/drakkan/sftpgo/v2/common"
+	"github.com/drakkan/sftpgo/v2/dataprovider"
+)
+
+func getRetentionChecks(w http.ResponseWriter, r *http.Request) {
+	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
+	render.JSON(w, r, common.RetentionChecks.Get())
+}
+
+func startRetentionCheck(w http.ResponseWriter, r *http.Request) {
+	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
+	username := getURLParam(r, "username")
+	user, err := dataprovider.UserExists(username)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", getRespStatus(err))
+		return
+	}
+	var check common.RetentionCheck
+	err = render.DecodeJSON(r.Body, &check.Folders)
+	if err != nil {
+		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
+		return
+	}
+	if err := check.Validate(); err != nil {
+		sendAPIResponse(w, r, err, "Invalid folders to check", http.StatusBadRequest)
+		return
+	}
+	c := common.RetentionChecks.Add(check, &user)
+	if c == nil {
+		sendAPIResponse(w, r, err, fmt.Sprintf("Another check is already in progress for user %#v", username),
+			http.StatusConflict)
+		return
+	}
+	go c.Start()
+	sendAPIResponse(w, r, err, "Check started", http.StatusAccepted)
+}

httpd/httpd.go

@@ -75,6 +75,8 @@ const (
 	userTOTPSavePath                       = "/api/v2/user/totp/save"
 	user2FARecoveryCodesPath               = "/api/v2/user/2fa/recoverycodes"
 	userManageAPIKeyPath                   = "/api/v2/user/apikeyauth"
+	retentionBasePath                      = "/api/v2/retention/users"
+	retentionChecksPath                    = "/api/v2/retention/users/checks"
 	healthzPath                            = "/healthz"
 	webRootPathDefault                     = "/"
 	webBasePathDefault                     = "/web"

httpd/httpd_test.go

@@ -99,6 +99,7 @@ const (
 	userTOTPSavePath                = "/api/v2/user/totp/save"
 	user2FARecoveryCodesPath        = "/api/v2/user/2fa/recoverycodes"
 	userManageAPIKeyPath            = "/api/v2/user/apikeyauth"
+	retentionBasePath               = "/api/v2/retention/users"
 	healthzPath                     = "/healthz"
 	webBasePath                     = "/web"
 	webBasePathAdmin                = "/web/admin"
@@ -1542,6 +1543,84 @@ func TestUserType(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestRetentionAPI(t *testing.T) {
+	user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated)
+	assert.NoError(t, err)
+
+	checks, _, err := httpdtest.GetRetentionChecks(http.StatusOK)
+	assert.NoError(t, err)
+	assert.Len(t, checks, 0)
+
+	localFilePath := filepath.Join(user.HomeDir, "testdir", "testfile")
+	err = os.MkdirAll(filepath.Dir(localFilePath), os.ModePerm)
+	assert.NoError(t, err)
+	err = os.WriteFile(localFilePath, []byte("test data"), os.ModePerm)
+	assert.NoError(t, err)
+
+	folderRetention := []common.FolderRetention{
+		{
+			Path:            "/",
+			Retention:       0,
+			DeleteEmptyDirs: true,
+		},
+	}
+
+	_, err = httpdtest.StartRetentionCheck(altAdminUsername, folderRetention, http.StatusNotFound)
+	assert.NoError(t, err)
+
+	resp, err := httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusBadRequest)
+	assert.NoError(t, err)
+	assert.Contains(t, string(resp), "Invalid folders to check")
+
+	folderRetention[0].Retention = 24
+	_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+	assert.NoError(t, err)
+
+	assert.Eventually(t, func() bool {
+		return len(common.RetentionChecks.Get()) == 0
+	}, 1000*time.Millisecond, 50*time.Millisecond)
+
+	assert.FileExists(t, localFilePath)
+
+	err = os.Chtimes(localFilePath, time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour))
+	assert.NoError(t, err)
+
+	_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+	assert.NoError(t, err)
+
+	assert.Eventually(t, func() bool {
+		return len(common.RetentionChecks.Get()) == 0
+	}, 1000*time.Millisecond, 50*time.Millisecond)
+
+	assert.NoFileExists(t, localFilePath)
+	assert.NoDirExists(t, filepath.Dir(localFilePath))
+
+	check := common.RetentionCheck{
+		Folders: folderRetention,
+	}
+	c := common.RetentionChecks.Add(check, &user)
+	assert.NotNil(t, c)
+
+	_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusConflict)
+	assert.NoError(t, err)
+
+	c.Start()
+	assert.Len(t, common.RetentionChecks.Get(), 0)
+
+	token, err := getJWTAPITokenFromTestServer(defaultTokenAuthUser, defaultTokenAuthPass)
+	assert.NoError(t, err)
+	req, _ := http.NewRequest(http.MethodPost, retentionBasePath+"/"+user.Username+"/check",
+		bytes.NewBuffer([]byte("invalid json")))
+	setBearerForReq(req, token)
+	rr := executeRequest(req)
+	checkResponseCode(t, http.StatusBadRequest, rr)
+
+	_, err = httpdtest.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+}
+
 func TestAddUserInvalidVirtualFolders(t *testing.T) {
 	u := getTestUser()
 	folderName := "fname"
@@ -9470,6 +9549,11 @@ func TestWebAdminSetupMock(t *testing.T) {
 	rr = executeRequest(req)
 	checkResponseCode(t, http.StatusFound, rr)
 	assert.Equal(t, webAdminSetupPath, rr.Header().Get("Location"))
+	req, err = http.NewRequest(http.MethodGet, webClientLoginPath, nil)
+	assert.NoError(t, err)
+	rr = executeRequest(req)
+	checkResponseCode(t, http.StatusFound, rr)
+	assert.Equal(t, webAdminSetupPath, rr.Header().Get("Location"))
 
 	csrfToken, err := getCSRFToken(httpBaseURL + webAdminSetupPath)
 	assert.NoError(t, err)

httpd/schema/openapi.yaml

@@ -11,6 +11,7 @@ tags:
   - name: folders
   - name: users
   - name: users API
+  - name: data retention
 info:
   title: SFTPGo
   description: |
@@ -745,6 +746,76 @@ paths:
           $ref: '#/components/responses/InternalServerError'
         default:
           $ref: '#/components/responses/DefaultResponse'
+  /retention/users/checks:
+    get:
+      tags:
+        - data retention
+      summary: Get retention checks
+      description: Returns the active retention checks
+      operationId: get_users_retention_checks
+      responses:
+        '200':
+          description: successful operation
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/RetentionCheck'
+        '401':
+          $ref: '#/components/responses/Unauthorized'
+        '403':
+          $ref: '#/components/responses/Forbidden'
+        '500':
+          $ref: '#/components/responses/InternalServerError'
+        default:
+          $ref: '#/components/responses/DefaultResponse'
+  /retention/users/{username}/check:
+    parameters:
+      - name: username
+        in: path
+        description: the username
+        required: true
+        schema:
+          type: string
+    post:
+      tags:
+        - data retention
+      summary: Start a retention check
+      description: 'Starts a new retention check for the given user. If a retention check for this user is already active a 409 status code is returned'
+      operationId: start_user_retention_check
+      requestBody:
+        required: true
+        description: 'Defines virtual paths to check and their retention time in hours'
+        content:
+          application/json:
+            schema:
+              type: array
+              items:
+                $ref: '#/components/schemas/FolderRetention'
+      responses:
+        '202':
+          description: successful operation
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ApiResponse'
+              example:
+                message: Check started
+        '400':
+          $ref: '#/components/responses/BadRequest'
+        '401':
+          $ref: '#/components/responses/Unauthorized'
+        '403':
+          $ref: '#/components/responses/Forbidden'
+        '404':
+          $ref: '#/components/responses/NotFound'
+        '409':
+          $ref: '#/components/responses/Conflict'
+        '500':
+          $ref: '#/components/responses/InternalServerError'
+        default:
+          $ref: '#/components/responses/DefaultResponse'
   /quotas/users/scans:
     get:
       tags:
@@ -3084,6 +3155,7 @@ components:
         - manage_system
         - manage_defender
         - view_defender
+        - retention_checks
       description: |
         Admin permissions:
           * `*` - all permissions are granted
@@ -3096,8 +3168,11 @@ components:
           * `view_status` - view the server status is allowed
           * `manage_admins` - manage other admins is allowed
           * `manage_apikeys` - manage API keys is allowed
+          * `quota_scans` - view and start quota scans is allowed
+          * `manage_system` - backups and restores are allowed
           * `manage_defender` - remove ip from the dynamic blocklist is allowed
           * `view_defender` - list the dynamic blocklist is allowed
+          * `retention_checks` - view and start retention checks is allowed
     LoginMethods:
       type: string
       enum:
@@ -3224,7 +3299,7 @@ components:
       properties:
         path:
           type: string
-          description: 'exposed virtual path, if no other specific filter is defined, the filter apply for sub directories too. For example if filters are defined for the paths "/" and "/sub" then the filters for "/" are applied for any file outside the "/sub" directory'
+          description: 'exposed virtual path, if no other specific filter is defined, the filter applies for sub directories too. For example if filters are defined for the paths "/" and "/sub" then the filters for "/" are applied for any file outside the "/sub" directory'
         allowed_patterns:
           type: array
           items:
@@ -3334,6 +3409,7 @@ components:
             - GCP
             - AWS
             - VaultTransit
+            - AzureKeyVault
             - Redacted
           description: 'Set to "Plain" to add or update an existing secret, set to "Redacted" to preserve the existing value'
         payload:
@@ -3546,7 +3622,7 @@ components:
           description: list of usernames associated with this virtual folder
         filesystem:
           $ref: '#/components/schemas/FilesystemConfig'
-      description: Defines the filesystem for the virtual folder and the used quota limits. The same folder can be shared among multiple users and each user can have different quota limits or a different virtual path.
+      description: 'Defines the filesystem for the virtual folder and the used quota limits. The same folder can be shared among multiple users and each user can have different quota limits or a different virtual path.'
     VirtualFolder:
       allOf:
         - $ref: '#/components/schemas/BaseVirtualFolder'
@@ -3861,12 +3937,43 @@ components:
           type: array
           items:
             $ref: '#/components/schemas/Transfer'
+    FolderRetention:
+      type: object
+      properties:
+        path:
+          type: string
+          description: 'exposed virtual directory path, if no other specific retention is defined, the retention applies for sub directories too. For example if retention is defined for the paths "/" and "/sub" then the retention for "/" is applied for any file outside the "/sub" directory'
+          example: '/'
+        retention:
+          type: integer
+          description: retention time in hours. All the files with a modification time older than the defined value will be deleted. 0 means exclude this path
+          example: 24
+        delete_empty_dirs:
+          type: boolean
+          description: if enabled, empty directories will be deleted
+        ignore_user_permissions:
+          type: boolean
+          description: 'if enabled, files will be deleted even if the user does not have the delete permission. The default is "false" which means that files will be skipped if the user does not have permission to delete them. File patterns filters will always be silently ignored'
+    RetentionCheck:
+      type: object
+      properties:
+        username:
+          type: string
+          description: username to which the retention check refers
+        folders:
+          type: array
+          items:
+            $ref: '#/components/schemas/FolderRetention'
+        start_time:
+          type: integer
+          format: int64
+          description: check start time as unix timestamp in milliseconds
     QuotaScan:
       type: object
       properties:
         username:
           type: string
-          description: username with an active scan
+          description: username to which the quota scan refers
         start_time:
           type: integer
           format: int64
@@ -3876,7 +3983,7 @@ components:
       properties:
         name:
           type: string
-          description: folder name with an active scan
+          description: folder name to which the quota scan refers
         start_time:
           type: integer
           format: int64

httpd/server.go

@@ -133,6 +133,10 @@ func (s *httpdServer) renderClientLoginPage(w http.ResponseWriter, error string)
 
 func (s *httpdServer) handleClientWebLogin(w http.ResponseWriter, r *http.Request) {
 	r.Body = http.MaxBytesReader(w, r.Body, maxLoginBodySize)
+	if !dataprovider.HasAdmin() {
+		http.Redirect(w, r, webAdminSetupPath, http.StatusFound)
+		return
+	}
 	s.renderClientLoginPage(w, "")
 }
 
@@ -970,6 +974,9 @@ func (s *httpdServer) initializeRouter() {
 		router.With(checkPerm(dataprovider.PermAdminManageAdmins)).Put(adminPath+"/{username}", updateAdmin)
 		router.With(checkPerm(dataprovider.PermAdminManageAdmins)).Delete(adminPath+"/{username}", deleteAdmin)
 		router.With(checkPerm(dataprovider.PermAdminManageAdmins)).Put(adminPath+"/{username}/2fa/disable", disableAdmin2FA)
+		router.With(checkPerm(dataprovider.PermAdminRetentionChecks)).Get(retentionChecksPath, getRetentionChecks)
+		router.With(checkPerm(dataprovider.PermAdminRetentionChecks)).Post(retentionBasePath+"/{username}/check",
+			startRetentionCheck)
 		router.With(forbidAPIKeyAuthentication, checkPerm(dataprovider.PermAdminManageAPIKeys)).
 			Get(apiKeysPath, getAPIKeys)
 		router.With(forbidAPIKeyAuthentication, checkPerm(dataprovider.PermAdminManageAPIKeys)).

httpdtest/httpdtest.go

@@ -45,6 +45,8 @@ const (
 	adminPath             = "/api/v2/admins"
 	adminPwdPath          = "/api/v2/admin/changepwd"
 	apiKeysPath           = "/api/v2/apikeys"
+	retentionBasePath     = "/api/v2/retention/users"
+	retentionChecksPath   = "/api/v2/retention/users/checks"
 )
 
 const (
@@ -527,7 +529,40 @@ func UpdateQuotaUsage(user dataprovider.User, mode string, expectedStatusCode in
 	if err != nil {
 		return body, err
 	}
-	resp, err := sendHTTPRequest(http.MethodPut, url.String(), bytes.NewBuffer(userAsJSON), "", getDefaultToken())
+	resp, err := sendHTTPRequest(http.MethodPut, url.String(), bytes.NewBuffer(userAsJSON), "application/json",
+		getDefaultToken())
+	if err != nil {
+		return body, err
+	}
+	defer resp.Body.Close()
+	body, _ = getResponseBody(resp)
+	return body, checkResponse(resp.StatusCode, expectedStatusCode)
+}
+
+// GetRetentionChecks returns the active retention checks
+func GetRetentionChecks(expectedStatusCode int) ([]common.ActiveRetentionChecks, []byte, error) {
+	var checks []common.ActiveRetentionChecks
+	var body []byte
+	resp, err := sendHTTPRequest(http.MethodGet, buildURLRelativeToBase(retentionChecksPath), nil, "", getDefaultToken())
+	if err != nil {
+		return checks, body, err
+	}
+	defer resp.Body.Close()
+	err = checkResponse(resp.StatusCode, expectedStatusCode)
+	if err == nil && expectedStatusCode == http.StatusOK {
+		err = render.DecodeJSON(resp.Body, &checks)
+	} else {
+		body, _ = getResponseBody(resp)
+	}
+	return checks, body, err
+}
+
+// StartRetentionCheck starts a new retention check
+func StartRetentionCheck(username string, retention []common.FolderRetention, expectedStatusCode int) ([]byte, error) {
+	var body []byte
+	asJSON, _ := json.Marshal(retention)
+	resp, err := sendHTTPRequest(http.MethodPost, buildURLRelativeToBase(retentionBasePath, username, "check"),
+		bytes.NewBuffer(asJSON), "application/json", getDefaultToken())
 	if err != nil {
 		return body, err
 	}

mfa/mfa_test.go

@@ -100,7 +100,7 @@ func TestCleanupPasscodes(t *testing.T) {
 	assert.Eventually(t, func() bool {
 		_, ok := usedPasscodes.Load("key")
 		return !ok
-	}, 300*time.Millisecond, 100*time.Millisecond)
+	}, 1000*time.Millisecond, 100*time.Millisecond)
 	stopCleanupTicker()
 }
 

sdk/user.go

@@ -62,7 +62,7 @@ func (d *DirectoryPermissions) HasPerm(perm string) bool {
 // and they are not aware about these restrictions so they are not allowed
 // inside paths with extensions filters
 type PatternsFilter struct {
-	// Virtual path, if no other specific filter is defined, the filter apply for
+	// Virtual path, if no other specific filter is defined, the filter applies for
 	// sub directories too.
 	// For example if filters are defined for the paths "/" and "/sub" then the
 	// filters for "/" are applied for any file outside the "/sub" directory