always check recently updated users

also fix the query to get users for quota check for sql based providers

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>

common/protocol_test.go

@@ -695,6 +695,9 @@ func TestRootDirVirtualFolder(t *testing.T) {
 	})
 	user, _, err := httpdtest.AddUser(u, http.StatusCreated)
 	assert.NoError(t, err)
+	f, err := user.GetVirtualFolderForPath("/")
+	assert.NoError(t, err)
+	assert.Equal(t, "/", f.VirtualPath)
 	conn, client, err := getSftpClient(user)
 	if assert.NoError(t, err) {
 		defer conn.Close()

common/transferschecker_test.go

@@ -23,14 +23,25 @@ import (
 func TestTransfersCheckerDiskQuota(t *testing.T) {
 	username := "transfers_check_username"
 	folderName := "test_transfers_folder"
+	groupName := "test_transfers_group"
 	vdirPath := "/vdir"
+	group := dataprovider.Group{
+		BaseGroup: sdk.BaseGroup{
+			Name: groupName,
+		},
+		UserSettings: dataprovider.GroupUserSettings{
+			BaseGroupUserSettings: sdk.BaseGroupUserSettings{
+				QuotaSize: 120,
+			},
+		},
+	}
 	user := dataprovider.User{
 		BaseUser: sdk.BaseUser{
 			Username:  username,
 			Password:  "testpwd",
 			HomeDir:   filepath.Join(os.TempDir(), username),
 			Status:    1,
-			QuotaSize: 120,
+			QuotaSize: 0, // the quota size defined for the group is used
 			Permissions: map[string][]string{
 				"/": {dataprovider.PermAny},
 			},
@@ -45,11 +56,21 @@ func TestTransfersCheckerDiskQuota(t *testing.T) {
 				QuotaSize:   100,
 			},
 		},
+		Groups: []sdk.GroupMapping{
+			{
+				Name: groupName,
+				Type: sdk.GroupTypePrimary,
+			},
+		},
 	}
-
-	err := dataprovider.AddUser(&user, "", "")
+	err := dataprovider.AddGroup(&group, "", "")
+	assert.NoError(t, err)
+	group, err = dataprovider.GroupExists(groupName)
+	assert.NoError(t, err)
+	assert.Equal(t, int64(120), group.UserSettings.QuotaSize)
+	err = dataprovider.AddUser(&user, "", "")
 	assert.NoError(t, err)
-	user, err = dataprovider.UserExists(username)
+	user, err = dataprovider.GetUserWithGroupSettings(username)
 	assert.NoError(t, err)
 
 	connID1 := xid.New().String()
@@ -113,14 +134,14 @@ func TestTransfersCheckerDiskQuota(t *testing.T) {
 	atomic.StoreInt32(&Connections.transfersCheckStatus, 0)
 
 	Connections.checkTransfers()
-	assert.True(t, conn1.IsQuotaExceededError(transfer1.errAbort))
-	assert.True(t, conn2.IsQuotaExceededError(transfer2.errAbort))
+	assert.True(t, conn1.IsQuotaExceededError(transfer1.errAbort), transfer1.errAbort)
+	assert.True(t, conn2.IsQuotaExceededError(transfer2.errAbort), transfer2.errAbort)
 	assert.True(t, conn1.IsQuotaExceededError(transfer1.GetAbortError()))
 	assert.Nil(t, transfer3.errAbort)
 	assert.True(t, conn3.IsQuotaExceededError(transfer3.GetAbortError()))
 	// update the user quota size
-	user.QuotaSize = 1000
-	err = dataprovider.UpdateUser(&user, "", "")
+	group.UserSettings.QuotaSize = 1000
+	err = dataprovider.UpdateGroup(&group, []string{username}, "", "")
 	assert.NoError(t, err)
 	transfer1.errAbort = nil
 	transfer2.errAbort = nil
@@ -129,8 +150,8 @@ func TestTransfersCheckerDiskQuota(t *testing.T) {
 	assert.Nil(t, transfer2.errAbort)
 	assert.Nil(t, transfer3.errAbort)
 
-	user.QuotaSize = 0
-	err = dataprovider.UpdateUser(&user, "", "")
+	group.UserSettings.QuotaSize = 0
+	err = dataprovider.UpdateGroup(&group, []string{username}, "", "")
 	assert.NoError(t, err)
 	Connections.checkTransfers()
 	assert.Nil(t, transfer1.errAbort)
@@ -221,6 +242,8 @@ func TestTransfersCheckerDiskQuota(t *testing.T) {
 	assert.NoError(t, err)
 	err = os.RemoveAll(filepath.Join(os.TempDir(), folderName))
 	assert.NoError(t, err)
+	err = dataprovider.DeleteGroup(groupName, "", "")
+	assert.NoError(t, err)
 }
 
 func TestTransferCheckerTransferQuota(t *testing.T) {

dataprovider/bolt.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"path/filepath"
+	"sync/atomic"
 	"time"
 
 	bolt "go.etcd.io/bbolt"
@@ -24,6 +25,7 @@ const (
 )
 
 var (
+	lastUserUpdate  int64
 	usersBucket     = []byte("users")
 	groupsBucket    = []byte("groups")
 	foldersBucket   = []byte("folders")
@@ -185,6 +187,7 @@ func (p *BoltProvider) setUpdatedAt(username string) {
 		err = bucket.Put([]byte(username), buf)
 		if err == nil {
 			providerLog(logger.LevelDebug, "updated at set for user %#v", username)
+			setLastUserUpdate()
 		} else {
 			providerLog(logger.LevelWarn, "error setting updated_at for user %#v: %v", username, err)
 		}
@@ -514,11 +517,11 @@ func (p *BoltProvider) userExists(username string) (User, error) {
 		if u == nil {
 			return util.NewRecordNotFoundError(fmt.Sprintf("username %#v does not exist", username))
 		}
-		folderBucket, err := p.getFoldersBucket(tx)
+		foldersBucket, err := p.getFoldersBucket(tx)
 		if err != nil {
 			return err
 		}
-		user, err = p.joinUserAndFolders(u, folderBucket)
+		user, err = p.joinUserAndFolders(u, foldersBucket)
 		return err
 	})
 	return user, err
@@ -534,7 +537,7 @@ func (p *BoltProvider) addUser(user *User) error {
 		if err != nil {
 			return err
 		}
-		folderBucket, err := p.getFoldersBucket(tx)
+		foldersBucket, err := p.getFoldersBucket(tx)
 		if err != nil {
 			return err
 		}
@@ -559,7 +562,7 @@ func (p *BoltProvider) addUser(user *User) error {
 		user.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
 		user.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
 		for idx := range user.VirtualFolders {
-			err = p.addRelationToFolderMapping(&user.VirtualFolders[idx].BaseVirtualFolder, user, nil, folderBucket)
+			err = p.addRelationToFolderMapping(&user.VirtualFolders[idx].BaseVirtualFolder, user, nil, foldersBucket)
 			if err != nil {
 				return err
 			}
@@ -613,7 +616,12 @@ func (p *BoltProvider) updateUser(user *User) error {
 		if err != nil {
 			return err
 		}
-		return bucket.Put([]byte(user.Username), buf)
+
+		err = bucket.Put([]byte(user.Username), buf)
+		if err == nil {
+			setLastUserUpdate()
+		}
+		return err
 	})
 }
 
@@ -629,12 +637,12 @@ func (p *BoltProvider) deleteUser(user User) error {
 		}
 
 		if len(user.VirtualFolders) > 0 {
-			folderBucket, err := p.getFoldersBucket(tx)
+			foldersBucket, err := p.getFoldersBucket(tx)
 			if err != nil {
 				return err
 			}
 			for idx := range user.VirtualFolders {
-				err = p.removeRelationFromFolderMapping(user.VirtualFolders[idx], user.Username, "", folderBucket)
+				err = p.removeRelationFromFolderMapping(user.VirtualFolders[idx], user.Username, "", foldersBucket)
 				if err != nil {
 					return err
 				}
@@ -693,13 +701,13 @@ func (p *BoltProvider) dumpUsers() ([]User, error) {
 		if err != nil {
 			return err
 		}
-		folderBucket, err := p.getFoldersBucket(tx)
+		foldersBucket, err := p.getFoldersBucket(tx)
 		if err != nil {
 			return err
 		}
 		cursor := bucket.Cursor()
 		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			user, err := p.joinUserAndFolders(v, folderBucket)
+			user, err := p.joinUserAndFolders(v, foldersBucket)
 			if err != nil {
 				return err
 			}
@@ -710,14 +718,11 @@ func (p *BoltProvider) dumpUsers() ([]User, error) {
 	return users, err
 }
 
-// bolt provider cannot be shared, so we always return no recently updated users
 func (p *BoltProvider) getRecentlyUpdatedUsers(after int64) ([]User, error) {
-	return nil, nil
-}
-
-func (p *BoltProvider) getUsersForQuotaCheck(toFetch map[string]bool) ([]User, error) {
-	users := make([]User, 0, 30)
-
+	if getLastUserUpdate() < after {
+		return nil, nil
+	}
+	users := make([]User, 0, 10)
 	err := p.dbHandle.View(func(tx *bolt.Tx) error {
 		bucket, err := p.getUsersBucket(tx)
 		if err != nil {
@@ -738,22 +743,10 @@ func (p *BoltProvider) getUsersForQuotaCheck(toFetch map[string]bool) ([]User, e
 			if err != nil {
 				return err
 			}
-			needFolders, ok := toFetch[user.Username]
-			if !ok {
+			if user.UpdatedAt < after {
 				continue
 			}
-			if len(user.Groups) > 0 {
-				groupMapping := make(map[string]Group)
-				for idx := range user.Groups {
-					group, err := p.groupExistsInternal(user.Groups[idx].Name, groupsBucket)
-					if err != nil {
-						continue
-					}
-					groupMapping[group.Name] = group
-				}
-				user.applyGroupSettings(groupMapping)
-			}
-			if needFolders && len(user.VirtualFolders) > 0 {
+			if len(user.VirtualFolders) > 0 {
 				var folders []vfs.VirtualFolder
 				for idx := range user.VirtualFolders {
 					folder := &user.VirtualFolders[idx]
@@ -766,11 +759,79 @@ func (p *BoltProvider) getUsersForQuotaCheck(toFetch map[string]bool) ([]User, e
 				}
 				user.VirtualFolders = folders
 			}
-
+			if len(user.Groups) > 0 {
+				groupMapping := make(map[string]Group)
+				for idx := range user.Groups {
+					group, err := p.groupExistsInternal(user.Groups[idx].Name, groupsBucket)
+					if err != nil {
+						continue
+					}
+					groupMapping[group.Name] = group
+				}
+				user.applyGroupSettings(groupMapping)
+			}
 			user.SetEmptySecretsIfNil()
-			user.PrepareForRendering()
 			users = append(users, user)
 		}
+		return err
+	})
+	return users, err
+}
+
+func (p *BoltProvider) getUsersForQuotaCheck(toFetch map[string]bool) ([]User, error) {
+	users := make([]User, 0, 10)
+
+	err := p.dbHandle.View(func(tx *bolt.Tx) error {
+		bucket, err := p.getUsersBucket(tx)
+		if err != nil {
+			return err
+		}
+		foldersBucket, err := p.getFoldersBucket(tx)
+		if err != nil {
+			return err
+		}
+		groupsBucket, err := p.getGroupsBucket(tx)
+		if err != nil {
+			return err
+		}
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var user User
+			err := json.Unmarshal(v, &user)
+			if err != nil {
+				return err
+			}
+			if needFolders, ok := toFetch[user.Username]; ok {
+				if needFolders && len(user.VirtualFolders) > 0 {
+					var folders []vfs.VirtualFolder
+					for idx := range user.VirtualFolders {
+						folder := &user.VirtualFolders[idx]
+						baseFolder, err := p.folderExistsInternal(folder.Name, foldersBucket)
+						if err != nil {
+							continue
+						}
+						folder.BaseVirtualFolder = baseFolder
+						folders = append(folders, *folder)
+					}
+					user.VirtualFolders = folders
+				}
+				if len(user.Groups) > 0 {
+					groupMapping := make(map[string]Group)
+					for idx := range user.Groups {
+						group, err := p.groupExistsInternal(user.Groups[idx].Name, groupsBucket)
+						if err != nil {
+							continue
+						}
+						groupMapping[group.Name] = group
+					}
+					user.applyGroupSettings(groupMapping)
+				}
+
+				user.SetEmptySecretsIfNil()
+				user.PrepareForRendering()
+				users = append(users, user)
+			}
+		}
 		return nil
 	})
 
@@ -788,7 +849,7 @@ func (p *BoltProvider) getUsers(limit int, offset int, order string) ([]User, er
 		if err != nil {
 			return err
 		}
-		folderBucket, err := p.getFoldersBucket(tx)
+		foldersBucket, err := p.getFoldersBucket(tx)
 		if err != nil {
 			return err
 		}
@@ -800,7 +861,7 @@ func (p *BoltProvider) getUsers(limit int, offset int, order string) ([]User, er
 				if itNum <= offset {
 					continue
 				}
-				user, err := p.joinUserAndFolders(v, folderBucket)
+				user, err := p.joinUserAndFolders(v, foldersBucket)
 				if err != nil {
 					return err
 				}
@@ -816,7 +877,7 @@ func (p *BoltProvider) getUsers(limit int, offset int, order string) ([]User, er
 				if itNum <= offset {
 					continue
 				}
-				user, err := p.joinUserAndFolders(v, folderBucket)
+				user, err := p.joinUserAndFolders(v, foldersBucket)
 				if err != nil {
 					return err
 				}
@@ -1112,7 +1173,7 @@ func (p *BoltProvider) getGroups(limit, offset int, order string, minimal bool)
 		if err != nil {
 			return err
 		}
-		folderBucket, err := p.getFoldersBucket(tx)
+		foldersBucket, err := p.getFoldersBucket(tx)
 		if err != nil {
 			return err
 		}
@@ -1125,7 +1186,7 @@ func (p *BoltProvider) getGroups(limit, offset int, order string, minimal bool)
 					continue
 				}
 				var group Group
-				group, err = p.joinGroupAndFolders(v, folderBucket)
+				group, err = p.joinGroupAndFolders(v, foldersBucket)
 				if err != nil {
 					return err
 				}
@@ -1142,7 +1203,7 @@ func (p *BoltProvider) getGroups(limit, offset int, order string, minimal bool)
 					continue
 				}
 				var group Group
-				group, err = p.joinGroupAndFolders(v, folderBucket)
+				group, err = p.joinGroupAndFolders(v, foldersBucket)
 				if err != nil {
 					return err
 				}
@@ -1165,7 +1226,7 @@ func (p *BoltProvider) getGroupsWithNames(names []string) ([]Group, error) {
 		if err != nil {
 			return err
 		}
-		folderBucket, err := p.getFoldersBucket(tx)
+		foldersBucket, err := p.getFoldersBucket(tx)
 		if err != nil {
 			return err
 		}
@@ -1174,7 +1235,7 @@ func (p *BoltProvider) getGroupsWithNames(names []string) ([]Group, error) {
 			if g == nil {
 				continue
 			}
-			group, err := p.joinGroupAndFolders(g, folderBucket)
+			group, err := p.joinGroupAndFolders(g, foldersBucket)
 			if err != nil {
 				return err
 			}
@@ -1220,11 +1281,11 @@ func (p *BoltProvider) groupExists(name string) (Group, error) {
 		if g == nil {
 			return util.NewRecordNotFoundError(fmt.Sprintf("group %#v does not exist", name))
 		}
-		folderBucket, err := p.getFoldersBucket(tx)
+		foldersBucket, err := p.getFoldersBucket(tx)
 		if err != nil {
 			return err
 		}
-		group, err = p.joinGroupAndFolders(g, folderBucket)
+		group, err = p.joinGroupAndFolders(g, foldersBucket)
 		return err
 	})
 	return group, err
@@ -1239,7 +1300,7 @@ func (p *BoltProvider) addGroup(group *Group) error {
 		if err != nil {
 			return err
 		}
-		folderBucket, err := p.getFoldersBucket(tx)
+		foldersBucket, err := p.getFoldersBucket(tx)
 		if err != nil {
 			return err
 		}
@@ -1254,7 +1315,7 @@ func (p *BoltProvider) addGroup(group *Group) error {
 		group.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
 		group.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
 		for idx := range group.VirtualFolders {
-			err = p.addRelationToFolderMapping(&group.VirtualFolders[idx].BaseVirtualFolder, nil, group, folderBucket)
+			err = p.addRelationToFolderMapping(&group.VirtualFolders[idx].BaseVirtualFolder, nil, group, foldersBucket)
 			if err != nil {
 				return err
 			}
@@ -1276,7 +1337,7 @@ func (p *BoltProvider) updateGroup(group *Group) error {
 		if err != nil {
 			return err
 		}
-		folderBucket, err := p.getFoldersBucket(tx)
+		foldersBucket, err := p.getFoldersBucket(tx)
 		if err != nil {
 			return err
 		}
@@ -1290,13 +1351,13 @@ func (p *BoltProvider) updateGroup(group *Group) error {
 			return err
 		}
 		for idx := range oldGroup.VirtualFolders {
-			err = p.removeRelationFromFolderMapping(oldGroup.VirtualFolders[idx], "", oldGroup.Name, folderBucket)
+			err = p.removeRelationFromFolderMapping(oldGroup.VirtualFolders[idx], "", oldGroup.Name, foldersBucket)
 			if err != nil {
 				return err
 			}
 		}
 		for idx := range group.VirtualFolders {
-			err = p.addRelationToFolderMapping(&group.VirtualFolders[idx].BaseVirtualFolder, nil, group, folderBucket)
+			err = p.addRelationToFolderMapping(&group.VirtualFolders[idx].BaseVirtualFolder, nil, group, foldersBucket)
 			if err != nil {
 				return err
 			}
@@ -1330,12 +1391,12 @@ func (p *BoltProvider) deleteGroup(group Group) error {
 		if len(oldGroup.Users) > 0 {
 			return util.NewValidationError(fmt.Sprintf("the group %#v is referenced, it cannot be removed", group.Name))
 		}
-		folderBucket, err := p.getFoldersBucket(tx)
+		foldersBucket, err := p.getFoldersBucket(tx)
 		if err != nil {
 			return err
 		}
 		for idx := range group.VirtualFolders {
-			err = p.removeRelationFromFolderMapping(group.VirtualFolders[idx], "", group.Name, folderBucket)
+			err = p.removeRelationFromFolderMapping(group.VirtualFolders[idx], "", group.Name, foldersBucket)
 			if err != nil {
 				return err
 			}
@@ -1352,13 +1413,13 @@ func (p *BoltProvider) dumpGroups() ([]Group, error) {
 		if err != nil {
 			return err
 		}
-		folderBucket, err := p.getFoldersBucket(tx)
+		foldersBucket, err := p.getFoldersBucket(tx)
 		if err != nil {
 			return err
 		}
 		cursor := bucket.Cursor()
 		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			group, err := p.joinGroupAndFolders(v, folderBucket)
+			group, err := p.joinGroupAndFolders(v, foldersBucket)
 			if err != nil {
 				return err
 			}
@@ -2144,7 +2205,7 @@ func (p *BoltProvider) removeRelationFromFolderMapping(folder vfs.VirtualFolder,
 }
 
 func (p *BoltProvider) updateUserRelations(tx *bolt.Tx, user *User, oldUser User) error {
-	folderBucket, err := p.getFoldersBucket(tx)
+	foldersBucket, err := p.getFoldersBucket(tx)
 	if err != nil {
 		return err
 	}
@@ -2153,7 +2214,7 @@ func (p *BoltProvider) updateUserRelations(tx *bolt.Tx, user *User, oldUser User
 		return err
 	}
 	for idx := range oldUser.VirtualFolders {
-		err = p.removeRelationFromFolderMapping(oldUser.VirtualFolders[idx], oldUser.Username, "", folderBucket)
+		err = p.removeRelationFromFolderMapping(oldUser.VirtualFolders[idx], oldUser.Username, "", foldersBucket)
 		if err != nil {
 			return err
 		}
@@ -2165,7 +2226,7 @@ func (p *BoltProvider) updateUserRelations(tx *bolt.Tx, user *User, oldUser User
 		}
 	}
 	for idx := range user.VirtualFolders {
-		err = p.addRelationToFolderMapping(&user.VirtualFolders[idx].BaseVirtualFolder, user, nil, folderBucket)
+		err = p.addRelationToFolderMapping(&user.VirtualFolders[idx].BaseVirtualFolder, user, nil, foldersBucket)
 		if err != nil {
 			return err
 		}
@@ -2320,6 +2381,14 @@ func (p *BoltProvider) getFoldersBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
 	return bucket, err
 }
 
+func setLastUserUpdate() {
+	atomic.StoreInt64(&lastUserUpdate, util.GetTimeAsMsSinceEpoch(time.Now()))
+}
+
+func getLastUserUpdate() int64 {
+	return atomic.LoadInt64(&lastUserUpdate)
+}
+
 func getBoltDatabaseVersion(dbHandle *bolt.DB) (schemaVersion, error) {
 	var dbVersion schemaVersion
 	err := dbHandle.View(func(tx *bolt.Tx) error {

dataprovider/memory.go

@@ -182,6 +182,7 @@ func (p *MemoryProvider) setUpdatedAt(username string) {
 	}
 	user.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
 	p.dbHandle.users[user.Username] = user
+	setLastUserUpdate()
 }
 
 func (p *MemoryProvider) updateLastLogin(username string) error {
@@ -360,6 +361,7 @@ func (p *MemoryProvider) updateUser(user *User) error {
 	user.ID = u.ID
 	// pre-login and external auth hook will use the passed *user so save a copy
 	p.dbHandle.users[user.Username] = user.getACopy()
+	setLastUserUpdate()
 	return nil
 }
 
@@ -439,9 +441,40 @@ func (p *MemoryProvider) dumpFolders() ([]vfs.BaseVirtualFolder, error) {
 	return folders, nil
 }
 
-// memory provider cannot be shared, so we always return no recently updated users
 func (p *MemoryProvider) getRecentlyUpdatedUsers(after int64) ([]User, error) {
-	return nil, nil
+	if getLastUserUpdate() < after {
+		return nil, nil
+	}
+	p.dbHandle.Lock()
+	defer p.dbHandle.Unlock()
+	if p.dbHandle.isClosed {
+		return nil, errMemoryProviderClosed
+	}
+	users := make([]User, 0, 10)
+	for _, username := range p.dbHandle.usernames {
+		u := p.dbHandle.users[username]
+		if u.UpdatedAt < after {
+			continue
+		}
+		user := u.getACopy()
+		p.addVirtualFoldersToUser(&user)
+		if len(user.Groups) > 0 {
+			groupMapping := make(map[string]Group)
+			for idx := range user.Groups {
+				group, err := p.groupExistsInternal(user.Groups[idx].Name)
+				if err != nil {
+					continue
+				}
+				groupMapping[group.Name] = group
+			}
+			user.applyGroupSettings(groupMapping)
+		}
+
+		user.SetEmptySecretsIfNil()
+		users = append(users, user)
+	}
+
+	return users, nil
 }
 
 func (p *MemoryProvider) getUsersForQuotaCheck(toFetch map[string]bool) ([]User, error) {
@@ -452,9 +485,12 @@ func (p *MemoryProvider) getUsersForQuotaCheck(toFetch map[string]bool) ([]User,
 		return users, errMemoryProviderClosed
 	}
 	for _, username := range p.dbHandle.usernames {
-		if val, ok := toFetch[username]; ok {
+		if needFolders, ok := toFetch[username]; ok {
 			u := p.dbHandle.users[username]
 			user := u.getACopy()
+			if needFolders {
+				p.addVirtualFoldersToUser(&user)
+			}
 			if len(user.Groups) > 0 {
 				groupMapping := make(map[string]Group)
 				for idx := range user.Groups {
@@ -466,9 +502,6 @@ func (p *MemoryProvider) getUsersForQuotaCheck(toFetch map[string]bool) ([]User,
 				}
 				user.applyGroupSettings(groupMapping)
 			}
-			if val {
-				p.addVirtualFoldersToUser(&user)
-			}
 			user.SetEmptySecretsIfNil()
 			user.PrepareForRendering()
 			users = append(users, user)

dataprovider/scheduler.go

@@ -49,13 +49,6 @@ func startScheduler() error {
 }
 
 func addScheduledCacheUpdates() error {
-	if config.IsShared == 0 {
-		return nil
-	}
-	if !util.IsStringInSlice(config.Driver, sharedProviders) {
-		providerLog(logger.LevelError, "update caches not supported for provider %v", config.Driver)
-		return nil
-	}
 	lastCachesUpdate = util.GetTimeAsMsSinceEpoch(time.Now())
 	_, err := scheduler.AddFunc("@every 10m", checkCacheUpdates)
 	if err != nil {

dataprovider/sqlcommon.go

@@ -1288,9 +1288,12 @@ func sqlCommonGetRecentlyUpdatedUsers(after int64, dbHandle sqlQuerier) ([]User,
 	if err != nil {
 		return users, err
 	}
+	if len(groups) == 0 {
+		return users, nil
+	}
 	groupsMapping := make(map[string]Group)
-	for _, g := range groups {
-		groupsMapping[g.Name] = g
+	for idx := range groups {
+		groupsMapping[groups[idx].Name] = groups[idx]
 	}
 	for idx := range users {
 		ref := &users[idx]
@@ -1332,9 +1335,6 @@ func sqlCommonGetUsersForQuotaCheck(toFetch map[string]bool, dbHandle sqlQuerier
 		}
 	}
 	users = users[:validIdx]
-	if len(usersWithFolders) == 0 {
-		return users, nil
-	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
 	defer cancel()
@@ -1355,13 +1355,16 @@ func sqlCommonGetUsersForQuotaCheck(toFetch map[string]bool, dbHandle sqlQuerier
 		}
 	}
 	groupNames = util.RemoveDuplicates(groupNames)
+	if len(groupNames) == 0 {
+		return users, nil
+	}
 	groups, err := sqlCommonGetGroupsWithNames(groupNames, dbHandle)
 	if err != nil {
 		return users, err
 	}
 	groupsMapping := make(map[string]Group)
-	for _, g := range groups {
-		groupsMapping[g.Name] = g
+	for idx := range groups {
+		groupsMapping[groups[idx].Name] = groups[idx]
 	}
 	for idx := range users {
 		ref := &users[idx]

dataprovider/sqlite.go

@@ -252,9 +252,8 @@ func (p *SQLiteProvider) dumpUsers() ([]User, error) {
 	return sqlCommonDumpUsers(p.dbHandle)
 }
 
-// SQLite provider cannot be shared, so we always return no recently updated users
 func (p *SQLiteProvider) getRecentlyUpdatedUsers(after int64) ([]User, error) {
-	return nil, nil
+	return sqlCommonGetRecentlyUpdatedUsers(after, p.dbHandle)
 }
 
 func (p *SQLiteProvider) getUsers(limit int, offset int, order string) ([]User, error) {

dataprovider/user.go

@@ -1676,6 +1676,7 @@ func (u *User) removeDuplicatesAfterGroupMerge() {
 	u.Filters.DeniedProtocols = util.RemoveDuplicates(u.Filters.DeniedProtocols)
 	u.Filters.WebClient = util.RemoveDuplicates(u.Filters.WebClient)
 	u.Filters.TwoFactorAuthProtocols = util.RemoveDuplicates(u.Filters.TwoFactorAuthProtocols)
+	u.SetEmptySecretsIfNil()
 	u.groupSettingsApplied = true
 }
 

docs/full-configuration.md

@@ -224,7 +224,7 @@ The configuration file contains the following sections:
   - `update_mode`, integer. Defines how the database will be initialized/updated. 0 means automatically. 1 means manually using the initprovider sub-command.
   - `create_default_admin`, boolean. Before you can use SFTPGo you need to create an admin account. If you open the admin web UI, a setup screen will guide you in creating the first admin account. You can automatically create the first admin account by enabling this setting and setting the environment variables `SFTPGO_DEFAULT_ADMIN_USERNAME` and `SFTPGO_DEFAULT_ADMIN_PASSWORD`. You can also create the first admin by loading initial data. This setting has no effect if an admin account is already found within the data provider. Default `false`.
   - `naming_rules`, integer. Naming rules for usernames and folder names. `0` means no rules. `1` means you can use any UTF-8 character. The names are used in URIs for REST API and Web admin. If not set only unreserved URI characters are allowed: ALPHA / DIGIT / "-" / "." / "_" / "~". `2` means names are converted to lowercase before saving/matching and so case insensitive matching is possible. `3` means trimming trailing and leading white spaces before saving/matching. Rules can be combined, for example `3` means both converting to lowercase and allowing any UTF-8 character. Enabling these options for existing installations could be backward incompatible, some users could be unable to login, for example existing users with mixed cases in their usernames. You have to ensure that all existing users respect the defined rules. Default: `0`.
-  - `is_shared`, integer. If the data provider is shared across multiple SFTPGo instances, set this parameter to `1`. `MySQL`, `PostgreSQL` and `CockroachDB` can be shared, this setting is ignored for other data providers. For shared data providers, SFTPGo periodically reloads the latest updated users, based on the `updated_at` field, and updates its internal caches if users are updated from a different instance. This check, if enabled, is executed every 10 minutes. For shared data providers, active transfers are persisted in the database and thus quota checks between ongoing transfers will work cross multiple instances. Default: `0`.
+  - `is_shared`, integer. If the data provider is shared across multiple SFTPGo instances, set this parameter to `1`. `MySQL`, `PostgreSQL` and `CockroachDB` can be shared, this setting is ignored for other data providers. For shared data providers, active transfers are persisted in the database and thus quota checks between ongoing transfers will work cross multiple instances. Default: `0`.
   - `backups_path`, string. Path to the backup directory. This can be an absolute path or a path relative to the config dir. We don't allow backups in arbitrary paths for security reasons.
   - `auto_backup`, struct. Defines the configuration for automatic data provider backups. Example: hour `0` and day_of_week `*` means a backup every day at midnight. The backup file name is in the format `backup_<day_of_week>_<hour>.json`, files with the same name will be overwritten. Note, this process will only backup provider data (users, folders, shars, admins, api keys) and will not backup the configuration file and users files.
     - `enabled`, boolean. Set to `true` to enable automatic backups. Default: `true`.

go.mod

@@ -67,9 +67,9 @@ require (
 	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4
 	golang.org/x/net v0.0.0-20220421235706-1d1ef9303861
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
-	golang.org/x/sys v0.0.0-20220422013727-9388b58f7150
+	golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32
 	golang.org/x/time v0.0.0-20220411224347-583f2d630306
-	google.golang.org/api v0.76.0
+	google.golang.org/api v0.77.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
@@ -152,7 +152,7 @@ require (
 	golang.org/x/tools v0.1.10 // indirect
 	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46 // indirect
+	google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e // indirect
 	google.golang.org/grpc v1.46.0 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect

go.sum

@@ -965,8 +965,8 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc=
-golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32 h1:Js08h5hqB5xyWR789+QqueR6sDE8mk+YvpETZ+F6X9Y=
+golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1102,8 +1102,8 @@ google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/S
 google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
 google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
 google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
-google.golang.org/api v0.76.0 h1:UkZl25bR1FHNqtK/EKs3vCdpZtUO6gea3YElTwc8pQg=
-google.golang.org/api v0.76.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
+google.golang.org/api v0.77.0 h1:msijLTxwkJ7Jub5tv9KBVCKtHOQwnvnvkX7ErFFCVxY=
+google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1203,8 +1203,8 @@ google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX
 google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
 google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
 google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46 h1:G1IeWbjrqEq9ChWxEuRPJu6laA67+XgTFHVSAvepr38=
-google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e h1:gMjH4zLGs9m+dGzR7qHCHaXMOwsJHJKKkHtyXhtOrJk=
+google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
 google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=