beenull/sftpgo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

always check recently updated users

Browse source 
also fix the query to get users for quota check for sql based providers

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2022-04-30 11:59:36 +02:00
parent dd9c5b2149
commit 14fb6c4038
No known key found for this signature in database
GPG key ID: 2F1FB59433D5A8CB
11 changed files with 227 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
common/protocol_test.go
View file

@ -695,6 +695,9 @@ func TestRootDirVirtualFolder(t *testing.T) {
})
user, _, err := httpdtest.AddUser(u, http.StatusCreated)
assert.NoError(t, err)
f, err := user.GetVirtualFolderForPath("/")
assert.NoError(t, err)
assert.Equal(t, "/", f.VirtualPath)
conn, client, err := getSftpClient(user)
if assert.NoError(t, err) {
defer conn.Close()

43
common/transferschecker_test.go
View file

@ -23,14 +23,25 @@ import (
func TestTransfersCheckerDiskQuota(t *testing.T) {
username := "transfers_check_username"
folderName := "test_transfers_folder"
groupName := "test_transfers_group"
vdirPath := "/vdir"
group := dataprovider.Group{
BaseGroup: sdk.BaseGroup{
Name: groupName,
},
UserSettings: dataprovider.GroupUserSettings{
BaseGroupUserSettings: sdk.BaseGroupUserSettings{
QuotaSize: 120,
},
},
}
user := dataprovider.User{
BaseUser: sdk.BaseUser{
Username: username,
Password: "testpwd",
HomeDir: filepath.Join(os.TempDir(), username),
Status: 1,
QuotaSize: 120,
QuotaSize: 0, // the quota size defined for the group is used
Permissions: map[string][]string{
"/": {dataprovider.PermAny},
},
@ -45,11 +56,21 @@ func TestTransfersCheckerDiskQuota(t *testing.T) {
QuotaSize: 100,
},
},
Groups: []sdk.GroupMapping{
{
Name: groupName,
Type: sdk.GroupTypePrimary,
},
},
}
err := dataprovider.AddUser(&user, "", "")
err := dataprovider.AddGroup(&group, "", "")
assert.NoError(t, err)
user, err = dataprovider.UserExists(username)
group, err = dataprovider.GroupExists(groupName)
assert.NoError(t, err)
assert.Equal(t, int64(120), group.UserSettings.QuotaSize)
err = dataprovider.AddUser(&user, "", "")
assert.NoError(t, err)
user, err = dataprovider.GetUserWithGroupSettings(username)
assert.NoError(t, err)
connID1 := xid.New().String()
@ -113,14 +134,14 @@ func TestTransfersCheckerDiskQuota(t *testing.T) {
atomic.StoreInt32(&Connections.transfersCheckStatus, 0)
Connections.checkTransfers()
assert.True(t, conn1.IsQuotaExceededError(transfer1.errAbort))
assert.True(t, conn2.IsQuotaExceededError(transfer2.errAbort))
assert.True(t, conn1.IsQuotaExceededError(transfer1.errAbort), transfer1.errAbort)
assert.True(t, conn2.IsQuotaExceededError(transfer2.errAbort), transfer2.errAbort)
assert.True(t, conn1.IsQuotaExceededError(transfer1.GetAbortError()))
assert.Nil(t, transfer3.errAbort)
assert.True(t, conn3.IsQuotaExceededError(transfer3.GetAbortError()))
// update the user quota size
user.QuotaSize = 1000
err = dataprovider.UpdateUser(&user, "", "")
group.UserSettings.QuotaSize = 1000
err = dataprovider.UpdateGroup(&group, []string{username}, "", "")
assert.NoError(t, err)
transfer1.errAbort = nil
transfer2.errAbort = nil
@ -129,8 +150,8 @@ func TestTransfersCheckerDiskQuota(t *testing.T) {
assert.Nil(t, transfer2.errAbort)
assert.Nil(t, transfer3.errAbort)
user.QuotaSize = 0
err = dataprovider.UpdateUser(&user, "", "")
group.UserSettings.QuotaSize = 0
err = dataprovider.UpdateGroup(&group, []string{username}, "", "")
assert.NoError(t, err)
Connections.checkTransfers()
assert.Nil(t, transfer1.errAbort)
@ -221,6 +242,8 @@ func TestTransfersCheckerDiskQuota(t *testing.T) {
assert.NoError(t, err)
err = os.RemoveAll(filepath.Join(os.TempDir(), folderName))
assert.NoError(t, err)
err = dataprovider.DeleteGroup(groupName, "", "")
assert.NoError(t, err)
}
func TestTransferCheckerTransferQuota(t *testing.T) {

165
dataprovider/bolt.go
View file

@ -9,6 +9,7 @@ import (
"errors"
"fmt"
"path/filepath"
"sync/atomic"
"time"
bolt "go.etcd.io/bbolt"
@ -24,6 +25,7 @@ const (
)
var (
lastUserUpdate int64
usersBucket = []byte("users")
groupsBucket = []byte("groups")
foldersBucket = []byte("folders")
@ -185,6 +187,7 @@ func (p *BoltProvider) setUpdatedAt(username string) {
err = bucket.Put([]byte(username), buf)
if err == nil {
providerLog(logger.LevelDebug, "updated at set for user %#v", username)
setLastUserUpdate()
} else {
providerLog(logger.LevelWarn, "error setting updated_at for user %#v: %v", username, err)
}
@ -514,11 +517,11 @@ func (p *BoltProvider) userExists(username string) (User, error) {
if u == nil {
return util.NewRecordNotFoundError(fmt.Sprintf("username %#v does not exist", username))
}
folderBucket, err := p.getFoldersBucket(tx)
foldersBucket, err := p.getFoldersBucket(tx)
if err != nil {
return err
}
user, err = p.joinUserAndFolders(u, folderBucket)
user, err = p.joinUserAndFolders(u, foldersBucket)
return err
})
return user, err
@ -534,7 +537,7 @@ func (p *BoltProvider) addUser(user *User) error {
if err != nil {
return err
}
folderBucket, err := p.getFoldersBucket(tx)
foldersBucket, err := p.getFoldersBucket(tx)
if err != nil {
return err
}
@ -559,7 +562,7 @@ func (p *BoltProvider) addUser(user *User) error {
user.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
user.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
for idx := range user.VirtualFolders {
err = p.addRelationToFolderMapping(&user.VirtualFolders[idx].BaseVirtualFolder, user, nil, folderBucket)
err = p.addRelationToFolderMapping(&user.VirtualFolders[idx].BaseVirtualFolder, user, nil, foldersBucket)
if err != nil {
return err
}
@ -613,7 +616,12 @@ func (p *BoltProvider) updateUser(user *User) error {
if err != nil {
return err
}
return bucket.Put([]byte(user.Username), buf)
err = bucket.Put([]byte(user.Username), buf)
if err == nil {
setLastUserUpdate()
}
return err
})
}
@ -629,12 +637,12 @@ func (p *BoltProvider) deleteUser(user User) error {
}
if len(user.VirtualFolders) > 0 {
folderBucket, err := p.getFoldersBucket(tx)
foldersBucket, err := p.getFoldersBucket(tx)
if err != nil {
return err
}
for idx := range user.VirtualFolders {
err = p.removeRelationFromFolderMapping(user.VirtualFolders[idx], user.Username, "", folderBucket)
err = p.removeRelationFromFolderMapping(user.VirtualFolders[idx], user.Username, "", foldersBucket)
if err != nil {
return err
}
@ -693,13 +701,13 @@ func (p *BoltProvider) dumpUsers() ([]User, error) {
if err != nil {
return err
}
folderBucket, err := p.getFoldersBucket(tx)
foldersBucket, err := p.getFoldersBucket(tx)
if err != nil {
return err
}
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
user, err := p.joinUserAndFolders(v, folderBucket)
user, err := p.joinUserAndFolders(v, foldersBucket)
if err != nil {
return err
}
@ -710,13 +718,68 @@ func (p *BoltProvider) dumpUsers() ([]User, error) {
return users, err
}
// bolt provider cannot be shared, so we always return no recently updated users
func (p *BoltProvider) getRecentlyUpdatedUsers(after int64) ([]User, error) {
if getLastUserUpdate() < after {
return nil, nil
}
users := make([]User, 0, 10)
err := p.dbHandle.View(func(tx *bolt.Tx) error {
bucket, err := p.getUsersBucket(tx)
if err != nil {
return err
}
foldersBucket, err := p.getFoldersBucket(tx)
if err != nil {
return err
}
groupsBucket, err := p.getGroupsBucket(tx)
if err != nil {
return err
}
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
var user User
err := json.Unmarshal(v, &user)
if err != nil {
return err
}
if user.UpdatedAt < after {
continue
}
if len(user.VirtualFolders) > 0 {
var folders []vfs.VirtualFolder
for idx := range user.VirtualFolders {
folder := &user.VirtualFolders[idx]
baseFolder, err := p.folderExistsInternal(folder.Name, foldersBucket)
if err != nil {
continue
}
folder.BaseVirtualFolder = baseFolder
folders = append(folders, *folder)
}
user.VirtualFolders = folders
}
if len(user.Groups) > 0 {
groupMapping := make(map[string]Group)
for idx := range user.Groups {
group, err := p.groupExistsInternal(user.Groups[idx].Name, groupsBucket)
if err != nil {
continue
}
groupMapping[group.Name] = group
}
user.applyGroupSettings(groupMapping)
}
user.SetEmptySecretsIfNil()
users = append(users, user)
}
return err
})
return users, err
}
func (p *BoltProvider) getUsersForQuotaCheck(toFetch map[string]bool) ([]User, error) {
users := make([]User, 0, 30)
users := make([]User, 0, 10)
err := p.dbHandle.View(func(tx *bolt.Tx) error {
bucket, err := p.getUsersBucket(tx)
@ -738,21 +801,7 @@ func (p *BoltProvider) getUsersForQuotaCheck(toFetch map[string]bool) ([]User, e
if err != nil {
return err
}
needFolders, ok := toFetch[user.Username]
if !ok {
continue
}
if len(user.Groups) > 0 {
groupMapping := make(map[string]Group)
for idx := range user.Groups {
group, err := p.groupExistsInternal(user.Groups[idx].Name, groupsBucket)
if err != nil {
continue
}
groupMapping[group.Name] = group
}
user.applyGroupSettings(groupMapping)
}
if needFolders, ok := toFetch[user.Username]; ok {
if needFolders && len(user.VirtualFolders) > 0 {
var folders []vfs.VirtualFolder
for idx := range user.VirtualFolders {
@ -766,11 +815,23 @@ func (p *BoltProvider) getUsersForQuotaCheck(toFetch map[string]bool) ([]User, e
}
user.VirtualFolders = folders
}
if len(user.Groups) > 0 {
groupMapping := make(map[string]Group)
for idx := range user.Groups {
group, err := p.groupExistsInternal(user.Groups[idx].Name, groupsBucket)
if err != nil {
continue
}
groupMapping[group.Name] = group
}
user.applyGroupSettings(groupMapping)
}
user.SetEmptySecretsIfNil()
user.PrepareForRendering()
users = append(users, user)
}
}
return nil
})
@ -788,7 +849,7 @@ func (p *BoltProvider) getUsers(limit int, offset int, order string) ([]User, er
if err != nil {
return err
}
folderBucket, err := p.getFoldersBucket(tx)
foldersBucket, err := p.getFoldersBucket(tx)
if err != nil {
return err
}
@ -800,7 +861,7 @@ func (p *BoltProvider) getUsers(limit int, offset int, order string) ([]User, er
if itNum <= offset {
continue
}
user, err := p.joinUserAndFolders(v, folderBucket)
user, err := p.joinUserAndFolders(v, foldersBucket)
if err != nil {
return err
}
@ -816,7 +877,7 @@ func (p *BoltProvider) getUsers(limit int, offset int, order string) ([]User, er
if itNum <= offset {
continue
}
user, err := p.joinUserAndFolders(v, folderBucket)
user, err := p.joinUserAndFolders(v, foldersBucket)
if err != nil {
return err
}
@ -1112,7 +1173,7 @@ func (p *BoltProvider) getGroups(limit, offset int, order string, minimal bool)
if err != nil {
return err
}
folderBucket, err := p.getFoldersBucket(tx)
foldersBucket, err := p.getFoldersBucket(tx)
if err != nil {
return err
}
@ -1125,7 +1186,7 @@ func (p *BoltProvider) getGroups(limit, offset int, order string, minimal bool)
continue
}
var group Group
group, err = p.joinGroupAndFolders(v, folderBucket)
group, err = p.joinGroupAndFolders(v, foldersBucket)
if err != nil {
return err
}
@ -1142,7 +1203,7 @@ func (p *BoltProvider) getGroups(limit, offset int, order string, minimal bool)
continue
}
var group Group
group, err = p.joinGroupAndFolders(v, folderBucket)
group, err = p.joinGroupAndFolders(v, foldersBucket)
if err != nil {
return err
}
@ -1165,7 +1226,7 @@ func (p *BoltProvider) getGroupsWithNames(names []string) ([]Group, error) {
if err != nil {
return err
}
folderBucket, err := p.getFoldersBucket(tx)
foldersBucket, err := p.getFoldersBucket(tx)
if err != nil {
return err
}
@ -1174,7 +1235,7 @@ func (p *BoltProvider) getGroupsWithNames(names []string) ([]Group, error) {
if g == nil {
continue
}
group, err := p.joinGroupAndFolders(g, folderBucket)
group, err := p.joinGroupAndFolders(g, foldersBucket)
if err != nil {
return err
}
@ -1220,11 +1281,11 @@ func (p *BoltProvider) groupExists(name string) (Group, error) {
if g == nil {
return util.NewRecordNotFoundError(fmt.Sprintf("group %#v does not exist", name))
}
folderBucket, err := p.getFoldersBucket(tx)
foldersBucket, err := p.getFoldersBucket(tx)
if err != nil {
return err
}
group, err = p.joinGroupAndFolders(g, folderBucket)
group, err = p.joinGroupAndFolders(g, foldersBucket)
return err
})
return group, err
@ -1239,7 +1300,7 @@ func (p *BoltProvider) addGroup(group *Group) error {
if err != nil {
return err
}
folderBucket, err := p.getFoldersBucket(tx)
foldersBucket, err := p.getFoldersBucket(tx)
if err != nil {
return err
}
@ -1254,7 +1315,7 @@ func (p *BoltProvider) addGroup(group *Group) error {
group.CreatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
group.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
for idx := range group.VirtualFolders {
err = p.addRelationToFolderMapping(&group.VirtualFolders[idx].BaseVirtualFolder, nil, group, folderBucket)
err = p.addRelationToFolderMapping(&group.VirtualFolders[idx].BaseVirtualFolder, nil, group, foldersBucket)
if err != nil {
return err
}
@ -1276,7 +1337,7 @@ func (p *BoltProvider) updateGroup(group *Group) error {
if err != nil {
return err
}
folderBucket, err := p.getFoldersBucket(tx)
foldersBucket, err := p.getFoldersBucket(tx)
if err != nil {
return err
}
@ -1290,13 +1351,13 @@ func (p *BoltProvider) updateGroup(group *Group) error {
return err
}
for idx := range oldGroup.VirtualFolders {
err = p.removeRelationFromFolderMapping(oldGroup.VirtualFolders[idx], "", oldGroup.Name, folderBucket)
err = p.removeRelationFromFolderMapping(oldGroup.VirtualFolders[idx], "", oldGroup.Name, foldersBucket)
if err != nil {
return err
}
}
for idx := range group.VirtualFolders {
err = p.addRelationToFolderMapping(&group.VirtualFolders[idx].BaseVirtualFolder, nil, group, folderBucket)
err = p.addRelationToFolderMapping(&group.VirtualFolders[idx].BaseVirtualFolder, nil, group, foldersBucket)
if err != nil {
return err
}
@ -1330,12 +1391,12 @@ func (p *BoltProvider) deleteGroup(group Group) error {
if len(oldGroup.Users) > 0 {
return util.NewValidationError(fmt.Sprintf("the group %#v is referenced, it cannot be removed", group.Name))
}
folderBucket, err := p.getFoldersBucket(tx)
foldersBucket, err := p.getFoldersBucket(tx)
if err != nil {
return err
}
for idx := range group.VirtualFolders {
err = p.removeRelationFromFolderMapping(group.VirtualFolders[idx], "", group.Name, folderBucket)
err = p.removeRelationFromFolderMapping(group.VirtualFolders[idx], "", group.Name, foldersBucket)
if err != nil {
return err
}
@ -1352,13 +1413,13 @@ func (p *BoltProvider) dumpGroups() ([]Group, error) {
if err != nil {
return err
}
folderBucket, err := p.getFoldersBucket(tx)
foldersBucket, err := p.getFoldersBucket(tx)
if err != nil {
return err
}
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
group, err := p.joinGroupAndFolders(v, folderBucket)
group, err := p.joinGroupAndFolders(v, foldersBucket)
if err != nil {
return err
}
@ -2144,7 +2205,7 @@ func (p *BoltProvider) removeRelationFromFolderMapping(folder vfs.VirtualFolder,
}
func (p *BoltProvider) updateUserRelations(tx *bolt.Tx, user *User, oldUser User) error {
folderBucket, err := p.getFoldersBucket(tx)
foldersBucket, err := p.getFoldersBucket(tx)
if err != nil {
return err
}
@ -2153,7 +2214,7 @@ func (p *BoltProvider) updateUserRelations(tx *bolt.Tx, user *User, oldUser User
return err
}
for idx := range oldUser.VirtualFolders {
err = p.removeRelationFromFolderMapping(oldUser.VirtualFolders[idx], oldUser.Username, "", folderBucket)
err = p.removeRelationFromFolderMapping(oldUser.VirtualFolders[idx], oldUser.Username, "", foldersBucket)
if err != nil {
return err
}
@ -2165,7 +2226,7 @@ func (p *BoltProvider) updateUserRelations(tx *bolt.Tx, user *User, oldUser User
}
}
for idx := range user.VirtualFolders {
err = p.addRelationToFolderMapping(&user.VirtualFolders[idx].BaseVirtualFolder, user, nil, folderBucket)
err = p.addRelationToFolderMapping(&user.VirtualFolders[idx].BaseVirtualFolder, user, nil, foldersBucket)
if err != nil {
return err
}
@ -2320,6 +2381,14 @@ func (p *BoltProvider) getFoldersBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
return bucket, err
}
func setLastUserUpdate() {
atomic.StoreInt64(&lastUserUpdate, util.GetTimeAsMsSinceEpoch(time.Now()))
}
func getLastUserUpdate() int64 {
return atomic.LoadInt64(&lastUserUpdate)
}
func getBoltDatabaseVersion(dbHandle *bolt.DB) (schemaVersion, error) {
var dbVersion schemaVersion
err := dbHandle.View(func(tx *bolt.Tx) error {

47
dataprovider/memory.go
View file

@ -182,6 +182,7 @@ func (p *MemoryProvider) setUpdatedAt(username string) {
}
user.UpdatedAt = util.GetTimeAsMsSinceEpoch(time.Now())
p.dbHandle.users[user.Username] = user
setLastUserUpdate()
}
func (p *MemoryProvider) updateLastLogin(username string) error {
@ -360,6 +361,7 @@ func (p *MemoryProvider) updateUser(user *User) error {
user.ID = u.ID
// pre-login and external auth hook will use the passed *user so save a copy
p.dbHandle.users[user.Username] = user.getACopy()
setLastUserUpdate()
return nil
}
@ -439,22 +441,23 @@ func (p *MemoryProvider) dumpFolders() ([]vfs.BaseVirtualFolder, error) {
return folders, nil
}
// memory provider cannot be shared, so we always return no recently updated users
func (p *MemoryProvider) getRecentlyUpdatedUsers(after int64) ([]User, error) {
if getLastUserUpdate() < after {
return nil, nil
}
func (p *MemoryProvider) getUsersForQuotaCheck(toFetch map[string]bool) ([]User, error) {
users := make([]User, 0, 30)
p.dbHandle.Lock()
defer p.dbHandle.Unlock()
if p.dbHandle.isClosed {
return users, errMemoryProviderClosed
return nil, errMemoryProviderClosed
}
users := make([]User, 0, 10)
for _, username := range p.dbHandle.usernames {
if val, ok := toFetch[username]; ok {
u := p.dbHandle.users[username]
if u.UpdatedAt < after {
continue
}
user := u.getACopy()
p.addVirtualFoldersToUser(&user)
if len(user.Groups) > 0 {
groupMapping := make(map[string]Group)
for idx := range user.Groups {
@ -466,9 +469,39 @@ func (p *MemoryProvider) getUsersForQuotaCheck(toFetch map[string]bool) ([]User,
}
user.applyGroupSettings(groupMapping)
}
if val {
user.SetEmptySecretsIfNil()
users = append(users, user)
}
return users, nil
}
func (p *MemoryProvider) getUsersForQuotaCheck(toFetch map[string]bool) ([]User, error) {
users := make([]User, 0, 30)
p.dbHandle.Lock()
defer p.dbHandle.Unlock()
if p.dbHandle.isClosed {
return users, errMemoryProviderClosed
}
for _, username := range p.dbHandle.usernames {
if needFolders, ok := toFetch[username]; ok {
u := p.dbHandle.users[username]
user := u.getACopy()
if needFolders {
p.addVirtualFoldersToUser(&user)
}
if len(user.Groups) > 0 {
groupMapping := make(map[string]Group)
for idx := range user.Groups {
group, err := p.groupExistsInternal(user.Groups[idx].Name)
if err != nil {
continue
}
groupMapping[group.Name] = group
}
user.applyGroupSettings(groupMapping)
}
user.SetEmptySecretsIfNil()
user.PrepareForRendering()
users = append(users, user)

7
dataprovider/scheduler.go
View file

@ -49,13 +49,6 @@ func startScheduler() error {
}
func addScheduledCacheUpdates() error {
if config.IsShared == 0 {
return nil
}
if !util.IsStringInSlice(config.Driver, sharedProviders) {
providerLog(logger.LevelError, "update caches not supported for provider %v", config.Driver)
return nil
}
lastCachesUpdate = util.GetTimeAsMsSinceEpoch(time.Now())
_, err := scheduler.AddFunc("@every 10m", checkCacheUpdates)
if err != nil {

17
dataprovider/sqlcommon.go
View file

@ -1288,9 +1288,12 @@ func sqlCommonGetRecentlyUpdatedUsers(after int64, dbHandle sqlQuerier) ([]User,
if err != nil {
return users, err
}
if len(groups) == 0 {
return users, nil
}
groupsMapping := make(map[string]Group)
for _, g := range groups {
groupsMapping[g.Name] = g
for idx := range groups {
groupsMapping[groups[idx].Name] = groups[idx]
}
for idx := range users {
ref := &users[idx]
@ -1332,9 +1335,6 @@ func sqlCommonGetUsersForQuotaCheck(toFetch map[string]bool, dbHandle sqlQuerier
}
}
users = users[:validIdx]
if len(usersWithFolders) == 0 {
return users, nil
}
ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
defer cancel()
@ -1355,13 +1355,16 @@ func sqlCommonGetUsersForQuotaCheck(toFetch map[string]bool, dbHandle sqlQuerier
}
}
groupNames = util.RemoveDuplicates(groupNames)
if len(groupNames) == 0 {
return users, nil
}
groups, err := sqlCommonGetGroupsWithNames(groupNames, dbHandle)
if err != nil {
return users, err
}
groupsMapping := make(map[string]Group)
for _, g := range groups {
groupsMapping[g.Name] = g
for idx := range groups {
groupsMapping[groups[idx].Name] = groups[idx]
}
for idx := range users {
ref := &users[idx]

3
dataprovider/sqlite.go
View file

@ -252,9 +252,8 @@ func (p *SQLiteProvider) dumpUsers() ([]User, error) {
return sqlCommonDumpUsers(p.dbHandle)
}
// SQLite provider cannot be shared, so we always return no recently updated users
func (p *SQLiteProvider) getRecentlyUpdatedUsers(after int64) ([]User, error) {
return nil, nil
return sqlCommonGetRecentlyUpdatedUsers(after, p.dbHandle)
}
func (p *SQLiteProvider) getUsers(limit int, offset int, order string) ([]User, error) {

1
dataprovider/user.go
View file

@ -1676,6 +1676,7 @@ func (u *User) removeDuplicatesAfterGroupMerge() {
u.Filters.DeniedProtocols = util.RemoveDuplicates(u.Filters.DeniedProtocols)
u.Filters.WebClient = util.RemoveDuplicates(u.Filters.WebClient)
u.Filters.TwoFactorAuthProtocols = util.RemoveDuplicates(u.Filters.TwoFactorAuthProtocols)
u.SetEmptySecretsIfNil()
u.groupSettingsApplied = true
}

2
docs/full-configuration.md
View file

@ -224,7 +224,7 @@ The configuration file contains the following sections:
- `update_mode`, integer. Defines how the database will be initialized/updated. 0 means automatically. 1 means manually using the initprovider sub-command.
- `create_default_admin`, boolean. Before you can use SFTPGo you need to create an admin account. If you open the admin web UI, a setup screen will guide you in creating the first admin account. You can automatically create the first admin account by enabling this setting and setting the environment variables `SFTPGO_DEFAULT_ADMIN_USERNAME` and `SFTPGO_DEFAULT_ADMIN_PASSWORD`. You can also create the first admin by loading initial data. This setting has no effect if an admin account is already found within the data provider. Default `false`.
- `naming_rules`, integer. Naming rules for usernames and folder names. `0` means no rules. `1` means you can use any UTF-8 character. The names are used in URIs for REST API and Web admin. If not set only unreserved URI characters are allowed: ALPHA / DIGIT / "-" / "." / "_" / "~". `2` means names are converted to lowercase before saving/matching and so case insensitive matching is possible. `3` means trimming trailing and leading white spaces before saving/matching. Rules can be combined, for example `3` means both converting to lowercase and allowing any UTF-8 character. Enabling these options for existing installations could be backward incompatible, some users could be unable to login, for example existing users with mixed cases in their usernames. You have to ensure that all existing users respect the defined rules. Default: `0`.
- `is_shared`, integer. If the data provider is shared across multiple SFTPGo instances, set this parameter to `1`. `MySQL`, `PostgreSQL` and `CockroachDB` can be shared, this setting is ignored for other data providers. For shared data providers, SFTPGo periodically reloads the latest updated users, based on the `updated_at` field, and updates its internal caches if users are updated from a different instance. This check, if enabled, is executed every 10 minutes. For shared data providers, active transfers are persisted in the database and thus quota checks between ongoing transfers will work cross multiple instances. Default: `0`.
- `is_shared`, integer. If the data provider is shared across multiple SFTPGo instances, set this parameter to `1`. `MySQL`, `PostgreSQL` and `CockroachDB` can be shared, this setting is ignored for other data providers. For shared data providers, active transfers are persisted in the database and thus quota checks between ongoing transfers will work cross multiple instances. Default: `0`.
- `backups_path`, string. Path to the backup directory. This can be an absolute path or a path relative to the config dir. We don't allow backups in arbitrary paths for security reasons.
- `auto_backup`, struct. Defines the configuration for automatic data provider backups. Example: hour `0` and day_of_week `*` means a backup every day at midnight. The backup file name is in the format `backup_<day_of_week>_<hour>.json`, files with the same name will be overwritten. Note, this process will only backup provider data (users, folders, shars, admins, api keys) and will not backup the configuration file and users files.
- `enabled`, boolean. Set to `true` to enable automatic backups. Default: `true`.

6
go.mod
View file

@ -67,9 +67,9 @@ require (
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4
golang.org/x/net v0.0.0-20220421235706-1d1ef9303861
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150
golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32
golang.org/x/time v0.0.0-20220411224347-583f2d630306
google.golang.org/api v0.76.0
google.golang.org/api v0.77.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0
)
@ -152,7 +152,7 @@ require (
golang.org/x/tools v0.1.10 // indirect
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46 // indirect
google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e // indirect
google.golang.org/grpc v1.46.0 // indirect
google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/ini.v1 v1.66.4 // indirect

12
go.sum
View file

@ -965,8 +965,8 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc=
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32 h1:Js08h5hqB5xyWR789+QqueR6sDE8mk+YvpETZ+F6X9Y=
golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@ -1102,8 +1102,8 @@ google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/S
google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
google.golang.org/api v0.76.0 h1:UkZl25bR1FHNqtK/EKs3vCdpZtUO6gea3YElTwc8pQg=
google.golang.org/api v0.76.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
google.golang.org/api v0.77.0 h1:msijLTxwkJ7Jub5tv9KBVCKtHOQwnvnvkX7ErFFCVxY=
google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -1203,8 +1203,8 @@ google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX
google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46 h1:G1IeWbjrqEq9ChWxEuRPJu6laA67+XgTFHVSAvepr38=
google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e h1:gMjH4zLGs9m+dGzR7qHCHaXMOwsJHJKKkHtyXhtOrJk=
google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=