beenull/sftpgo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

virtual folders: change dataprovider structure

Browse source 
This way we no longer depend on the local file system path and so we can
add support for cloud backends in future updates
This commit is contained in:
Nicola Murino 2021-02-01 19:04:15 +01:00
parent afe1da92c5
commit 78bf808322
No known key found for this signature in database
GPG key ID: 2F1FB59433D5A8CB
37 changed files with 2197 additions and 1231 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -39,7 +39,7 @@ Several storage backends are supported: local filesystem, encrypted local filesy
- SCP and rsync are supported.
- FTP/S is supported. You can configure the FTP service to require TLS for both control and data connections.
- [WebDAV](./docs/webdav.md) is supported.
- Two-Way TLS authentication, aka TLS with client certificate authentication, is supported for FTPS and WebDAV over HTTPS.
- Two-Way TLS authentication, aka TLS with client certificate authentication, is supported for REST API/Web Admin, FTPS and WebDAV over HTTPS.
- Support for serving local filesystem, encrypted local filesystem, S3 Compatible Object Storage, Google Cloud Storage, Azure Blob Storage or other SFTP accounts over SFTP/SCP/FTP/WebDAV.
- Per user protocols restrictions. You can configure the allowed protocols (SSH/FTP/WebDAV) for each user.
- [Prometheus metrics](./docs/metrics.md) are exposed.

16
common/common.go
View file

@ -723,8 +723,8 @@ type ActiveQuotaScan struct {
// ActiveVirtualFolderQuotaScan defines an active quota scan for a virtual folder
type ActiveVirtualFolderQuotaScan struct {
// folder path to which the quota scan refers
MappedPath string `json:"mapped_path"`
// folder name to which the quota scan refers
Name string `json:"name"`
// quota scan start time as unix timestamp in milliseconds
StartTime int64 `json:"start_time"`
}
@ -796,31 +796,31 @@ func (s *ActiveScans) GetVFoldersQuotaScans() []ActiveVirtualFolderQuotaScan {
// AddVFolderQuotaScan adds a virtual folder to the ones with active quota scans.
// Returns false if the folder has a quota scan already running
func (s *ActiveScans) AddVFolderQuotaScan(folderPath string) bool {
func (s *ActiveScans) AddVFolderQuotaScan(folderName string) bool {
s.Lock()
defer s.Unlock()
for _, scan := range s.FolderScans {
if scan.MappedPath == folderPath {
if scan.Name == folderName {
return false
}
}
s.FolderScans = append(s.FolderScans, ActiveVirtualFolderQuotaScan{
MappedPath: folderPath,
StartTime: utils.GetTimeAsMsSinceEpoch(time.Now()),
Name: folderName,
StartTime: utils.GetTimeAsMsSinceEpoch(time.Now()),
})
return true
}
// RemoveVFolderQuotaScan removes a folder from the ones with active quota scans.
// Returns false if the folder has no active quota scans
func (s *ActiveScans) RemoveVFolderQuotaScan(folderPath string) bool {
func (s *ActiveScans) RemoveVFolderQuotaScan(folderName string) bool {
s.Lock()
defer s.Unlock()
indexToRemove := -1
for i, scan := range s.FolderScans {
if scan.MappedPath == folderPath {
if scan.Name == folderName {
indexToRemove = i
break
}

30
common/common_test.go
View file

@ -6,6 +6,7 @@ import (
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"sync/atomic"
@ -21,6 +22,7 @@ import (
"github.com/drakkan/sftpgo/httpclient"
"github.com/drakkan/sftpgo/kms"
"github.com/drakkan/sftpgo/logger"
"github.com/drakkan/sftpgo/utils"
"github.com/drakkan/sftpgo/vfs"
)
@ -528,11 +530,11 @@ func TestQuotaScans(t *testing.T) {
assert.False(t, QuotaScans.RemoveUserQuotaScan(username))
assert.Len(t, QuotaScans.GetUsersQuotaScans(), 0)
folderName := "/folder"
folderName := "folder"
assert.True(t, QuotaScans.AddVFolderQuotaScan(folderName))
assert.False(t, QuotaScans.AddVFolderQuotaScan(folderName))
if assert.Len(t, QuotaScans.GetVFoldersQuotaScans(), 1) {
assert.Equal(t, QuotaScans.GetVFoldersQuotaScans()[0].MappedPath, folderName)
assert.Equal(t, QuotaScans.GetVFoldersQuotaScans()[0].Name, folderName)
}
assert.True(t, QuotaScans.RemoveVFolderQuotaScan(folderName))
@ -624,3 +626,27 @@ func TestCryptoConvertFileInfo(t *testing.T) {
info = vfs.NewFileInfo(name, false, 1, time.Now(), false)
assert.Equal(t, int64(0), cryptFs.ConvertFileInfo(info).Size())
}
func TestFolderCopy(t *testing.T) {
folder := vfs.BaseVirtualFolder{
ID: 1,
Name: "name",
MappedPath: filepath.Clean(os.TempDir()),
UsedQuotaSize: 4096,
UsedQuotaFiles: 2,
LastQuotaUpdate: utils.GetTimeAsMsSinceEpoch(time.Now()),
Users: []string{"user1", "user2"},
}
folderCopy := folder.GetACopy()
folder.ID = 2
folder.Users = []string{"user3"}
require.Len(t, folderCopy.Users, 2)
require.True(t, utils.IsStringInSlice("user1", folderCopy.Users))
require.True(t, utils.IsStringInSlice("user2", folderCopy.Users))
require.Equal(t, int64(1), folderCopy.ID)
require.Equal(t, folder.Name, folderCopy.Name)
require.Equal(t, folder.MappedPath, folderCopy.MappedPath)
require.Equal(t, folder.UsedQuotaSize, folderCopy.UsedQuotaSize)
require.Equal(t, folder.UsedQuotaFiles, folderCopy.UsedQuotaFiles)
require.Equal(t, folder.LastQuotaUpdate, folderCopy.LastQuotaUpdate)
}

2
common/connection.go
View file

@ -797,7 +797,7 @@ func (c *BaseConnection) HasSpace(checkFiles bool, requestPath string) vfs.Quota
}
result.QuotaSize = vfolder.QuotaSize
result.QuotaFiles = vfolder.QuotaFiles
result.UsedFiles, result.UsedSize, err = dataprovider.GetUsedVirtualFolderQuota(vfolder.MappedPath)
result.UsedFiles, result.UsedSize, err = dataprovider.GetUsedVirtualFolderQuota(vfolder.Name)
} else {
if c.User.HasNoQuotaRestrictions(checkFiles) {
return result

30
common/connection_test.go
View file

@ -1013,11 +1013,13 @@ func TestHasSpace(t *testing.T) {
Password: userTestPwd,
}
mappedPath := filepath.Join(os.TempDir(), "vdir")
folderName := "testFolder"
user.Permissions = make(map[string][]string)
user.Permissions["/"] = []string{dataprovider.PermAny}
user.VirtualFolders = append(user.VirtualFolders, vfs.VirtualFolder{
BaseVirtualFolder: vfs.BaseVirtualFolder{
MappedPath: mappedPath,
Name: folderName,
},
VirtualPath: "/vdir",
QuotaFiles: -1,
@ -1050,7 +1052,7 @@ func TestHasSpace(t *testing.T) {
quotaResult = c.HasSpace(true, "/file")
assert.True(t, quotaResult.HasSpace)
folder, err := dataprovider.GetFolderByPath(mappedPath)
folder, err := dataprovider.GetFolderByName(folderName)
assert.NoError(t, err)
err = dataprovider.UpdateVirtualFolderQuota(folder, 10, 1048576, true)
assert.NoError(t, err)
@ -1060,7 +1062,7 @@ func TestHasSpace(t *testing.T) {
err = dataprovider.DeleteUser(user.Username)
assert.NoError(t, err)
err = dataprovider.DeleteFolder(folder.MappedPath)
err = dataprovider.DeleteFolder(folder.Name)
assert.NoError(t, err)
}
@ -1071,6 +1073,8 @@ func TestUpdateQuotaMoveVFolders(t *testing.T) {
Password: userTestPwd,
QuotaFiles: 100,
}
folderName1 := "testFolder1"
folderName2 := "testFolder2"
mappedPath1 := filepath.Join(os.TempDir(), "vdir1")
mappedPath2 := filepath.Join(os.TempDir(), "vdir2")
user.Permissions = make(map[string][]string)
@ -1078,6 +1082,7 @@ func TestUpdateQuotaMoveVFolders(t *testing.T) {
user.VirtualFolders = append(user.VirtualFolders, vfs.VirtualFolder{
BaseVirtualFolder: vfs.BaseVirtualFolder{
MappedPath: mappedPath1,
Name: folderName1,
},
VirtualPath: "/vdir1",
QuotaFiles: -1,
@ -1086,6 +1091,7 @@ func TestUpdateQuotaMoveVFolders(t *testing.T) {
user.VirtualFolders = append(user.VirtualFolders, vfs.VirtualFolder{
BaseVirtualFolder: vfs.BaseVirtualFolder{
MappedPath: mappedPath2,
Name: folderName2,
},
VirtualPath: "/vdir2",
QuotaFiles: -1,
@ -1095,9 +1101,9 @@ func TestUpdateQuotaMoveVFolders(t *testing.T) {
assert.NoError(t, err)
user, err = dataprovider.UserExists(user.Username)
assert.NoError(t, err)
folder1, err := dataprovider.GetFolderByPath(mappedPath1)
folder1, err := dataprovider.GetFolderByName(folderName1)
assert.NoError(t, err)
folder2, err := dataprovider.GetFolderByPath(mappedPath2)
folder2, err := dataprovider.GetFolderByName(folderName2)
assert.NoError(t, err)
err = dataprovider.UpdateVirtualFolderQuota(folder1, 1, 100, true)
assert.NoError(t, err)
@ -1107,21 +1113,21 @@ func TestUpdateQuotaMoveVFolders(t *testing.T) {
assert.NoError(t, err)
c := NewBaseConnection("", ProtocolSFTP, user, fs)
c.updateQuotaMoveBetweenVFolders(user.VirtualFolders[0], user.VirtualFolders[1], -1, 100, 1)
folder1, err = dataprovider.GetFolderByPath(mappedPath1)
folder1, err = dataprovider.GetFolderByName(folderName1)
assert.NoError(t, err)
assert.Equal(t, 0, folder1.UsedQuotaFiles)
assert.Equal(t, int64(0), folder1.UsedQuotaSize)
folder2, err = dataprovider.GetFolderByPath(mappedPath2)
folder2, err = dataprovider.GetFolderByName(folderName2)
assert.NoError(t, err)
assert.Equal(t, 3, folder2.UsedQuotaFiles)
assert.Equal(t, int64(250), folder2.UsedQuotaSize)
c.updateQuotaMoveBetweenVFolders(user.VirtualFolders[1], user.VirtualFolders[0], 10, 100, 1)
folder1, err = dataprovider.GetFolderByPath(mappedPath1)
folder1, err = dataprovider.GetFolderByName(folderName1)
assert.NoError(t, err)
assert.Equal(t, 0, folder1.UsedQuotaFiles)
assert.Equal(t, int64(90), folder1.UsedQuotaSize)
folder2, err = dataprovider.GetFolderByPath(mappedPath2)
folder2, err = dataprovider.GetFolderByName(folderName2)
assert.NoError(t, err)
assert.Equal(t, 2, folder2.UsedQuotaFiles)
assert.Equal(t, int64(150), folder2.UsedQuotaSize)
@ -1129,7 +1135,7 @@ func TestUpdateQuotaMoveVFolders(t *testing.T) {
err = dataprovider.UpdateUserQuota(user, 1, 100, true)
assert.NoError(t, err)
c.updateQuotaMoveFromVFolder(user.VirtualFolders[1], -1, 50, 1)
folder2, err = dataprovider.GetFolderByPath(mappedPath2)
folder2, err = dataprovider.GetFolderByName(folderName2)
assert.NoError(t, err)
assert.Equal(t, 1, folder2.UsedQuotaFiles)
assert.Equal(t, int64(100), folder2.UsedQuotaSize)
@ -1139,7 +1145,7 @@ func TestUpdateQuotaMoveVFolders(t *testing.T) {
assert.Equal(t, int64(100), user.UsedQuotaSize)
c.updateQuotaMoveToVFolder(user.VirtualFolders[1], -1, 100, 1)
folder2, err = dataprovider.GetFolderByPath(mappedPath2)
folder2, err = dataprovider.GetFolderByName(folderName2)
assert.NoError(t, err)
assert.Equal(t, 2, folder2.UsedQuotaFiles)
assert.Equal(t, int64(200), folder2.UsedQuotaSize)
@ -1150,9 +1156,9 @@ func TestUpdateQuotaMoveVFolders(t *testing.T) {
err = dataprovider.DeleteUser(user.Username)
assert.NoError(t, err)
err = dataprovider.DeleteFolder(folder1.MappedPath)
err = dataprovider.DeleteFolder(folder1.Name)
assert.NoError(t, err)
err = dataprovider.DeleteFolder(folder2.MappedPath)
err = dataprovider.DeleteFolder(folder2.Name)
assert.NoError(t, err)
}

237
dataprovider/bolt.go
View file

@ -18,7 +18,7 @@ import (
)
const (
boltDatabaseVersion = 5
boltDatabaseVersion = 6
)
var (
@ -612,22 +612,12 @@ func (p *BoltProvider) dumpFolders() ([]vfs.BaseVirtualFolder, error) {
return folders, err
}
func (p *BoltProvider) getFolders(limit, offset int, order, folderPath string) ([]vfs.BaseVirtualFolder, error) {
func (p *BoltProvider) getFolders(limit, offset int, order string) ([]vfs.BaseVirtualFolder, error) {
folders := make([]vfs.BaseVirtualFolder, 0, limit)
var err error
if limit <= 0 {
return folders, err
}
if len(folderPath) > 0 {
if offset == 0 {
var folder vfs.BaseVirtualFolder
folder, err = p.getFolderByPath(folderPath)
if err == nil {
folders = append(folders, folder)
}
}
return folders, err
}
err = p.dbHandle.View(func(tx *bolt.Tx) error {
bucket, err := getFolderBucket(tx)
if err != nil {
@ -673,7 +663,7 @@ func (p *BoltProvider) getFolders(limit, offset int, order, folderPath string) (
return folders, err
}
func (p *BoltProvider) getFolderByPath(name string) (vfs.BaseVirtualFolder, error) {
func (p *BoltProvider) getFolderByName(name string) (vfs.BaseVirtualFolder, error) {
var folder vfs.BaseVirtualFolder
err := p.dbHandle.View(func(tx *bolt.Tx) error {
bucket, err := getFolderBucket(tx)
@ -696,14 +686,49 @@ func (p *BoltProvider) addFolder(folder *vfs.BaseVirtualFolder) error {
if err != nil {
return err
}
if f := bucket.Get([]byte(folder.MappedPath)); f != nil {
return fmt.Errorf("folder %v already exists", folder.MappedPath)
if f := bucket.Get([]byte(folder.Name)); f != nil {
return fmt.Errorf("folder %v already exists", folder.Name)
}
folder.Users = nil
_, err = addFolderInternal(*folder, bucket)
return err
})
}
func (p *BoltProvider) updateFolder(folder *vfs.BaseVirtualFolder) error {
err := validateFolder(folder)
if err != nil {
return err
}
return p.dbHandle.Update(func(tx *bolt.Tx) error {
bucket, err := getFolderBucket(tx)
if err != nil {
return err
}
var f []byte
if f = bucket.Get([]byte(folder.Name)); f == nil {
return &RecordNotFoundError{err: fmt.Sprintf("folder %v does not exist", folder.Name)}
}
var oldFolder vfs.BaseVirtualFolder
err = json.Unmarshal(f, &oldFolder)
if err != nil {
return err
}
folder.ID = oldFolder.ID
folder.LastQuotaUpdate = oldFolder.LastQuotaUpdate
folder.UsedQuotaFiles = oldFolder.UsedQuotaFiles
folder.UsedQuotaSize = oldFolder.UsedQuotaSize
folder.Users = oldFolder.Users
buf, err := json.Marshal(folder)
if err != nil {
return err
}
return bucket.Put([]byte(folder.Name), buf)
})
}
func (p *BoltProvider) deleteFolder(folder *vfs.BaseVirtualFolder) error {
return p.dbHandle.Update(func(tx *bolt.Tx) error {
bucket, err := getFolderBucket(tx)
@ -715,8 +740,8 @@ func (p *BoltProvider) deleteFolder(folder *vfs.BaseVirtualFolder) error {
return err
}
var f []byte
if f = bucket.Get([]byte(folder.MappedPath)); f == nil {
return &RecordNotFoundError{err: fmt.Sprintf("folder %v does not exist", folder.MappedPath)}
if f = bucket.Get([]byte(folder.Name)); f == nil {
return &RecordNotFoundError{err: fmt.Sprintf("folder %v does not exist", folder.Name)}
}
var folder vfs.BaseVirtualFolder
err = json.Unmarshal(f, &folder)
@ -735,7 +760,7 @@ func (p *BoltProvider) deleteFolder(folder *vfs.BaseVirtualFolder) error {
}
var folders []vfs.VirtualFolder
for _, userFolder := range user.VirtualFolders {
if folder.MappedPath != userFolder.MappedPath {
if folder.Name != userFolder.Name {
folders = append(folders, userFolder)
}
}
@ -750,19 +775,19 @@ func (p *BoltProvider) deleteFolder(folder *vfs.BaseVirtualFolder) error {
}
}
return bucket.Delete([]byte(folder.MappedPath))
return bucket.Delete([]byte(folder.Name))
})
}
func (p *BoltProvider) updateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64, reset bool) error {
func (p *BoltProvider) updateFolderQuota(name string, filesAdd int, sizeAdd int64, reset bool) error {
return p.dbHandle.Update(func(tx *bolt.Tx) error {
bucket, err := getFolderBucket(tx)
if err != nil {
return err
}
var f []byte
if f = bucket.Get([]byte(mappedPath)); f == nil {
return &RecordNotFoundError{err: fmt.Sprintf("folder %v does not exist, unable to update quota", mappedPath)}
if f = bucket.Get([]byte(name)); f == nil {
return &RecordNotFoundError{err: fmt.Sprintf("folder %#v does not exist, unable to update quota", name)}
}
var folder vfs.BaseVirtualFolder
err = json.Unmarshal(f, &folder)
@ -781,14 +806,14 @@ func (p *BoltProvider) updateFolderQuota(mappedPath string, filesAdd int, sizeAd
if err != nil {
return err
}
return bucket.Put([]byte(folder.MappedPath), buf)
return bucket.Put([]byte(folder.Name), buf)
})
}
func (p *BoltProvider) getUsedFolderQuota(mappedPath string) (int, int64, error) {
folder, err := p.getFolderByPath(mappedPath)
func (p *BoltProvider) getUsedFolderQuota(name string) (int, int64, error) {
folder, err := p.getFolderByName(name)
if err != nil {
providerLog(logger.LevelWarn, "unable to get quota for folder %#v error: %v", mappedPath, err)
providerLog(logger.LevelWarn, "unable to get quota for folder %#v error: %v", name, err)
return 0, 0, err
}
return folder.UsedQuotaFiles, folder.UsedQuotaSize, err
@ -825,6 +850,8 @@ func (p *BoltProvider) migrateDatabase() error {
return updateBoltDatabaseFromV3(p.dbHandle)
case 4:
return updateBoltDatabaseFromV4(p.dbHandle)
case 5:
return updateBoltDatabaseFromV5(p.dbHandle)
default:
if dbVersion.Version > boltDatabaseVersion {
providerLog(logger.LevelWarn, "database version %v is newer than the supported: %v", dbVersion.Version,
@ -848,6 +875,12 @@ func (p *BoltProvider) revertDatabase(targetVersion int) error {
switch dbVersion.Version {
case 5:
return downgradeBoltDatabaseFrom5To4(p.dbHandle)
case 6:
err := downgradeBoltDatabaseFrom6To5(p.dbHandle)
if err != nil {
return err
}
return downgradeBoltDatabaseFrom5To4(p.dbHandle)
default:
return fmt.Errorf("Database version not handled: %v", dbVersion.Version)
}
@ -878,7 +911,15 @@ func updateBoltDatabaseFromV3(dbHandle *bolt.DB) error {
}
func updateBoltDatabaseFromV4(dbHandle *bolt.DB) error {
return updateDatabaseFrom4To5(dbHandle)
err := updateDatabaseFrom4To5(dbHandle)
if err != nil {
return err
}
return updateBoltDatabaseFromV5(dbHandle)
}
func updateBoltDatabaseFromV5(dbHandle *bolt.DB) error {
return updateDatabaseFrom5To6(dbHandle)
}
func joinUserAndFolders(u []byte, foldersBucket *bolt.Bucket) (User, error) {
@ -890,10 +931,11 @@ func joinUserAndFolders(u []byte, foldersBucket *bolt.Bucket) (User, error) {
if len(user.VirtualFolders) > 0 {
var folders []vfs.VirtualFolder
for _, folder := range user.VirtualFolders {
baseFolder, err := folderExistsInternal(folder.MappedPath, foldersBucket)
baseFolder, err := folderExistsInternal(folder.Name, foldersBucket)
if err != nil {
continue
}
folder.MappedPath = baseFolder.MappedPath
folder.UsedQuotaFiles = baseFolder.UsedQuotaFiles
folder.UsedQuotaSize = baseFolder.UsedQuotaSize
folder.LastQuotaUpdate = baseFolder.LastQuotaUpdate
@ -927,15 +969,18 @@ func addFolderInternal(folder vfs.BaseVirtualFolder, bucket *bolt.Bucket) (vfs.B
if err != nil {
return folder, err
}
err = bucket.Put([]byte(folder.MappedPath), buf)
err = bucket.Put([]byte(folder.Name), buf)
return folder, err
}
func addUserToFolderMapping(folder vfs.VirtualFolder, user *User, bucket *bolt.Bucket) error {
var baseFolder vfs.BaseVirtualFolder
var err error
if f := bucket.Get([]byte(folder.MappedPath)); f == nil {
if f := bucket.Get([]byte(folder.Name)); f == nil {
// folder does not exists, try to create
folder.LastQuotaUpdate = 0
folder.UsedQuotaFiles = 0
folder.UsedQuotaSize = 0
baseFolder, err = addFolderInternal(folder.BaseVirtualFolder, bucket)
} else {
err = json.Unmarshal(f, &baseFolder)
@ -949,7 +994,7 @@ func addUserToFolderMapping(folder vfs.VirtualFolder, user *User, bucket *bolt.B
if err != nil {
return err
}
err = bucket.Put([]byte(folder.MappedPath), buf)
err = bucket.Put([]byte(folder.Name), buf)
if err != nil {
return err
}
@ -959,7 +1004,7 @@ func addUserToFolderMapping(folder vfs.VirtualFolder, user *User, bucket *bolt.B
func removeUserFromFolderMapping(folder vfs.VirtualFolder, user *User, bucket *bolt.Bucket) error {
var f []byte
if f = bucket.Get([]byte(folder.MappedPath)); f == nil {
if f = bucket.Get([]byte(folder.Name)); f == nil {
// the folder does not exists so there is no associated user
return nil
}
@ -980,7 +1025,7 @@ func removeUserFromFolderMapping(folder vfs.VirtualFolder, user *User, bucket *b
if err != nil {
return err
}
return bucket.Put([]byte(folder.MappedPath), buf)
return bucket.Put([]byte(folder.Name), buf)
}
return err
}
@ -1186,11 +1231,11 @@ func updateDatabaseFrom3To4(dbHandle *bolt.DB) error {
}
}
err = updateBoltDatabaseVersion(dbHandle, 4)
if err == nil {
return updateBoltDatabaseVersion(dbHandle, 4)
/*if err == nil {
go updateVFoldersQuotaAfterRestore(foldersToScan)
}
return err
return err*/
}
//nolint:dupl
@ -1275,6 +1320,126 @@ func updateDatabaseFrom4To5(dbHandle *bolt.DB) error {
return updateBoltDatabaseVersion(dbHandle, 5)
}
// this compat code will be removed after 2.0.0, ignore the lint warning for now
//nolint:gocyclo
func updateDatabaseFrom5To6(dbHandle *bolt.DB) error {
logger.InfoToConsole("updating bolt database version: 5 -> 6")
providerLog(logger.LevelInfo, "updating bolt database version: 5 -> 6")
err := dbHandle.Update(func(tx *bolt.Tx) error {
bucket, err := getFolderBucket(tx)
if err != nil {
return err
}
usersBucket, err := getUsersBucket(tx)
if err != nil {
return err
}
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
if filepath.IsAbs(string(k)) {
var folder vfs.BaseVirtualFolder
err = json.Unmarshal(v, &folder)
if err != nil {
return err
}
folder.Name = fmt.Sprintf("Folder%v", folder.ID)
buf, err := json.Marshal(folder)
if err != nil {
return err
}
// insert the folder with the new structure
err = bucket.Put([]byte(folder.Name), buf)
if err != nil {
return err
}
// delete the folder with the old structure
err = bucket.Delete(k)
if err != nil {
return err
}
// update users mapping
for _, username := range folder.Users {
var u []byte
if u = usersBucket.Get([]byte(username)); u == nil {
continue
}
var user User
err = json.Unmarshal(u, &user)
if err != nil {
return err
}
var folders []vfs.VirtualFolder
for _, userFolder := range user.VirtualFolders {
if folder.MappedPath == userFolder.MappedPath {
userFolder.Name = folder.Name
}
folders = append(folders, userFolder)
}
user.VirtualFolders = folders
buf, err := json.Marshal(user)
if err != nil {
return err
}
err = usersBucket.Put([]byte(user.Username), buf)
if err != nil {
return err
}
}
}
}
return nil
})
if err != nil {
return err
}
return updateBoltDatabaseVersion(dbHandle, 6)
}
func downgradeBoltDatabaseFrom6To5(dbHandle *bolt.DB) error {
logger.InfoToConsole("downgrading bolt database version: 6 -> 5")
providerLog(logger.LevelInfo, "downgrading bolt database version: 6 -> 5")
// best effort we'll remove this code soon
err := dbHandle.Update(func(tx *bolt.Tx) error {
// just update the folder keys
bucket, err := getFolderBucket(tx)
if err != nil {
return err
}
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
if !filepath.IsAbs(string(k)) {
var folder vfs.BaseVirtualFolder
err = json.Unmarshal(v, &folder)
if err != nil {
return err
}
if filepath.IsAbs(folder.MappedPath) {
buf, err := json.Marshal(folder)
if err != nil {
return err
}
// insert the folder with the old key
err = bucket.Put([]byte(folder.MappedPath), buf)
if err != nil {
return err
}
// delete the folder with the new key
err = bucket.Delete(k)
if err != nil {
return err
}
}
}
}
return nil
})
if err != nil {
return err
}
return updateBoltDatabaseVersion(dbHandle, 5)
}
func getBoltAvailableUsernames(dbHandle *bolt.DB) ([]string, error) {
usernames := []string{}
err := dbHandle.View(func(tx *bolt.Tx) error {

136
dataprovider/dataprovider.go
View file

@ -281,6 +281,56 @@ type BackupData struct {
Version int `json:"version"`
}
// HasFolder returns true if the folder with the given name is included
func (d *BackupData) HasFolder(name string) bool {
for _, folder := range d.Folders {
if folder.Name == name {
return true
}
}
return false
}
func (d *BackupData) checkFolderNames() {
if len(d.Folders) == 0 {
return
}
if d.Folders[0].Name != "" {
return
}
logger.WarnToConsole("You are loading folders without a name, please update to the latest supported format. This compatibility layer will be removed soon.")
providerLog(logger.LevelWarn, "You are loading folders without a name, please update to the latest supported format. This compatibility layer will be removed soon.")
folders := make([]vfs.BaseVirtualFolder, 0, len(d.Folders))
for idx, folder := range d.Folders {
if folder.Name == "" {
folder.Name = fmt.Sprintf("Folder%v", idx)
}
folders = append(folders, folder)
}
d.Folders = folders
users := make([]User, 0, len(d.Users))
for _, user := range d.Users {
if len(user.VirtualFolders) > 0 {
vfolders := make([]vfs.VirtualFolder, 0, len(user.VirtualFolders))
for _, vfolder := range user.VirtualFolders {
if vfolder.Name == "" {
for _, f := range d.Folders {
if f.MappedPath == vfolder.MappedPath {
vfolder.Name = f.Name
}
}
}
if vfolder.Name != "" {
vfolders = append(vfolders, vfolder)
}
}
user.VirtualFolders = vfolders
}
users = append(users, user)
}
d.Users = users
}
type keyboardAuthHookRequest struct {
RequestID string `json:"request_id"`
Username string `json:"username,omitempty"`
@ -381,12 +431,13 @@ type Provider interface {
getUsers(limit int, offset int, order string) ([]User, error)
dumpUsers() ([]User, error)
updateLastLogin(username string) error
getFolders(limit, offset int, order, folderPath string) ([]vfs.BaseVirtualFolder, error)
getFolderByPath(mappedPath string) (vfs.BaseVirtualFolder, error)
getFolders(limit, offset int, order string) ([]vfs.BaseVirtualFolder, error)
getFolderByName(name string) (vfs.BaseVirtualFolder, error)
addFolder(folder *vfs.BaseVirtualFolder) error
updateFolder(folder *vfs.BaseVirtualFolder) error
deleteFolder(folder *vfs.BaseVirtualFolder) error
updateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64, reset bool) error
getUsedFolderQuota(mappedPath string) (int, int64, error)
updateFolderQuota(name string, filesAdd int, sizeAdd int64, reset bool) error
getUsedFolderQuota(name string) (int, int64, error)
dumpFolders() ([]vfs.BaseVirtualFolder, error)
adminExists(username string) (Admin, error)
addAdmin(admin *Admin) error
@ -660,7 +711,7 @@ func UpdateVirtualFolderQuota(vfolder vfs.BaseVirtualFolder, filesAdd int, sizeA
if filesAdd == 0 && sizeAdd == 0 && !reset {
return nil
}
return provider.updateFolderQuota(vfolder.MappedPath, filesAdd, sizeAdd, reset)
return provider.updateFolderQuota(vfolder.Name, filesAdd, sizeAdd, reset)
}
// GetUsedQuota returns the used quota for the given SFTP user.
@ -672,11 +723,11 @@ func GetUsedQuota(username string) (int, int64, error) {
}
// GetUsedVirtualFolderQuota returns the used quota for the given virtual folder.
func GetUsedVirtualFolderQuota(mappedPath string) (int, int64, error) {
func GetUsedVirtualFolderQuota(name string) (int, int64, error) {
if config.TrackQuota == 0 {
return 0, 0, &MethodDisabledError{err: trackQuotaDisabledError}
}
return provider.getUsedFolderQuota(mappedPath)
return provider.getUsedFolderQuota(name)
}
// AddAdmin adds a new SFTPGo admin
@ -763,23 +814,28 @@ func AddFolder(folder *vfs.BaseVirtualFolder) error {
return provider.addFolder(folder)
}
// UpdateFolder updates the specified virtual folder
func UpdateFolder(folder *vfs.BaseVirtualFolder) error {
return provider.updateFolder(folder)
}
// DeleteFolder deletes an existing folder.
func DeleteFolder(folderPath string) error {
folder, err := provider.getFolderByPath(folderPath)
func DeleteFolder(folderName string) error {
folder, err := provider.getFolderByName(folderName)
if err != nil {
return err
}
return provider.deleteFolder(&folder)
}
// GetFolderByPath returns the folder with the specified path if any
func GetFolderByPath(mappedPath string) (vfs.BaseVirtualFolder, error) {
return provider.getFolderByPath(mappedPath)
// GetFolderByName returns the folder with the specified name if any
func GetFolderByName(name string) (vfs.BaseVirtualFolder, error) {
return provider.getFolderByName(name)
}
// GetFolders returns an array of folders respecting limit and offset
func GetFolders(limit, offset int, order, folderPath string) ([]vfs.BaseVirtualFolder, error) {
return provider.getFolders(limit, offset, order, folderPath)
func GetFolders(limit, offset int, order string) ([]vfs.BaseVirtualFolder, error) {
return provider.getFolders(limit, offset, order)
}
// DumpData returns all users and folders
@ -809,6 +865,7 @@ func ParseDumpData(data []byte) (BackupData, error) {
var dump BackupData
err := json.Unmarshal(data, &dump)
if err == nil {
dump.checkFolderNames()
return dump, err
}
dump = BackupData{}
@ -828,6 +885,7 @@ func ParseDumpData(data []byte) (BackupData, error) {
}
dump.Users = append(dump.Users, createUserFromV4(compatUser, fsConfig))
}
dump.checkFolderNames()
return dump, err
}
@ -929,7 +987,7 @@ func validateFolderQuotaLimits(folder vfs.VirtualFolder) error {
return &ValidationError{err: fmt.Sprintf("invalid quota_size: %v folder path %#v", folder.QuotaSize, folder.MappedPath)}
}
if folder.QuotaFiles < -1 {
return &ValidationError{err: fmt.Sprintf("invalid quota_file: %v folder path %#v", folder.QuotaSize, folder.MappedPath)}
return &ValidationError{err: fmt.Sprintf("invalid quota_file: %v folder path %#v", folder.QuotaFiles, folder.MappedPath)}
}
if (folder.QuotaSize == -1 && folder.QuotaFiles != -1) || (folder.QuotaFiles == -1 && folder.QuotaSize != -1) {
return &ValidationError{err: fmt.Sprintf("virtual folder quota_size and quota_files must be both -1 or >= 0, quota_size: %v quota_files: %v",
@ -938,6 +996,23 @@ func validateFolderQuotaLimits(folder vfs.VirtualFolder) error {
return nil
}
func getVirtualFolderIfInvalid(folder *vfs.BaseVirtualFolder) *vfs.BaseVirtualFolder {
if err := validateFolder(folder); err == nil {
return folder
}
// we try to get the folder from the data provider if only the Name is populated
if folder.MappedPath != "" {
return folder
}
if folder.Name == "" {
return folder
}
if f, err := GetFolderByName(folder.Name); err == nil {
return &f
}
return folder
}
func validateUserVirtualFolders(user *User) error {
if len(user.VirtualFolders) == 0 || user.FsConfig.Provider != LocalFilesystemProvider {
user.VirtualFolders = []vfs.VirtualFolder{}
@ -953,21 +1028,20 @@ func validateUserVirtualFolders(user *User) error {
if err := validateFolderQuotaLimits(v); err != nil {
return err
}
cleanedMPath := filepath.Clean(v.MappedPath)
if !filepath.IsAbs(cleanedMPath) {
return &ValidationError{err: fmt.Sprintf("invalid mapped folder %#v", v.MappedPath)}
folder := getVirtualFolderIfInvalid(&v.BaseVirtualFolder)
if err := validateFolder(folder); err != nil {
return err
}
cleanedMPath := folder.MappedPath
if isMappedDirOverlapped(cleanedMPath, user.GetHomeDir()) {
return &ValidationError{err: fmt.Sprintf("invalid mapped folder %#v cannot be inside or contain the user home dir %#v",
v.MappedPath, user.GetHomeDir())}
folder.MappedPath, user.GetHomeDir())}
}
virtualFolders = append(virtualFolders, vfs.VirtualFolder{
BaseVirtualFolder: vfs.BaseVirtualFolder{
MappedPath: cleanedMPath,
},
VirtualPath: cleanedVPath,
QuotaSize: v.QuotaSize,
QuotaFiles: v.QuotaFiles,
BaseVirtualFolder: *folder,
VirtualPath: cleanedVPath,
QuotaSize: v.QuotaSize,
QuotaFiles: v.QuotaFiles,
})
for k, virtual := range mappedPaths {
if GetQuotaTracking() > 0 {
@ -1315,9 +1389,15 @@ func createUserPasswordHash(user *User) error {
}
func validateFolder(folder *vfs.BaseVirtualFolder) error {
if folder.Name == "" {
return &ValidationError{err: "folder name is mandatory"}
}
if !usernameRegex.MatchString(folder.Name) {
return &ValidationError{err: fmt.Sprintf("folder name %#v is not valid", folder.Name)}
}
cleanedMPath := filepath.Clean(folder.MappedPath)
if !filepath.IsAbs(cleanedMPath) {
return &ValidationError{err: fmt.Sprintf("invalid mapped folder %#v", folder.MappedPath)}
return &ValidationError{err: fmt.Sprintf("invalid folder mapped path %#v", folder.MappedPath)}
}
folder.MappedPath = cleanedMPath
return nil
@ -2205,7 +2285,7 @@ func executeAction(operation string, user *User) {
}
// after migrating database to v4 we have to update the quota for the imported folders
func updateVFoldersQuotaAfterRestore(foldersToScan []string) {
/*func updateVFoldersQuotaAfterRestore(foldersToScan []string) {
fs := vfs.NewOsFs("", "", nil).(*vfs.OsFs)
for _, folder := range foldersToScan {
providerLog(logger.LevelDebug, "starting quota scan after migration for folder %#v", folder)
@ -2222,7 +2302,7 @@ func updateVFoldersQuotaAfterRestore(foldersToScan []string) {
err = UpdateVirtualFolderQuota(vfolder, numFiles, size, true)
providerLog(logger.LevelDebug, "quota updated for virtual folder %#v, error: %v", vfolder.MappedPath, err)
}
}
}*/
func updateWebDavCachedUserLastLogin(username string) {
result, ok := webDAVUsersCache.Load(username)

264
dataprovider/memory.go
View file

@ -28,10 +28,10 @@ type memoryProviderHandle struct {
usernames []string
// map for users, username is the key
users map[string]User
// map for virtual folders, MappedPath is the key
// map for virtual folders, folder name is the key
vfolders map[string]vfs.BaseVirtualFolder
// slice with ordered folders mapped path
vfoldersPaths []string
// slice with ordered folder names
vfoldersNames []string
// map for admins, username is the key
admins map[string]Admin
// slice with ordered admins
@ -58,7 +58,7 @@ func initializeMemoryProvider(basePath string) {
usernames: []string{},
users: make(map[string]User),
vfolders: make(map[string]vfs.BaseVirtualFolder),
vfoldersPaths: []string{},
vfoldersNames: []string{},
admins: make(map[string]Admin),
adminsUsernames: []string{},
configFile: configFile,
@ -180,15 +180,19 @@ func (p *MemoryProvider) getUsedQuota(username string) (int, int64, error) {
}
func (p *MemoryProvider) addUser(user *User) error {
// we can query virtual folder while validating a user
// so we have to check without holding the lock
err := ValidateUser(user)
if err != nil {
return err
}
p.dbHandle.Lock()
defer p.dbHandle.Unlock()
if p.dbHandle.isClosed {
return errMemoryProviderClosed
}
err := ValidateUser(user)
if err != nil {
return err
}
_, err = p.userExistsInternal(user.Username)
if err == nil {
return fmt.Errorf("username %#v already exists", user.Username)
@ -206,21 +210,25 @@ func (p *MemoryProvider) addUser(user *User) error {
}
func (p *MemoryProvider) updateUser(user *User) error {
// we can query virtual folder while validating a user
// so we have to check without holding the lock
err := ValidateUser(user)
if err != nil {
return err
}
p.dbHandle.Lock()
defer p.dbHandle.Unlock()
if p.dbHandle.isClosed {
return errMemoryProviderClosed
}
err := ValidateUser(user)
if err != nil {
return err
}
u, err := p.userExistsInternal(user.Username)
if err != nil {
return err
}
for _, oldFolder := range u.VirtualFolders {
p.removeUserFromFolderMapping(oldFolder.MappedPath, u.Username)
p.removeUserFromFolderMapping(oldFolder.Name, u.Username)
}
user.VirtualFolders = p.joinVirtualFoldersFields(user)
user.LastQuotaUpdate = u.LastQuotaUpdate
@ -244,7 +252,7 @@ func (p *MemoryProvider) deleteUser(user *User) error {
return err
}
for _, oldFolder := range u.VirtualFolders {
p.removeUserFromFolderMapping(oldFolder.MappedPath, u.Username)
p.removeUserFromFolderMapping(oldFolder.Name, u.Username)
}
delete(p.dbHandle.users, user.Username)
// this could be more efficient
@ -279,7 +287,7 @@ func (p *MemoryProvider) dumpUsers() ([]User, error) {
func (p *MemoryProvider) dumpFolders() ([]vfs.BaseVirtualFolder, error) {
p.dbHandle.Lock()
defer p.dbHandle.Unlock()
folders := make([]vfs.BaseVirtualFolder, 0, len(p.dbHandle.vfoldersPaths))
folders := make([]vfs.BaseVirtualFolder, 0, len(p.dbHandle.vfoldersNames))
if p.dbHandle.isClosed {
return folders, errMemoryProviderClosed
}
@ -488,15 +496,15 @@ func (p *MemoryProvider) getAdmins(limit int, offset int, order string) ([]Admin
return admins, nil
}
func (p *MemoryProvider) updateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64, reset bool) error {
func (p *MemoryProvider) updateFolderQuota(name string, filesAdd int, sizeAdd int64, reset bool) error {
p.dbHandle.Lock()
defer p.dbHandle.Unlock()
if p.dbHandle.isClosed {
return errMemoryProviderClosed
}
folder, err := p.folderExistsInternal(mappedPath)
folder, err := p.folderExistsInternal(name)
if err != nil {
providerLog(logger.LevelWarn, "unable to update quota for folder %#v error: %v", mappedPath, err)
providerLog(logger.LevelWarn, "unable to update quota for folder %#v error: %v", name, err)
return err
}
if reset {
@ -507,19 +515,19 @@ func (p *MemoryProvider) updateFolderQuota(mappedPath string, filesAdd int, size
folder.UsedQuotaFiles += filesAdd
}
folder.LastQuotaUpdate = utils.GetTimeAsMsSinceEpoch(time.Now())
p.dbHandle.vfolders[mappedPath] = folder
p.dbHandle.vfolders[name] = folder
return nil
}
func (p *MemoryProvider) getUsedFolderQuota(mappedPath string) (int, int64, error) {
func (p *MemoryProvider) getUsedFolderQuota(name string) (int, int64, error) {
p.dbHandle.Lock()
defer p.dbHandle.Unlock()
if p.dbHandle.isClosed {
return 0, 0, errMemoryProviderClosed
}
folder, err := p.folderExistsInternal(mappedPath)
folder, err := p.folderExistsInternal(name)
if err != nil {
providerLog(logger.LevelWarn, "unable to get quota for folder %#v error: %v", mappedPath, err)
providerLog(logger.LevelWarn, "unable to get quota for folder %#v error: %v", name, err)
return 0, 0, err
}
return folder.UsedQuotaFiles, folder.UsedQuotaSize, err
@ -528,21 +536,21 @@ func (p *MemoryProvider) getUsedFolderQuota(mappedPath string) (int, int64, erro
func (p *MemoryProvider) joinVirtualFoldersFields(user *User) []vfs.VirtualFolder {
var folders []vfs.VirtualFolder
for _, folder := range user.VirtualFolders {
f, err := p.addOrGetFolderInternal(folder.MappedPath, user.Username, folder.UsedQuotaSize, folder.UsedQuotaFiles,
folder.LastQuotaUpdate)
f, err := p.addOrGetFolderInternal(folder.Name, folder.MappedPath, user.Username)
if err == nil {
folder.UsedQuotaFiles = f.UsedQuotaFiles
folder.UsedQuotaSize = f.UsedQuotaSize
folder.LastQuotaUpdate = f.LastQuotaUpdate
folder.ID = f.ID
folder.MappedPath = f.MappedPath
folders = append(folders, folder)
}
}
return folders
}
func (p *MemoryProvider) removeUserFromFolderMapping(mappedPath, username string) {
folder, err := p.folderExistsInternal(mappedPath)
func (p *MemoryProvider) removeUserFromFolderMapping(folderName, username string) {
folder, err := p.folderExistsInternal(folderName)
if err == nil {
var usernames []string
for _, user := range folder.Users {
@ -551,27 +559,28 @@ func (p *MemoryProvider) removeUserFromFolderMapping(mappedPath, username string
}
}
folder.Users = usernames
p.dbHandle.vfolders[folder.MappedPath] = folder
p.dbHandle.vfolders[folder.Name] = folder
}
}
func (p *MemoryProvider) updateFoldersMappingInternal(folder vfs.BaseVirtualFolder) {
p.dbHandle.vfolders[folder.MappedPath] = folder
if !utils.IsStringInSlice(folder.MappedPath, p.dbHandle.vfoldersPaths) {
p.dbHandle.vfoldersPaths = append(p.dbHandle.vfoldersPaths, folder.MappedPath)
sort.Strings(p.dbHandle.vfoldersPaths)
p.dbHandle.vfolders[folder.Name] = folder
if !utils.IsStringInSlice(folder.Name, p.dbHandle.vfoldersNames) {
p.dbHandle.vfoldersNames = append(p.dbHandle.vfoldersNames, folder.Name)
sort.Strings(p.dbHandle.vfoldersNames)
}
}
func (p *MemoryProvider) addOrGetFolderInternal(mappedPath, username string, usedQuotaSize int64, usedQuotaFiles int, lastQuotaUpdate int64) (vfs.BaseVirtualFolder, error) {
folder, err := p.folderExistsInternal(mappedPath)
func (p *MemoryProvider) addOrGetFolderInternal(folderName, folderMappedPath, username string) (vfs.BaseVirtualFolder, error) {
folder, err := p.folderExistsInternal(folderName)
if _, ok := err.(*RecordNotFoundError); ok {
folder := vfs.BaseVirtualFolder{
ID: p.getNextFolderID(),
MappedPath: mappedPath,
UsedQuotaSize: usedQuotaSize,
UsedQuotaFiles: usedQuotaFiles,
LastQuotaUpdate: lastQuotaUpdate,
Name: folderName,
MappedPath: folderMappedPath,
UsedQuotaSize: 0,
UsedQuotaFiles: 0,
LastQuotaUpdate: 0,
Users: []string{username},
}
p.updateFoldersMappingInternal(folder)
@ -584,14 +593,14 @@ func (p *MemoryProvider) addOrGetFolderInternal(mappedPath, username string, use
return folder, err
}
func (p *MemoryProvider) folderExistsInternal(mappedPath string) (vfs.BaseVirtualFolder, error) {
if val, ok := p.dbHandle.vfolders[mappedPath]; ok {
func (p *MemoryProvider) folderExistsInternal(name string) (vfs.BaseVirtualFolder, error) {
if val, ok := p.dbHandle.vfolders[name]; ok {
return val, nil
}
return vfs.BaseVirtualFolder{}, &RecordNotFoundError{err: fmt.Sprintf("folder %#v does not exist", mappedPath)}
return vfs.BaseVirtualFolder{}, &RecordNotFoundError{err: fmt.Sprintf("folder %#v does not exist", name)}
}
func (p *MemoryProvider) getFolders(limit, offset int, order, folderPath string) ([]vfs.BaseVirtualFolder, error) {
func (p *MemoryProvider) getFolders(limit, offset int, order string) ([]vfs.BaseVirtualFolder, error) {
folders := make([]vfs.BaseVirtualFolder, 0, limit)
var err error
p.dbHandle.Lock()
@ -602,37 +611,27 @@ func (p *MemoryProvider) getFolders(limit, offset int, order, folderPath string)
if limit <= 0 {
return folders, err
}
if len(folderPath) > 0 {
if offset == 0 {
var folder vfs.BaseVirtualFolder
folder, err = p.folderExistsInternal(folderPath)
if err == nil {
folders = append(folders, folder)
}
}
return folders, err
}
itNum := 0
if order == OrderASC {
for _, mappedPath := range p.dbHandle.vfoldersPaths {
for _, name := range p.dbHandle.vfoldersNames {
itNum++
if itNum <= offset {
continue
}
folder := p.dbHandle.vfolders[mappedPath]
folder := p.dbHandle.vfolders[name]
folders = append(folders, folder)
if len(folders) >= limit {
break
}
}
} else {
for i := len(p.dbHandle.vfoldersPaths) - 1; i >= 0; i-- {
for i := len(p.dbHandle.vfoldersNames) - 1; i >= 0; i-- {
itNum++
if itNum <= offset {
continue
}
mappedPath := p.dbHandle.vfoldersPaths[i]
folder := p.dbHandle.vfolders[mappedPath]
name := p.dbHandle.vfoldersNames[i]
folder := p.dbHandle.vfolders[name]
folders = append(folders, folder)
if len(folders) >= limit {
break
@ -642,33 +641,60 @@ func (p *MemoryProvider) getFolders(limit, offset int, order, folderPath string)
return folders, err
}
func (p *MemoryProvider) getFolderByPath(mappedPath string) (vfs.BaseVirtualFolder, error) {
func (p *MemoryProvider) getFolderByName(name string) (vfs.BaseVirtualFolder, error) {
p.dbHandle.Lock()
defer p.dbHandle.Unlock()
if p.dbHandle.isClosed {
return vfs.BaseVirtualFolder{}, errMemoryProviderClosed
}
return p.folderExistsInternal(mappedPath)
return p.folderExistsInternal(name)
}
func (p *MemoryProvider) addFolder(folder *vfs.BaseVirtualFolder) error {
err := validateFolder(folder)
if err != nil {
return err
}
p.dbHandle.Lock()
defer p.dbHandle.Unlock()
if p.dbHandle.isClosed {
return errMemoryProviderClosed
}
_, err = p.folderExistsInternal(folder.Name)
if err == nil {
return fmt.Errorf("folder %#v already exists", folder.Name)
}
folder.ID = p.getNextFolderID()
folder.Users = nil
p.dbHandle.vfolders[folder.Name] = folder.GetACopy()
p.dbHandle.vfoldersNames = append(p.dbHandle.vfoldersNames, folder.Name)
sort.Strings(p.dbHandle.vfoldersNames)
return nil
}
func (p *MemoryProvider) updateFolder(folder *vfs.BaseVirtualFolder) error {
err := validateFolder(folder)
if err != nil {
return err
}
_, err = p.folderExistsInternal(folder.MappedPath)
if err == nil {
return fmt.Errorf("folder %#v already exists", folder.MappedPath)
p.dbHandle.Lock()
defer p.dbHandle.Unlock()
if p.dbHandle.isClosed {
return errMemoryProviderClosed
}
folder.ID = p.getNextFolderID()
p.dbHandle.vfolders[folder.MappedPath] = *folder
p.dbHandle.vfoldersPaths = append(p.dbHandle.vfoldersPaths, folder.MappedPath)
sort.Strings(p.dbHandle.vfoldersPaths)
f, err := p.folderExistsInternal(folder.Name)
if err != nil {
return err
}
folder.ID = f.ID
folder.LastQuotaUpdate = f.LastQuotaUpdate
folder.UsedQuotaFiles = f.UsedQuotaFiles
folder.UsedQuotaSize = f.UsedQuotaSize
folder.Users = f.Users
p.dbHandle.vfolders[folder.Name] = folder.GetACopy()
return nil
}
@ -679,7 +705,7 @@ func (p *MemoryProvider) deleteFolder(folder *vfs.BaseVirtualFolder) error {
return errMemoryProviderClosed
}
_, err := p.folderExistsInternal(folder.MappedPath)
_, err := p.folderExistsInternal(folder.Name)
if err != nil {
return err
}
@ -688,7 +714,7 @@ func (p *MemoryProvider) deleteFolder(folder *vfs.BaseVirtualFolder) error {
if err == nil {
var folders []vfs.VirtualFolder
for _, userFolder := range user.VirtualFolders {
if folder.MappedPath != userFolder.MappedPath {
if folder.Name != userFolder.Name {
folders = append(folders, userFolder)
}
}
@ -696,12 +722,12 @@ func (p *MemoryProvider) deleteFolder(folder *vfs.BaseVirtualFolder) error {
p.dbHandle.users[user.Username] = user
}
}
delete(p.dbHandle.vfolders, folder.MappedPath)
p.dbHandle.vfoldersPaths = []string{}
for mappedPath := range p.dbHandle.vfolders {
p.dbHandle.vfoldersPaths = append(p.dbHandle.vfoldersPaths, mappedPath)
delete(p.dbHandle.vfolders, folder.Name)
p.dbHandle.vfoldersNames = []string{}
for name := range p.dbHandle.vfolders {
p.dbHandle.vfoldersNames = append(p.dbHandle.vfoldersNames, name)
}
sort.Strings(p.dbHandle.vfoldersPaths)
sort.Strings(p.dbHandle.vfoldersNames)
return nil
}
@ -740,7 +766,7 @@ func (p *MemoryProvider) clear() {
defer p.dbHandle.Unlock()
p.dbHandle.usernames = []string{}
p.dbHandle.users = make(map[string]User)
p.dbHandle.vfoldersPaths = []string{}
p.dbHandle.vfoldersNames = []string{}
p.dbHandle.vfolders = make(map[string]vfs.BaseVirtualFolder)
p.dbHandle.admins = make(map[string]Admin)
p.dbHandle.adminsUsernames = []string{}
@ -748,53 +774,102 @@ func (p *MemoryProvider) clear() {
func (p *MemoryProvider) reloadConfig() error {
if p.dbHandle.configFile == "" {
providerLog(logger.LevelDebug, "no users configuration file defined")
providerLog(logger.LevelDebug, "no dump configuration file defined")
return nil
}
providerLog(logger.LevelDebug, "loading users from file: %#v", p.dbHandle.configFile)
providerLog(logger.LevelDebug, "loading dump from file: %#v", p.dbHandle.configFile)
fi, err := os.Stat(p.dbHandle.configFile)
if err != nil {
providerLog(logger.LevelWarn, "error loading users: %v", err)
providerLog(logger.LevelWarn, "error loading dump: %v", err)
return err
}
if fi.Size() == 0 {
err = errors.New("users configuration file is invalid, its size must be > 0")
providerLog(logger.LevelWarn, "error loading users: %v", err)
err = errors.New("dump configuration file is invalid, its size must be > 0")
providerLog(logger.LevelWarn, "error loading dump: %v", err)
return err
}
if fi.Size() > 10485760 {
err = errors.New("users configuration file is invalid, its size must be <= 10485760 bytes")
providerLog(logger.LevelWarn, "error loading users: %v", err)
err = errors.New("dump configuration file is invalid, its size must be <= 10485760 bytes")
providerLog(logger.LevelWarn, "error loading dump: %v", err)
return err
}
content, err := ioutil.ReadFile(p.dbHandle.configFile)
if err != nil {
providerLog(logger.LevelWarn, "error loading users: %v", err)
providerLog(logger.LevelWarn, "error loading dump: %v", err)
return err
}
dump, err := ParseDumpData(content)
if err != nil {
providerLog(logger.LevelWarn, "error loading users: %v", err)
providerLog(logger.LevelWarn, "error loading dump: %v", err)
return err
}
p.clear()
for _, folder := range dump.Folders {
_, err := p.getFolderByPath(folder.MappedPath)
if err := p.restoreFolders(&dump); err != nil {
return err
}
if err := p.restoreUsers(&dump); err != nil {
return err
}
if err := p.restoreAdmins(&dump); err != nil {
return err
}
providerLog(logger.LevelDebug, "config loaded from file: %#v", p.dbHandle.configFile)
return nil
}
func (p *MemoryProvider) restoreAdmins(dump *BackupData) error {
for _, admin := range dump.Admins {
a, err := p.adminExists(admin.Username)
admin := admin // pin
if err == nil {
logger.Debug(logSender, "", "folder %#v already exists, restore not needed", folder.MappedPath)
continue
}
folder := folder // pin
folder.Users = nil
err = p.addFolder(&folder)
if err != nil {
providerLog(logger.LevelWarn, "error adding folder %#v: %v", folder.MappedPath, err)
return err
admin.ID = a.ID
err = p.updateAdmin(&admin)
if err != nil {
providerLog(logger.LevelWarn, "error updating admin %#v: %v", admin.Username, err)
return err
}
} else {
err = p.addAdmin(&admin)
if err != nil {
providerLog(logger.LevelWarn, "error adding admin %#v: %v", admin.Username, err)
return err
}
}
}
return nil
}
func (p *MemoryProvider) restoreFolders(dump *BackupData) error {
for _, folder := range dump.Folders {
folder := folder // pin
f, err := p.getFolderByName(folder.Name)
if err == nil {
folder.ID = f.ID
err = p.updateFolder(&folder)
if err != nil {
providerLog(logger.LevelWarn, "error updating folder %#v: %v", folder.Name, err)
return err
}
} else {
folder.Users = nil
err = p.addFolder(&folder)
if err != nil {
providerLog(logger.LevelWarn, "error adding folder %#v: %v", folder.Name, err)
return err
}
}
}
return nil
}
func (p *MemoryProvider) restoreUsers(dump *BackupData) error {
for _, user := range dump.Users {
u, err := p.userExists(user.Username)
user := user // pin
u, err := p.userExists(user.Username)
if err == nil {
user.ID = u.ID
err = p.updateUser(&user)
@ -810,7 +885,6 @@ func (p *MemoryProvider) reloadConfig() error {
}
}
}
providerLog(logger.LevelDebug, "user and folders loaded from file: %#v", p.dbHandle.configFile)
return nil
}

70
dataprovider/mysql.go
View file

@ -44,6 +44,15 @@ const (
"`password` varchar(255) NOT NULL, `email` varchar(255) NULL, `status` integer NOT NULL, `permissions` longtext NOT NULL, " +
"`filters` longtext NULL, `additional_info` longtext NULL);"
mysqlV7DownSQL = "DROP TABLE `{{admins}}` CASCADE;"
mysqlV8SQL = "ALTER TABLE `{{folders}}` ADD COLUMN `name` varchar(255) NULL;" +
"ALTER TABLE `{{folders}}` MODIFY `path` varchar(512) NULL;" +
"ALTER TABLE `{{folders}}` DROP INDEX `path`;" +
"UPDATE `{{folders}}` f1 SET name = (SELECT CONCAT('folder',f2.id) FROM `{{folders}}` f2 WHERE f2.id = f1.id);" +
"ALTER TABLE `{{folders}}` MODIFY `name` varchar(255) NOT NULL;" +
"ALTER TABLE `folders` ADD CONSTRAINT `name` UNIQUE (`name`);"
mysqlV8DownSQL = "ALTER TABLE `{{folders}}` DROP COLUMN `name`;" +
"ALTER TABLE `{{folders}}` MODIFY `path` varchar(512) NOT NULL;" +
"ALTER TABLE `{{folders}}` ADD CONSTRAINT `path` UNIQUE (`path`);"
)
// MySQLProvider auth provider for MySQL/MariaDB database
@ -143,30 +152,34 @@ func (p *MySQLProvider) dumpFolders() ([]vfs.BaseVirtualFolder, error) {
return sqlCommonDumpFolders(p.dbHandle)
}
func (p *MySQLProvider) getFolders(limit, offset int, order, folderPath string) ([]vfs.BaseVirtualFolder, error) {
return sqlCommonGetFolders(limit, offset, order, folderPath, p.dbHandle)
func (p *MySQLProvider) getFolders(limit, offset int, order string) ([]vfs.BaseVirtualFolder, error) {
return sqlCommonGetFolders(limit, offset, order, p.dbHandle)
}
func (p *MySQLProvider) getFolderByPath(mappedPath string) (vfs.BaseVirtualFolder, error) {
func (p *MySQLProvider) getFolderByName(name string) (vfs.BaseVirtualFolder, error) {
ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
defer cancel()
return sqlCommonCheckFolderExists(ctx, mappedPath, p.dbHandle)
return sqlCommonGetFolderByName(ctx, name, p.dbHandle)
}
func (p *MySQLProvider) addFolder(folder *vfs.BaseVirtualFolder) error {
return sqlCommonAddFolder(folder, p.dbHandle)
}
func (p *MySQLProvider) updateFolder(folder *vfs.BaseVirtualFolder) error {
return sqlCommonUpdateFolder(folder, p.dbHandle)
}
func (p *MySQLProvider) deleteFolder(folder *vfs.BaseVirtualFolder) error {
return sqlCommonDeleteFolder(folder, p.dbHandle)
}
func (p *MySQLProvider) updateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64, reset bool) error {
return sqlCommonUpdateFolderQuota(mappedPath, filesAdd, sizeAdd, reset, p.dbHandle)
func (p *MySQLProvider) updateFolderQuota(name string, filesAdd int, sizeAdd int64, reset bool) error {
return sqlCommonUpdateFolderQuota(name, filesAdd, sizeAdd, reset, p.dbHandle)
}
func (p *MySQLProvider) getUsedFolderQuota(mappedPath string) (int, int64, error) {
return sqlCommonGetFolderUsedQuota(mappedPath, p.dbHandle)
func (p *MySQLProvider) getUsedFolderQuota(name string) (int, int64, error) {
return sqlCommonGetFolderUsedQuota(name, p.dbHandle)
}
func (p *MySQLProvider) adminExists(username string) (Admin, error) {
@ -256,6 +269,8 @@ func (p *MySQLProvider) migrateDatabase() error {
return updateMySQLDatabaseFromV5(p.dbHandle)
case 6:
return updateMySQLDatabaseFromV6(p.dbHandle)
case 7:
return updateMySQLDatabaseFromV7(p.dbHandle)
default:
if dbVersion.Version > sqlDatabaseVersion {
providerLog(logger.LevelWarn, "database version %v is newer than the supported: %v", dbVersion.Version,
@ -268,6 +283,7 @@ func (p *MySQLProvider) migrateDatabase() error {
}
}
//nolint:dupl
func (p *MySQLProvider) revertDatabase(targetVersion int) error {
dbVersion, err := sqlCommonGetDatabaseVersion(p.dbHandle, true)
if err != nil {
@ -277,6 +293,20 @@ func (p *MySQLProvider) revertDatabase(targetVersion int) error {
return fmt.Errorf("current version match target version, nothing to do")
}
switch dbVersion.Version {
case 8:
err = downgradeMySQLDatabaseFrom8To7(p.dbHandle)
if err != nil {
return err
}
err = downgradeMySQLDatabaseFrom7To6(p.dbHandle)
if err != nil {
return err
}
err = downgradeMySQLDatabaseFrom6To5(p.dbHandle)
if err != nil {
return err
}
return downgradeMySQLDatabaseFrom5To4(p.dbHandle)
case 7:
err = downgradeMySQLDatabaseFrom7To6(p.dbHandle)
if err != nil {
@ -341,7 +371,15 @@ func updateMySQLDatabaseFromV5(dbHandle *sql.DB) error {
}
func updateMySQLDatabaseFromV6(dbHandle *sql.DB) error {
return updateMySQLDatabaseFrom6To7(dbHandle)
err := updateMySQLDatabaseFrom6To7(dbHandle)
if err != nil {
return err
}
return updateMySQLDatabaseFromV7(dbHandle)
}
func updateMySQLDatabaseFromV7(dbHandle *sql.DB) error {
return updateMySQLDatabaseFrom7To8(dbHandle)
}
func updateMySQLDatabaseFrom1To2(dbHandle *sql.DB) error {
@ -380,6 +418,20 @@ func updateMySQLDatabaseFrom6To7(dbHandle *sql.DB) error {
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 7)
}
func updateMySQLDatabaseFrom7To8(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database version: 7 -> 8")
providerLog(logger.LevelInfo, "updating database version: 7 -> 8")
sql := strings.ReplaceAll(mysqlV8SQL, "{{folders}}", sqlTableFolders)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, strings.Split(sql, ";"), 8)
}
func downgradeMySQLDatabaseFrom8To7(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database version: 8 -> 7")
providerLog(logger.LevelInfo, "downgrading database version: 8 -> 7")
sql := strings.ReplaceAll(mysqlV8DownSQL, "{{folders}}", sqlTableFolders)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 7)
}
func downgradeMySQLDatabaseFrom7To6(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database version: 7 -> 6")
providerLog(logger.LevelInfo, "downgrading database version: 7 -> 6")

72
dataprovider/pgsql.go
View file

@ -45,6 +45,17 @@ CREATE INDEX "folders_mapping_user_id_idx" ON "{{folders_mapping}}" ("user_id");
"filters" text NULL, "additional_info" text NULL);
`
pgsqlV7DownSQL = `DROP TABLE "{{admins}}" CASCADE;`
pgsqlV8SQL = `ALTER TABLE "{{folders}}" ADD COLUMN "name" varchar(255) NULL;
ALTER TABLE "folders" ALTER COLUMN "path" DROP NOT NULL;
ALTER TABLE "{{folders}}" DROP CONSTRAINT IF EXISTS folders_path_key;
UPDATE "{{folders}}" f1 SET name = (SELECT CONCAT('folder',f2.id) FROM "{{folders}}" f2 WHERE f2.id = f1.id);
ALTER TABLE "{{folders}}" ALTER COLUMN "name" SET NOT NULL;
ALTER TABLE "{{folders}}" ADD CONSTRAINT "folders_name_uniq" UNIQUE ("name");
`
pgsqlV8DownSQL = `ALTER TABLE "{{folders}}" DROP COLUMN "name" CASCADE;
ALTER TABLE "{{folders}}" ALTER COLUMN "path" SET NOT NULL;
ALTER TABLE "{{folders}}" ADD CONSTRAINT folders_path_key UNIQUE (path);
`
)
// PGSQLProvider auth provider for PostgreSQL database
@ -145,30 +156,34 @@ func (p *PGSQLProvider) dumpFolders() ([]vfs.BaseVirtualFolder, error) {
return sqlCommonDumpFolders(p.dbHandle)
}
func (p *PGSQLProvider) getFolders(limit, offset int, order, folderPath string) ([]vfs.BaseVirtualFolder, error) {
return sqlCommonGetFolders(limit, offset, order, folderPath, p.dbHandle)
func (p *PGSQLProvider) getFolders(limit, offset int, order string) ([]vfs.BaseVirtualFolder, error) {
return sqlCommonGetFolders(limit, offset, order, p.dbHandle)
}
func (p *PGSQLProvider) getFolderByPath(mappedPath string) (vfs.BaseVirtualFolder, error) {
func (p *PGSQLProvider) getFolderByName(name string) (vfs.BaseVirtualFolder, error) {
ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
defer cancel()
return sqlCommonCheckFolderExists(ctx, mappedPath, p.dbHandle)
return sqlCommonGetFolderByName(ctx, name, p.dbHandle)
}
func (p *PGSQLProvider) addFolder(folder *vfs.BaseVirtualFolder) error {
return sqlCommonAddFolder(folder, p.dbHandle)
}
func (p *PGSQLProvider) updateFolder(folder *vfs.BaseVirtualFolder) error {
return sqlCommonUpdateFolder(folder, p.dbHandle)
}
func (p *PGSQLProvider) deleteFolder(folder *vfs.BaseVirtualFolder) error {
return sqlCommonDeleteFolder(folder, p.dbHandle)
}
func (p *PGSQLProvider) updateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64, reset bool) error {
return sqlCommonUpdateFolderQuota(mappedPath, filesAdd, sizeAdd, reset, p.dbHandle)
func (p *PGSQLProvider) updateFolderQuota(name string, filesAdd int, sizeAdd int64, reset bool) error {
return sqlCommonUpdateFolderQuota(name, filesAdd, sizeAdd, reset, p.dbHandle)
}
func (p *PGSQLProvider) getUsedFolderQuota(mappedPath string) (int, int64, error) {
return sqlCommonGetFolderUsedQuota(mappedPath, p.dbHandle)
func (p *PGSQLProvider) getUsedFolderQuota(name string) (int, int64, error) {
return sqlCommonGetFolderUsedQuota(name, p.dbHandle)
}
func (p *PGSQLProvider) adminExists(username string) (Admin, error) {
@ -258,6 +273,8 @@ func (p *PGSQLProvider) migrateDatabase() error {
return updatePGSQLDatabaseFromV5(p.dbHandle)
case 6:
return updatePGSQLDatabaseFromV6(p.dbHandle)
case 7:
return updatePGSQLDatabaseFromV7(p.dbHandle)
default:
if dbVersion.Version > sqlDatabaseVersion {
providerLog(logger.LevelWarn, "database version %v is newer than the supported: %v", dbVersion.Version,
@ -270,6 +287,7 @@ func (p *PGSQLProvider) migrateDatabase() error {
}
}
//nolint:dupl
func (p *PGSQLProvider) revertDatabase(targetVersion int) error {
dbVersion, err := sqlCommonGetDatabaseVersion(p.dbHandle, true)
if err != nil {
@ -279,6 +297,20 @@ func (p *PGSQLProvider) revertDatabase(targetVersion int) error {
return fmt.Errorf("current version match target version, nothing to do")
}
switch dbVersion.Version {
case 8:
err = downgradePGSQLDatabaseFrom8To7(p.dbHandle)
if err != nil {
return err
}
err = downgradePGSQLDatabaseFrom7To6(p.dbHandle)
if err != nil {
return err
}
err = downgradePGSQLDatabaseFrom6To5(p.dbHandle)
if err != nil {
return err
}
return downgradePGSQLDatabaseFrom5To4(p.dbHandle)
case 7:
err = downgradePGSQLDatabaseFrom7To6(p.dbHandle)
if err != nil {
@ -343,7 +375,15 @@ func updatePGSQLDatabaseFromV5(dbHandle *sql.DB) error {
}
func updatePGSQLDatabaseFromV6(dbHandle *sql.DB) error {
return updatePGSQLDatabaseFrom6To7(dbHandle)
err := updatePGSQLDatabaseFrom6To7(dbHandle)
if err != nil {
return err
}
return updatePGSQLDatabaseFromV7(dbHandle)
}
func updatePGSQLDatabaseFromV7(dbHandle *sql.DB) error {
return updatePGSQLDatabaseFrom7To8(dbHandle)
}
func updatePGSQLDatabaseFrom1To2(dbHandle *sql.DB) error {
@ -382,6 +422,20 @@ func updatePGSQLDatabaseFrom6To7(dbHandle *sql.DB) error {
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 7)
}
func updatePGSQLDatabaseFrom7To8(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database version: 7 -> 8")
providerLog(logger.LevelInfo, "updating database version: 7 -> 8")
sql := strings.ReplaceAll(pgsqlV8SQL, "{{folders}}", sqlTableFolders)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 8)
}
func downgradePGSQLDatabaseFrom8To7(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database version: 8 -> 7")
providerLog(logger.LevelInfo, "downgrading database version: 8 -> 7")
sql := strings.ReplaceAll(pgsqlV8DownSQL, "{{folders}}", sqlTableAdmins)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 7)
}
func downgradePGSQLDatabaseFrom7To6(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database version: 7 -> 6")
providerLog(logger.LevelInfo, "downgrading database version: 7 -> 6")

119
dataprovider/sqlcommon.go
View file

@ -5,6 +5,7 @@ import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
@ -14,7 +15,7 @@ import (
)
const (
sqlDatabaseVersion = 7
sqlDatabaseVersion = 8
initialDBVersionSQL = "INSERT INTO {{schema_version}} (version) VALUES (1);"
defaultSQLQueryTimeout = 10 * time.Second
longSQLQueryTimeout = 60 * time.Second
@ -613,7 +614,7 @@ func getUserFromDbRow(row sqlScanner) (User, error) {
func sqlCommonCheckFolderExists(ctx context.Context, name string, dbHandle sqlQuerier) (vfs.BaseVirtualFolder, error) {
var folder vfs.BaseVirtualFolder
q := getFolderByPathQuery()
q := getFolderByNameQuery()
stmt, err := dbHandle.PrepareContext(ctx, q)
if err != nil {
providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
@ -621,18 +622,39 @@ func sqlCommonCheckFolderExists(ctx context.Context, name string, dbHandle sqlQu
}
defer stmt.Close()
row := stmt.QueryRowContext(ctx, name)
err = row.Scan(&folder.ID, &folder.MappedPath, &folder.UsedQuotaSize, &folder.UsedQuotaFiles, &folder.LastQuotaUpdate)
var mappedPath sql.NullString
err = row.Scan(&folder.ID, &mappedPath, &folder.UsedQuotaSize, &folder.UsedQuotaFiles, &folder.LastQuotaUpdate,
&folder.Name)
if err == sql.ErrNoRows {
return folder, &RecordNotFoundError{err: err.Error()}
}
if mappedPath.Valid {
folder.MappedPath = mappedPath.String
}
return folder, err
}
func sqlCommonAddOrGetFolder(ctx context.Context, name string, usedQuotaSize int64, usedQuotaFiles int, lastQuotaUpdate int64, dbHandle sqlQuerier) (vfs.BaseVirtualFolder, error) {
func sqlCommonGetFolderByName(ctx context.Context, name string, dbHandle sqlQuerier) (vfs.BaseVirtualFolder, error) {
folder, err := sqlCommonCheckFolderExists(ctx, name, dbHandle)
if err != nil {
return folder, err
}
folders, err := getVirtualFoldersWithUsers([]vfs.BaseVirtualFolder{folder}, dbHandle)
if err != nil {
return folder, err
}
if len(folders) != 1 {
return folder, fmt.Errorf("unable to associate users with folder %#v", name)
}
return folders[0], nil
}
func sqlCommonAddOrGetFolder(ctx context.Context, baseFolder vfs.BaseVirtualFolder, usedQuotaSize int64, usedQuotaFiles int, lastQuotaUpdate int64, dbHandle sqlQuerier) (vfs.BaseVirtualFolder, error) {
folder, err := sqlCommonCheckFolderExists(ctx, baseFolder.Name, dbHandle)
if _, ok := err.(*RecordNotFoundError); ok {
f := &vfs.BaseVirtualFolder{
MappedPath: name,
Name: baseFolder.Name,
MappedPath: baseFolder.MappedPath,
UsedQuotaSize: usedQuotaSize,
UsedQuotaFiles: usedQuotaFiles,
LastQuotaUpdate: lastQuotaUpdate,
@ -641,7 +663,7 @@ func sqlCommonAddOrGetFolder(ctx context.Context, name string, usedQuotaSize int
if err != nil {
return folder, err
}
return sqlCommonCheckFolderExists(ctx, name, dbHandle)
return sqlCommonCheckFolderExists(ctx, baseFolder.Name, dbHandle)
}
return folder, err
}
@ -660,7 +682,26 @@ func sqlCommonAddFolder(folder *vfs.BaseVirtualFolder, dbHandle sqlQuerier) erro
return err
}
defer stmt.Close()
_, err = stmt.ExecContext(ctx, folder.MappedPath, folder.UsedQuotaSize, folder.UsedQuotaFiles, folder.LastQuotaUpdate)
_, err = stmt.ExecContext(ctx, folder.MappedPath, folder.UsedQuotaSize, folder.UsedQuotaFiles,
folder.LastQuotaUpdate, folder.Name)
return err
}
func sqlCommonUpdateFolder(folder *vfs.BaseVirtualFolder, dbHandle *sql.DB) error {
err := validateFolder(folder)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
defer cancel()
q := getUpdateFolderQuery()
stmt, err := dbHandle.PrepareContext(ctx, q)
if err != nil {
providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
return err
}
defer stmt.Close()
_, err = stmt.ExecContext(ctx, folder.MappedPath, folder.Name)
return err
}
@ -696,10 +737,15 @@ func sqlCommonDumpFolders(dbHandle sqlQuerier) ([]vfs.BaseVirtualFolder, error)
defer rows.Close()
for rows.Next() {
var folder vfs.BaseVirtualFolder
err = rows.Scan(&folder.ID, &folder.MappedPath, &folder.UsedQuotaSize, &folder.UsedQuotaFiles, &folder.LastQuotaUpdate)
var mappedPath sql.NullString
err = rows.Scan(&folder.ID, &mappedPath, &folder.UsedQuotaSize, &folder.UsedQuotaFiles,
&folder.LastQuotaUpdate, &folder.Name)
if err != nil {
return folders, err
}
if mappedPath.Valid {
folder.MappedPath = mappedPath.String
}
folders = append(folders, folder)
}
err = rows.Err()
@ -709,33 +755,34 @@ func sqlCommonDumpFolders(dbHandle sqlQuerier) ([]vfs.BaseVirtualFolder, error)
return getVirtualFoldersWithUsers(folders, dbHandle)
}
func sqlCommonGetFolders(limit, offset int, order, folderPath string, dbHandle sqlQuerier) ([]vfs.BaseVirtualFolder, error) {
func sqlCommonGetFolders(limit, offset int, order string, dbHandle sqlQuerier) ([]vfs.BaseVirtualFolder, error) {
folders := make([]vfs.BaseVirtualFolder, 0, limit)
ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
defer cancel()
q := getFoldersQuery(order, folderPath)
q := getFoldersQuery(order)
stmt, err := dbHandle.PrepareContext(ctx, q)
if err != nil {
providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
return nil, err
}
defer stmt.Close()
var rows *sql.Rows
if len(folderPath) > 0 {
rows, err = stmt.QueryContext(ctx, folderPath, limit, offset) //nolint:rowserrcheck // rows.Err() is checked
} else {
rows, err = stmt.QueryContext(ctx, limit, offset) //nolint:rowserrcheck // rows.Err() is checked
}
rows, err := stmt.QueryContext(ctx, limit, offset)
if err != nil {
return folders, err
}
defer rows.Close()
for rows.Next() {
var folder vfs.BaseVirtualFolder
err = rows.Scan(&folder.ID, &folder.MappedPath, &folder.UsedQuotaSize, &folder.UsedQuotaFiles, &folder.LastQuotaUpdate)
var mappedPath sql.NullString
err = rows.Scan(&folder.ID, &mappedPath, &folder.UsedQuotaSize, &folder.UsedQuotaFiles,
&folder.LastQuotaUpdate, &folder.Name)
if err != nil {
return folders, err
}
if mappedPath.Valid {
folder.MappedPath = mappedPath.String
}
folders = append(folders, folder)
}
@ -776,7 +823,7 @@ func generateVirtualFoldersMapping(ctx context.Context, user *User, dbHandle sql
return err
}
for _, vfolder := range user.VirtualFolders {
f, err := sqlCommonAddOrGetFolder(ctx, vfolder.MappedPath, 0, 0, 0, dbHandle)
f, err := sqlCommonAddOrGetFolder(ctx, vfolder.BaseVirtualFolder, 0, 0, 0, dbHandle)
if err != nil {
return err
}
@ -823,11 +870,15 @@ func getUsersWithVirtualFolders(users []User, dbHandle sqlQuerier) ([]User, erro
for rows.Next() {
var folder vfs.VirtualFolder
var userID int64
err = rows.Scan(&folder.ID, &folder.MappedPath, &folder.UsedQuotaSize, &folder.UsedQuotaFiles,
var mappedPath sql.NullString
err = rows.Scan(&folder.ID, &folder.Name, &mappedPath, &folder.UsedQuotaSize, &folder.UsedQuotaFiles,
&folder.LastQuotaUpdate, &folder.VirtualPath, &folder.QuotaSize, &folder.QuotaFiles, &userID)
if err != nil {
return users, err
}
if mappedPath.Valid {
folder.MappedPath = mappedPath.String
}
usersVirtualFolders[userID] = append(usersVirtualFolders[userID], folder)
}
err = rows.Err()
@ -887,7 +938,7 @@ func getVirtualFoldersWithUsers(folders []vfs.BaseVirtualFolder, dbHandle sqlQue
return folders, err
}
func sqlCommonUpdateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64, reset bool, dbHandle *sql.DB) error {
func sqlCommonUpdateFolderQuota(name string, filesAdd int, sizeAdd int64, reset bool, dbHandle *sql.DB) error {
ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
defer cancel()
q := getUpdateFolderQuotaQuery(reset)
@ -897,12 +948,12 @@ func sqlCommonUpdateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64,
return err
}
defer stmt.Close()
_, err = stmt.ExecContext(ctx, sizeAdd, filesAdd, utils.GetTimeAsMsSinceEpoch(time.Now()), mappedPath)
_, err = stmt.ExecContext(ctx, sizeAdd, filesAdd, utils.GetTimeAsMsSinceEpoch(time.Now()), name)
if err == nil {
providerLog(logger.LevelDebug, "quota updated for folder %#v, files increment: %v size increment: %v is reset? %v",
mappedPath, filesAdd, sizeAdd, reset)
name, filesAdd, sizeAdd, reset)
} else {
providerLog(logger.LevelWarn, "error updating quota for folder %#v: %v", mappedPath, err)
providerLog(logger.LevelWarn, "error updating quota for folder %#v: %v", name, err)
}
return err
}
@ -991,7 +1042,7 @@ func sqlCommonExecSQLAndUpdateDBVersion(dbHandle *sql.DB, sql []string, newVersi
return tx.Commit()
}
func sqlCommonGetCompatVirtualFolders(dbHandle *sql.DB) ([]userCompactVFolders, error) {
/*func sqlCommonGetCompatVirtualFolders(dbHandle *sql.DB) ([]userCompactVFolders, error) {
users := []userCompactVFolders{}
ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
defer cancel()
@ -1024,9 +1075,9 @@ func sqlCommonGetCompatVirtualFolders(dbHandle *sql.DB) ([]userCompactVFolders,
}
}
return users, rows.Err()
}
}*/
func sqlCommonRestoreCompatVirtualFolders(ctx context.Context, users []userCompactVFolders, dbHandle sqlQuerier) ([]string, error) {
/*func sqlCommonRestoreCompatVirtualFolders(ctx context.Context, users []userCompactVFolders, dbHandle sqlQuerier) ([]string, error) {
foldersToScan := []string{}
for _, user := range users {
for _, vfolder := range user.VirtualFolders {
@ -1065,15 +1116,15 @@ func sqlCommonRestoreCompatVirtualFolders(ctx context.Context, users []userCompa
}
}
return foldersToScan, nil
}
}*/
func sqlCommonUpdateDatabaseFrom3To4(sqlV4 string, dbHandle *sql.DB) error {
logger.InfoToConsole("updating database version: 3 -> 4")
providerLog(logger.LevelInfo, "updating database version: 3 -> 4")
users, err := sqlCommonGetCompatVirtualFolders(dbHandle)
/*users, err := sqlCommonGetCompatVirtualFolders(dbHandle)
if err != nil {
return err
}
}*/
sql := strings.ReplaceAll(sqlV4, "{{users}}", sqlTableUsers)
sql = strings.ReplaceAll(sql, "{{folders}}", sqlTableFolders)
sql = strings.ReplaceAll(sql, "{{folders_mapping}}", sqlTableFoldersMapping)
@ -1093,21 +1144,21 @@ func sqlCommonUpdateDatabaseFrom3To4(sqlV4 string, dbHandle *sql.DB) error {
return err
}
}
foldersToScan, err := sqlCommonRestoreCompatVirtualFolders(ctx, users, tx)
/*_, err = sqlCommonRestoreCompatVirtualFolders(ctx, users, tx)
if err != nil {
sqlCommonRollbackTransaction(tx)
return err
}
}*/
err = sqlCommonUpdateDatabaseVersion(ctx, tx, 4)
if err != nil {
sqlCommonRollbackTransaction(tx)
return err
}
err = tx.Commit()
if err == nil {
return tx.Commit()
/*if err == nil {
go updateVFoldersQuotaAfterRestore(foldersToScan)
}
return err
return err*/
}
//nolint:dupl

99
dataprovider/sqlite.go
View file

@ -82,6 +82,22 @@ ALTER TABLE "new__users" RENAME TO "{{users}}";
"password" varchar(255) NOT NULL, "email" varchar(255) NULL, "status" integer NOT NULL, "permissions" text NOT NULL, "filters" text NULL,
"additional_info" text NULL);`
sqliteV7DownSQL = `DROP TABLE "{{admins}}";`
sqliteV8SQL = `CREATE TABLE "new__folders" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"name" varchar(255) NOT NULL UNIQUE, "path" varchar(512) NULL, "used_quota_size" bigint NOT NULL,
"used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL);
INSERT INTO "new__folders" ("id", "path", "used_quota_size", "used_quota_files", "last_quota_update", "name")
SELECT "id", "path", "used_quota_size", "used_quota_files", "last_quota_update", ('folder' || "id") FROM "{{folders}}";
DROP TABLE "{{folders}}";
ALTER TABLE "new__folders" RENAME TO "{{folders}}";
`
sqliteV8DownSQL = `CREATE TABLE "new__folders" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"path" varchar(512) NOT NULL UNIQUE, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL,
"last_quota_update" bigint NOT NULL);
INSERT INTO "new__folders" ("id", "path", "used_quota_size", "used_quota_files", "last_quota_update")
SELECT "id", "path", "used_quota_size", "used_quota_files", "last_quota_update" FROM "{{folders}}";
DROP TABLE "{{folders}}";
ALTER TABLE "new__folders" RENAME TO "{{folders}}";
`
)
// SQLiteProvider auth provider for SQLite database
@ -173,30 +189,34 @@ func (p *SQLiteProvider) dumpFolders() ([]vfs.BaseVirtualFolder, error) {
return sqlCommonDumpFolders(p.dbHandle)
}
func (p *SQLiteProvider) getFolders(limit, offset int, order, folderPath string) ([]vfs.BaseVirtualFolder, error) {
return sqlCommonGetFolders(limit, offset, order, folderPath, p.dbHandle)
func (p *SQLiteProvider) getFolders(limit, offset int, order string) ([]vfs.BaseVirtualFolder, error) {
return sqlCommonGetFolders(limit, offset, order, p.dbHandle)
}
func (p *SQLiteProvider) getFolderByPath(mappedPath string) (vfs.BaseVirtualFolder, error) {
func (p *SQLiteProvider) getFolderByName(name string) (vfs.BaseVirtualFolder, error) {
ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
defer cancel()
return sqlCommonCheckFolderExists(ctx, mappedPath, p.dbHandle)
return sqlCommonGetFolderByName(ctx, name, p.dbHandle)
}
func (p *SQLiteProvider) addFolder(folder *vfs.BaseVirtualFolder) error {
return sqlCommonAddFolder(folder, p.dbHandle)
}
func (p *SQLiteProvider) updateFolder(folder *vfs.BaseVirtualFolder) error {
return sqlCommonUpdateFolder(folder, p.dbHandle)
}
func (p *SQLiteProvider) deleteFolder(folder *vfs.BaseVirtualFolder) error {
return sqlCommonDeleteFolder(folder, p.dbHandle)
}
func (p *SQLiteProvider) updateFolderQuota(mappedPath string, filesAdd int, sizeAdd int64, reset bool) error {
return sqlCommonUpdateFolderQuota(mappedPath, filesAdd, sizeAdd, reset, p.dbHandle)
func (p *SQLiteProvider) updateFolderQuota(name string, filesAdd int, sizeAdd int64, reset bool) error {
return sqlCommonUpdateFolderQuota(name, filesAdd, sizeAdd, reset, p.dbHandle)
}
func (p *SQLiteProvider) getUsedFolderQuota(mappedPath string) (int, int64, error) {
return sqlCommonGetFolderUsedQuota(mappedPath, p.dbHandle)
func (p *SQLiteProvider) getUsedFolderQuota(name string) (int, int64, error) {
return sqlCommonGetFolderUsedQuota(name, p.dbHandle)
}
func (p *SQLiteProvider) adminExists(username string) (Admin, error) {
@ -286,6 +306,8 @@ func (p *SQLiteProvider) migrateDatabase() error {
return updateSQLiteDatabaseFromV5(p.dbHandle)
case 6:
return updateSQLiteDatabaseFromV6(p.dbHandle)
case 7:
return updateSQLiteDatabaseFromV7(p.dbHandle)
default:
if dbVersion.Version > sqlDatabaseVersion {
providerLog(logger.LevelWarn, "database version %v is newer than the supported: %v", dbVersion.Version,
@ -298,6 +320,7 @@ func (p *SQLiteProvider) migrateDatabase() error {
}
}
//nolint:dupl
func (p *SQLiteProvider) revertDatabase(targetVersion int) error {
dbVersion, err := sqlCommonGetDatabaseVersion(p.dbHandle, true)
if err != nil {
@ -307,6 +330,20 @@ func (p *SQLiteProvider) revertDatabase(targetVersion int) error {
return fmt.Errorf("current version match target version, nothing to do")
}
switch dbVersion.Version {
case 8:
err = downgradeSQLiteDatabaseFrom8To7(p.dbHandle)
if err != nil {
return err
}
err = downgradeSQLiteDatabaseFrom7To6(p.dbHandle)
if err != nil {
return err
}
err = downgradeSQLiteDatabaseFrom6To5(p.dbHandle)
if err != nil {
return err
}
return downgradeSQLiteDatabaseFrom5To4(p.dbHandle)
case 7:
err = downgradeSQLiteDatabaseFrom7To6(p.dbHandle)
if err != nil {
@ -371,7 +408,15 @@ func updateSQLiteDatabaseFromV5(dbHandle *sql.DB) error {
}
func updateSQLiteDatabaseFromV6(dbHandle *sql.DB) error {
return updateSQLiteDatabaseFrom6To7(dbHandle)
err := updateSQLiteDatabaseFrom6To7(dbHandle)
if err != nil {
return err
}
return updateSQLiteDatabaseFromV7(dbHandle)
}
func updateSQLiteDatabaseFromV7(dbHandle *sql.DB) error {
return updateSQLiteDatabaseFrom7To8(dbHandle)
}
func updateSQLiteDatabaseFrom1To2(dbHandle *sql.DB) error {
@ -410,6 +455,42 @@ func updateSQLiteDatabaseFrom6To7(dbHandle *sql.DB) error {
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 7)
}
func updateSQLiteDatabaseFrom7To8(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database version: 7 -> 8")
providerLog(logger.LevelInfo, "updating database version: 7 -> 8")
if err := setPragmaFK(dbHandle, "OFF"); err != nil {
return err
}
sql := strings.ReplaceAll(sqliteV8SQL, "{{folders}}", sqlTableFolders)
if err := sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 8); err != nil {
return err
}
return setPragmaFK(dbHandle, "ON")
}
func setPragmaFK(dbHandle *sql.DB, value string) error {
ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout)
defer cancel()
sql := fmt.Sprintf("PRAGMA foreign_keys=%v;", value)
_, err := dbHandle.ExecContext(ctx, sql)
return err
}
func downgradeSQLiteDatabaseFrom8To7(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database version: 8 -> 7")
providerLog(logger.LevelInfo, "downgrading database version: 8 -> 7")
if err := setPragmaFK(dbHandle, "OFF"); err != nil {
return err
}
sql := strings.ReplaceAll(sqliteV8DownSQL, "{{folders}}", sqlTableFolders)
if err := sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 7); err != nil {
return err
}
return setPragmaFK(dbHandle, "ON")
}
func downgradeSQLiteDatabaseFrom7To6(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database version: 7 -> 6")
providerLog(logger.LevelInfo, "downgrading database version: 7 -> 6")

34
dataprovider/sqlqueries.go
View file

@ -11,7 +11,7 @@ import (
const (
selectUserFields = "id,username,password,public_keys,home_dir,uid,gid,max_sessions,quota_size,quota_files,permissions,used_quota_size," +
"used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,expiration_date,last_login,status,filters,filesystem,additional_info"
selectFolderFields = "id,path,used_quota_size,used_quota_files,last_quota_update"
selectFolderFields = "id,path,used_quota_size,used_quota_files,last_quota_update,name"
selectAdminFields = "id,username,password,status,email,permissions,filters,additional_info"
)
@ -114,13 +114,17 @@ func getDeleteUserQuery() string {
return fmt.Sprintf(`DELETE FROM %v WHERE id = %v`, sqlTableUsers, sqlPlaceholders[0])
}
func getFolderByPathQuery() string {
return fmt.Sprintf(`SELECT %v FROM %v WHERE path = %v`, selectFolderFields, sqlTableFolders, sqlPlaceholders[0])
func getFolderByNameQuery() string {
return fmt.Sprintf(`SELECT %v FROM %v WHERE name = %v`, selectFolderFields, sqlTableFolders, sqlPlaceholders[0])
}
func getAddFolderQuery() string {
return fmt.Sprintf(`INSERT INTO %v (path,used_quota_size,used_quota_files,last_quota_update) VALUES (%v,%v,%v,%v)`,
sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
return fmt.Sprintf(`INSERT INTO %v (path,used_quota_size,used_quota_files,last_quota_update,name) VALUES (%v,%v,%v,%v,%v)`,
sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4])
}
func getUpdateFolderQuery() string {
return fmt.Sprintf(`UPDATE %v SET path = %v WHERE name = %v`, sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1])
}
func getDeleteFolderQuery() string {
@ -138,26 +142,22 @@ func getAddFolderMappingQuery() string {
sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlTableUsers, sqlPlaceholders[4])
}
func getFoldersQuery(order, folderPath string) string {
if len(folderPath) > 0 {
return fmt.Sprintf(`SELECT %v FROM %v WHERE path = %v ORDER BY path %v LIMIT %v OFFSET %v`,
selectFolderFields, sqlTableFolders, sqlPlaceholders[0], order, sqlPlaceholders[1], sqlPlaceholders[2])
}
return fmt.Sprintf(`SELECT %v FROM %v ORDER BY path %v LIMIT %v OFFSET %v`, selectFolderFields, sqlTableFolders,
func getFoldersQuery(order string) string {
return fmt.Sprintf(`SELECT %v FROM %v ORDER BY name %v LIMIT %v OFFSET %v`, selectFolderFields, sqlTableFolders,
order, sqlPlaceholders[0], sqlPlaceholders[1])
}
func getUpdateFolderQuotaQuery(reset bool) string {
if reset {
return fmt.Sprintf(`UPDATE %v SET used_quota_size = %v,used_quota_files = %v,last_quota_update = %v
WHERE path = %v`, sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
WHERE name = %v`, sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
}
return fmt.Sprintf(`UPDATE %v SET used_quota_size = used_quota_size + %v,used_quota_files = used_quota_files + %v,last_quota_update = %v
WHERE path = %v`, sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
WHERE name = %v`, sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3])
}
func getQuotaFolderQuery() string {
return fmt.Sprintf(`SELECT used_quota_size,used_quota_files FROM %v WHERE path = %v`, sqlTableFolders,
return fmt.Sprintf(`SELECT used_quota_size,used_quota_files FROM %v WHERE name = %v`, sqlTableFolders,
sqlPlaceholders[0])
}
@ -174,7 +174,7 @@ func getRelatedFoldersForUsersQuery(users []User) string {
if sb.Len() > 0 {
sb.WriteString(")")
}
return fmt.Sprintf(`SELECT f.id,f.path,f.used_quota_size,f.used_quota_files,f.last_quota_update,fm.virtual_path,fm.quota_size,fm.quota_files,fm.user_id
return fmt.Sprintf(`SELECT f.id,f.name,f.path,f.used_quota_size,f.used_quota_files,f.last_quota_update,fm.virtual_path,fm.quota_size,fm.quota_files,fm.user_id
FROM %v f INNER JOIN %v fm ON f.id = fm.folder_id WHERE fm.user_id IN %v ORDER BY fm.user_id`, sqlTableFolders,
sqlTableFoldersMapping, sb.String())
}
@ -204,9 +204,9 @@ func getUpdateDBVersionQuery() string {
return fmt.Sprintf(`UPDATE %v SET version=%v`, sqlTableSchemaVersion, sqlPlaceholders[0])
}
func getCompatVirtualFoldersQuery() string {
/*func getCompatVirtualFoldersQuery() string {
return fmt.Sprintf(`SELECT id,username,virtual_folders FROM %v`, sqlTableUsers)
}
}*/
func getCompatV4FsConfigQuery() string {
return fmt.Sprintf(`SELECT id,username,filesystem FROM %v`, sqlTableUsers)

2
docs/kms.md
View file

@ -1,6 +1,6 @@
# Key Management Services
SFTPGo stores sensitive data such as Cloud accounts credentials or passphrases to derive per-object encryption keys. These data are stored as ciphertext and only loaded to RAM in plaintext when needed.
SFTPGo stores sensitive data such as Cloud account credentials or passphrases to derive per-object encryption keys. These data are stored as ciphertext and only loaded to RAM in plaintext when needed.
## Supported Services for encryption and decryption

2
docs/rest-api.md
View file

@ -4,7 +4,7 @@ SFTPGo exposes REST API to manage, backup, and restore users and folders, and to
If quota tracking is enabled in the configuration file, then the used size and number of files are updated each time a file is added/removed. If files are added/removed not using SFTP/SCP, or if you change `track_quota` from `2` to `1`, you can rescan the users home dir and update the used quota using the REST API.
REST API are protected using JSON Web Tokens (JWT) authentication and can be exposed over HTTPS.
REST API are protected using JSON Web Tokens (JWT) authentication and can be exposed over HTTPS. You can also configure client certificate authentication in addition to JWT.
The default credentials are:

26
ftpd/ftpd_test.go
View file

@ -1398,8 +1398,10 @@ func TestUploadOverwriteVfolder(t *testing.T) {
u := getTestUser()
vdir := "/vdir"
mappedPath := filepath.Join(os.TempDir(), "vdir")
folderName := filepath.Base(mappedPath)
u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{
BaseVirtualFolder: vfs.BaseVirtualFolder{
Name: folderName,
MappedPath: mappedPath,
},
VirtualPath: vdir,
@ -1418,22 +1420,16 @@ func TestUploadOverwriteVfolder(t *testing.T) {
assert.NoError(t, err)
err = ftpUploadFile(testFilePath, path.Join(vdir, testFileName), testFileSize, client, 0)
assert.NoError(t, err)
folder, _, err := httpdtest.GetFolders(0, 0, mappedPath, http.StatusOK)
folder, _, err := httpdtest.GetFolderByName(folderName, http.StatusOK)
assert.NoError(t, err)
if assert.Len(t, folder, 1) {
f := folder[0]
assert.Equal(t, testFileSize, f.UsedQuotaSize)
assert.Equal(t, 1, f.UsedQuotaFiles)
}
assert.Equal(t, testFileSize, folder.UsedQuotaSize)
assert.Equal(t, 1, folder.UsedQuotaFiles)
err = ftpUploadFile(testFilePath, path.Join(vdir, testFileName), testFileSize, client, 0)
assert.NoError(t, err)
folder, _, err = httpdtest.GetFolders(0, 0, mappedPath, http.StatusOK)
folder, _, err = httpdtest.GetFolderByName(folderName, http.StatusOK)
assert.NoError(t, err)
if assert.Len(t, folder, 1) {
f := folder[0]
assert.Equal(t, testFileSize, f.UsedQuotaSize)
assert.Equal(t, 1, f.UsedQuotaFiles)
}
assert.Equal(t, testFileSize, folder.UsedQuotaSize)
assert.Equal(t, 1, folder.UsedQuotaFiles)
err = client.Quit()
assert.NoError(t, err)
err = os.Remove(testFilePath)
@ -1441,7 +1437,7 @@ func TestUploadOverwriteVfolder(t *testing.T) {
}
_, err = httpdtest.RemoveUser(user, http.StatusOK)
assert.NoError(t, err)
_, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{MappedPath: mappedPath}, http.StatusOK)
_, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName}, http.StatusOK)
assert.NoError(t, err)
err = os.RemoveAll(user.GetHomeDir())
assert.NoError(t, err)
@ -1452,8 +1448,10 @@ func TestUploadOverwriteVfolder(t *testing.T) {
func TestAllocateAvailable(t *testing.T) {
u := getTestUser()
mappedPath := filepath.Join(os.TempDir(), "vdir")
folderName := filepath.Base(mappedPath)
u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{
BaseVirtualFolder: vfs.BaseVirtualFolder{
Name: folderName,
MappedPath: mappedPath,
},
VirtualPath: "/vdir",
@ -1579,7 +1577,7 @@ func TestAllocateAvailable(t *testing.T) {
_, err = httpdtest.RemoveUser(user, http.StatusOK)
assert.NoError(t, err)
_, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{MappedPath: mappedPath}, http.StatusOK)
_, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName}, http.StatusOK)
assert.NoError(t, err)
err = os.RemoveAll(user.GetHomeDir())
assert.NoError(t, err)

33
go.mod
View file

@ -5,34 +5,35 @@ go 1.15
require (
cloud.google.com/go v0.75.0 // indirect
cloud.google.com/go/storage v1.12.0
github.com/Azure/azure-storage-blob-go v0.12.0
github.com/Azure/azure-storage-blob-go v0.13.0
github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
github.com/alexedwards/argon2id v0.0.0-20201228115903-cf543ebc1f7b
github.com/aws/aws-sdk-go v1.36.28
github.com/aws/aws-sdk-go v1.37.1
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d
github.com/fclairamb/ftpserverlib v0.12.0
github.com/frankban/quicktest v1.11.2 // indirect
github.com/go-chi/chi v1.5.1
github.com/go-chi/jwtauth v1.1.1
github.com/go-chi/jwtauth v1.2.0
github.com/go-chi/render v1.0.1
github.com/go-ole/go-ole v1.2.5 // indirect
github.com/go-sql-driver/mysql v1.5.0
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/google/uuid v1.1.5 // indirect
github.com/google/uuid v1.2.0 // indirect
github.com/google/wire v0.5.0 // indirect
github.com/grandcat/zeroconf v1.0.0
github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126
github.com/lestrrat-go/jwx v1.0.8
github.com/lestrrat-go/jwx v1.1.0
github.com/lib/pq v1.9.0
github.com/magiconair/properties v1.8.4 // indirect
github.com/mattn/go-sqlite3 v1.14.6
github.com/miekg/dns v1.1.35 // indirect
github.com/miekg/dns v1.1.37 // indirect
github.com/minio/sha256-simd v0.1.1
github.com/minio/sio v0.2.1
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/otiai10/copy v1.4.2
github.com/pelletier/go-toml v1.8.1 // indirect
github.com/pires/go-proxyproto v0.3.3
github.com/pires/go-proxyproto v0.4.2
github.com/pkg/sftp v1.12.1-0.20201128220914-b5b6f3393fe9
github.com/prometheus/client_golang v1.9.0
github.com/prometheus/procfs v0.3.0 // indirect
@ -40,7 +41,7 @@ require (
github.com/rs/xid v1.2.1
github.com/rs/zerolog v1.20.0
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shirou/gopsutil/v3 v3.20.12
github.com/shirou/gopsutil/v3 v3.21.1
github.com/spf13/afero v1.5.1
github.com/spf13/cast v1.3.1 // indirect
github.com/spf13/cobra v1.1.1
@ -50,19 +51,19 @@ require (
github.com/studio-b12/gowebdav v0.0.0-20200929080739-bdacfab94796
github.com/yl2chen/cidranger v1.0.2
go.etcd.io/bbolt v1.3.5
go.opencensus.io v0.22.6 // indirect
go.uber.org/automaxprocs v1.3.0
gocloud.dev v0.21.0
gocloud.dev/secrets/hashivault v0.21.0
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
golang.org/x/mod v0.4.1 // indirect
golang.org/x/net v0.0.0-20201224014010-6772e930b67b
golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3 // indirect
golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78
golang.org/x/net v0.0.0-20210119194325-5f4716e94777
golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013 // indirect
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
golang.org/x/text v0.3.5 // indirect
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
golang.org/x/tools v0.0.0-20210115202250-e0d201561e39 // indirect
google.golang.org/api v0.36.0
google.golang.org/genproto v0.0.0-20210114201628-6edceaf6022f // indirect
golang.org/x/tools v0.1.0 // indirect
google.golang.org/api v0.38.0
google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506 // indirect
google.golang.org/grpc v1.35.0 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0
@ -74,5 +75,5 @@ replace (
github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9
github.com/pkg/sftp => github.com/drakkan/sftp v0.0.0-20201211115031-0b6bbc64f191
golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20201217113543-470e61ed2598
golang.org/x/net => github.com/drakkan/net v0.0.0-20201217113732-2a124bb1694b
golang.org/x/net => github.com/drakkan/net v0.0.0-20210201075003-5fb2b186574d
)

134
go.sum
View file

@ -17,6 +17,7 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko=
cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
cloud.google.com/go v0.75.0 h1:XgtDnVJRCPEUG21gjFiRPz4zI1Mjg16R+NYQjfmU4XY=
cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
@ -53,29 +54,36 @@ github.com/Azure/azure-sdk-for-go v37.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9mo
github.com/Azure/azure-sdk-for-go v49.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
github.com/Azure/azure-service-bus-go v0.10.7/go.mod h1:o5z/3lDG1iT/T/G7vgIwIqVDTx9Qa2wndf5OdzSzpF8=
github.com/Azure/azure-storage-blob-go v0.11.0/go.mod h1:A0u4VjtpgZJ7Y7um/+ix2DHBuEKFC6sEIlj0xc13a4Q=
github.com/Azure/azure-storage-blob-go v0.12.0 h1:7bFXA1QB+lOK2/ASWHhp6/vnxjaeeZq6t8w1Jyp0Iaw=
github.com/Azure/azure-storage-blob-go v0.12.0/go.mod h1:A0u4VjtpgZJ7Y7um/+ix2DHBuEKFC6sEIlj0xc13a4Q=
github.com/Azure/azure-storage-blob-go v0.13.0 h1:lgWHvFh+UYBNVQLFHXkvul2f6yOPA9PIH82RTG2cSwc=
github.com/Azure/azure-storage-blob-go v0.13.0/go.mod h1:pA9kNqtjUeQF2zOSu4s//nUdBD+e64lEuc4sVnuOfNs=
github.com/Azure/go-amqp v0.13.0/go.mod h1:qj+o8xPCz9tMSbQ83Vp8boHahuRDl5mkNHyt1xlxUTs=
github.com/Azure/go-amqp v0.13.1/go.mod h1:qj+o8xPCz9tMSbQ83Vp8boHahuRDl5mkNHyt1xlxUTs=
github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
github.com/Azure/go-autorest/autorest v0.11.3/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
github.com/Azure/go-autorest/autorest v0.11.7/go.mod h1:V6p3pKZx1KKkJubbxnDWrzNhEIfOy/pTGasLqzHIPHs=
github.com/Azure/go-autorest/autorest v0.11.9/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
github.com/Azure/go-autorest/autorest v0.11.12 h1:gI8ytXbxMfI+IVbI9mP2JGCTXIuhHLgRlvQ9X4PsnHE=
github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE=
github.com/Azure/go-autorest/autorest/adal v0.9.4/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE=
github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
github.com/Azure/go-autorest/autorest/adal v0.9.6 h1:d3pSDwvBWBLqdA91u+keH1zs1cCEzrQdHKY6iqbQNkE=
github.com/Azure/go-autorest/autorest/adal v0.9.6/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
github.com/Azure/go-autorest/autorest/azure/auth v0.5.3/go.mod h1:4bJZhUhcq8LB20TruwHbAQsmUs2Xh+QR7utuJpLXX3A=
github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM=
github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk=
github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE=
github.com/Azure/go-autorest/autorest/validation v0.3.0/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E=
github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 h1:KeNholpO2xKjgaaSyd+DyQRrsQjhbSeS7qe4nEw8aQw=
@ -85,6 +93,7 @@ github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
@ -106,8 +115,8 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo
github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.36.1/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.36.28 h1:JVRN7BZgwQ31SQCBwG5QM445+ynJU0ruKu+miFIijYY=
github.com/aws/aws-sdk-go v1.36.28/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.37.1 h1:BTHmuN+gzhxkvU9sac2tZvaY0gV9ihbHw+KxZOecYvY=
github.com/aws/aws-sdk-go v1.37.1/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@ -161,8 +170,8 @@ github.com/drakkan/crypto v0.0.0-20201217113543-470e61ed2598 h1:JawLnfGaG2uL0eFT
github.com/drakkan/crypto v0.0.0-20201217113543-470e61ed2598/go.mod h1:HCh3rfXxsHzqOEbzc/nqz6WnUhb7Nv19n/o64V0Zmbg=
github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 h1:LPH1dEblAOO/LoG7yHPMtBLXhQmjaga91/DDjWk9jWA=
github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
github.com/drakkan/net v0.0.0-20201217113732-2a124bb1694b h1:v7XxoSGerTbxqH6aX6Rgq8EAKKGLQwUS/gIlKH29T4o=
github.com/drakkan/net v0.0.0-20201217113732-2a124bb1694b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
github.com/drakkan/net v0.0.0-20210201075003-5fb2b186574d h1:h2rU/lTUkEYB3y4k6+FgQNMajf4uE+sbMRn85kT+VTQ=
github.com/drakkan/net v0.0.0-20210201075003-5fb2b186574d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
github.com/drakkan/sftp v0.0.0-20201211115031-0b6bbc64f191 h1:c+RLqMs6Aqc8IDc5MWTf+zqNlO4+5WfiJqZzHFlr4a8=
github.com/drakkan/sftp v0.0.0-20201211115031-0b6bbc64f191/go.mod h1:fUqqXB5vEgVCZ131L+9say31RAri6aF6KDViawhxKK8=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@ -183,11 +192,13 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/fclairamb/ftpserverlib v0.12.0 h1:vud3Q4v/rLZU5CfIDFaXq7ST2+V9BF5cKjzNWPN18c4=
github.com/fclairamb/ftpserverlib v0.12.0/go.mod h1:X6sAMSYtN0YDPu+nHfyE9dsKPUOrEZ8O5EMgt1xvPwk=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y=
github.com/frankban/quicktest v1.11.2 h1:mjwHjStlXWibxOohM7HYieIViKyh56mmt3+6viyhDDI=
github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
@ -198,8 +209,8 @@ github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmC
github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
github.com/go-chi/chi v1.5.1 h1:kfTK3Cxd/dkMu/rKs5ZceWYp+t5CtiE7vmaTv3LjC6w=
github.com/go-chi/chi v1.5.1/go.mod h1:REp24E+25iKvxgeTfHmdUoL5x15kBiDBlnIl5bCwe2k=
github.com/go-chi/jwtauth v1.1.1 h1:CtUHwzvXUfZeZSbASLgzaTZQ8mL7p+vitX59NBTL1vY=
github.com/go-chi/jwtauth v1.1.1/go.mod h1:znOWz9e5/GfBOKiZlOUoEfjSjUF+cLZO3GcpkoGXvFI=
github.com/go-chi/jwtauth v1.2.0 h1:Z116SPpevIABBYsv8ih/AHYBHmd4EufKSKsLUnWdrTM=
github.com/go-chi/jwtauth v1.2.0/go.mod h1:NTUpKoTQV6o25UwYE6w/VaLUu83hzrVKYTVo+lE6qDA=
github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@ -208,12 +219,15 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2
github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo=
github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o=
github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=
github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
@ -224,11 +238,14 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
github.com/goccy/go-json v0.3.5 h1:HqrLjEWx7hD62JRhBh+mHv+rEEzBANIu6O0kbDlaLzU=
github.com/goccy/go-json v0.3.5/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@ -277,14 +294,19 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-replayers/grpcreplay v1.0.0 h1:B5kVOzJ1hBgnevTgIWhSTatQ3608yu/2NnU0Ta1d0kY=
github.com/google/go-replayers/grpcreplay v1.0.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE=
github.com/google/go-replayers/httpreplay v0.1.2 h1:HCfx+dQzwN9XbGTHF8qJ+67WN8glL9FTWV5rraCJ/jU=
github.com/google/go-replayers/httpreplay v0.1.2/go.mod h1:YKZViNhiGgqdBlUbI2MwGpq4pXxNmhJLPHQ7cv2b5no=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE=
github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60=
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@ -295,6 +317,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
@ -303,15 +326,17 @@ github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.5 h1:kxhtnfFVi+rYdOALN0B3k9UT86zVJKfBimRaciULW4I=
github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/wire v0.4.0 h1:kXcsA/rIGzJImVqPdhfnr6q0xsS9gU0515q1EPpJ9fE=
github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8=
github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww=
github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
@ -338,6 +363,7 @@ github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVo
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI=
github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
@ -379,12 +405,14 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@ -395,7 +423,9 @@ github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
@ -410,24 +440,27 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/lestrrat-go/backoff/v2 v2.0.3 h1:2ABaTa5ifB1L90aoRMjaPa97p0WzzVe93Vggv8oZftw=
github.com/lestrrat-go/backoff/v2 v2.0.3/go.mod h1:mU93bMXuG27/Y5erI5E9weqavpTX5qiVFZI4uXAX0xk=
github.com/lestrrat-go/httpcc v0.0.0-20210101035852-e7e8fea419e3 h1:e52qvXxpJPV/Kb2ovtuYgcRFjNmf9ntcn8BPIbpRM4k=
github.com/lestrrat-go/httpcc v0.0.0-20210101035852-e7e8fea419e3/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++0Gf8MBnAvE=
github.com/lestrrat-go/iter v0.0.0-20200422075355-fc1769541911 h1:FvnrqecqX4zT0wOIbYK1gNgTm0677INEWiFY8UEYggY=
github.com/lestrrat-go/iter v0.0.0-20200422075355-fc1769541911/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc=
github.com/lestrrat-go/jwx v1.0.6-0.20201127121120-26218808f029/go.mod h1:TPF17WiSFegZo+c20fdpw49QD+/7n4/IsGvEmCSWwT0=
github.com/lestrrat-go/jwx v1.0.8 h1:Mj/2Ey9rkGx4w5IMQ2Q+9KLZn4cZoMgKrnMxi9eXE3k=
github.com/lestrrat-go/jwx v1.0.8/go.mod h1:6XJ5sxHF5U116AxYxeHfTnfsZRMgmeKY214zwZDdvho=
github.com/lestrrat-go/option v0.0.0-20210103042652-6f1ecfceda35 h1:lea8Wt+1ePkVrI2/WD+NgQT5r/XsLAzxeqtyFLcEs10=
github.com/lestrrat-go/backoff/v2 v2.0.7 h1:i2SeK33aOFJlUNJZzf2IpXRBvqBBnaGXfY5Xaop/GsE=
github.com/lestrrat-go/backoff/v2 v2.0.7/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
github.com/lestrrat-go/codegen v1.0.0/go.mod h1:JhJw6OQAuPEfVKUCLItpaVLumDGWQznd1VaXrBk9TdM=
github.com/lestrrat-go/httpcc v1.0.0 h1:FszVC6cKfDvBKcJv646+lkh4GydQg2Z29scgUfkOpYc=
github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++0Gf8MBnAvE=
github.com/lestrrat-go/iter v1.0.0 h1:QD+hHQPDSHC4rCJkZYY/yXChYr/vjfBopKekTc+7l4Q=
github.com/lestrrat-go/iter v1.0.0/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc=
github.com/lestrrat-go/jwx v1.1.0 h1:gerfaQK3mEIL8X8oJ5MFvsB/JuxXoGryLtTlNmPi3/k=
github.com/lestrrat-go/jwx v1.1.0/go.mod h1:vn9FzD6gJtKkgYs7RTKV7CjWtEka8F/voUollhnn4QE=
github.com/lestrrat-go/option v0.0.0-20210103042652-6f1ecfceda35/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
github.com/lestrrat-go/pdebug v0.0.0-20200204225717-4d6bd78da58d/go.mod h1:B06CSso/AWxiPejj+fheUINGeBKeeEZNt8w+EoU7+L8=
github.com/lestrrat-go/pdebug/v3 v3.0.0-20210111091911-ec4f5c88c087/go.mod h1:za+m+Ve24yCxTEhR59N7UlnJomWwCiIqbJRmKeiADU4=
github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFeEO4=
github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
github.com/lestrrat-go/pdebug/v3 v3.0.1 h1:3G5sX/aw/TbMTtVc9U7IHBWRZtMvwvBziF1e4HoQtv8=
github.com/lestrrat-go/pdebug/v3 v3.0.1/go.mod h1:za+m+Ve24yCxTEhR59N7UlnJomWwCiIqbJRmKeiADU4=
github.com/lib/pq v1.9.0 h1:L8nSXQQzAYByakOFMTwpjRoHsMJklur4Gi59b6VivR8=
github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
@ -450,8 +483,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.35 h1:oTfOaDH+mZkdcgdIjH6yBajRGtIwcwcaR+rt23ZSrJs=
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.37 h1:+kky2ArpBqk0S/74RkwFjmKM9jja7AB1RN7VUuVq0iM=
github.com/miekg/dns v1.1.37/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/minio/sio v0.2.1 h1:NjzKiIMSMcHediVQR0AFVx2tp7Wxh9tKPfDI3kH7aHQ=
@ -459,6 +492,7 @@ github.com/minio/sio v0.2.1/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebh
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
@ -485,6 +519,7 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE
github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
@ -505,8 +540,10 @@ github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh
github.com/otiai10/copy v1.4.2 h1:RTiz2sol3eoXPLF4o+YWqEybwfUa/Q2Nkc4ZIUs3fwI=
github.com/otiai10/copy v1.4.2/go.mod h1:XWfuS3CrI0R6IE0FbgHsEazaXO8G0LpMp9o8tos0x4E=
github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI=
github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
github.com/otiai10/mint v1.3.2 h1:VYWnrP5fXmz1MXvjuUvcBrXSjGE6xjON+axB/UrpO3E=
github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@ -520,8 +557,8 @@ github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A=
github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pires/go-proxyproto v0.3.3 h1:jOXGrsAfSQVFiD1hWg1aiHpLYsd6SJw/8cLN594sB7Q=
github.com/pires/go-proxyproto v0.3.3/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
github.com/pires/go-proxyproto v0.4.2 h1:VRAvsUCTrmiahoU5fqQqkbY0GWcJ1Q0F7b7CkFaipSU=
github.com/pires/go-proxyproto v0.4.2/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@ -581,14 +618,17 @@ github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkB
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo=
github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY=
github.com/shirou/gopsutil/v3 v3.20.12 h1:abpcjSQRHdb3thCge/UyJty9CnvvmUHljTSrjtFU+Og=
github.com/shirou/gopsutil/v3 v3.20.12/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4=
github.com/shirou/gopsutil/v3 v3.21.1 h1:dA72XXj5WOXIZkAL2iYTKRVcNOOqh4yfLn9Rm7t8BMM=
github.com/shirou/gopsutil/v3 v3.21.1/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
@ -655,8 +695,9 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opencensus.io v0.22.6 h1:BdkrbWrzDlV9dnbzoP7sfN+dHheJ4J9JOaYxcUDL+ok=
go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
@ -693,6 +734,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI=
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
@ -703,6 +745,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -713,8 +756,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20201203001011-0b49973bad19/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3 h1:BaN3BAqnopnKjvl+15DYP6LLrbBHfbfmlFYzmFj/Q9Q=
golang.org/x/oauth2 v0.0.0-20210113205817-d3ed898aa8a3/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013 h1:55H5j7lotzuFCEOKDsMch+fRNUQ9DgtyHOUP31FNqKc=
golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -724,6 +767,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -778,9 +823,12 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78 h1:nVuTkr9L6Bq62qpUqKo/RnZCFfzDBL0bYo6w9OJUqZY=
golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -842,7 +890,6 @@ golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapK
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/tools v0.0.0-20200417140056-c07e33ef3290/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
@ -858,8 +905,12 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201203202102-a1a1cbeaa516/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210115202250-e0d201561e39/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -887,8 +938,9 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513
google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo=
google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
google.golang.org/api v0.36.0 h1:l2Nfbl2GPXdWorv+dT2XfinX2jOOw4zv1VhLstx+6rE=
google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
google.golang.org/api v0.38.0 h1:vDyWk6eup8eQAidaZ31sNWIn8tZEL8qpbtGkBD4ytQo=
google.golang.org/api v0.38.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@ -897,6 +949,7 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww
google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@ -937,9 +990,11 @@ google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6D
google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201203001206-6486ece9c497/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210114201628-6edceaf6022f h1:izedQ6yVIc5mZsRuXzmSreCOlzI0lCU1HpG8yEdMiKw=
google.golang.org/genproto v0.0.0-20210114201628-6edceaf6022f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506 h1:uLBY0yHDCj2PMQ98KWDSIDFwn9zK2zh+tgWtbvPPBjI=
google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
@ -981,6 +1036,7 @@ gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUy
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=

98
httpd/api_folder.go
View file

@ -2,9 +2,7 @@ package httpd
import (
"context"
"errors"
"net/http"
"strconv"
"github.com/go-chi/render"
@ -13,42 +11,12 @@ import (
)
func getFolders(w http.ResponseWriter, r *http.Request) {
var err error
limit := 100
offset := 0
order := dataprovider.OrderASC
folderPath := ""
if _, ok := r.URL.Query()["limit"]; ok {
limit, err = strconv.Atoi(r.URL.Query().Get("limit"))
if err != nil {
err = errors.New("Invalid limit")
sendAPIResponse(w, r, err, "", http.StatusBadRequest)
return
}
if limit > 500 {
limit = 500
}
limit, offset, order, err := getSearchFilters(w, r)
if err != nil {
return
}
if _, ok := r.URL.Query()["offset"]; ok {
offset, err = strconv.Atoi(r.URL.Query().Get("offset"))
if err != nil {
err = errors.New("Invalid offset")
sendAPIResponse(w, r, err, "", http.StatusBadRequest)
return
}
}
if _, ok := r.URL.Query()["order"]; ok {
order = r.URL.Query().Get("order")
if order != dataprovider.OrderASC && order != dataprovider.OrderDESC {
err = errors.New("Invalid order")
sendAPIResponse(w, r, err, "", http.StatusBadRequest)
return
}
}
if _, ok := r.URL.Query()["folder-path"]; ok {
folderPath = r.URL.Query().Get("folder-path")
}
folders, err := dataprovider.GetFolders(limit, offset, order, folderPath)
folders, err := dataprovider.GetFolders(limit, offset, order)
if err == nil {
render.JSON(w, r, folders)
} else {
@ -69,31 +37,57 @@ func addFolder(w http.ResponseWriter, r *http.Request) {
sendAPIResponse(w, r, err, "", getRespStatus(err))
return
}
renderFolder(w, r, folder.MappedPath)
renderFolder(w, r, folder.Name, http.StatusCreated)
}
func renderFolder(w http.ResponseWriter, r *http.Request, mappedPath string) {
folder, err := dataprovider.GetFolderByPath(mappedPath)
func updateFolder(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
var err error
name := getURLParam(r, "name")
folder, err := dataprovider.GetFolderByName(name)
if err != nil {
sendAPIResponse(w, r, err, "", getRespStatus(err))
return
}
ctx := context.WithValue(r.Context(), render.StatusCtxKey, http.StatusCreated)
render.JSON(w, r.WithContext(ctx), folder)
}
func deleteFolderByPath(w http.ResponseWriter, r *http.Request) {
var folderPath string
if _, ok := r.URL.Query()["folder-path"]; ok {
folderPath = r.URL.Query().Get("folder-path")
}
if folderPath == "" {
err := errors.New("a non-empty folder path is required")
folderID := folder.ID
err = render.DecodeJSON(r.Body, &folder)
if err != nil {
sendAPIResponse(w, r, err, "", http.StatusBadRequest)
return
}
folder.ID = folderID
folder.Name = name
err = dataprovider.UpdateFolder(&folder)
if err != nil {
sendAPIResponse(w, r, err, "", getRespStatus(err))
return
}
sendAPIResponse(w, r, nil, "Folder updated", http.StatusOK)
}
err := dataprovider.DeleteFolder(folderPath)
func renderFolder(w http.ResponseWriter, r *http.Request, name string, status int) {
folder, err := dataprovider.GetFolderByName(name)
if err != nil {
sendAPIResponse(w, r, err, "", getRespStatus(err))
return
}
if status != http.StatusOK {
ctx := context.WithValue(r.Context(), render.StatusCtxKey, status)
render.JSON(w, r.WithContext(ctx), folder)
} else {
render.JSON(w, r, folder)
}
}
func getFolderByName(w http.ResponseWriter, r *http.Request) {
name := getURLParam(r, "name")
renderFolder(w, r, name, http.StatusOK)
}
func deleteFolder(w http.ResponseWriter, r *http.Request) {
name := getURLParam(r, "name")
err := dataprovider.DeleteFolder(name)
if err != nil {
sendAPIResponse(w, r, err, "", getRespStatus(err))
return

30
httpd/api_maintenance.go
View file

@ -153,7 +153,7 @@ func restoreBackup(content []byte, inputFile string, scanQuota, mode int) error
return dataprovider.NewValidationError(fmt.Sprintf("Unable to parse backup content: %v", err))
}
if err = RestoreFolders(dump.Folders, inputFile, scanQuota); err != nil {
if err = RestoreFolders(dump.Folders, inputFile, mode, scanQuota); err != nil {
return err
}
@ -197,23 +197,29 @@ func getLoaddataOptions(r *http.Request) (string, int, int, error) {
}
// RestoreFolders restores the specified folders
func RestoreFolders(folders []vfs.BaseVirtualFolder, inputFile string, scanQuota int) error {
func RestoreFolders(folders []vfs.BaseVirtualFolder, inputFile string, mode, scanQuota int) error {
for _, folder := range folders {
_, err := dataprovider.GetFolderByPath(folder.MappedPath)
if err == nil {
logger.Debug(logSender, "", "folder %#v already exists, restore not needed", folder.MappedPath)
continue
}
folder := folder // pin
folder.Users = nil
err = dataprovider.AddFolder(&folder)
logger.Debug(logSender, "", "adding new folder: %+v, dump file: %#v, error: %v", folder, inputFile, err)
f, err := dataprovider.GetFolderByName(folder.Name)
if err == nil {
if mode == 1 {
logger.Debug(logSender, "", "loaddata mode 1, existing folder %#v not updated", folder.Name)
continue
}
folder.ID = f.ID
err = dataprovider.UpdateFolder(&folder)
logger.Debug(logSender, "", "restoring existing folder: %+v, dump file: %#v, error: %v", folder, inputFile, err)
} else {
folder.Users = nil
err = dataprovider.AddFolder(&folder)
logger.Debug(logSender, "", "adding new folder: %+v, dump file: %#v, error: %v", folder, inputFile, err)
}
if err != nil {
return err
}
if scanQuota >= 1 {
if common.QuotaScans.AddVFolderQuotaScan(folder.MappedPath) {
logger.Debug(logSender, "", "starting quota scan for restored folder: %#v", folder.MappedPath)
if common.QuotaScans.AddVFolderQuotaScan(folder.Name) {
logger.Debug(logSender, "", "starting quota scan for restored folder: %#v", folder.Name)
go doFolderQuotaScan(folder) //nolint:errcheck
}
}

14
httpd/api_quota.go
View file

@ -84,16 +84,16 @@ func updateVFolderQuotaUsage(w http.ResponseWriter, r *http.Request) {
sendAPIResponse(w, r, err, "", http.StatusBadRequest)
return
}
folder, err := dataprovider.GetFolderByPath(f.MappedPath)
folder, err := dataprovider.GetFolderByName(f.Name)
if err != nil {
sendAPIResponse(w, r, err, "", getRespStatus(err))
return
}
if !common.QuotaScans.AddVFolderQuotaScan(folder.MappedPath) {
if !common.QuotaScans.AddVFolderQuotaScan(folder.Name) {
sendAPIResponse(w, r, err, "A quota scan is in progress for this folder", http.StatusConflict)
return
}
defer common.QuotaScans.RemoveVFolderQuotaScan(folder.MappedPath)
defer common.QuotaScans.RemoveVFolderQuotaScan(folder.Name)
err = dataprovider.UpdateVirtualFolderQuota(folder, f.UsedQuotaFiles, f.UsedQuotaSize, mode == quotaUpdateModeReset)
if err != nil {
sendAPIResponse(w, r, err, "", getRespStatus(err))
@ -139,12 +139,12 @@ func startVFolderQuotaScan(w http.ResponseWriter, r *http.Request) {
sendAPIResponse(w, r, err, "", http.StatusBadRequest)
return
}
folder, err := dataprovider.GetFolderByPath(f.MappedPath)
folder, err := dataprovider.GetFolderByName(f.Name)
if err != nil {
sendAPIResponse(w, r, err, "", getRespStatus(err))
return
}
if common.QuotaScans.AddVFolderQuotaScan(folder.MappedPath) {
if common.QuotaScans.AddVFolderQuotaScan(folder.Name) {
go doFolderQuotaScan(folder) //nolint:errcheck
sendAPIResponse(w, r, err, "Scan started", http.StatusAccepted)
} else {
@ -171,7 +171,7 @@ func doQuotaScan(user dataprovider.User) error {
}
func doFolderQuotaScan(folder vfs.BaseVirtualFolder) error {
defer common.QuotaScans.RemoveVFolderQuotaScan(folder.MappedPath)
defer common.QuotaScans.RemoveVFolderQuotaScan(folder.Name)
fs := vfs.NewOsFs("", "", nil).(*vfs.OsFs)
numFiles, size, err := fs.GetDirSize(folder.MappedPath)
if err != nil {
@ -179,7 +179,7 @@ func doFolderQuotaScan(folder vfs.BaseVirtualFolder) error {
return err
}
err = dataprovider.UpdateVirtualFolderQuota(folder, numFiles, size, true)
logger.Debug(logSender, "", "virtual folder %#v scanned, error: %v", folder.MappedPath, err)
logger.Debug(logSender, "", "virtual folder %#v scanned, error: %v", folder.Name, err)
return err
}

37
httpd/api_user.go
View file

@ -16,38 +16,11 @@ import (
)
func getUsers(w http.ResponseWriter, r *http.Request) {
var err error
limit, offset, order, err := getSearchFilters(w, r)
if err != nil {
return
}
limit := 100
offset := 0
order := dataprovider.OrderASC
if _, ok := r.URL.Query()["limit"]; ok {
limit, err = strconv.Atoi(r.URL.Query().Get("limit"))
if err != nil {
err = errors.New("Invalid limit")
sendAPIResponse(w, r, err, "", http.StatusBadRequest)
return
}
if limit > 500 {
limit = 500
}
}
if _, ok := r.URL.Query()["offset"]; ok {
offset, err = strconv.Atoi(r.URL.Query().Get("offset"))
if err != nil {
err = errors.New("Invalid offset")
sendAPIResponse(w, r, err, "", http.StatusBadRequest)
return
}
}
if _, ok := r.URL.Query()["order"]; ok {
order = r.URL.Query().Get("order")
if order != dataprovider.OrderASC && order != dataprovider.OrderDESC {
err = errors.New("Invalid order")
sendAPIResponse(w, r, err, "", http.StatusBadRequest)
return
}
}
users, err := dataprovider.GetUsers(limit, offset, order)
if err == nil {
render.JSON(w, r, users)
@ -69,7 +42,7 @@ func renderUser(w http.ResponseWriter, r *http.Request, username string, status
}
user.HideConfidentialData()
if status != http.StatusOK {
ctx := context.WithValue(r.Context(), render.StatusCtxKey, http.StatusCreated)
ctx := context.WithValue(r.Context(), render.StatusCtxKey, status)
render.JSON(w, r.WithContext(ctx), user)
} else {
render.JSON(w, r, user)

38
httpd/api_utils.go
View file

@ -2,8 +2,10 @@ package httpd
import (
"context"
"errors"
"net/http"
"os"
"strconv"
"github.com/go-chi/render"
@ -52,3 +54,39 @@ func handleCloseConnection(w http.ResponseWriter, r *http.Request) {
sendAPIResponse(w, r, nil, "Not Found", http.StatusNotFound)
}
}
func getSearchFilters(w http.ResponseWriter, r *http.Request) (int, int, string, error) {
var err error
limit := 100
offset := 0
order := dataprovider.OrderASC
if _, ok := r.URL.Query()["limit"]; ok {
limit, err = strconv.Atoi(r.URL.Query().Get("limit"))
if err != nil {
err = errors.New("Invalid limit")
sendAPIResponse(w, r, err, "", http.StatusBadRequest)
return limit, offset, order, err
}
if limit > 500 {
limit = 500
}
}
if _, ok := r.URL.Query()["offset"]; ok {
offset, err = strconv.Atoi(r.URL.Query().Get("offset"))
if err != nil {
err = errors.New("Invalid offset")
sendAPIResponse(w, r, err, "", http.StatusBadRequest)
return limit, offset, order, err
}
}
if _, ok := r.URL.Query()["order"]; ok {
order = r.URL.Query().Get("order")
if order != dataprovider.OrderASC && order != dataprovider.OrderDESC {
err = errors.New("Invalid order")
sendAPIResponse(w, r, err, "", http.StatusBadRequest)
return limit, offset, order, err
}
}
return limit, offset, order, err
}

718
httpd/httpd_test.go
View file

File diff suppressed because it is too large Load diff

6
httpd/internal_test.go
View file

@ -608,7 +608,7 @@ func TestChangePwdValidationErrors(t *testing.T) {
func TestRenderUnexistingFolder(t *testing.T) {
rr := httptest.NewRecorder()
req, _ := http.NewRequest(http.MethodPost, folderPath, nil)
renderFolder(rr, req, "path not mapped")
renderFolder(rr, req, "path not mapped", http.StatusOK)
assert.Equal(t, http.StatusNotFound, rr.Code)
}
@ -714,7 +714,7 @@ func TestGetUserFromTemplate(t *testing.T) {
}
user.VirtualFolders = append(user.VirtualFolders, vfs.VirtualFolder{
BaseVirtualFolder: vfs.BaseVirtualFolder{
MappedPath: "dir%username%",
Name: "Folder%username%",
},
})
@ -727,7 +727,7 @@ func TestGetUserFromTemplate(t *testing.T) {
userTemplate := getUserFromTemplate(user, templateFields)
require.Len(t, userTemplate.VirtualFolders, 1)
require.Equal(t, "dir"+username, userTemplate.VirtualFolders[0].MappedPath)
require.Equal(t, "Folder"+username, userTemplate.VirtualFolders[0].Name)
user.FsConfig.Provider = dataprovider.CryptedFilesystemProvider
user.FsConfig.CryptConfig.Passphrase = kms.NewPlainSecret("%password%")

148
httpd/schema/openapi.yaml
View file

@ -2,7 +2,7 @@ openapi: 3.0.3
info:
title: SFTPGo
description: SFTPGo REST API
version: 2.4.3
version: 2.4.4
servers:
- url: /api/v2
@ -517,12 +517,6 @@ paths:
- ASC
- DESC
example: ASC
- in: query
name: folder-path
required: false
description: Filter by folder path, extact match case sensitive
schema:
type: string
responses:
200:
description: successful operation
@ -571,18 +565,50 @@ paths:
$ref: '#/components/responses/InternalServerError'
default:
$ref: '#/components/responses/DefaultResponse'
delete:
/folders/{name}:
parameters:
- name: name
in: path
description: folder name
required: true
schema:
type: string
get:
tags:
- folders
summary: Delete an existing folder
operationId: delete_folder
parameters:
- name: folder-path
in: query
description: path to the folder to delete
required: true
schema:
type: string
summary: Find user by name
description: For security reasons the hashed password is omitted in the response
operationId: get_folder_by_name
responses:
200:
description: successful operation
content:
application/json:
schema:
$ref : '#/components/schemas/BaseVirtualFolder'
400:
$ref: '#/components/responses/BadRequest'
401:
$ref: '#/components/responses/Unauthorized'
403:
$ref: '#/components/responses/Forbidden'
404:
$ref: '#/components/responses/NotFound'
500:
$ref: '#/components/responses/InternalServerError'
default:
$ref: '#/components/responses/DefaultResponse'
put:
tags:
- folders
summary: Update an existing user
operationId: update_folder
requestBody:
required: true
content:
application/json:
schema:
$ref : '#/components/schemas/BaseVirtualFolder'
responses:
200:
description: successful operation
@ -591,7 +617,33 @@ paths:
schema:
$ref : '#/components/schemas/ApiResponse'
example:
message: "Folder deleted"
message: "User updated"
400:
$ref: '#/components/responses/BadRequest'
401:
$ref: '#/components/responses/Unauthorized'
403:
$ref: '#/components/responses/Forbidden'
404:
$ref: '#/components/responses/NotFound'
500:
$ref: '#/components/responses/InternalServerError'
default:
$ref: '#/components/responses/DefaultResponse'
delete:
tags:
- folders
summary: Delete an existing folder
operationId: delete_folder
responses:
200:
description: successful operation
content:
application/json:
schema:
$ref : '#/components/schemas/ApiResponse'
example:
message: "User deleted"
400:
$ref: '#/components/responses/BadRequest'
401:
@ -686,19 +738,19 @@ paths:
default:
$ref: '#/components/responses/DefaultResponse'
/admins/{username}:
parameters:
- name: username
in: path
description: the admin username
required: true
schema:
type: string
get:
tags:
- admins
summary: Find admin by username
description: For security reasons the hashed password is omitted in the response
operationId: get_admin_by_username
parameters:
- name: username
in: path
description: username of the admin to retrieve
required: true
schema:
type: string
responses:
200:
description: successful operation
@ -723,13 +775,6 @@ paths:
- admins
summary: Update an existing admin
operationId: update_admin
parameters:
- name: username
in: path
description: username of the admin to update
required: true
schema:
type: string
requestBody:
required: true
content:
@ -762,13 +807,6 @@ paths:
- admins
summary: Delete an existing admin
operationId: delete_admin
parameters:
- name: username
in: path
description: username of the admin to delete
required: true
schema:
type: string
responses:
200:
description: successful operation
@ -872,19 +910,19 @@ paths:
default:
$ref: '#/components/responses/DefaultResponse'
/users/{username}:
parameters:
- name: username
in: path
description: the username
required: true
schema:
type: string
get:
tags:
- users
summary: Find user by username
description: For security reasons the hashed password is omitted in the response
operationId: get_user_by_username
parameters:
- name: username
in: path
description: username of the user to retrieve
required: true
schema:
type: string
responses:
200:
description: successful operation
@ -910,12 +948,6 @@ paths:
summary: Update an existing user
operationId: update_user
parameters:
- name: username
in: path
description: username of the user to update
required: true
schema:
type: string
- in: query
name: disconnect
schema:
@ -959,13 +991,6 @@ paths:
- users
summary: Delete an existing user
operationId: delete_user
parameters:
- name: username
in: path
description: username of the user to delete
required: true
schema:
type: string
responses:
200:
description: successful operation
@ -1529,9 +1554,12 @@ components:
type: integer
format: int32
minimum: 1
name:
type: string
description: unique name for this virtual folder
mapped_path:
type: string
description: absolute filesystem path to use as virtual folder. This field is unique
description: absolute filesystem path to use as virtual folder
used_quota_size:
type: integer
format: int64

9
httpd/server.go
View file

@ -304,8 +304,10 @@ func (s *httpdServer) initializeRouter() {
router.With(checkPerm(dataprovider.PermAdminChangeUsers)).Put(userPath+"/{username}", updateUser)
router.With(checkPerm(dataprovider.PermAdminDeleteUsers)).Delete(userPath+"/{username}", deleteUser)
router.With(checkPerm(dataprovider.PermAdminViewUsers)).Get(folderPath, getFolders)
router.With(checkPerm(dataprovider.PermAdminViewUsers)).Get(folderPath+"/{name}", getFolderByName)
router.With(checkPerm(dataprovider.PermAdminAddUsers)).Post(folderPath, addFolder)
router.With(checkPerm(dataprovider.PermAdminDeleteUsers)).Delete(folderPath, deleteFolderByPath)
router.With(checkPerm(dataprovider.PermAdminChangeUsers)).Put(folderPath+"/{name}", updateFolder)
router.With(checkPerm(dataprovider.PermAdminDeleteUsers)).Delete(folderPath+"/{name}", deleteFolder)
router.With(checkPerm(dataprovider.PermAdminManageSystem)).Get(dumpDataPath, dumpData)
router.With(checkPerm(dataprovider.PermAdminManageSystem)).Get(loadDataPath, loadData)
router.With(checkPerm(dataprovider.PermAdminManageSystem)).Post(loadDataPath, loadDataFromRequest)
@ -368,7 +370,10 @@ func (s *httpdServer) initializeRouter() {
router.With(checkPerm(dataprovider.PermAdminManageAdmins)).Delete(webAdminPath+"/{username}", deleteAdmin)
router.With(checkPerm(dataprovider.PermAdminCloseConnections)).
Delete(webConnectionsPath+"/{connectionID}", handleCloseConnection)
router.With(checkPerm(dataprovider.PermAdminDeleteUsers)).Delete(webFolderPath, deleteFolderByPath)
router.With(checkPerm(dataprovider.PermAdminChangeUsers), s.refreshCookie).
Get(webFolderPath+"/{name}", handleWebUpdateFolderGet)
router.With(checkPerm(dataprovider.PermAdminChangeUsers)).Post(webFolderPath+"/{name}", handleWebUpdateFolderPost)
router.With(checkPerm(dataprovider.PermAdminDeleteUsers)).Delete(webFolderPath+"/{name}", deleteFolder)
router.With(checkPerm(dataprovider.PermAdminQuotaScans)).Post(webScanVFolderPath, startVFolderQuotaScan)
router.With(checkPerm(dataprovider.PermAdminDeleteUsers)).Delete(webUserPath+"/{username}", deleteUser)
router.With(checkPerm(dataprovider.PermAdminQuotaScans)).Post(webQuotaScanPath, startQuotaScan)

118
httpd/web.go
View file

@ -31,6 +31,13 @@ const (
userPageModeTemplate
)
type folderPageMode int
const (
folderPageModeAdd folderPageMode = iota + 1
folderPageModeUpdate
)
const (
templateBase = "base.html"
templateUsers = "users.html"
@ -155,6 +162,7 @@ type folderPage struct {
basePage
Folder vfs.BaseVirtualFolder
Error string
Mode folderPageMode
}
type messagePage struct {
@ -383,11 +391,21 @@ func renderUserPage(w http.ResponseWriter, r *http.Request, user *dataprovider.U
renderTemplate(w, templateUser, data)
}
func renderAddFolderPage(w http.ResponseWriter, r *http.Request, folder vfs.BaseVirtualFolder, error string) {
func renderFolderPage(w http.ResponseWriter, r *http.Request, folder vfs.BaseVirtualFolder, mode folderPageMode, error string) {
var title, currentURL string
switch mode {
case folderPageModeAdd:
title = "Add a new folder"
currentURL = webFolderPath
case folderPageModeUpdate:
title = "Update folder"
currentURL = fmt.Sprintf("%v/%v", webFolderPath, url.PathEscape(folder.Name))
}
data := folderPage{
basePage: getBasePageData("Add a new folder", webFolderPath, r),
basePage: getBasePageData(title, currentURL, r),
Error: error,
Folder: folder,
Mode: mode,
}
renderTemplate(w, templateFolder, data)
}
@ -395,6 +413,7 @@ func renderAddFolderPage(w http.ResponseWriter, r *http.Request, folder vfs.Base
func getUsersForTemplate(r *http.Request) []userTemplateFields {
var res []userTemplateFields
formValue := r.Form.Get("users")
users := make(map[string]bool)
for _, cleaned := range getSliceFromDelimitedValues(formValue, "\n") {
if strings.Contains(cleaned, "::") {
mapping := strings.Split(cleaned, "::")
@ -408,6 +427,11 @@ func getUsersForTemplate(r *http.Request) []userTemplateFields {
if username == "" || (password == "" && publicKey == "") {
continue
}
if _, ok := users[username]; ok {
continue
}
users[username] = true
res = append(res, userTemplateFields{
Username: username,
Password: password,
@ -428,7 +452,7 @@ func getVirtualFoldersFromPostFields(r *http.Request) []vfs.VirtualFolder {
if len(mapping) > 1 {
vfolder := vfs.VirtualFolder{
BaseVirtualFolder: vfs.BaseVirtualFolder{
MappedPath: strings.TrimSpace(mapping[1]),
Name: strings.TrimSpace(mapping[1]),
},
VirtualPath: strings.TrimSpace(mapping[0]),
QuotaFiles: -1,
@ -816,7 +840,8 @@ func getUserFromTemplate(user dataprovider.User, template userTemplateFields) da
user.HomeDir = replacePlaceholders(user.HomeDir, replacements)
var vfolders []vfs.VirtualFolder
for _, vfolder := range user.VirtualFolders {
vfolder.MappedPath = replacePlaceholders(vfolder.MappedPath, replacements)
vfolder.Name = replacePlaceholders(vfolder.Name, replacements)
vfolder.VirtualPath = replacePlaceholders(vfolder.VirtualPath, replacements)
vfolders = append(vfolders, vfolder)
}
user.VirtualFolders = vfolders
@ -838,39 +863,6 @@ func getUserFromTemplate(user dataprovider.User, template userTemplateFields) da
return user
}
/*func decryptSecretsForTemplateUser(user *dataprovider.User) error {
user.SetEmptySecretsIfNil()
switch user.FsConfig.Provider {
case dataprovider.CryptedFilesystemProvider:
if user.FsConfig.CryptConfig.Passphrase.IsEncrypted() {
return user.FsConfig.CryptConfig.Passphrase.Decrypt()
}
case dataprovider.S3FilesystemProvider:
if user.FsConfig.S3Config.AccessSecret.IsEncrypted() {
return user.FsConfig.S3Config.AccessSecret.Decrypt()
}
case dataprovider.GCSFilesystemProvider:
if user.FsConfig.GCSConfig.Credentials.IsEncrypted() {
return user.FsConfig.GCSConfig.Credentials.Decrypt()
}
case dataprovider.AzureBlobFilesystemProvider:
if user.FsConfig.AzBlobConfig.AccountKey.IsEncrypted() {
return user.FsConfig.AzBlobConfig.AccountKey.Decrypt()
}
case dataprovider.SFTPFilesystemProvider:
if user.FsConfig.SFTPConfig.Password.IsEncrypted() {
if err := user.FsConfig.SFTPConfig.Password.Decrypt(); err != nil {
return err
}
}
if user.FsConfig.SFTPConfig.PrivateKey.IsEncrypted() {
return user.FsConfig.SFTPConfig.PrivateKey.Decrypt()
}
}
return nil
}*/
func getUserFromPostFields(r *http.Request) (dataprovider.User, error) {
var user dataprovider.User
err := r.ParseMultipartForm(maxRequestSize)
@ -1204,6 +1196,11 @@ func handleWebTemplateUserPost(w http.ResponseWriter, r *http.Request) {
return
}
dump.Users = append(dump.Users, u)
for _, folder := range u.VirtualFolders {
if !dump.HasFolder(folder.Name) {
dump.Folders = append(dump.Folders, folder.BaseVirtualFolder)
}
}
}
if len(dump.Users) == 0 {
@ -1317,7 +1314,7 @@ func handleWebGetConnections(w http.ResponseWriter, r *http.Request) {
}
func handleWebAddFolderGet(w http.ResponseWriter, r *http.Request) {
renderAddFolderPage(w, r, vfs.BaseVirtualFolder{}, "")
renderFolderPage(w, r, vfs.BaseVirtualFolder{}, folderPageModeAdd, "")
}
func handleWebAddFolderPost(w http.ResponseWriter, r *http.Request) {
@ -1325,19 +1322,58 @@ func handleWebAddFolderPost(w http.ResponseWriter, r *http.Request) {
folder := vfs.BaseVirtualFolder{}
err := r.ParseForm()
if err != nil {
renderAddFolderPage(w, r, folder, err.Error())
renderFolderPage(w, r, folder, folderPageModeAdd, err.Error())
return
}
folder.MappedPath = r.Form.Get("mapped_path")
folder.Name = r.Form.Get("name")
err = dataprovider.AddFolder(&folder)
if err == nil {
http.Redirect(w, r, webFoldersPath, http.StatusSeeOther)
} else {
renderAddFolderPage(w, r, folder, err.Error())
renderFolderPage(w, r, folder, folderPageModeAdd, err.Error())
}
}
func handleWebUpdateFolderGet(w http.ResponseWriter, r *http.Request) {
name := getURLParam(r, "name")
folder, err := dataprovider.GetFolderByName(name)
if err == nil {
renderFolderPage(w, r, folder, folderPageModeUpdate, "")
} else if _, ok := err.(*dataprovider.RecordNotFoundError); ok {
renderNotFoundPage(w, r, err)
} else {
renderInternalServerErrorPage(w, r, err)
}
}
func handleWebUpdateFolderPost(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
name := getURLParam(r, "name")
folder, err := dataprovider.GetFolderByName(name)
if _, ok := err.(*dataprovider.RecordNotFoundError); ok {
renderNotFoundPage(w, r, err)
return
} else if err != nil {
renderInternalServerErrorPage(w, r, err)
return
}
err = r.ParseForm()
if err != nil {
renderFolderPage(w, r, folder, folderPageModeUpdate, err.Error())
return
}
folder.MappedPath = r.Form.Get("mapped_path")
err = dataprovider.UpdateFolder(&folder)
if err != nil {
renderFolderPage(w, r, folder, folderPageModeUpdate, err.Error())
return
}
http.Redirect(w, r, webFoldersPath, http.StatusSeeOther)
}
func handleWebGetFolders(w http.ResponseWriter, r *http.Request) {
limit := defaultQueryLimit
if _, ok := r.URL.Query()["qlimit"]; ok {
@ -1349,7 +1385,7 @@ func handleWebGetFolders(w http.ResponseWriter, r *http.Request) {
}
folders := make([]vfs.BaseVirtualFolder, 0, limit)
for {
f, err := dataprovider.GetFolders(limit, len(folders), dataprovider.OrderASC, "")
f, err := dataprovider.GetFolders(limit, len(folders), dataprovider.OrderASC)
if err != nil {
renderInternalServerErrorPage(w, r, err)
return

72
httpdtest/httpdtest.go
View file

@ -264,15 +264,14 @@ func AddAdmin(admin dataprovider.Admin, expectedStatusCode int) (dataprovider.Ad
return newAdmin, body, err
}
// UpdateAdmin updates an existing admin and checks the received HTTP Status code against expectedStatusCode.
// UpdateAdmin updates an existing admin and checks the received HTTP Status code against expectedStatusCode
func UpdateAdmin(admin dataprovider.Admin, expectedStatusCode int) (dataprovider.Admin, []byte, error) {
var newAdmin dataprovider.Admin
var body []byte
asJSON, _ := json.Marshal(admin)
resp, err := sendHTTPRequest(http.MethodPut, buildURLRelativeToBase(adminPath, url.PathEscape(admin.Username)),
bytes.NewBuffer(asJSON), "application/json",
getDefaultToken())
bytes.NewBuffer(asJSON), "application/json", getDefaultToken())
if err != nil {
return newAdmin, body, err
}
@ -477,18 +476,38 @@ func AddFolder(folder vfs.BaseVirtualFolder, expectedStatusCode int) (vfs.BaseVi
return newFolder, body, err
}
// UpdateFolder updates an existing folder and checks the received HTTP Status code against expectedStatusCode.
func UpdateFolder(folder vfs.BaseVirtualFolder, expectedStatusCode int) (vfs.BaseVirtualFolder, []byte, error) {
var updatedFolder vfs.BaseVirtualFolder
var body []byte
folderAsJSON, _ := json.Marshal(folder)
resp, err := sendHTTPRequest(http.MethodPut, buildURLRelativeToBase(folderPath, url.PathEscape(folder.Name)),
bytes.NewBuffer(folderAsJSON), "application/json", getDefaultToken())
if err != nil {
return updatedFolder, body, err
}
defer resp.Body.Close()
body, _ = getResponseBody(resp)
err = checkResponse(resp.StatusCode, expectedStatusCode)
if expectedStatusCode != http.StatusOK {
return updatedFolder, body, err
}
if err == nil {
updatedFolder, body, err = GetFolderByName(folder.Name, expectedStatusCode)
}
if err == nil {
err = checkFolder(&folder, &updatedFolder)
}
return updatedFolder, body, err
}
// RemoveFolder removes an existing user and checks the received HTTP Status code against expectedStatusCode.
func RemoveFolder(folder vfs.BaseVirtualFolder, expectedStatusCode int) ([]byte, error) {
var body []byte
baseURL := buildURLRelativeToBase(folderPath)
url, err := url.Parse(baseURL)
if err != nil {
return body, err
}
q := url.Query()
q.Add("folder-path", folder.MappedPath)
url.RawQuery = q.Encode()
resp, err := sendHTTPRequest(http.MethodDelete, url.String(), nil, "", getDefaultToken())
resp, err := sendHTTPRequest(http.MethodDelete, buildURLRelativeToBase(folderPath, url.PathEscape(folder.Name)),
nil, "", getDefaultToken())
if err != nil {
return body, err
}
@ -497,22 +516,36 @@ func RemoveFolder(folder vfs.BaseVirtualFolder, expectedStatusCode int) ([]byte,
return body, checkResponse(resp.StatusCode, expectedStatusCode)
}
// GetFolderByName gets a folder by name and checks the received HTTP Status code against expectedStatusCode.
func GetFolderByName(name string, expectedStatusCode int) (vfs.BaseVirtualFolder, []byte, error) {
var folder vfs.BaseVirtualFolder
var body []byte
resp, err := sendHTTPRequest(http.MethodGet, buildURLRelativeToBase(folderPath, url.PathEscape(name)),
nil, "", getDefaultToken())
if err != nil {
return folder, body, err
}
defer resp.Body.Close()
err = checkResponse(resp.StatusCode, expectedStatusCode)
if err == nil && expectedStatusCode == http.StatusOK {
err = render.DecodeJSON(resp.Body, &folder)
} else {
body, _ = getResponseBody(resp)
}
return folder, body, err
}
// GetFolders returns a list of folders and checks the received HTTP Status code against expectedStatusCode.
// The number of results can be limited specifying a limit.
// Some results can be skipped specifying an offset.
// The results can be filtered specifying a folder path, the folder path filter is an exact match
func GetFolders(limit int64, offset int64, mappedPath string, expectedStatusCode int) ([]vfs.BaseVirtualFolder, []byte, error) {
func GetFolders(limit int64, offset int64, expectedStatusCode int) ([]vfs.BaseVirtualFolder, []byte, error) {
var folders []vfs.BaseVirtualFolder
var body []byte
url, err := addLimitAndOffsetQueryParams(buildURLRelativeToBase(folderPath), limit, offset)
if err != nil {
return folders, body, err
}
if len(mappedPath) > 0 {
q := url.Query()
q.Add("folder-path", mappedPath)
url.RawQuery = q.Encode()
}
resp, err := sendHTTPRequest(http.MethodGet, url.String(), nil, "", getDefaultToken())
if err != nil {
return folders, body, err
@ -792,6 +825,9 @@ func checkFolder(expected *vfs.BaseVirtualFolder, actual *vfs.BaseVirtualFolder)
return errors.New("folder ID mismatch")
}
}
if expected.Name != actual.Name {
return errors.New("name mismatch")
}
if expected.MappedPath != actual.MappedPath {
return errors.New("mapped path mismatch")
}

2
service/service.go
View file

@ -248,7 +248,7 @@ func (s *Service) loadInitialData() error {
if err != nil {
return fmt.Errorf("unable to parse file to restore %#v: %v", s.LoadDataFrom, err)
}
err = httpd.RestoreFolders(dump.Folders, s.LoadDataFrom, s.LoadDataQuotaScan)
err = httpd.RestoreFolders(dump.Folders, s.LoadDataFrom, s.LoadDataMode, s.LoadDataQuotaScan)
if err != nil {
return fmt.Errorf("unable to restore folders from file %#v: %v", s.LoadDataFrom, err)
}

745
sftpd/sftpd_test.go
View file

File diff suppressed because it is too large Load diff

2
sftpgo.json
View file

@ -115,7 +115,7 @@
"driver": "sqlite",
"name": "sftpgo.db",
"host": "",
"port": 5432,
"port": 0,
"username": "",
"password": "",
"sslmode": 0,

7
templates/folder.html
View file

@ -14,6 +14,13 @@
</div>
{{end}}
<form id="folder_form" action="{{.CurrentURL}}" method="POST" autocomplete="off">
<div class="form-group row">
<label for="idFolderName" class="col-sm-2 col-form-label">Name</label>
<div class="col-sm-10">
<input type="text" class="form-control" id="idFolderName" name="name" placeholder=""
value="{{.Folder.Name}}" maxlength="255" autocomplete="nope" required {{if ge .Mode 2}}readonly{{end}}>
</div>
</div>
<div class="form-group row">
<label for="idMappedPath" class="col-sm-2 col-form-label">Absolute Path</label>
<div class="col-sm-10">

30
templates/folders.html
View file

@ -27,6 +27,7 @@
<table class="table table-striped table-bordered" id="dataTable" width="100%" cellspacing="0">
<thead>
<tr>
<th>Name</th>
<th>Path</th>
<th>Quota</th>
<th>Used by</th>
@ -35,6 +36,7 @@
<tbody>
{{range .Folders}}
<tr>
<td>{{.Name}}</td>
<td>{{.MappedPath}}</td>
<td>{{.GetQuotaSummary}}</td>
<td>{{.GetUsersAsString}}</td>
@ -88,11 +90,11 @@
function deleteAction() {
var table = $('#dataTable').DataTable();
table.button('delete:name').enable(false);
var folderPath = table.row({ selected: true }).data()[0];
var path = '{{.FolderURL}}' + "?folder-path=" + encodeURIComponent(folderPath);
var folderName = table.row({ selected: true }).data()[0];
var path = '{{.FolderURL}}' + "/" + folderName;
$('#deleteModal').modal('hide');
$.ajax({
url: path,
url: encodeURI(path),
type: 'DELETE',
dataType: 'json',
timeout: 15000,
@ -127,6 +129,17 @@ function deleteAction() {
}
};
$.fn.dataTable.ext.buttons.edit = {
text: 'Edit',
name: 'edit',
action: function (e, dt, node, config) {
var folderName = table.row({ selected: true }).data()[0];
var path = '{{.FolderURL}}' + "/" + folderName;
window.location.href = encodeURI(path);
},
enabled: false
};
$.fn.dataTable.ext.buttons.delete = {
text: 'Delete',
name: 'delete',
@ -141,13 +154,13 @@ function deleteAction() {
name: 'quota_scan',
action: function (e, dt, node, config) {
dt.button('quota_scan:name').enable(false);
var folderPath = dt.row({ selected: true }).data()[0];
var folderName = dt.row({ selected: true }).data()[0];
var path = '{{.FolderQuotaScanURL}}'
$.ajax({
url: path,
type: 'POST',
dataType: 'json',
data: JSON.stringify({ "mapped_path": folderPath }),
data: JSON.stringify({ "name": folderName }),
timeout: 15000,
success: function (result) {
dt.button('quota_scan:name').enable(true);
@ -202,6 +215,10 @@ function deleteAction() {
table.button().add(0,'delete');
{{end}}
{{if .LoggedAdmin.HasPermission "edit_users"}}
table.button().add(0,'edit');
{{end}}
{{if .LoggedAdmin.HasPermission "add_users"}}
table.button().add(0,'add');
{{end}}
@ -211,6 +228,9 @@ function deleteAction() {
{{if .LoggedAdmin.HasPermission "del_users"}}
table.button('delete:name').enable(selectedRows == 1);
{{end}}
{{if .LoggedAdmin.HasPermission "edit_users"}}
table.button('edit:name').enable(selectedRows == 1);
{{end}}
{{if .LoggedAdmin.HasPermission "quota_scans"}}
table.button('quota_scan:name').enable(selectedRows == 1);
{{end}}

6
templates/user.html
View file

@ -169,11 +169,11 @@
<div class="col-sm-10">
<textarea class="form-control" id="idVirtualFolders" name="virtual_folders" rows="3"
aria-describedby="vfHelpBlock">{{range $index, $mapping := .User.VirtualFolders -}}
{{$mapping.VirtualPath}}::{{$mapping.MappedPath}}::{{$mapping.QuotaFiles}}::{{$mapping.QuotaSize}}&#10;
{{$mapping.VirtualPath}}::{{$mapping.Name}}::{{$mapping.QuotaFiles}}::{{$mapping.QuotaSize}}&#10;
{{- end}}</textarea>
<small id="vfHelpBlock" class="form-text text-muted">
One mapping per line as vpath::fspath::[quota_files]::[quota_size(bytes)], for example
/vdir::/home/adir or /vdir::C:\adir::10::104857600. Quota -1 means included inside user quota.
One mapping per line as vpath::folder-name::[quota_files]::[quota_size(bytes)], for example
/vdir::afolder or /vdir::afolder::10::104857600. Quota -1 means included inside user quota.
Ignored for non local filesystems
</small>
</div>

18
vfs/folder.go
View file

@ -13,7 +13,8 @@ import (
// quota limits or a different virtual path.
type BaseVirtualFolder struct {
ID int64 `json:"id"`
MappedPath string `json:"mapped_path"`
Name string `json:"name"`
MappedPath string `json:"mapped_path,omitempty"`
UsedQuotaSize int64 `json:"used_quota_size"`
// Used quota as number of files
UsedQuotaFiles int `json:"used_quota_files"`
@ -23,6 +24,21 @@ type BaseVirtualFolder struct {
Users []string `json:"users,omitempty"`
}
// GetACopy returns a copy
func (v *BaseVirtualFolder) GetACopy() BaseVirtualFolder {
users := make([]string, len(v.Users))
copy(users, v.Users)
return BaseVirtualFolder{
ID: v.ID,
Name: v.Name,
MappedPath: v.MappedPath,
UsedQuotaSize: v.UsedQuotaSize,
UsedQuotaFiles: v.UsedQuotaFiles,
LastQuotaUpdate: v.LastQuotaUpdate,
Users: users,
}
}
// GetUsersAsString returns the list of users as comma separated string
func (v *BaseVirtualFolder) GetUsersAsString() string {
return strings.Join(v.Users, ",")

22
webdavd/webdavd_test.go
View file

@ -1414,8 +1414,10 @@ func TestUploadOverwriteVfolder(t *testing.T) {
u := getTestUser()
vdir := "/vdir"
mappedPath := filepath.Join(os.TempDir(), "mappedDir")
folderName := filepath.Base(mappedPath)
u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{
BaseVirtualFolder: vfs.BaseVirtualFolder{
Name: folderName,
MappedPath: mappedPath,
},
VirtualPath: vdir,
@ -1448,27 +1450,21 @@ func TestUploadOverwriteVfolder(t *testing.T) {
assert.NoError(t, err)
err = uploadFile(testFilePath, path.Join(vdir, testFileName), testFileSize, client)
assert.NoError(t, err)
folder, _, err := httpdtest.GetFolders(0, 0, mappedPath, http.StatusOK)
folder, _, err := httpdtest.GetFolderByName(folderName, http.StatusOK)
assert.NoError(t, err)
if assert.Len(t, folder, 1) {
f := folder[0]
assert.Equal(t, testFileSize, f.UsedQuotaSize)
assert.Equal(t, 1, f.UsedQuotaFiles)
}
assert.Equal(t, testFileSize, folder.UsedQuotaSize)
assert.Equal(t, 1, folder.UsedQuotaFiles)
err = uploadFile(testFilePath, path.Join(vdir, testFileName), testFileSize, client)
assert.NoError(t, err)
folder, _, err = httpdtest.GetFolders(0, 0, mappedPath, http.StatusOK)
folder, _, err = httpdtest.GetFolderByName(folderName, http.StatusOK)
assert.NoError(t, err)
if assert.Len(t, folder, 1) {
f := folder[0]
assert.Equal(t, testFileSize, f.UsedQuotaSize)
assert.Equal(t, 1, f.UsedQuotaFiles)
}
assert.Equal(t, testFileSize, folder.UsedQuotaSize)
assert.Equal(t, 1, folder.UsedQuotaFiles)
err = os.Remove(testFilePath)
assert.NoError(t, err)
_, err = httpdtest.RemoveUser(user, http.StatusOK)
assert.NoError(t, err)
_, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{MappedPath: mappedPath}, http.StatusOK)
_, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName}, http.StatusOK)
assert.NoError(t, err)
err = os.RemoveAll(user.GetHomeDir())
assert.NoError(t, err)