From 9d3d7db29ce325a34839e45c2722a026cb28401c Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Fri, 11 Jun 2021 22:27:36 +0200 Subject: [PATCH] azblob: store SAS URL as kms.Secret --- cmd/portable.go | 2 +- common/actions.go | 4 +- common/actions_test.go | 4 +- dataprovider/bolt.go | 366 +++++++++++++++++++++++++++++-- dataprovider/compat.go | 118 ++++++++++ dataprovider/dataprovider.go | 2 +- dataprovider/mysql.go | 28 ++- dataprovider/pgsql.go | 28 ++- dataprovider/sqlcommon.go | 312 +++++++++++++++++++++++++- dataprovider/sqlite.go | 28 ++- dataprovider/sqlqueries.go | 16 ++ dataprovider/user.go | 5 + docs/custom-actions.md | 4 +- httpd/api_folder.go | 3 +- httpd/api_user.go | 14 +- httpd/httpd_test.go | 77 ++++++- httpd/schema/openapi.yaml | 3 +- httpd/webadmin.go | 10 +- httpdtest/httpdtest.go | 4 +- service/service_portable.go | 5 + sftpd/sftpd_test.go | 2 +- templates/webadmin/fsconfig.html | 4 +- vfs/azblobfs.go | 11 +- vfs/filesystem.go | 8 +- vfs/folder.go | 4 + vfs/vfs.go | 36 ++- 26 files changed, 1026 insertions(+), 72 deletions(-) create mode 100644 dataprovider/compat.go diff --git a/cmd/portable.go b/cmd/portable.go index 54224754..c5d2c614 100644 --- a/cmd/portable.go +++ b/cmd/portable.go @@ -176,7 +176,7 @@ Please take a look at the usage below to customize the serving parameters`, AccountKey: kms.NewPlainSecret(portableAzAccountKey), Endpoint: portableAzEndpoint, AccessTier: portableAzAccessTier, - SASURL: portableAzSASURL, + SASURL: kms.NewPlainSecret(portableAzSASURL), KeyPrefix: portableAzKeyPrefix, UseEmulator: portableAzUseEmulator, UploadPartSize: int64(portableAzULPartSize), diff --git a/common/actions.go b/common/actions.go index ac967309..aaf4391f 100644 --- a/common/actions.go +++ b/common/actions.go @@ -117,9 +117,7 @@ func newActionNotification( bucket = fsConfig.GCSConfig.Bucket case vfs.AzureBlobFilesystemProvider: bucket = fsConfig.AzBlobConfig.Container - if fsConfig.AzBlobConfig.SASURL != "" { - endpoint = fsConfig.AzBlobConfig.SASURL - } else { + if fsConfig.AzBlobConfig.Endpoint != "" { endpoint = fsConfig.AzBlobConfig.Endpoint } case vfs.SFTPFilesystemProvider: diff --git a/common/actions_test.go b/common/actions_test.go index f6e6713b..81028421 100644 --- a/common/actions_test.go +++ b/common/actions_test.go @@ -29,7 +29,6 @@ func TestNewActionNotification(t *testing.T) { } user.FsConfig.AzBlobConfig = vfs.AzBlobFsConfig{ Container: "azcontainer", - SASURL: "azsasurl", Endpoint: "azendpoint", } user.FsConfig.SFTPConfig = vfs.SFTPFsConfig{ @@ -56,10 +55,9 @@ func TestNewActionNotification(t *testing.T) { user.FsConfig.Provider = vfs.AzureBlobFilesystemProvider a = newActionNotification(user, operationDownload, "path", "vpath", "target", "", ProtocolSCP, 123, 0, nil) assert.Equal(t, "azcontainer", a.Bucket) - assert.Equal(t, "azsasurl", a.Endpoint) + assert.Equal(t, "azendpoint", a.Endpoint) assert.Equal(t, 1, a.Status) - user.FsConfig.AzBlobConfig.SASURL = "" a = newActionNotification(user, operationDownload, "path", "vpath", "target", "", ProtocolSCP, 123, os.O_APPEND, nil) assert.Equal(t, "azcontainer", a.Bucket) assert.Equal(t, "azendpoint", a.Endpoint) diff --git a/dataprovider/bolt.go b/dataprovider/bolt.go index 22600abd..472fb97d 100644 --- a/dataprovider/bolt.go +++ b/dataprovider/bolt.go @@ -19,7 +19,7 @@ import ( ) const ( - boltDatabaseVersion = 6 + boltDatabaseVersion = 10 ) var ( @@ -229,7 +229,7 @@ func (p *BoltProvider) adminExists(username string) (Admin, error) { var admin Admin err := p.dbHandle.View(func(tx *bolt.Tx) error { - bucket, err := getAdminBucket(tx) + bucket, err := getAdminsBucket(tx) if err != nil { return err } @@ -249,7 +249,7 @@ func (p *BoltProvider) addAdmin(admin *Admin) error { return err } return p.dbHandle.Update(func(tx *bolt.Tx) error { - bucket, err := getAdminBucket(tx) + bucket, err := getAdminsBucket(tx) if err != nil { return err } @@ -275,7 +275,7 @@ func (p *BoltProvider) updateAdmin(admin *Admin) error { return err } return p.dbHandle.Update(func(tx *bolt.Tx) error { - bucket, err := getAdminBucket(tx) + bucket, err := getAdminsBucket(tx) if err != nil { return err } @@ -301,7 +301,7 @@ func (p *BoltProvider) updateAdmin(admin *Admin) error { func (p *BoltProvider) deleteAdmin(admin *Admin) error { return p.dbHandle.Update(func(tx *bolt.Tx) error { - bucket, err := getAdminBucket(tx) + bucket, err := getAdminsBucket(tx) if err != nil { return err } @@ -318,7 +318,7 @@ func (p *BoltProvider) getAdmins(limit int, offset int, order string) ([]Admin, admins := make([]Admin, 0, limit) err := p.dbHandle.View(func(tx *bolt.Tx) error { - bucket, err := getAdminBucket(tx) + bucket, err := getAdminsBucket(tx) if err != nil { return err } @@ -368,7 +368,7 @@ func (p *BoltProvider) getAdmins(limit int, offset int, order string) ([]Admin, func (p *BoltProvider) dumpAdmins() ([]Admin, error) { admins := make([]Admin, 0, 30) err := p.dbHandle.View(func(tx *bolt.Tx) error { - bucket, err := getAdminBucket(tx) + bucket, err := getAdminsBucket(tx) if err != nil { return err } @@ -399,7 +399,7 @@ func (p *BoltProvider) userExists(username string) (User, error) { if u == nil { return &RecordNotFoundError{err: fmt.Sprintf("username %#v does not exist", username)} } - folderBucket, err := getFolderBucket(tx) + folderBucket, err := getFoldersBucket(tx) if err != nil { return err } @@ -419,7 +419,7 @@ func (p *BoltProvider) addUser(user *User) error { if err != nil { return err } - folderBucket, err := getFolderBucket(tx) + folderBucket, err := getFoldersBucket(tx) if err != nil { return err } @@ -459,7 +459,7 @@ func (p *BoltProvider) updateUser(user *User) error { if err != nil { return err } - folderBucket, err := getFolderBucket(tx) + folderBucket, err := getFoldersBucket(tx) if err != nil { return err } @@ -504,7 +504,7 @@ func (p *BoltProvider) deleteUser(user *User) error { return err } if len(user.VirtualFolders) > 0 { - folderBucket, err := getFolderBucket(tx) + folderBucket, err := getFoldersBucket(tx) if err != nil { return err } @@ -530,7 +530,7 @@ func (p *BoltProvider) dumpUsers() ([]User, error) { if err != nil { return err } - folderBucket, err := getFolderBucket(tx) + folderBucket, err := getFoldersBucket(tx) if err != nil { return err } @@ -562,7 +562,7 @@ func (p *BoltProvider) getUsers(limit int, offset int, order string) ([]User, er if err != nil { return err } - folderBucket, err := getFolderBucket(tx) + folderBucket, err := getFoldersBucket(tx) if err != nil { return err } @@ -607,7 +607,7 @@ func (p *BoltProvider) getUsers(limit int, offset int, order string) ([]User, er func (p *BoltProvider) dumpFolders() ([]vfs.BaseVirtualFolder, error) { folders := make([]vfs.BaseVirtualFolder, 0, 50) err := p.dbHandle.View(func(tx *bolt.Tx) error { - bucket, err := getFolderBucket(tx) + bucket, err := getFoldersBucket(tx) if err != nil { return err } @@ -632,7 +632,7 @@ func (p *BoltProvider) getFolders(limit, offset int, order string) ([]vfs.BaseVi return folders, err } err = p.dbHandle.View(func(tx *bolt.Tx) error { - bucket, err := getFolderBucket(tx) + bucket, err := getFoldersBucket(tx) if err != nil { return err } @@ -681,7 +681,7 @@ func (p *BoltProvider) getFolders(limit, offset int, order string) ([]vfs.BaseVi func (p *BoltProvider) getFolderByName(name string) (vfs.BaseVirtualFolder, error) { var folder vfs.BaseVirtualFolder err := p.dbHandle.View(func(tx *bolt.Tx) error { - bucket, err := getFolderBucket(tx) + bucket, err := getFoldersBucket(tx) if err != nil { return err } @@ -697,7 +697,7 @@ func (p *BoltProvider) addFolder(folder *vfs.BaseVirtualFolder) error { return err } return p.dbHandle.Update(func(tx *bolt.Tx) error { - bucket, err := getFolderBucket(tx) + bucket, err := getFoldersBucket(tx) if err != nil { return err } @@ -715,7 +715,7 @@ func (p *BoltProvider) updateFolder(folder *vfs.BaseVirtualFolder) error { return err } return p.dbHandle.Update(func(tx *bolt.Tx) error { - bucket, err := getFolderBucket(tx) + bucket, err := getFoldersBucket(tx) if err != nil { return err } @@ -745,7 +745,7 @@ func (p *BoltProvider) updateFolder(folder *vfs.BaseVirtualFolder) error { func (p *BoltProvider) deleteFolder(folder *vfs.BaseVirtualFolder) error { return p.dbHandle.Update(func(tx *bolt.Tx) error { - bucket, err := getFolderBucket(tx) + bucket, err := getFoldersBucket(tx) if err != nil { return err } @@ -795,7 +795,7 @@ func (p *BoltProvider) deleteFolder(folder *vfs.BaseVirtualFolder) error { func (p *BoltProvider) updateFolderQuota(name string, filesAdd int, sizeAdd int64, reset bool) error { return p.dbHandle.Update(func(tx *bolt.Tx) error { - bucket, err := getFolderBucket(tx) + bucket, err := getFoldersBucket(tx) if err != nil { return err } @@ -860,6 +860,8 @@ func (p *BoltProvider) migrateDatabase() error { providerLog(logger.LevelError, "%v", err) logger.ErrorToConsole("%v", err) return err + case version == 6: + return updateBoltDatabaseFrom6To10(p.dbHandle) default: if version > boltDatabaseVersion { providerLog(logger.LevelWarn, "database version %v is newer than the supported one: %v", version, @@ -883,6 +885,9 @@ func (p *BoltProvider) revertDatabase(targetVersion int) error { if dbVersion.Version == targetVersion { return errors.New("current version match target version, nothing to do") } + if dbVersion.Version == 10 { + return downgradeBoltDatabaseFrom10To6(p.dbHandle) + } return errors.New("the current version cannot be reverted") } @@ -991,7 +996,7 @@ func removeUserFromFolderMapping(folder *vfs.VirtualFolder, user *User, bucket * return err } -func getAdminBucket(tx *bolt.Tx) (*bolt.Bucket, error) { +func getAdminsBucket(tx *bolt.Tx) (*bolt.Bucket, error) { var err error bucket := tx.Bucket(adminsBucket) @@ -1010,7 +1015,7 @@ func getUsersBucket(tx *bolt.Tx) (*bolt.Bucket, error) { return bucket, err } -func getFolderBucket(tx *bolt.Tx) (*bolt.Bucket, error) { +func getFoldersBucket(tx *bolt.Tx) (*bolt.Bucket, error) { var err error bucket := tx.Bucket(foldersBucket) if bucket == nil { @@ -1038,7 +1043,7 @@ func getBoltDatabaseVersion(dbHandle *bolt.DB) (schemaVersion, error) { return dbVersion, err } -/*func updateBoltDatabaseVersion(dbHandle *bolt.DB, version int) error { +func updateBoltDatabaseVersion(dbHandle *bolt.DB, version int) error { err := dbHandle.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket(dbVersionBucket) if bucket == nil { @@ -1054,4 +1059,317 @@ func getBoltDatabaseVersion(dbHandle *bolt.DB) (schemaVersion, error) { return bucket.Put(dbVersionKey, buf) }) return err -}*/ +} + +func updateBoltDatabaseFrom6To10(dbHandle *bolt.DB) error { + logger.InfoToConsole("updating database version: 6 -> 10") + providerLog(logger.LevelInfo, "updating database version: 6 -> 10") + + if err := boltUpdateV7Folders(dbHandle); err != nil { + return err + } + if err := boltUpdateV7Users(dbHandle); err != nil { + return err + } + return updateBoltDatabaseVersion(dbHandle, 10) +} + +func downgradeBoltDatabaseFrom10To6(dbHandle *bolt.DB) error { + logger.InfoToConsole("downgrading database version: 10 -> 6") + providerLog(logger.LevelInfo, "downgrading database version: 10 -> 6") + + if err := boltDowngradeV7Folders(dbHandle); err != nil { + return err + } + if err := boltDowngradeV7Users(dbHandle); err != nil { + return err + } + return updateBoltDatabaseVersion(dbHandle, 6) +} + +func boltUpdateV7Folders(dbHandle *bolt.DB) error { + var folders []map[string]interface{} + err := dbHandle.View(func(tx *bolt.Tx) error { + bucket, err := getFoldersBucket(tx) + if err != nil { + return err + } + cursor := bucket.Cursor() + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { + var folderMap map[string]interface{} + err = json.Unmarshal(v, &folderMap) + if err != nil { + return err + } + fsBytes, err := json.Marshal(folderMap["filesystem"]) + if err != nil { + continue + } + var compatFsConfig compatFilesystemV9 + err = json.Unmarshal(fsBytes, &compatFsConfig) + if err != nil { + logger.WarnToConsole("failed to unmarshal v9 fsconfig for folder %#v, is it already migrated?", folderMap["name"]) + continue + } + if compatFsConfig.AzBlobConfig.SASURL != "" { + folder := vfs.BaseVirtualFolder{ + Name: folderMap["name"].(string), + } + fsConfig, err := convertFsConfigFromV9(compatFsConfig, folder.GetEncrytionAdditionalData()) + if err != nil { + return err + } + folderMap["filesystem"] = fsConfig + folders = append(folders, folderMap) + } + } + return err + }) + + if err != nil { + return err + } + + return dbHandle.Update(func(tx *bolt.Tx) error { + bucket, err := getFoldersBucket(tx) + if err != nil { + return err + } + for _, folder := range folders { + buf, err := json.Marshal(folder) + if err != nil { + return err + } + err = bucket.Put([]byte(folder["name"].(string)), buf) + if err != nil { + return err + } + } + return nil + }) +} + +//nolint:gocyclo +func boltUpdateV7Users(dbHandle *bolt.DB) error { + var users []map[string]interface{} + err := dbHandle.View(func(tx *bolt.Tx) error { + bucket, err := getUsersBucket(tx) + if err != nil { + return err + } + cursor := bucket.Cursor() + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { + var userMap map[string]interface{} + err = json.Unmarshal(v, &userMap) + if err != nil { + return err + } + fsBytes, err := json.Marshal(userMap["filesystem"]) + if err != nil { + continue + } + foldersBytes, err := json.Marshal(userMap["virtual_folders"]) + if err != nil { + continue + } + var compatFsConfig compatFilesystemV9 + err = json.Unmarshal(fsBytes, &compatFsConfig) + if err != nil { + logger.WarnToConsole("failed to unmarshal v9 fsconfig for user %#v, is it already migrated?", userMap["name"]) + continue + } + var compatFolders []compatFolderV9 + err = json.Unmarshal(foldersBytes, &compatFolders) + if err != nil { + logger.WarnToConsole("failed to unmarshal v9 folders for user %#v, is it already migrated?", userMap["name"]) + continue + } + toConvert := false + for idx := range compatFolders { + f := &compatFolders[idx] + if f.FsConfig.AzBlobConfig.SASURL != "" { + f.FsConfig.AzBlobConfig = compatAzBlobFsConfigV9{} + toConvert = true + } + } + if compatFsConfig.AzBlobConfig.SASURL != "" { + user := User{ + Username: userMap["username"].(string), + } + fsConfig, err := convertFsConfigFromV9(compatFsConfig, user.GetEncrytionAdditionalData()) + if err != nil { + return err + } + userMap["filesystem"] = fsConfig + toConvert = true + } + if toConvert { + userMap["virtual_folders"] = compatFolders + users = append(users, userMap) + } + } + return err + }) + + if err != nil { + return err + } + + return dbHandle.Update(func(tx *bolt.Tx) error { + bucket, err := getUsersBucket(tx) + if err != nil { + return err + } + for _, user := range users { + buf, err := json.Marshal(user) + if err != nil { + return err + } + err = bucket.Put([]byte(user["username"].(string)), buf) + if err != nil { + return err + } + } + return nil + }) +} + +//nolint:dupl +func boltDowngradeV7Folders(dbHandle *bolt.DB) error { + var folders []map[string]interface{} + err := dbHandle.View(func(tx *bolt.Tx) error { + bucket, err := getFoldersBucket(tx) + if err != nil { + return err + } + cursor := bucket.Cursor() + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { + var folderMap map[string]interface{} + err = json.Unmarshal(v, &folderMap) + if err != nil { + return err + } + fsBytes, err := json.Marshal(folderMap["filesystem"]) + if err != nil { + continue + } + var fsConfig vfs.Filesystem + err = json.Unmarshal(fsBytes, &fsConfig) + if err != nil { + logger.WarnToConsole("failed to unmarshal v10 fsconfig for folder %#v, is it already migrated?", folderMap["name"]) + continue + } + if fsConfig.AzBlobConfig.SASURL != nil && !fsConfig.AzBlobConfig.SASURL.IsEmpty() { + fsV9, err := convertFsConfigToV9(fsConfig) + if err != nil { + return err + } + folderMap["filesystem"] = fsV9 + folders = append(folders, folderMap) + } + } + return err + }) + + if err != nil { + return err + } + + return dbHandle.Update(func(tx *bolt.Tx) error { + bucket, err := getFoldersBucket(tx) + if err != nil { + return err + } + for _, folder := range folders { + buf, err := json.Marshal(folder) + if err != nil { + return err + } + err = bucket.Put([]byte(folder["name"].(string)), buf) + if err != nil { + return err + } + } + return nil + }) +} + +//nolint:dupl,gocyclo +func boltDowngradeV7Users(dbHandle *bolt.DB) error { + var users []map[string]interface{} + err := dbHandle.View(func(tx *bolt.Tx) error { + bucket, err := getUsersBucket(tx) + if err != nil { + return err + } + cursor := bucket.Cursor() + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { + var userMap map[string]interface{} + err = json.Unmarshal(v, &userMap) + if err != nil { + return err + } + fsBytes, err := json.Marshal(userMap["filesystem"]) + if err != nil { + continue + } + foldersBytes, err := json.Marshal(userMap["virtual_folders"]) + if err != nil { + continue + } + var fsConfig vfs.Filesystem + err = json.Unmarshal(fsBytes, &fsConfig) + if err != nil { + logger.WarnToConsole("failed to unmarshal v10 fsconfig for user %#v, is it already migrated?", userMap["username"]) + continue + } + var folders []vfs.VirtualFolder + err = json.Unmarshal(foldersBytes, &folders) + if err != nil { + logger.WarnToConsole("failed to unmarshal v9 folders for user %#v, is it already migrated?", userMap["name"]) + continue + } + toConvert := false + for idx := range folders { + f := &folders[idx] + f.FsConfig.AzBlobConfig = vfs.AzBlobFsConfig{} + toConvert = true + } + if fsConfig.AzBlobConfig.SASURL != nil && !fsConfig.AzBlobConfig.SASURL.IsEmpty() { + fsV9, err := convertFsConfigToV9(fsConfig) + if err != nil { + return err + } + userMap["filesystem"] = fsV9 + toConvert = true + } + if toConvert { + userMap["virtual_folders"] = folders + users = append(users, userMap) + } + } + return err + }) + + if err != nil { + return err + } + + return dbHandle.Update(func(tx *bolt.Tx) error { + bucket, err := getUsersBucket(tx) + if err != nil { + return err + } + for _, user := range users { + buf, err := json.Marshal(user) + if err != nil { + return err + } + err = bucket.Put([]byte(user["username"].(string)), buf) + if err != nil { + return err + } + } + return nil + }) +} diff --git a/dataprovider/compat.go b/dataprovider/compat.go new file mode 100644 index 00000000..c2d6d9ab --- /dev/null +++ b/dataprovider/compat.go @@ -0,0 +1,118 @@ +package dataprovider + +import ( + "github.com/drakkan/sftpgo/kms" + "github.com/drakkan/sftpgo/vfs" +) + +type compatAzBlobFsConfigV9 struct { + Container string `json:"container,omitempty"` + AccountName string `json:"account_name,omitempty"` + AccountKey *kms.Secret `json:"account_key,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + SASURL string `json:"sas_url,omitempty"` + KeyPrefix string `json:"key_prefix,omitempty"` + UploadPartSize int64 `json:"upload_part_size,omitempty"` + UploadConcurrency int `json:"upload_concurrency,omitempty"` + UseEmulator bool `json:"use_emulator,omitempty"` + AccessTier string `json:"access_tier,omitempty"` +} + +type compatFilesystemV9 struct { + Provider vfs.FilesystemProvider `json:"provider"` + S3Config vfs.S3FsConfig `json:"s3config,omitempty"` + GCSConfig vfs.GCSFsConfig `json:"gcsconfig,omitempty"` + AzBlobConfig compatAzBlobFsConfigV9 `json:"azblobconfig,omitempty"` + CryptConfig vfs.CryptFsConfig `json:"cryptconfig,omitempty"` + SFTPConfig vfs.SFTPFsConfig `json:"sftpconfig,omitempty"` +} + +type compatBaseFolderV9 struct { + ID int64 `json:"id"` + Name string `json:"name"` + MappedPath string `json:"mapped_path,omitempty"` + Description string `json:"description,omitempty"` + UsedQuotaSize int64 `json:"used_quota_size"` + UsedQuotaFiles int `json:"used_quota_files"` + LastQuotaUpdate int64 `json:"last_quota_update"` + Users []string `json:"users,omitempty"` + FsConfig compatFilesystemV9 `json:"filesystem"` +} + +type compatFolderV9 struct { + compatBaseFolderV9 + VirtualPath string `json:"virtual_path"` + QuotaSize int64 `json:"quota_size"` + QuotaFiles int `json:"quota_files"` +} + +type compatUserV9 struct { + ID int64 `json:"id"` + Username string `json:"username"` + FsConfig compatFilesystemV9 `json:"filesystem"` +} + +func convertFsConfigFromV9(compatFs compatFilesystemV9, aead string) (vfs.Filesystem, error) { + fsConfig := vfs.Filesystem{ + Provider: compatFs.Provider, + S3Config: compatFs.S3Config, + GCSConfig: compatFs.GCSConfig, + CryptConfig: compatFs.CryptConfig, + SFTPConfig: compatFs.SFTPConfig, + } + azSASURL := kms.NewEmptySecret() + if compatFs.Provider == vfs.AzureBlobFilesystemProvider && compatFs.AzBlobConfig.SASURL != "" { + azSASURL = kms.NewPlainSecret(compatFs.AzBlobConfig.SASURL) + } + if compatFs.AzBlobConfig.AccountKey == nil { + compatFs.AzBlobConfig.AccountKey = kms.NewEmptySecret() + } + fsConfig.AzBlobConfig = vfs.AzBlobFsConfig{ + Container: compatFs.AzBlobConfig.Container, + AccountName: compatFs.AzBlobConfig.AccountName, + AccountKey: compatFs.AzBlobConfig.AccountKey, + Endpoint: compatFs.AzBlobConfig.Endpoint, + SASURL: azSASURL, + KeyPrefix: compatFs.AzBlobConfig.KeyPrefix, + UploadPartSize: compatFs.AzBlobConfig.UploadPartSize, + UploadConcurrency: compatFs.AzBlobConfig.UploadConcurrency, + UseEmulator: compatFs.AzBlobConfig.UseEmulator, + AccessTier: compatFs.AzBlobConfig.AccessTier, + } + err := fsConfig.AzBlobConfig.EncryptCredentials(aead) + return fsConfig, err +} + +func convertFsConfigToV9(fs vfs.Filesystem) (compatFilesystemV9, error) { + azSASURL := "" + if fs.Provider == vfs.AzureBlobFilesystemProvider { + if fs.AzBlobConfig.SASURL != nil && fs.AzBlobConfig.SASURL.IsEncrypted() { + err := fs.AzBlobConfig.SASURL.Decrypt() + if err != nil { + return compatFilesystemV9{}, err + } + azSASURL = fs.AzBlobConfig.SASURL.GetPayload() + } + } + azFsCompat := compatAzBlobFsConfigV9{ + Container: fs.AzBlobConfig.Container, + AccountName: fs.AzBlobConfig.AccountName, + AccountKey: fs.AzBlobConfig.AccountKey, + Endpoint: fs.AzBlobConfig.Endpoint, + SASURL: azSASURL, + KeyPrefix: fs.AzBlobConfig.KeyPrefix, + UploadPartSize: fs.AzBlobConfig.UploadPartSize, + UploadConcurrency: fs.AzBlobConfig.UploadConcurrency, + UseEmulator: fs.AzBlobConfig.UseEmulator, + AccessTier: fs.AzBlobConfig.AccessTier, + } + fsV9 := compatFilesystemV9{ + Provider: fs.Provider, + S3Config: fs.S3Config, + GCSConfig: fs.GCSConfig, + AzBlobConfig: azFsCompat, + CryptConfig: fs.CryptConfig, + SFTPConfig: fs.SFTPConfig, + } + return fsV9, nil +} diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go index 0c792db6..ab67af69 100644 --- a/dataprovider/dataprovider.go +++ b/dataprovider/dataprovider.go @@ -66,7 +66,7 @@ const ( CockroachDataProviderName = "cockroachdb" // DumpVersion defines the version for the dump. // For restore/load we support the current version and the previous one - DumpVersion = 7 + DumpVersion = 8 argonPwdPrefix = "$argon2id$" bcryptPwdPrefix = "$2a$" diff --git a/dataprovider/mysql.go b/dataprovider/mysql.go index 8379c37b..55614f12 100644 --- a/dataprovider/mysql.go +++ b/dataprovider/mysql.go @@ -250,6 +250,8 @@ func (p *MySQLProvider) migrateDatabase() error { return err case version == 8: return updateMySQLDatabaseFromV8(p.dbHandle) + case version == 9: + return updateMySQLDatabaseFromV9(p.dbHandle) default: if version > sqlDatabaseVersion { providerLog(logger.LevelWarn, "database version %v is newer than the supported one: %v", version, @@ -274,19 +276,35 @@ func (p *MySQLProvider) revertDatabase(targetVersion int) error { switch dbVersion.Version { case 9: return downgradeMySQLDatabaseFromV9(p.dbHandle) + case 10: + return downgradeMySQLDatabaseFromV10(p.dbHandle) default: return fmt.Errorf("database version not handled: %v", dbVersion.Version) } } func updateMySQLDatabaseFromV8(dbHandle *sql.DB) error { - return updateMySQLDatabaseFrom8To9(dbHandle) + if err := updateMySQLDatabaseFrom8To9(dbHandle); err != nil { + return err + } + return updateMySQLDatabaseFromV9(dbHandle) +} + +func updateMySQLDatabaseFromV9(dbHandle *sql.DB) error { + return updateMySQLDatabaseFrom9To10(dbHandle) } func downgradeMySQLDatabaseFromV9(dbHandle *sql.DB) error { return downgradeMySQLDatabaseFrom9To8(dbHandle) } +func downgradeMySQLDatabaseFromV10(dbHandle *sql.DB) error { + if err := downgradeMySQLDatabaseFrom10To9(dbHandle); err != nil { + return err + } + return downgradeMySQLDatabaseFromV9(dbHandle) +} + func updateMySQLDatabaseFrom8To9(dbHandle *sql.DB) error { logger.InfoToConsole("updating database version: 8 -> 9") providerLog(logger.LevelInfo, "updating database version: 8 -> 9") @@ -304,3 +322,11 @@ func downgradeMySQLDatabaseFrom9To8(dbHandle *sql.DB) error { sql = strings.ReplaceAll(sql, "{{folders}}", sqlTableFolders) return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, strings.Split(sql, ";"), 8) } + +func updateMySQLDatabaseFrom9To10(dbHandle *sql.DB) error { + return sqlCommonUpdateDatabaseFrom9To10(dbHandle) +} + +func downgradeMySQLDatabaseFrom10To9(dbHandle *sql.DB) error { + return sqlCommonDowngradeDatabaseFrom10To9(dbHandle) +} diff --git a/dataprovider/pgsql.go b/dataprovider/pgsql.go index 7192586a..55684db6 100644 --- a/dataprovider/pgsql.go +++ b/dataprovider/pgsql.go @@ -263,6 +263,8 @@ func (p *PGSQLProvider) migrateDatabase() error { return err case version == 8: return updatePGSQLDatabaseFromV8(p.dbHandle) + case version == 9: + return updatePGSQLDatabaseFromV9(p.dbHandle) default: if version > sqlDatabaseVersion { providerLog(logger.LevelWarn, "database version %v is newer than the supported one: %v", version, @@ -287,19 +289,35 @@ func (p *PGSQLProvider) revertDatabase(targetVersion int) error { switch dbVersion.Version { case 9: return downgradePGSQLDatabaseFromV9(p.dbHandle) + case 10: + return downgradePGSQLDatabaseFromV10(p.dbHandle) default: return fmt.Errorf("database version not handled: %v", dbVersion.Version) } } func updatePGSQLDatabaseFromV8(dbHandle *sql.DB) error { - return updatePGSQLDatabaseFrom8To9(dbHandle) + if err := updatePGSQLDatabaseFrom8To9(dbHandle); err != nil { + return err + } + return updatePGSQLDatabaseFromV9(dbHandle) +} + +func updatePGSQLDatabaseFromV9(dbHandle *sql.DB) error { + return updatePGSQLDatabaseFrom9To10(dbHandle) } func downgradePGSQLDatabaseFromV9(dbHandle *sql.DB) error { return downgradePGSQLDatabaseFrom9To8(dbHandle) } +func downgradePGSQLDatabaseFromV10(dbHandle *sql.DB) error { + if err := downgradePGSQLDatabaseFrom10To9(dbHandle); err != nil { + return err + } + return downgradePGSQLDatabaseFromV9(dbHandle) +} + func updatePGSQLDatabaseFrom8To9(dbHandle *sql.DB) error { logger.InfoToConsole("updating database version: 8 -> 9") providerLog(logger.LevelInfo, "updating database version: 8 -> 9") @@ -317,3 +335,11 @@ func downgradePGSQLDatabaseFrom9To8(dbHandle *sql.DB) error { sql = strings.ReplaceAll(sql, "{{folders}}", sqlTableFolders) return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 8) } + +func updatePGSQLDatabaseFrom9To10(dbHandle *sql.DB) error { + return sqlCommonUpdateDatabaseFrom9To10(dbHandle) +} + +func downgradePGSQLDatabaseFrom10To9(dbHandle *sql.DB) error { + return sqlCommonDowngradeDatabaseFrom10To9(dbHandle) +} diff --git a/dataprovider/sqlcommon.go b/dataprovider/sqlcommon.go index 08b2da54..edeba718 100644 --- a/dataprovider/sqlcommon.go +++ b/dataprovider/sqlcommon.go @@ -18,7 +18,7 @@ import ( ) const ( - sqlDatabaseVersion = 9 + sqlDatabaseVersion = 10 defaultSQLQueryTimeout = 10 * time.Second longSQLQueryTimeout = 60 * time.Second ) @@ -1096,3 +1096,313 @@ func sqlCommonExecuteTx(ctx context.Context, dbHandle *sql.DB, txFn func(*sql.Tx } return tx.Commit() } + +func sqlCommonUpdateDatabaseFrom9To10(dbHandle *sql.DB) error { + logger.InfoToConsole("updating database version: 9 -> 10") + providerLog(logger.LevelInfo, "updating database version: 9 -> 10") + + if err := sqlCommonUpdateV10Folders(dbHandle); err != nil { + return err + } + + if err := sqlCommonUpdateV10Users(dbHandle); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) + defer cancel() + + return sqlCommonUpdateDatabaseVersion(ctx, dbHandle, 10) +} + +func sqlCommonDowngradeDatabaseFrom10To9(dbHandle *sql.DB) error { + logger.InfoToConsole("downgrading database version: 10 -> 9") + providerLog(logger.LevelInfo, "downgrading database version: 10 -> 9") + + if err := sqlCommonDowngradeV10Folders(dbHandle); err != nil { + return err + } + + if err := sqlCommonDowngradeV10Users(dbHandle); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) + defer cancel() + + return sqlCommonUpdateDatabaseVersion(ctx, dbHandle, 9) +} + +//nolint:dupl +func sqlCommonDowngradeV10Folders(dbHandle *sql.DB) error { + ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout) + defer cancel() + + q := getCompatFolderV10FsConfigQuery() + stmt, err := dbHandle.PrepareContext(ctx, q) + if err != nil { + providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err) + return err + } + defer stmt.Close() + rows, err := stmt.QueryContext(ctx) + if err != nil { + return err + } + defer rows.Close() + + var folders []compatBaseFolderV9 + for rows.Next() { + var folder compatBaseFolderV9 + var fsConfigString sql.NullString + err = rows.Scan(&folder.ID, &folder.Name, &fsConfigString) + if err != nil { + return err + } + if fsConfigString.Valid { + var fsConfig vfs.Filesystem + err = json.Unmarshal([]byte(fsConfigString.String), &fsConfig) + if err != nil { + logger.WarnToConsole("failed to unmarshal v10 fsconfig for folder %#v, is it already migrated?", folder.Name) + continue + } + if fsConfig.AzBlobConfig.SASURL != nil && !fsConfig.AzBlobConfig.SASURL.IsEmpty() { + fsV9, err := convertFsConfigToV9(fsConfig) + if err != nil { + return err + } + folder.FsConfig = fsV9 + folders = append(folders, folder) + } + } + } + if err := rows.Err(); err != nil { + return err + } + // update fsconfig for affected folders + for _, folder := range folders { + q := updateCompatFolderV10FsConfigQuery() + stmt, err := dbHandle.PrepareContext(ctx, q) + if err != nil { + providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err) + return err + } + defer stmt.Close() + cfg, err := json.Marshal(folder.FsConfig) + if err != nil { + return err + } + + _, err = stmt.ExecContext(ctx, string(cfg), folder.ID) + if err != nil { + return err + } + } + + return nil +} + +//nolint:dupl +func sqlCommonDowngradeV10Users(dbHandle *sql.DB) error { + ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout) + defer cancel() + + q := getCompatUserV10FsConfigQuery() + stmt, err := dbHandle.PrepareContext(ctx, q) + if err != nil { + providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err) + return err + } + defer stmt.Close() + rows, err := stmt.QueryContext(ctx) + if err != nil { + return err + } + defer rows.Close() + + var users []compatUserV9 + for rows.Next() { + var user compatUserV9 + var fsConfigString sql.NullString + err = rows.Scan(&user.ID, &user.Username, &fsConfigString) + if err != nil { + return err + } + if fsConfigString.Valid { + var fsConfig vfs.Filesystem + err = json.Unmarshal([]byte(fsConfigString.String), &fsConfig) + if err != nil { + logger.WarnToConsole("failed to unmarshal v10 fsconfig for user %#v, is it already migrated?", user.Username) + continue + } + if fsConfig.AzBlobConfig.SASURL != nil && !fsConfig.AzBlobConfig.SASURL.IsEmpty() { + fsV9, err := convertFsConfigToV9(fsConfig) + if err != nil { + return err + } + user.FsConfig = fsV9 + users = append(users, user) + } + } + } + if err := rows.Err(); err != nil { + return err + } + // update fsconfig for affected users + for _, user := range users { + q := updateCompatUserV10FsConfigQuery() + stmt, err := dbHandle.PrepareContext(ctx, q) + if err != nil { + providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err) + return err + } + defer stmt.Close() + cfg, err := json.Marshal(user.FsConfig) + if err != nil { + return err + } + + _, err = stmt.ExecContext(ctx, string(cfg), user.ID) + if err != nil { + return err + } + } + + return nil +} + +func sqlCommonUpdateV10Folders(dbHandle *sql.DB) error { + ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout) + defer cancel() + + q := getCompatFolderV10FsConfigQuery() + stmt, err := dbHandle.PrepareContext(ctx, q) + if err != nil { + providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err) + return err + } + defer stmt.Close() + rows, err := stmt.QueryContext(ctx) + if err != nil { + return err + } + defer rows.Close() + + var folders []vfs.BaseVirtualFolder + for rows.Next() { + var folder vfs.BaseVirtualFolder + var fsConfigString sql.NullString + err = rows.Scan(&folder.ID, &folder.Name, &fsConfigString) + if err != nil { + return err + } + if fsConfigString.Valid { + var compatFsConfig compatFilesystemV9 + err = json.Unmarshal([]byte(fsConfigString.String), &compatFsConfig) + if err != nil { + logger.WarnToConsole("failed to unmarshal v9 fsconfig for folder %#v, is it already migrated?", folder.Name) + continue + } + if compatFsConfig.AzBlobConfig.SASURL != "" { + fsConfig, err := convertFsConfigFromV9(compatFsConfig, folder.GetEncrytionAdditionalData()) + if err != nil { + return err + } + folder.FsConfig = fsConfig + folders = append(folders, folder) + } + } + } + if err := rows.Err(); err != nil { + return err + } + // update fsconfig for affected folders + for _, folder := range folders { + q := updateCompatFolderV10FsConfigQuery() + stmt, err := dbHandle.PrepareContext(ctx, q) + if err != nil { + providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err) + return err + } + defer stmt.Close() + cfg, err := json.Marshal(folder.FsConfig) + if err != nil { + return err + } + + _, err = stmt.ExecContext(ctx, string(cfg), folder.ID) + if err != nil { + return err + } + } + + return nil +} + +func sqlCommonUpdateV10Users(dbHandle *sql.DB) error { + ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout) + defer cancel() + + q := getCompatUserV10FsConfigQuery() + stmt, err := dbHandle.PrepareContext(ctx, q) + if err != nil { + providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err) + return err + } + defer stmt.Close() + rows, err := stmt.QueryContext(ctx) + if err != nil { + return err + } + defer rows.Close() + + var users []User + for rows.Next() { + var user User + var fsConfigString sql.NullString + err = rows.Scan(&user.ID, &user.Username, &fsConfigString) + if err != nil { + return err + } + if fsConfigString.Valid { + var compatFsConfig compatFilesystemV9 + err = json.Unmarshal([]byte(fsConfigString.String), &compatFsConfig) + if err != nil { + logger.WarnToConsole("failed to unmarshal v9 fsconfig for user %#v, is it already migrated?", user.Username) + continue + } + if compatFsConfig.AzBlobConfig.SASURL != "" { + fsConfig, err := convertFsConfigFromV9(compatFsConfig, user.GetEncrytionAdditionalData()) + if err != nil { + return err + } + user.FsConfig = fsConfig + users = append(users, user) + } + } + } + if err := rows.Err(); err != nil { + return err + } + // update fsconfig for affected users + for _, user := range users { + q := updateCompatUserV10FsConfigQuery() + stmt, err := dbHandle.PrepareContext(ctx, q) + if err != nil { + providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err) + return err + } + defer stmt.Close() + cfg, err := json.Marshal(user.FsConfig) + if err != nil { + return err + } + + _, err = stmt.ExecContext(ctx, string(cfg), user.ID) + if err != nil { + return err + } + } + + return nil +} diff --git a/dataprovider/sqlite.go b/dataprovider/sqlite.go index 68a38af9..83cb11f3 100644 --- a/dataprovider/sqlite.go +++ b/dataprovider/sqlite.go @@ -271,6 +271,8 @@ func (p *SQLiteProvider) migrateDatabase() error { return err case version == 8: return updateSQLiteDatabaseFromV8(p.dbHandle) + case version == 9: + return updateSQLiteDatabaseFromV9(p.dbHandle) default: if version > sqlDatabaseVersion { providerLog(logger.LevelWarn, "database version %v is newer than the supported one: %v", version, @@ -295,19 +297,35 @@ func (p *SQLiteProvider) revertDatabase(targetVersion int) error { switch dbVersion.Version { case 9: return downgradeSQLiteDatabaseFromV9(p.dbHandle) + case 10: + return downgradeSQLiteDatabaseFromV10(p.dbHandle) default: return fmt.Errorf("database version not handled: %v", dbVersion.Version) } } func updateSQLiteDatabaseFromV8(dbHandle *sql.DB) error { - return updateSQLiteDatabaseFrom8To9(dbHandle) + if err := updateSQLiteDatabaseFrom8To9(dbHandle); err != nil { + return err + } + return updateSQLiteDatabaseFromV9(dbHandle) +} + +func updateSQLiteDatabaseFromV9(dbHandle *sql.DB) error { + return updateSQLiteDatabaseFrom9To10(dbHandle) } func downgradeSQLiteDatabaseFromV9(dbHandle *sql.DB) error { return downgradeSQLiteDatabaseFrom9To8(dbHandle) } +func downgradeSQLiteDatabaseFromV10(dbHandle *sql.DB) error { + if err := downgradeSQLiteDatabaseFrom10To9(dbHandle); err != nil { + return err + } + return downgradeSQLiteDatabaseFromV9(dbHandle) +} + func updateSQLiteDatabaseFrom8To9(dbHandle *sql.DB) error { logger.InfoToConsole("updating database version: 8 -> 9") providerLog(logger.LevelInfo, "updating database version: 8 -> 9") @@ -332,6 +350,14 @@ func downgradeSQLiteDatabaseFrom9To8(dbHandle *sql.DB) error { return setPragmaFK(dbHandle, "ON") } +func updateSQLiteDatabaseFrom9To10(dbHandle *sql.DB) error { + return sqlCommonUpdateDatabaseFrom9To10(dbHandle) +} + +func downgradeSQLiteDatabaseFrom10To9(dbHandle *sql.DB) error { + return sqlCommonDowngradeDatabaseFrom10To9(dbHandle) +} + func setPragmaFK(dbHandle *sql.DB, value string) error { ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout) defer cancel() diff --git a/dataprovider/sqlqueries.go b/dataprovider/sqlqueries.go index fe716251..bae6c689 100644 --- a/dataprovider/sqlqueries.go +++ b/dataprovider/sqlqueries.go @@ -210,3 +210,19 @@ func getDatabaseVersionQuery() string { func getUpdateDBVersionQuery() string { return fmt.Sprintf(`UPDATE %v SET version=%v`, sqlTableSchemaVersion, sqlPlaceholders[0]) } + +func getCompatUserV10FsConfigQuery() string { + return fmt.Sprintf(`SELECT id,username,filesystem FROM %v`, sqlTableUsers) +} + +func updateCompatUserV10FsConfigQuery() string { + return fmt.Sprintf(`UPDATE %v SET filesystem=%v WHERE id=%v`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1]) +} + +func getCompatFolderV10FsConfigQuery() string { + return fmt.Sprintf(`SELECT id,name,filesystem FROM %v`, sqlTableFolders) +} + +func updateCompatFolderV10FsConfigQuery() string { + return fmt.Sprintf(`UPDATE %v SET filesystem=%v WHERE id=%v`, sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1]) +} diff --git a/dataprovider/user.go b/dataprovider/user.go index 5be0010c..a1789a01 100644 --- a/dataprovider/user.go +++ b/dataprovider/user.go @@ -349,6 +349,7 @@ func (u *User) hideConfidentialData() { u.FsConfig.GCSConfig.Credentials.Hide() case vfs.AzureBlobFilesystemProvider: u.FsConfig.AzBlobConfig.AccountKey.Hide() + u.FsConfig.AzBlobConfig.SASURL.Hide() case vfs.CryptedFilesystemProvider: u.FsConfig.CryptConfig.Passphrase.Hide() case vfs.SFTPFilesystemProvider: @@ -399,6 +400,9 @@ func (u *User) hasRedactedSecret() bool { if u.FsConfig.AzBlobConfig.AccountKey.IsRedacted() { return true } + if u.FsConfig.AzBlobConfig.SASURL.IsRedacted() { + return true + } case vfs.CryptedFilesystemProvider: if u.FsConfig.CryptConfig.Passphrase.IsRedacted() { return true @@ -457,6 +461,7 @@ func (u *User) SetEmptySecrets() { u.FsConfig.S3Config.AccessSecret = kms.NewEmptySecret() u.FsConfig.GCSConfig.Credentials = kms.NewEmptySecret() u.FsConfig.AzBlobConfig.AccountKey = kms.NewEmptySecret() + u.FsConfig.AzBlobConfig.SASURL = kms.NewEmptySecret() u.FsConfig.CryptConfig.Passphrase = kms.NewEmptySecret() u.FsConfig.SFTPConfig.Password = kms.NewEmptySecret() u.FsConfig.SFTPConfig.PrivateKey = kms.NewEmptySecret() diff --git a/docs/custom-actions.md b/docs/custom-actions.md index dbd9a85b..ad89ab60 100644 --- a/docs/custom-actions.md +++ b/docs/custom-actions.md @@ -40,7 +40,7 @@ The external program can also read the following environment variables: - `SFTPGO_ACTION_FILE_SIZE`, non-zero for `pre-upload`,`upload`, `download` and `delete` actions if the file size is greater than `0` - `SFTPGO_ACTION_FS_PROVIDER`, `0` for local filesystem, `1` for S3 backend, `2` for Google Cloud Storage (GCS) backend, `3` for Azure Blob Storage backend, `4` for local encrypted backend, `5` for SFTP backend - `SFTPGO_ACTION_BUCKET`, non-empty for S3, GCS and Azure backends -- `SFTPGO_ACTION_ENDPOINT`, non-empty for S3, SFTP and Azure backend if configured. For Azure this is the SAS URL, if configured otherwise the endpoint +- `SFTPGO_ACTION_ENDPOINT`, non-empty for S3, SFTP and Azure backend if configured. For Azure this is the endpoint, if configured - `SFTPGO_ACTION_STATUS`, integer. Status for `upload`, `download` and `ssh_cmd` actions. 0 means a generic error occurred. 1 means no error, 2 means quota exceeded error - `SFTPGO_ACTION_PROTOCOL`, string. Possible values are `SSH`, `SFTP`, `SCP`, `FTP`, `DAV`, `HTTP` - `SFTPGO_ACTION_OPEN_FLAGS`, integer. File open flags, can be non-zero for `pre-upload` action. If `SFTPGO_ACTION_FILE_SIZE` is greater than zero and `SFTPGO_ACTION_OPEN_FLAGS&512 == 0` the target file will not be truncated @@ -58,7 +58,7 @@ If the `hook` defines an HTTP URL then this URL will be invoked as HTTP POST. Th - `file_size`, included for `pre-upload`, `upload`, `download`, `delete` actions if the file size is greater than `0` - `fs_provider`, `0` for local filesystem, `1` for S3 backend, `2` for Google Cloud Storage (GCS) backend, `3` for Azure Blob Storage backend, `4` for local encrypted backend, `5` for SFTP backend - `bucket`, inlcuded for S3, GCS and Azure backends -- `endpoint`, included for S3, SFTP and Azure backend if configured. For Azure this is the SAS URL, if configured, otherwise the endpoint +- `endpoint`, included for S3, SFTP and Azure backend if configured. For Azure this is the endpoint, if configured - `status`, integer. Status for `upload`, `download` and `ssh_cmd` actions. 0 means a generic error occurred. 1 means no error, 2 means quota exceeded error - `protocol`, string. Possible values are `SSH`, `SFTP`, `SCP`, `FTP`, `DAV`, `HTTP` - `open_flags`, integer. File open flags, can be non-zero for `pre-upload` action. If `file_size` is greater than zero and `file_size&512 == 0` the target file will not be truncated diff --git a/httpd/api_folder.go b/httpd/api_folder.go index 8d6e9e15..e2238b4f 100644 --- a/httpd/api_folder.go +++ b/httpd/api_folder.go @@ -54,6 +54,7 @@ func updateFolder(w http.ResponseWriter, r *http.Request) { folderID := folder.ID currentS3AccessSecret := folder.FsConfig.S3Config.AccessSecret currentAzAccountKey := folder.FsConfig.AzBlobConfig.AccountKey + currentAzSASUrl := folder.FsConfig.AzBlobConfig.SASURL currentGCSCredentials := folder.FsConfig.GCSConfig.Credentials currentCryptoPassphrase := folder.FsConfig.CryptConfig.Passphrase currentSFTPPassword := folder.FsConfig.SFTPConfig.Password @@ -72,7 +73,7 @@ func updateFolder(w http.ResponseWriter, r *http.Request) { folder.ID = folderID folder.Name = name folder.FsConfig.SetEmptySecretsIfNil() - updateEncryptedSecrets(&folder.FsConfig, currentS3AccessSecret, currentAzAccountKey, currentGCSCredentials, + updateEncryptedSecrets(&folder.FsConfig, currentS3AccessSecret, currentAzAccountKey, currentAzSASUrl, currentGCSCredentials, currentCryptoPassphrase, currentSFTPPassword, currentSFTPKey) err = dataprovider.UpdateFolder(&folder, users) if err != nil { diff --git a/httpd/api_user.go b/httpd/api_user.go index 495079dd..a0225760 100644 --- a/httpd/api_user.go +++ b/httpd/api_user.go @@ -74,6 +74,10 @@ func addUser(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, errors.New("invalid account_key"), "", http.StatusBadRequest) return } + if user.FsConfig.AzBlobConfig.SASURL.IsRedacted() { + sendAPIResponse(w, r, errors.New("invalid sas_url"), "", http.StatusBadRequest) + return + } case vfs.CryptedFilesystemProvider: if user.FsConfig.CryptConfig.Passphrase.IsRedacted() { sendAPIResponse(w, r, errors.New("invalid passphrase"), "", http.StatusBadRequest) @@ -120,6 +124,7 @@ func updateUser(w http.ResponseWriter, r *http.Request) { currentPermissions := user.Permissions currentS3AccessSecret := user.FsConfig.S3Config.AccessSecret currentAzAccountKey := user.FsConfig.AzBlobConfig.AccountKey + currentAzSASUrl := user.FsConfig.AzBlobConfig.SASURL currentGCSCredentials := user.FsConfig.GCSConfig.Credentials currentCryptoPassphrase := user.FsConfig.CryptConfig.Passphrase currentSFTPPassword := user.FsConfig.SFTPConfig.Password @@ -144,8 +149,8 @@ func updateUser(w http.ResponseWriter, r *http.Request) { if len(user.Permissions) == 0 { user.Permissions = currentPermissions } - updateEncryptedSecrets(&user.FsConfig, currentS3AccessSecret, currentAzAccountKey, currentGCSCredentials, currentCryptoPassphrase, - currentSFTPPassword, currentSFTPKey) + updateEncryptedSecrets(&user.FsConfig, currentS3AccessSecret, currentAzAccountKey, currentAzSASUrl, + currentGCSCredentials, currentCryptoPassphrase, currentSFTPPassword, currentSFTPKey) err = dataprovider.UpdateUser(&user) if err != nil { sendAPIResponse(w, r, err, "", getRespStatus(err)) @@ -176,7 +181,7 @@ func disconnectUser(username string) { } } -func updateEncryptedSecrets(fsConfig *vfs.Filesystem, currentS3AccessSecret, currentAzAccountKey, +func updateEncryptedSecrets(fsConfig *vfs.Filesystem, currentS3AccessSecret, currentAzAccountKey, currentAzSASUrl, currentGCSCredentials, currentCryptoPassphrase, currentSFTPPassword, currentSFTPKey *kms.Secret) { // we use the new access secret if plain or empty, otherwise the old value switch fsConfig.Provider { @@ -188,6 +193,9 @@ func updateEncryptedSecrets(fsConfig *vfs.Filesystem, currentS3AccessSecret, cur if fsConfig.AzBlobConfig.AccountKey.IsNotPlainAndNotEmpty() { fsConfig.AzBlobConfig.AccountKey = currentAzAccountKey } + if fsConfig.AzBlobConfig.SASURL.IsNotPlainAndNotEmpty() { + fsConfig.AzBlobConfig.SASURL = currentAzSASUrl + } case vfs.GCSFilesystemProvider: if fsConfig.GCSConfig.Credentials.IsNotPlainAndNotEmpty() { fsConfig.GCSConfig.Credentials = currentGCSCredentials diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index e2e8918b..8cf53541 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -984,10 +984,13 @@ func TestAddUserInvalidFsConfig(t *testing.T) { u = getTestUser() u.FsConfig.Provider = vfs.AzureBlobFilesystemProvider - u.FsConfig.AzBlobConfig.SASURL = "http://foo\x7f.com/" + u.FsConfig.AzBlobConfig.SASURL = kms.NewPlainSecret("http://foo\x7f.com/") _, _, err = httpdtest.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) - u.FsConfig.AzBlobConfig.SASURL = "" + u.FsConfig.AzBlobConfig.SASURL = kms.NewSecret(kms.SecretStatusRedacted, "key", "", "") + _, _, err = httpdtest.AddUser(u, http.StatusBadRequest) + assert.NoError(t, err) + u.FsConfig.AzBlobConfig.SASURL = kms.NewEmptySecret() u.FsConfig.AzBlobConfig.AccountName = "name" _, _, err = httpdtest.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) @@ -1802,9 +1805,9 @@ func TestUserAzureBlobConfig(t *testing.T) { assert.Equal(t, initialPayload, user.FsConfig.AzBlobConfig.AccountKey.GetPayload()) assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.GetAdditionalData()) assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.GetKey()) - // test user without access key and access secret (sas) + // test user without access key and access secret (SAS) user.FsConfig.Provider = vfs.AzureBlobFilesystemProvider - user.FsConfig.AzBlobConfig.SASURL = "https://myaccount.blob.core.windows.net/pictures/profile.jpg?sv=2012-02-12&st=2009-02-09&se=2009-02-10&sr=c&sp=r&si=YWJjZGVmZw%3d%3d&sig=dD80ihBh5jfNpymO5Hg1IdiJIEvHcJpCMiCMnN%2fRnbI%3d" + user.FsConfig.AzBlobConfig.SASURL = kms.NewPlainSecret("https://myaccount.blob.core.windows.net/pictures/profile.jpg?sv=2012-02-12&st=2009-02-09&se=2009-02-10&sr=c&sp=r&si=YWJjZGVmZw%3d%3d&sig=dD80ihBh5jfNpymO5Hg1IdiJIEvHcJpCMiCMnN%2fRnbI%3d") user.FsConfig.AzBlobConfig.KeyPrefix = "somedir/subdir" user.FsConfig.AzBlobConfig.AccountName = "" user.FsConfig.AzBlobConfig.AccountKey = kms.NewEmptySecret() @@ -1813,14 +1816,34 @@ func TestUserAzureBlobConfig(t *testing.T) { user, _, err = httpdtest.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) assert.Nil(t, user.FsConfig.AzBlobConfig.AccountKey) + assert.NotNil(t, user.FsConfig.AzBlobConfig.SASURL) _, err = httpdtest.RemoveUser(user, http.StatusOK) assert.NoError(t, err) user.Password = defaultPassword user.ID = 0 // sas test for add instead of update + user.FsConfig.AzBlobConfig = vfs.AzBlobFsConfig{ + Container: user.FsConfig.AzBlobConfig.Container, + SASURL: kms.NewPlainSecret("http://127.0.0.1/fake/sass/url"), + } user, _, err = httpdtest.AddUser(user, http.StatusCreated) assert.NoError(t, err) assert.Nil(t, user.FsConfig.AzBlobConfig.AccountKey) + initialPayload = user.FsConfig.AzBlobConfig.SASURL.GetPayload() + assert.Equal(t, kms.SecretStatusSecretBox, user.FsConfig.AzBlobConfig.SASURL.GetStatus()) + assert.NotEmpty(t, initialPayload) + assert.Empty(t, user.FsConfig.AzBlobConfig.SASURL.GetAdditionalData()) + assert.Empty(t, user.FsConfig.AzBlobConfig.SASURL.GetKey()) + user.FsConfig.AzBlobConfig.SASURL.SetStatus(kms.SecretStatusSecretBox) + user.FsConfig.AzBlobConfig.SASURL.SetAdditionalData("data") + user.FsConfig.AzBlobConfig.SASURL.SetKey("fake key") + user, _, err = httpdtest.UpdateUser(user, http.StatusOK, "") + assert.NoError(t, err) + assert.Equal(t, kms.SecretStatusSecretBox, user.FsConfig.AzBlobConfig.SASURL.GetStatus()) + assert.Equal(t, initialPayload, user.FsConfig.AzBlobConfig.SASURL.GetPayload()) + assert.Empty(t, user.FsConfig.AzBlobConfig.SASURL.GetAdditionalData()) + assert.Empty(t, user.FsConfig.AzBlobConfig.SASURL.GetKey()) + _, err = httpdtest.RemoveUser(user, http.StatusOK) assert.NoError(t, err) } @@ -7718,6 +7741,7 @@ func TestWebUserGCSMock(t *testing.T) { err = os.Remove(credentialsFilePath) assert.NoError(t, err) } + func TestWebUserAzureBlobMock(t *testing.T) { webToken, err := getJWTWebTokenFromTestServer(defaultTokenAuthUser, defaultTokenAuthPass) assert.NoError(t, err) @@ -7763,7 +7787,6 @@ func TestWebUserAzureBlobMock(t *testing.T) { form.Set("az_container", user.FsConfig.AzBlobConfig.Container) form.Set("az_account_name", user.FsConfig.AzBlobConfig.AccountName) form.Set("az_account_key", user.FsConfig.AzBlobConfig.AccountKey.GetPayload()) - form.Set("az_sas_url", user.FsConfig.AzBlobConfig.SASURL) form.Set("az_endpoint", user.FsConfig.AzBlobConfig.Endpoint) form.Set("az_key_prefix", user.FsConfig.AzBlobConfig.KeyPrefix) form.Set("az_use_emulator", "checked") @@ -7810,7 +7833,6 @@ func TestWebUserAzureBlobMock(t *testing.T) { assert.Equal(t, updateUser.FsConfig.AzBlobConfig.Container, user.FsConfig.AzBlobConfig.Container) assert.Equal(t, updateUser.FsConfig.AzBlobConfig.AccountName, user.FsConfig.AzBlobConfig.AccountName) assert.Equal(t, updateUser.FsConfig.AzBlobConfig.Endpoint, user.FsConfig.AzBlobConfig.Endpoint) - assert.Equal(t, updateUser.FsConfig.AzBlobConfig.SASURL, user.FsConfig.AzBlobConfig.SASURL) assert.Equal(t, updateUser.FsConfig.AzBlobConfig.KeyPrefix, user.FsConfig.AzBlobConfig.KeyPrefix) assert.Equal(t, updateUser.FsConfig.AzBlobConfig.UploadPartSize, user.FsConfig.AzBlobConfig.UploadPartSize) assert.Equal(t, updateUser.FsConfig.AzBlobConfig.UploadConcurrency, user.FsConfig.AzBlobConfig.UploadConcurrency) @@ -7838,6 +7860,49 @@ func TestWebUserAzureBlobMock(t *testing.T) { assert.Equal(t, updateUser.FsConfig.AzBlobConfig.AccountKey.GetPayload(), lastUpdatedUser.FsConfig.AzBlobConfig.AccountKey.GetPayload()) assert.Empty(t, lastUpdatedUser.FsConfig.AzBlobConfig.AccountKey.GetKey()) assert.Empty(t, lastUpdatedUser.FsConfig.AzBlobConfig.AccountKey.GetAdditionalData()) + // test SAS url + user.FsConfig.AzBlobConfig.SASURL = kms.NewPlainSecret("sasurl") + form.Set("az_account_name", "") + form.Set("az_account_key", "") + form.Set("az_container", "") + form.Set("az_sas_url", user.FsConfig.AzBlobConfig.SASURL.GetPayload()) + b, contentType, _ = getMultipartFormData(form, "", "") + req, _ = http.NewRequest(http.MethodPost, path.Join(webUserPath, user.Username), &b) + setJWTCookieForReq(req, webToken) + req.Header.Set("Content-Type", contentType) + rr = executeRequest(req) + checkResponseCode(t, http.StatusSeeOther, rr) + req, _ = http.NewRequest(http.MethodGet, path.Join(userPath, user.Username), nil) + setBearerForReq(req, apiToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + updateUser = dataprovider.User{} + err = render.DecodeJSON(rr.Body, &updateUser) + assert.NoError(t, err) + assert.Equal(t, kms.SecretStatusSecretBox, updateUser.FsConfig.AzBlobConfig.SASURL.GetStatus()) + assert.NotEmpty(t, updateUser.FsConfig.AzBlobConfig.SASURL.GetPayload()) + assert.Empty(t, updateUser.FsConfig.AzBlobConfig.SASURL.GetKey()) + assert.Empty(t, updateUser.FsConfig.AzBlobConfig.SASURL.GetAdditionalData()) + // now check that a redacted sas url is not saved + form.Set("az_sas_url", redactedSecret) + b, contentType, _ = getMultipartFormData(form, "", "") + req, _ = http.NewRequest(http.MethodPost, path.Join(webUserPath, user.Username), &b) + setJWTCookieForReq(req, webToken) + req.Header.Set("Content-Type", contentType) + rr = executeRequest(req) + checkResponseCode(t, http.StatusSeeOther, rr) + req, _ = http.NewRequest(http.MethodGet, path.Join(userPath, user.Username), nil) + setBearerForReq(req, apiToken) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + lastUpdatedUser = dataprovider.User{} + err = render.DecodeJSON(rr.Body, &lastUpdatedUser) + assert.NoError(t, err) + assert.Equal(t, kms.SecretStatusSecretBox, lastUpdatedUser.FsConfig.AzBlobConfig.SASURL.GetStatus()) + assert.Equal(t, updateUser.FsConfig.AzBlobConfig.SASURL.GetPayload(), lastUpdatedUser.FsConfig.AzBlobConfig.SASURL.GetPayload()) + assert.Empty(t, lastUpdatedUser.FsConfig.AzBlobConfig.SASURL.GetKey()) + assert.Empty(t, lastUpdatedUser.FsConfig.AzBlobConfig.SASURL.GetAdditionalData()) + req, _ = http.NewRequest(http.MethodDelete, path.Join(userPath, user.Username), nil) setBearerForReq(req, apiToken) rr = executeRequest(req) diff --git a/httpd/schema/openapi.yaml b/httpd/schema/openapi.yaml index e2f0b3ef..cca7cac9 100644 --- a/httpd/schema/openapi.yaml +++ b/httpd/schema/openapi.yaml @@ -2194,8 +2194,7 @@ components: account_key: $ref: '#/components/schemas/Secret' sas_url: - type: string - description: 'Shared access signature URL, leave blank if using account/key' + $ref: '#/components/schemas/Secret' endpoint: type: string description: 'optional endpoint. Default is "blob.core.windows.net". If you use the emulator the endpoint must include the protocol, for example "http://127.0.0.1:10000"' diff --git a/httpd/webadmin.go b/httpd/webadmin.go index 55924ca8..b8a6502c 100644 --- a/httpd/webadmin.go +++ b/httpd/webadmin.go @@ -739,7 +739,7 @@ func getAzureConfig(r *http.Request) (vfs.AzBlobFsConfig, error) { config.Container = r.Form.Get("az_container") config.AccountName = r.Form.Get("az_account_name") config.AccountKey = getSecretFromFormField(r, "az_account_key") - config.SASURL = r.Form.Get("az_sas_url") + config.SASURL = getSecretFromFormField(r, "az_sas_url") config.Endpoint = r.Form.Get("az_endpoint") config.KeyPrefix = r.Form.Get("az_key_prefix") config.AccessTier = r.Form.Get("az_access_tier") @@ -1457,8 +1457,8 @@ func handleWebUpdateUserPost(w http.ResponseWriter, r *http.Request) { updatedUser.Password = user.Password } updateEncryptedSecrets(&updatedUser.FsConfig, user.FsConfig.S3Config.AccessSecret, user.FsConfig.AzBlobConfig.AccountKey, - user.FsConfig.GCSConfig.Credentials, user.FsConfig.CryptConfig.Passphrase, user.FsConfig.SFTPConfig.Password, - user.FsConfig.SFTPConfig.PrivateKey) + user.FsConfig.AzBlobConfig.SASURL, user.FsConfig.GCSConfig.Credentials, user.FsConfig.CryptConfig.Passphrase, + user.FsConfig.SFTPConfig.Password, user.FsConfig.SFTPConfig.PrivateKey) err = dataprovider.UpdateUser(&updatedUser) if err == nil { @@ -1569,8 +1569,8 @@ func handleWebUpdateFolderPost(w http.ResponseWriter, r *http.Request) { updatedFolder.FsConfig = fsConfig updatedFolder.FsConfig.SetEmptySecretsIfNil() updateEncryptedSecrets(&updatedFolder.FsConfig, folder.FsConfig.S3Config.AccessSecret, folder.FsConfig.AzBlobConfig.AccountKey, - folder.FsConfig.GCSConfig.Credentials, folder.FsConfig.CryptConfig.Passphrase, folder.FsConfig.SFTPConfig.Password, - folder.FsConfig.SFTPConfig.PrivateKey) + folder.FsConfig.AzBlobConfig.SASURL, folder.FsConfig.GCSConfig.Credentials, folder.FsConfig.CryptConfig.Passphrase, + folder.FsConfig.SFTPConfig.Password, folder.FsConfig.SFTPConfig.PrivateKey) err = dataprovider.UpdateFolder(updatedFolder, folder.Users) if err != nil { diff --git a/httpdtest/httpdtest.go b/httpdtest/httpdtest.go index fdf2f26b..f56f31e3 100644 --- a/httpdtest/httpdtest.go +++ b/httpdtest/httpdtest.go @@ -1132,8 +1132,8 @@ func compareAzBlobConfig(expected *vfs.Filesystem, actual *vfs.Filesystem) error if expected.AzBlobConfig.Endpoint != actual.AzBlobConfig.Endpoint { return errors.New("azure Blob endpoint mismatch") } - if expected.AzBlobConfig.SASURL != actual.AzBlobConfig.SASURL { - return errors.New("azure Blob SASL URL mismatch") + if err := checkEncryptedSecret(expected.AzBlobConfig.SASURL, actual.AzBlobConfig.SASURL); err != nil { + return fmt.Errorf("azure Blob SAS URL mismatch: %v", err) } if expected.AzBlobConfig.UploadPartSize != actual.AzBlobConfig.UploadPartSize { return errors.New("azure Blob upload part size mismatch") diff --git a/service/service_portable.go b/service/service_portable.go index d79bcef9..9ad47d16 100644 --- a/service/service_portable.go +++ b/service/service_portable.go @@ -282,6 +282,11 @@ func (s *Service) configurePortableSecrets() { if payload != "" { s.PortableUser.FsConfig.AzBlobConfig.AccountKey = kms.NewPlainSecret(payload) } + payload = s.PortableUser.FsConfig.AzBlobConfig.SASURL.GetPayload() + s.PortableUser.FsConfig.AzBlobConfig.SASURL = kms.NewEmptySecret() + if payload != "" { + s.PortableUser.FsConfig.AzBlobConfig.SASURL = kms.NewPlainSecret(payload) + } case vfs.CryptedFilesystemProvider: payload := s.PortableUser.FsConfig.CryptConfig.Passphrase.GetPayload() s.PortableUser.FsConfig.CryptConfig.Passphrase = kms.NewEmptySecret() diff --git a/sftpd/sftpd_test.go b/sftpd/sftpd_test.go index b174a4b3..eb49f589 100644 --- a/sftpd/sftpd_test.go +++ b/sftpd/sftpd_test.go @@ -7243,7 +7243,7 @@ func TestStatVFSCloudBackend(t *testing.T) { usePubKey := true u := getTestUser(usePubKey) u.FsConfig.Provider = vfs.AzureBlobFilesystemProvider - u.FsConfig.AzBlobConfig.SASURL = "https://myaccount.blob.core.windows.net/sasurl" + u.FsConfig.AzBlobConfig.SASURL = kms.NewPlainSecret("https://myaccount.blob.core.windows.net/sasurl") user, _, err := httpdtest.AddUser(u, http.StatusCreated) assert.NoError(t, err) conn, client, err := getSftpClient(user, usePubKey) diff --git a/templates/webadmin/fsconfig.html b/templates/webadmin/fsconfig.html index 557427c6..27cff232 100644 --- a/templates/webadmin/fsconfig.html +++ b/templates/webadmin/fsconfig.html @@ -161,8 +161,8 @@
- +
diff --git a/vfs/azblobfs.go b/vfs/azblobfs.go index 08ca05fd..d52f2db7 100644 --- a/vfs/azblobfs.go +++ b/vfs/azblobfs.go @@ -75,13 +75,16 @@ func NewAzBlobFs(connectionID, localTempDir, mountPath string, config AzBlobFsCo if err := fs.config.AccountKey.TryDecrypt(); err != nil { return fs, err } + if err := fs.config.SASURL.TryDecrypt(); err != nil { + return fs, err + } fs.setConfigDefaults() version := version.Get() telemetryValue := fmt.Sprintf("SFTPGo-%v_%v", version.Version, version.CommitHash) - if fs.config.SASURL != "" { - u, err := url.Parse(fs.config.SASURL) + if fs.config.SASURL.GetPayload() != "" { + u, err := url.Parse(fs.config.SASURL.GetPayload()) if err != nil { return fs, fmt.Errorf("invalid credentials: %v", err) } @@ -144,8 +147,8 @@ func NewAzBlobFs(connectionID, localTempDir, mountPath string, config AzBlobFsCo // Name returns the name for the Fs implementation func (fs *AzureBlobFs) Name() string { - if fs.config.SASURL != "" { - return fmt.Sprintf("Azure Blob SAS URL %#v", fs.config.Container) + if !fs.config.SASURL.IsEmpty() { + return fmt.Sprintf("Azure Blob with SAS URL, container %#v", fs.config.Container) } return fmt.Sprintf("Azure Blob container %#v", fs.config.Container) } diff --git a/vfs/filesystem.go b/vfs/filesystem.go index 5388ee3a..0978cb5c 100644 --- a/vfs/filesystem.go +++ b/vfs/filesystem.go @@ -37,6 +37,9 @@ func (f *Filesystem) SetEmptySecretsIfNil() { if f.AzBlobConfig.AccountKey == nil { f.AzBlobConfig.AccountKey = kms.NewEmptySecret() } + if f.AzBlobConfig.SASURL == nil { + f.AzBlobConfig.SASURL = kms.NewEmptySecret() + } if f.CryptConfig.Passphrase == nil { f.CryptConfig.Passphrase = kms.NewEmptySecret() } @@ -61,6 +64,9 @@ func (f *Filesystem) SetNilSecretsIfEmpty() { if f.AzBlobConfig.AccountKey != nil && f.AzBlobConfig.AccountKey.IsEmpty() { f.AzBlobConfig.AccountKey = nil } + if f.AzBlobConfig.SASURL != nil && f.AzBlobConfig.SASURL.IsEmpty() { + f.AzBlobConfig.SASURL = nil + } if f.CryptConfig.Passphrase != nil && f.CryptConfig.Passphrase.IsEmpty() { f.CryptConfig.Passphrase = nil } @@ -122,7 +128,7 @@ func (f *Filesystem) GetACopy() Filesystem { AccountName: f.AzBlobConfig.AccountName, AccountKey: f.AzBlobConfig.AccountKey.Clone(), Endpoint: f.AzBlobConfig.Endpoint, - SASURL: f.AzBlobConfig.SASURL, + SASURL: f.AzBlobConfig.SASURL.Clone(), KeyPrefix: f.AzBlobConfig.KeyPrefix, UploadPartSize: f.AzBlobConfig.UploadPartSize, UploadConcurrency: f.AzBlobConfig.UploadConcurrency, diff --git a/vfs/folder.go b/vfs/folder.go index c443b35c..73090afb 100644 --- a/vfs/folder.go +++ b/vfs/folder.go @@ -108,6 +108,7 @@ func (v *BaseVirtualFolder) hideConfidentialData() { v.FsConfig.GCSConfig.Credentials.Hide() case AzureBlobFilesystemProvider: v.FsConfig.AzBlobConfig.AccountKey.Hide() + v.FsConfig.AzBlobConfig.SASURL.Hide() case CryptedFilesystemProvider: v.FsConfig.CryptConfig.Passphrase.Hide() case SFTPFilesystemProvider: @@ -139,6 +140,9 @@ func (v *BaseVirtualFolder) HasRedactedSecret() bool { if v.FsConfig.AzBlobConfig.AccountKey.IsRedacted() { return true } + if v.FsConfig.AzBlobConfig.SASURL.IsRedacted() { + return true + } case CryptedFilesystemProvider: if v.FsConfig.CryptConfig.Passphrase.IsRedacted() { return true diff --git a/vfs/vfs.go b/vfs/vfs.go index 13f2486a..47e022c2 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -342,9 +342,9 @@ type AzBlobFsConfig struct { // for example "http://127.0.0.1:10000" Endpoint string `json:"endpoint,omitempty"` // Shared access signature URL, leave blank if using account/key - SASURL string `json:"sas_url,omitempty"` + SASURL *kms.Secret `json:"sas_url,omitempty"` // KeyPrefix is similar to a chroot directory for local filesystem. - // If specified then the SFTPGo userd will only see objects that starts + // If specified then the SFTPGo user will only see objects that starts // with this prefix and so you can restrict access to a specific // folder. The prefix, if not empty, must not start with "/" and must // end with "/". @@ -376,7 +376,13 @@ func (c *AzBlobFsConfig) isEqual(other *AzBlobFsConfig) bool { if c.Endpoint != other.Endpoint { return false } - if c.SASURL != other.SASURL { + if c.SASURL.IsEmpty() { + c.SASURL = kms.NewEmptySecret() + } + if other.SASURL.IsEmpty() { + other.SASURL = kms.NewEmptySecret() + } + if !c.SASURL.IsEqual(other.SASURL) { return false } if c.KeyPrefix != other.KeyPrefix { @@ -411,10 +417,26 @@ func (c *AzBlobFsConfig) EncryptCredentials(additionalData string) error { return err } } + if c.SASURL.IsPlain() { + c.SASURL.SetAdditionalData(additionalData) + if err := c.SASURL.Encrypt(); err != nil { + return err + } + } return nil } func (c *AzBlobFsConfig) checkCredentials() error { + if c.SASURL.IsPlain() { + _, err := url.Parse(c.SASURL.GetPayload()) + return err + } + if c.SASURL.IsEncrypted() && !c.SASURL.IsValid() { + return errors.New("invalid encrypted sas_url") + } + if !c.SASURL.IsEmpty() { + return nil + } if c.AccountName == "" || !c.AccountKey.IsValidInput() { return errors.New("credentials cannot be empty or invalid") } @@ -429,11 +451,11 @@ func (c *AzBlobFsConfig) Validate() error { if c.AccountKey == nil { c.AccountKey = kms.NewEmptySecret() } - if c.SASURL != "" { - _, err := url.Parse(c.SASURL) - return err + if c.SASURL == nil { + c.SASURL = kms.NewEmptySecret() } - if c.Container == "" { + // container could be embedded within SAS URL we check this at runtime + if c.SASURL.IsEmpty() && c.Container == "" { return errors.New("container cannot be empty") } if err := c.checkCredentials(); err != nil {