diff --git a/cmd/portable.go b/cmd/portable.go index 2c585f8b..27b5559c 100644 --- a/cmd/portable.go +++ b/cmd/portable.go @@ -143,10 +143,13 @@ Please take a look at the usage below to customize the serving parameters`, FsConfig: dataprovider.Filesystem{ Provider: dataprovider.FilesystemProvider(portableFsProvider), S3Config: vfs.S3FsConfig{ - Bucket: portableS3Bucket, - Region: portableS3Region, - AccessKey: portableS3AccessKey, - AccessSecret: portableS3AccessSecret, + Bucket: portableS3Bucket, + Region: portableS3Region, + AccessKey: portableS3AccessKey, + AccessSecret: vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: portableS3AccessSecret, + }, Endpoint: portableS3Endpoint, StorageClass: portableS3StorageClass, KeyPrefix: portableS3KeyPrefix, @@ -154,16 +157,22 @@ Please take a look at the usage below to customize the serving parameters`, UploadConcurrency: portableS3ULConcurrency, }, GCSConfig: vfs.GCSFsConfig{ - Bucket: portableGCSBucket, - Credentials: portableGCSCredentials, + Bucket: portableGCSBucket, + Credentials: vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: string(portableGCSCredentials), + }, AutomaticCredentials: portableGCSAutoCredentials, StorageClass: portableGCSStorageClass, KeyPrefix: portableGCSKeyPrefix, }, AzBlobConfig: vfs.AzBlobFsConfig{ - Container: portableAzContainer, - AccountName: portableAzAccountName, - AccountKey: portableAzAccountKey, + Container: portableAzContainer, + AccountName: portableAzAccountName, + AccountKey: vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: portableAzAccountKey, + }, Endpoint: portableAzEndpoint, AccessTier: portableAzAccessTier, SASURL: portableAzSASURL, diff --git a/dataprovider/bolt.go b/dataprovider/bolt.go index 33b0e2dc..debda928 100644 --- a/dataprovider/bolt.go +++ b/dataprovider/bolt.go @@ -19,7 +19,7 @@ import ( ) const ( - boltDatabaseVersion = 4 + boltDatabaseVersion = 5 ) var ( @@ -35,28 +35,6 @@ type BoltProvider struct { dbHandle *bolt.DB } -type compatUserV2 struct { - ID int64 `json:"id"` - Username string `json:"username"` - Password string `json:"password,omitempty"` - PublicKeys []string `json:"public_keys,omitempty"` - HomeDir string `json:"home_dir"` - UID int `json:"uid"` - GID int `json:"gid"` - MaxSessions int `json:"max_sessions"` - QuotaSize int64 `json:"quota_size"` - QuotaFiles int `json:"quota_files"` - Permissions []string `json:"permissions"` - UsedQuotaSize int64 `json:"used_quota_size"` - UsedQuotaFiles int `json:"used_quota_files"` - LastQuotaUpdate int64 `json:"last_quota_update"` - UploadBandwidth int64 `json:"upload_bandwidth"` - DownloadBandwidth int64 `json:"download_bandwidth"` - ExpirationDate int64 `json:"expiration_date"` - LastLogin int64 `json:"last_login"` - Status int `json:"status"` -} - func init() { version.AddFeature("+bolt") } @@ -425,7 +403,8 @@ func (p BoltProvider) getUserWithUsername(username string) ([]User, error) { var user User user, err := p.userExists(username) if err == nil { - users = append(users, HideUserSensitiveData(&user)) + user.HideConfidentialData() + users = append(users, user) return users, nil } if _, ok := err.(*RecordNotFoundError); ok { @@ -465,7 +444,8 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str } user, err := joinUserAndFolders(v, folderBucket) if err == nil { - users = append(users, HideUserSensitiveData(&user)) + user.HideConfidentialData() + users = append(users, user) } if len(users) >= limit { break @@ -479,7 +459,8 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str } user, err := joinUserAndFolders(v, folderBucket) if err == nil { - users = append(users, HideUserSensitiveData(&user)) + user.HideConfidentialData() + users = append(users, user) } if len(users) >= limit { break @@ -718,28 +699,46 @@ func (p BoltProvider) migrateDatabase() error { } switch dbVersion.Version { case 1: - err = updateDatabaseFrom1To2(p.dbHandle) - if err != nil { - return err - } - err = updateDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updateDatabaseFrom3To4(p.dbHandle) + return updateBoltDatabaseFromV1(p.dbHandle) case 2: - err = updateDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updateDatabaseFrom3To4(p.dbHandle) + return updateBoltDatabaseFromV2(p.dbHandle) case 3: - return updateDatabaseFrom3To4(p.dbHandle) + return updateBoltDatabaseFromV3(p.dbHandle) + case 4: + return updateBoltDatabaseFromV4(p.dbHandle) default: return fmt.Errorf("Database version not handled: %v", dbVersion.Version) } } +func updateBoltDatabaseFromV1(dbHandle *bolt.DB) error { + err := updateDatabaseFrom1To2(dbHandle) + if err != nil { + return err + } + return updateBoltDatabaseFromV2(dbHandle) +} + +func updateBoltDatabaseFromV2(dbHandle *bolt.DB) error { + err := updateDatabaseFrom2To3(dbHandle) + if err != nil { + return err + } + return updateBoltDatabaseFromV3(dbHandle) +} + +func updateBoltDatabaseFromV3(dbHandle *bolt.DB) error { + err := updateDatabaseFrom3To4(dbHandle) + if err != nil { + return err + } + return updateBoltDatabaseFromV4(dbHandle) +} + +func updateBoltDatabaseFromV4(dbHandle *bolt.DB) error { + return updateDatabaseFrom4To5(dbHandle) +} + // itob returns an 8-byte big endian representation of v. func itob(v int64) []byte { b := make([]byte, 8) @@ -847,6 +846,27 @@ func removeUserFromFolderMapping(folder vfs.VirtualFolder, user User, bucket *bo return err } +func updateV4BoltUser(dbHandle *bolt.DB, user User) error { + err := validateUser(&user) + if err != nil { + return err + } + return dbHandle.Update(func(tx *bolt.Tx) error { + bucket, _, err := getBuckets(tx) + if err != nil { + return err + } + if u := bucket.Get([]byte(user.Username)); u == nil { + return &RecordNotFoundError{err: fmt.Sprintf("username %v does not exist", user.Username)} + } + buf, err := json.Marshal(user) + if err != nil { + return err + } + return bucket.Put([]byte(user.Username), buf) + }) +} + func getBuckets(tx *bolt.Tx) (*bolt.Bucket, *bolt.Bucket, error) { var err error bucket := tx.Bucket(usersBucket) @@ -1007,6 +1027,46 @@ func updateDatabaseFrom3To4(dbHandle *bolt.DB) error { return err } +func updateDatabaseFrom4To5(dbHandle *bolt.DB) error { + logger.InfoToConsole("updating bolt database version: 4 -> 5") + providerLog(logger.LevelInfo, "updating bolt database version: 4 -> 5") + users := []User{} + err := dbHandle.View(func(tx *bolt.Tx) error { + bucket, _, err := getBuckets(tx) + if err != nil { + return err + } + cursor := bucket.Cursor() + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { + var compatUser compatUserV4 + err = json.Unmarshal(v, &compatUser) + if err != nil { + logger.WarnToConsole("failed to unmarshal v4 user %#v, is it already migrated?", string(k)) + continue + } + fsConfig, err := convertFsConfigFromV4(compatUser.FsConfig, compatUser.Username) + if err != nil { + return err + } + users = append(users, createUserFromV4(compatUser, fsConfig)) + } + return nil + }) + if err != nil { + return err + } + + for _, user := range users { + err = updateV4BoltUser(dbHandle, user) + if err != nil { + return err + } + providerLog(logger.LevelInfo, "filesystem config updated for user %#v", user.Username) + } + + return updateBoltDatabaseVersion(dbHandle, 5) +} + func getBoltAvailableUsernames(dbHandle *bolt.DB) ([]string, error) { usernames := []string{} err := dbHandle.View(func(tx *bolt.Tx) error { diff --git a/dataprovider/compat.go b/dataprovider/compat.go new file mode 100644 index 00000000..1a31af50 --- /dev/null +++ b/dataprovider/compat.go @@ -0,0 +1,222 @@ +package dataprovider + +import ( + "fmt" + "io/ioutil" + "path/filepath" + + "github.com/drakkan/sftpgo/logger" + "github.com/drakkan/sftpgo/vfs" +) + +type compatUserV2 struct { + ID int64 `json:"id"` + Username string `json:"username"` + Password string `json:"password,omitempty"` + PublicKeys []string `json:"public_keys,omitempty"` + HomeDir string `json:"home_dir"` + UID int `json:"uid"` + GID int `json:"gid"` + MaxSessions int `json:"max_sessions"` + QuotaSize int64 `json:"quota_size"` + QuotaFiles int `json:"quota_files"` + Permissions []string `json:"permissions"` + UsedQuotaSize int64 `json:"used_quota_size"` + UsedQuotaFiles int `json:"used_quota_files"` + LastQuotaUpdate int64 `json:"last_quota_update"` + UploadBandwidth int64 `json:"upload_bandwidth"` + DownloadBandwidth int64 `json:"download_bandwidth"` + ExpirationDate int64 `json:"expiration_date"` + LastLogin int64 `json:"last_login"` + Status int `json:"status"` +} + +type compatS3FsConfigV4 struct { + Bucket string `json:"bucket,omitempty"` + KeyPrefix string `json:"key_prefix,omitempty"` + Region string `json:"region,omitempty"` + AccessKey string `json:"access_key,omitempty"` + AccessSecret string `json:"access_secret,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + StorageClass string `json:"storage_class,omitempty"` + UploadPartSize int64 `json:"upload_part_size,omitempty"` + UploadConcurrency int `json:"upload_concurrency,omitempty"` +} + +type compatGCSFsConfigV4 struct { + Bucket string `json:"bucket,omitempty"` + KeyPrefix string `json:"key_prefix,omitempty"` + CredentialFile string `json:"-"` + Credentials []byte `json:"credentials,omitempty"` + AutomaticCredentials int `json:"automatic_credentials,omitempty"` + StorageClass string `json:"storage_class,omitempty"` +} + +type compatAzBlobFsConfigV4 struct { + Container string `json:"container,omitempty"` + AccountName string `json:"account_name,omitempty"` + AccountKey string `json:"account_key,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + SASURL string `json:"sas_url,omitempty"` + KeyPrefix string `json:"key_prefix,omitempty"` + UploadPartSize int64 `json:"upload_part_size,omitempty"` + UploadConcurrency int `json:"upload_concurrency,omitempty"` + UseEmulator bool `json:"use_emulator,omitempty"` + AccessTier string `json:"access_tier,omitempty"` +} + +type compatFilesystemV4 struct { + Provider FilesystemProvider `json:"provider"` + S3Config compatS3FsConfigV4 `json:"s3config,omitempty"` + GCSConfig compatGCSFsConfigV4 `json:"gcsconfig,omitempty"` + AzBlobConfig compatAzBlobFsConfigV4 `json:"azblobconfig,omitempty"` +} + +type compatUserV4 struct { + ID int64 `json:"id"` + Status int `json:"status"` + Username string `json:"username"` + ExpirationDate int64 `json:"expiration_date"` + Password string `json:"password,omitempty"` + PublicKeys []string `json:"public_keys,omitempty"` + HomeDir string `json:"home_dir"` + VirtualFolders []vfs.VirtualFolder `json:"virtual_folders,omitempty"` + UID int `json:"uid"` + GID int `json:"gid"` + MaxSessions int `json:"max_sessions"` + QuotaSize int64 `json:"quota_size"` + QuotaFiles int `json:"quota_files"` + Permissions map[string][]string `json:"permissions"` + UsedQuotaSize int64 `json:"used_quota_size"` + UsedQuotaFiles int `json:"used_quota_files"` + LastQuotaUpdate int64 `json:"last_quota_update"` + UploadBandwidth int64 `json:"upload_bandwidth"` + DownloadBandwidth int64 `json:"download_bandwidth"` + LastLogin int64 `json:"last_login"` + Filters UserFilters `json:"filters"` + FsConfig compatFilesystemV4 `json:"filesystem"` +} + +type backupDataV4Compat struct { + Users []compatUserV4 `json:"users"` + Folders []vfs.BaseVirtualFolder `json:"folders"` +} + +func createUserFromV4(u compatUserV4, fsConfig Filesystem) User { + user := User{ + ID: u.ID, + Status: u.Status, + Username: u.Username, + ExpirationDate: u.ExpirationDate, + Password: u.Password, + PublicKeys: u.PublicKeys, + HomeDir: u.HomeDir, + VirtualFolders: u.VirtualFolders, + UID: u.UID, + GID: u.GID, + MaxSessions: u.MaxSessions, + QuotaSize: u.QuotaSize, + QuotaFiles: u.QuotaFiles, + Permissions: u.Permissions, + UsedQuotaSize: u.UsedQuotaSize, + UsedQuotaFiles: u.UsedQuotaFiles, + LastQuotaUpdate: u.LastQuotaUpdate, + UploadBandwidth: u.UploadBandwidth, + DownloadBandwidth: u.DownloadBandwidth, + LastLogin: u.LastLogin, + Filters: u.Filters, + } + user.FsConfig = fsConfig + return user +} + +func getCGSCredentialsFromV4(config compatGCSFsConfigV4) (vfs.Secret, error) { + var secret vfs.Secret + var err error + if len(config.Credentials) > 0 { + secret.Status = vfs.SecretStatusPlain + secret.Payload = string(config.Credentials) + return secret, nil + } + if config.CredentialFile != "" { + creds, err := ioutil.ReadFile(config.CredentialFile) + if err != nil { + return secret, err + } + secret.Status = vfs.SecretStatusPlain + secret.Payload = string(creds) + return secret, nil + } + return secret, err +} + +func convertFsConfigFromV4(compatFs compatFilesystemV4, username string) (Filesystem, error) { + fsConfig := Filesystem{ + Provider: compatFs.Provider, + S3Config: vfs.S3FsConfig{}, + AzBlobConfig: vfs.AzBlobFsConfig{}, + GCSConfig: vfs.GCSFsConfig{}, + } + switch compatFs.Provider { + case S3FilesystemProvider: + fsConfig.S3Config = vfs.S3FsConfig{ + Bucket: compatFs.S3Config.Bucket, + KeyPrefix: compatFs.S3Config.KeyPrefix, + Region: compatFs.S3Config.Region, + AccessKey: compatFs.S3Config.AccessKey, + AccessSecret: vfs.Secret{}, + Endpoint: compatFs.S3Config.Endpoint, + StorageClass: compatFs.S3Config.StorageClass, + UploadPartSize: compatFs.S3Config.UploadPartSize, + UploadConcurrency: compatFs.S3Config.UploadConcurrency, + } + if compatFs.S3Config.AccessSecret != "" { + secret, err := vfs.GetSecretFromCompatString(compatFs.S3Config.AccessSecret) + if err != nil { + providerLog(logger.LevelError, "unable to convert v4 filesystem for user %#v: %v", username, err) + return fsConfig, err + } + fsConfig.S3Config.AccessSecret = secret + } + case AzureBlobFilesystemProvider: + fsConfig.AzBlobConfig = vfs.AzBlobFsConfig{ + Container: compatFs.AzBlobConfig.Container, + AccountName: compatFs.AzBlobConfig.AccountName, + AccountKey: vfs.Secret{}, + Endpoint: compatFs.AzBlobConfig.Endpoint, + SASURL: compatFs.AzBlobConfig.SASURL, + KeyPrefix: compatFs.AzBlobConfig.KeyPrefix, + UploadPartSize: compatFs.AzBlobConfig.UploadPartSize, + UploadConcurrency: compatFs.AzBlobConfig.UploadConcurrency, + UseEmulator: compatFs.AzBlobConfig.UseEmulator, + AccessTier: compatFs.AzBlobConfig.AccessTier, + } + if compatFs.AzBlobConfig.AccountKey != "" { + secret, err := vfs.GetSecretFromCompatString(compatFs.AzBlobConfig.AccountKey) + if err != nil { + providerLog(logger.LevelError, "unable to convert v4 filesystem for user %#v: %v", username, err) + return fsConfig, err + } + fsConfig.AzBlobConfig.AccountKey = secret + } + case GCSFilesystemProvider: + fsConfig.GCSConfig = vfs.GCSFsConfig{ + Bucket: compatFs.GCSConfig.Bucket, + KeyPrefix: compatFs.GCSConfig.KeyPrefix, + CredentialFile: compatFs.GCSConfig.CredentialFile, + AutomaticCredentials: compatFs.GCSConfig.AutomaticCredentials, + StorageClass: compatFs.GCSConfig.StorageClass, + } + if compatFs.GCSConfig.AutomaticCredentials == 0 { + compatFs.GCSConfig.CredentialFile = filepath.Join(credentialsDirPath, fmt.Sprintf("%v_gcs_credentials.json", + username)) + } + secret, err := getCGSCredentialsFromV4(compatFs.GCSConfig) + if err != nil { + providerLog(logger.LevelError, "unable to convert v4 filesystem for user %#v: %v", username, err) + return fsConfig, err + } + fsConfig.GCSConfig.Credentials = secret + } + return fsConfig, nil +} diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go index 3eda62b3..9dfd0e22 100644 --- a/dataprovider/dataprovider.go +++ b/dataprovider/dataprovider.go @@ -59,6 +59,9 @@ const ( BoltDataProviderName = "bolt" // MemoryDataProviderName name for memory provider MemoryDataProviderName = "memory" + // DumpVersion defines the version for the dump. + // For restore/load we support the current version and the previous one + DumpVersion = 5 argonPwdPrefix = "$argon2id$" bcryptPwdPrefix = "$2a$" @@ -265,6 +268,7 @@ type Config struct { type BackupData struct { Users []User `json:"users"` Folders []vfs.BaseVirtualFolder `json:"folders"` + Version int `json:"version"` } type keyboardAuthHookRequest struct { @@ -384,10 +388,8 @@ func Initialize(cnf Config, basePath string) error { if err = validateHooks(); err != nil { return err } - if !cnf.PreferDatabaseCredentials { - if err = validateCredentialsDir(basePath); err != nil { - return err - } + if err = validateCredentialsDir(basePath, cnf.PreferDatabaseCredentials); err != nil { + return err } err = createProvider(basePath) if err != nil { @@ -689,6 +691,7 @@ func GetFolders(limit, offset int, order, folderPath string) ([]vfs.BaseVirtualF // DumpData returns all users and folders func DumpData() (BackupData, error) { var data BackupData + data.Version = DumpVersion users, err := provider.dumpUsers() if err != nil { return data, err @@ -702,6 +705,33 @@ func DumpData() (BackupData, error) { return data, err } +// ParseDumpData tries to parse data as BackupData +func ParseDumpData(data []byte) (BackupData, error) { + var dump BackupData + err := json.Unmarshal(data, &dump) + if err == nil { + return dump, err + } + dump = BackupData{} + // try to parse as version 4 + var dumpCompat backupDataV4Compat + err = json.Unmarshal(data, &dumpCompat) + if err != nil { + return dump, err + } + logger.WarnToConsole("You are loading data from an old format, please update to the latest supported one. We only support the current and the previous format.") + providerLog(logger.LevelWarn, "You are loading data from an old format, please update to the latest supported one. We only support the current and the previous format.") + dump.Folders = dumpCompat.Folders + for _, compatUser := range dumpCompat.Users { + fsConfig, err := convertFsConfigFromV4(compatUser.FsConfig, compatUser.Username) + if err != nil { + return dump, err + } + dump.Users = append(dump.Users, createUserFromV4(compatUser, fsConfig)) + } + return dump, err +} + // GetProviderStatus returns an error if the provider is not available func GetProviderStatus() error { return provider.checkAvailability() @@ -1038,17 +1068,35 @@ func saveGCSCredentials(user *User) error { if user.FsConfig.Provider != GCSFilesystemProvider { return nil } - if len(user.FsConfig.GCSConfig.Credentials) == 0 { + if user.FsConfig.GCSConfig.Credentials.Payload == "" { return nil } if config.PreferDatabaseCredentials { + if user.FsConfig.GCSConfig.Credentials.IsPlain() { + user.FsConfig.GCSConfig.Credentials.AdditionalData = user.Username + err := user.FsConfig.GCSConfig.Credentials.Encrypt() + if err != nil { + return err + } + } return nil } - err := ioutil.WriteFile(user.getGCSCredentialsFilePath(), user.FsConfig.GCSConfig.Credentials, 0600) + if user.FsConfig.GCSConfig.Credentials.IsPlain() { + user.FsConfig.GCSConfig.Credentials.AdditionalData = user.Username + err := user.FsConfig.GCSConfig.Credentials.Encrypt() + if err != nil { + return &ValidationError{err: fmt.Sprintf("could not encrypt GCS credentials: %v", err)} + } + } + creds, err := json.Marshal(user.FsConfig.GCSConfig.Credentials) + if err != nil { + return &ValidationError{err: fmt.Sprintf("could not marshal GCS credentials: %v", err)} + } + err = ioutil.WriteFile(user.getGCSCredentialsFilePath(), creds, 0600) if err != nil { return &ValidationError{err: fmt.Sprintf("could not save GCS credentials: %v", err)} } - user.FsConfig.GCSConfig.Credentials = nil + user.FsConfig.GCSConfig.Credentials = vfs.Secret{} return nil } @@ -1058,38 +1106,38 @@ func validateFilesystemConfig(user *User) error { if err != nil { return &ValidationError{err: fmt.Sprintf("could not validate s3config: %v", err)} } - if user.FsConfig.S3Config.AccessSecret != "" { - vals := strings.Split(user.FsConfig.S3Config.AccessSecret, "$") - if !strings.HasPrefix(user.FsConfig.S3Config.AccessSecret, "$aes$") || len(vals) != 4 { - accessSecret, err := utils.EncryptData(user.FsConfig.S3Config.AccessSecret) - if err != nil { - return &ValidationError{err: fmt.Sprintf("could not encrypt s3 access secret: %v", err)} - } - user.FsConfig.S3Config.AccessSecret = accessSecret + if user.FsConfig.S3Config.AccessSecret.IsPlain() { + user.FsConfig.S3Config.AccessSecret.AdditionalData = user.Username + err = user.FsConfig.S3Config.AccessSecret.Encrypt() + if err != nil { + return &ValidationError{err: fmt.Sprintf("could not encrypt s3 access secret: %v", err)} } } + user.FsConfig.GCSConfig = vfs.GCSFsConfig{} + user.FsConfig.AzBlobConfig = vfs.AzBlobFsConfig{} return nil } else if user.FsConfig.Provider == GCSFilesystemProvider { err := vfs.ValidateGCSFsConfig(&user.FsConfig.GCSConfig, user.getGCSCredentialsFilePath()) if err != nil { return &ValidationError{err: fmt.Sprintf("could not validate GCS config: %v", err)} } + user.FsConfig.S3Config = vfs.S3FsConfig{} + user.FsConfig.AzBlobConfig = vfs.AzBlobFsConfig{} return nil } else if user.FsConfig.Provider == AzureBlobFilesystemProvider { err := vfs.ValidateAzBlobFsConfig(&user.FsConfig.AzBlobConfig) if err != nil { return &ValidationError{err: fmt.Sprintf("could not validate Azure Blob config: %v", err)} } - if user.FsConfig.AzBlobConfig.AccountKey != "" { - vals := strings.Split(user.FsConfig.AzBlobConfig.AccountKey, "$") - if !strings.HasPrefix(user.FsConfig.AzBlobConfig.AccountKey, "$aes$") || len(vals) != 4 { - accountKey, err := utils.EncryptData(user.FsConfig.AzBlobConfig.AccountKey) - if err != nil { - return &ValidationError{err: fmt.Sprintf("could not encrypt Azure blob account key: %v", err)} - } - user.FsConfig.AzBlobConfig.AccountKey = accountKey + if user.FsConfig.AzBlobConfig.AccountKey.IsPlain() { + user.FsConfig.AzBlobConfig.AccountKey.AdditionalData = user.Username + err = user.FsConfig.AzBlobConfig.AccountKey.Encrypt() + if err != nil { + return &ValidationError{err: fmt.Sprintf("could not encrypt Azure blob account key: %v", err)} } } + user.FsConfig.S3Config = vfs.S3FsConfig{} + user.FsConfig.GCSConfig = vfs.GCSFsConfig{} return nil } user.FsConfig.Provider = LocalFilesystemProvider @@ -1321,19 +1369,6 @@ func comparePbkdf2PasswordAndHash(password, hashedPassword string) (bool, error) return subtle.ConstantTimeCompare(df, expected) == 1, nil } -// HideUserSensitiveData hides user sensitive data -func HideUserSensitiveData(user *User) User { - user.Password = "" - if user.FsConfig.Provider == S3FilesystemProvider { - user.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(user.FsConfig.S3Config.AccessSecret) - } else if user.FsConfig.Provider == GCSFilesystemProvider { - user.FsConfig.GCSConfig.Credentials = nil - } else if user.FsConfig.Provider == AzureBlobFilesystemProvider { - user.FsConfig.AzBlobConfig.AccountKey = utils.RemoveDecryptionKey(user.FsConfig.AzBlobConfig.AccountKey) - } - return *user -} - func addCredentialsToUser(user *User) error { if user.FsConfig.Provider != GCSFilesystemProvider { return nil @@ -1343,7 +1378,7 @@ func addCredentialsToUser(user *User) error { } // Don't read from file if credentials have already been set - if len(user.FsConfig.GCSConfig.Credentials) > 0 { + if user.FsConfig.GCSConfig.Credentials.IsValid() { return nil } @@ -1351,8 +1386,7 @@ func addCredentialsToUser(user *User) error { if err != nil { return err } - user.FsConfig.GCSConfig.Credentials = cred - return nil + return json.Unmarshal(cred, &user.FsConfig.GCSConfig.Credentials) } func getSSLMode() string { @@ -1396,12 +1430,18 @@ func startAvailabilityTimer() { }() } -func validateCredentialsDir(basePath string) error { +func validateCredentialsDir(basePath string, preferDbCredentials bool) error { if filepath.IsAbs(config.CredentialsPath) { credentialsDirPath = config.CredentialsPath } else { credentialsDirPath = filepath.Join(basePath, config.CredentialsPath) } + // if we want to store credentials inside the database just stop here + // we just populate credentialsDirPath to be able to use existing users + // with credential files + if preferDbCredentials { + return nil + } fi, err := os.Stat(credentialsDirPath) if err == nil { if !fi.IsDir() { @@ -2013,7 +2053,7 @@ func executeAction(operation string, user User) { q := url.Query() q.Add("action", operation) url.RawQuery = q.Encode() - HideUserSensitiveData(&user) + user.HideConfidentialData() userAsJSON, err := json.Marshal(user) if err != nil { return diff --git a/dataprovider/memory.go b/dataprovider/memory.go index 1ad36a91..923f36bb 100644 --- a/dataprovider/memory.go +++ b/dataprovider/memory.go @@ -1,7 +1,6 @@ package dataprovider import ( - "encoding/json" "errors" "fmt" "io/ioutil" @@ -300,7 +299,8 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s if offset == 0 { user, err := p.userExistsInternal(username) if err == nil { - users = append(users, HideUserSensitiveData(&user)) + user.HideConfidentialData() + users = append(users, user) } } return users, err @@ -313,7 +313,8 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s continue } user := p.dbHandle.users[username] - users = append(users, HideUserSensitiveData(&user)) + user.HideConfidentialData() + users = append(users, user) if len(users) >= limit { break } @@ -326,7 +327,8 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s } username := p.dbHandle.usernames[i] user := p.dbHandle.users[username] - users = append(users, HideUserSensitiveData(&user)) + user.HideConfidentialData() + users = append(users, user) if len(users) >= limit { break } @@ -624,8 +626,7 @@ func (p MemoryProvider) reloadConfig() error { providerLog(logger.LevelWarn, "error loading users: %v", err) return err } - var dump BackupData - err = json.Unmarshal(content, &dump) + dump, err := ParseDumpData(content) if err != nil { providerLog(logger.LevelWarn, "error loading users: %v", err) return err diff --git a/dataprovider/mysql.go b/dataprovider/mysql.go index c5f11d3c..93d2d741 100644 --- a/dataprovider/mysql.go +++ b/dataprovider/mysql.go @@ -210,28 +210,46 @@ func (p MySQLProvider) migrateDatabase() error { } switch dbVersion.Version { case 1: - err = updateMySQLDatabaseFrom1To2(p.dbHandle) - if err != nil { - return err - } - err = updateMySQLDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updateMySQLDatabaseFrom3To4(p.dbHandle) + return updateMySQLDatabaseFromV1(p.dbHandle) case 2: - err = updateMySQLDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updateMySQLDatabaseFrom3To4(p.dbHandle) + return updateMySQLDatabaseFromV2(p.dbHandle) case 3: - return updateMySQLDatabaseFrom3To4(p.dbHandle) + return updateMySQLDatabaseFromV3(p.dbHandle) + case 4: + return updateMySQLDatabaseFromV4(p.dbHandle) default: return fmt.Errorf("Database version not handled: %v", dbVersion.Version) } } +func updateMySQLDatabaseFromV1(dbHandle *sql.DB) error { + err := updateMySQLDatabaseFrom1To2(dbHandle) + if err != nil { + return err + } + return updateMySQLDatabaseFromV2(dbHandle) +} + +func updateMySQLDatabaseFromV2(dbHandle *sql.DB) error { + err := updateMySQLDatabaseFrom2To3(dbHandle) + if err != nil { + return err + } + return updateMySQLDatabaseFromV3(dbHandle) +} + +func updateMySQLDatabaseFromV3(dbHandle *sql.DB) error { + err := updateMySQLDatabaseFrom3To4(dbHandle) + if err != nil { + return err + } + return updateMySQLDatabaseFromV4(dbHandle) +} + +func updateMySQLDatabaseFromV4(dbHandle *sql.DB) error { + return updateMySQLDatabaseFrom4To5(dbHandle) +} + func updateMySQLDatabaseFrom1To2(dbHandle *sql.DB) error { logger.InfoToConsole("updating database version: 1 -> 2") providerLog(logger.LevelInfo, "updating database version: 1 -> 2") @@ -249,3 +267,7 @@ func updateMySQLDatabaseFrom2To3(dbHandle *sql.DB) error { func updateMySQLDatabaseFrom3To4(dbHandle *sql.DB) error { return sqlCommonUpdateDatabaseFrom3To4(mysqlV4SQL, dbHandle) } + +func updateMySQLDatabaseFrom4To5(dbHandle *sql.DB) error { + return sqlCommonUpdateDatabaseFrom4To5(dbHandle) +} diff --git a/dataprovider/pgsql.go b/dataprovider/pgsql.go index dd9b055d..1bdad1d4 100644 --- a/dataprovider/pgsql.go +++ b/dataprovider/pgsql.go @@ -209,28 +209,46 @@ func (p PGSQLProvider) migrateDatabase() error { } switch dbVersion.Version { case 1: - err = updatePGSQLDatabaseFrom1To2(p.dbHandle) - if err != nil { - return err - } - err = updatePGSQLDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updatePGSQLDatabaseFrom3To4(p.dbHandle) + return updatePGSQLDatabaseFromV1(p.dbHandle) case 2: - err = updatePGSQLDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updatePGSQLDatabaseFrom3To4(p.dbHandle) + return updatePGSQLDatabaseFromV2(p.dbHandle) case 3: - return updatePGSQLDatabaseFrom3To4(p.dbHandle) + return updatePGSQLDatabaseFromV3(p.dbHandle) + case 4: + return updatePGSQLDatabaseFromV4(p.dbHandle) default: return fmt.Errorf("Database version not handled: %v", dbVersion.Version) } } +func updatePGSQLDatabaseFromV1(dbHandle *sql.DB) error { + err := updatePGSQLDatabaseFrom1To2(dbHandle) + if err != nil { + return err + } + return updatePGSQLDatabaseFromV2(dbHandle) +} + +func updatePGSQLDatabaseFromV2(dbHandle *sql.DB) error { + err := updatePGSQLDatabaseFrom2To3(dbHandle) + if err != nil { + return err + } + return updatePGSQLDatabaseFromV3(dbHandle) +} + +func updatePGSQLDatabaseFromV3(dbHandle *sql.DB) error { + err := updatePGSQLDatabaseFrom3To4(dbHandle) + if err != nil { + return err + } + return updatePGSQLDatabaseFromV4(dbHandle) +} + +func updatePGSQLDatabaseFromV4(dbHandle *sql.DB) error { + return updatePGSQLDatabaseFrom4To5(dbHandle) +} + func updatePGSQLDatabaseFrom1To2(dbHandle *sql.DB) error { logger.InfoToConsole("updating database version: 1 -> 2") providerLog(logger.LevelInfo, "updating database version: 1 -> 2") @@ -248,3 +266,7 @@ func updatePGSQLDatabaseFrom2To3(dbHandle *sql.DB) error { func updatePGSQLDatabaseFrom3To4(dbHandle *sql.DB) error { return sqlCommonUpdateDatabaseFrom3To4(pgsqlV4SQL, dbHandle) } + +func updatePGSQLDatabaseFrom4To5(dbHandle *sql.DB) error { + return sqlCommonUpdateDatabaseFrom4To5(dbHandle) +} diff --git a/dataprovider/sqlcommon.go b/dataprovider/sqlcommon.go index cca11bed..a5111798 100644 --- a/dataprovider/sqlcommon.go +++ b/dataprovider/sqlcommon.go @@ -14,7 +14,7 @@ import ( ) const ( - sqlDatabaseVersion = 4 + sqlDatabaseVersion = 5 initialDBVersionSQL = "INSERT INTO {{schema_version}} (version) VALUES (1);" defaultSQLQueryTimeout = 10 * time.Second longSQLQueryTimeout = 60 * time.Second @@ -354,7 +354,8 @@ func sqlCommonGetUsers(limit int, offset int, order string, username string, dbH if err != nil { return users, err } - users = append(users, HideUserSensitiveData(&u)) + u.HideConfidentialData() + users = append(users, u) } } err = rows.Err() @@ -940,3 +941,88 @@ func sqlCommonUpdateDatabaseFrom3To4(sqlV4 string, dbHandle *sql.DB) error { } return err } + +func sqlCommonUpdateDatabaseFrom4To5(dbHandle *sql.DB) error { + logger.InfoToConsole("updating database version: 4 -> 5") + providerLog(logger.LevelInfo, "updating database version: 4 -> 5") + ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout) + defer cancel() + q := getCompatV4FsConfigQuery() + stmt, err := dbHandle.PrepareContext(ctx, q) + if err != nil { + providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err) + return err + } + defer stmt.Close() + rows, err := stmt.QueryContext(ctx) + if err != nil { + return err + } + defer rows.Close() + + users := []User{} + for rows.Next() { + var compatUser compatUserV4 + var fsConfigString sql.NullString + err = rows.Scan(&compatUser.ID, &compatUser.Username, &fsConfigString) + if err != nil { + return err + } + if fsConfigString.Valid { + err = json.Unmarshal([]byte(fsConfigString.String), &compatUser.FsConfig) + if err != nil { + logger.WarnToConsole("failed to unmarshal v4 user %#v, is it already migrated?", compatUser.Username) + continue + } + fsConfig, err := convertFsConfigFromV4(compatUser.FsConfig, compatUser.Username) + if err != nil { + return err + } + users = append(users, createUserFromV4(compatUser, fsConfig)) + } + } + if err := rows.Err(); err != nil { + return err + } + + for _, user := range users { + err = sqlCommonUpdateV4User(dbHandle, user) + if err != nil { + return err + } + providerLog(logger.LevelInfo, "filesystem config updated for user %#v", user.Username) + } + + ctxVersion, cancelVersion := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) + defer cancelVersion() + + return sqlCommonUpdateDatabaseVersion(ctxVersion, dbHandle, 5) +} + +func sqlCommonUpdateV4User(dbHandle *sql.DB, user User) error { + err := validateFilesystemConfig(&user) + if err != nil { + return err + } + err = saveGCSCredentials(&user) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) + defer cancel() + + q := updateCompatV4FsConfigQuery() + stmt, err := dbHandle.PrepareContext(ctx, q) + if err != nil { + providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err) + return err + } + defer stmt.Close() + + fsConfig, err := user.GetFsConfigAsJSON() + if err != nil { + return err + } + _, err = stmt.ExecContext(ctx, string(fsConfig), user.ID) + return err +} diff --git a/dataprovider/sqlite.go b/dataprovider/sqlite.go index d96f5aca..00374448 100644 --- a/dataprovider/sqlite.go +++ b/dataprovider/sqlite.go @@ -232,28 +232,46 @@ func (p SQLiteProvider) migrateDatabase() error { } switch dbVersion.Version { case 1: - err = updateSQLiteDatabaseFrom1To2(p.dbHandle) - if err != nil { - return err - } - err = updateSQLiteDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updateSQLiteDatabaseFrom3To4(p.dbHandle) + return updateSQLiteDatabaseFromV1(p.dbHandle) case 2: - err = updateSQLiteDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updateSQLiteDatabaseFrom3To4(p.dbHandle) + return updateSQLiteDatabaseFromV2(p.dbHandle) case 3: - return updateSQLiteDatabaseFrom3To4(p.dbHandle) + return updateSQLiteDatabaseFromV3(p.dbHandle) + case 4: + return updateSQLiteDatabaseFromV4(p.dbHandle) default: return fmt.Errorf("Database version not handled: %v", dbVersion.Version) } } +func updateSQLiteDatabaseFromV1(dbHandle *sql.DB) error { + err := updateSQLiteDatabaseFrom1To2(dbHandle) + if err != nil { + return err + } + return updateSQLiteDatabaseFromV2(dbHandle) +} + +func updateSQLiteDatabaseFromV2(dbHandle *sql.DB) error { + err := updateSQLiteDatabaseFrom2To3(dbHandle) + if err != nil { + return err + } + return updateSQLiteDatabaseFromV3(dbHandle) +} + +func updateSQLiteDatabaseFromV3(dbHandle *sql.DB) error { + err := updateSQLiteDatabaseFrom3To4(dbHandle) + if err != nil { + return err + } + return updateSQLiteDatabaseFromV4(dbHandle) +} + +func updateSQLiteDatabaseFromV4(dbHandle *sql.DB) error { + return updateSQLiteDatabaseFrom4To5(dbHandle) +} + func updateSQLiteDatabaseFrom1To2(dbHandle *sql.DB) error { logger.InfoToConsole("updating database version: 1 -> 2") providerLog(logger.LevelInfo, "updating database version: 1 -> 2") @@ -271,3 +289,7 @@ func updateSQLiteDatabaseFrom2To3(dbHandle *sql.DB) error { func updateSQLiteDatabaseFrom3To4(dbHandle *sql.DB) error { return sqlCommonUpdateDatabaseFrom3To4(sqliteV4SQL, dbHandle) } + +func updateSQLiteDatabaseFrom4To5(dbHandle *sql.DB) error { + return sqlCommonUpdateDatabaseFrom4To5(dbHandle) +} diff --git a/dataprovider/sqlqueries.go b/dataprovider/sqlqueries.go index 193b3044..564531fb 100644 --- a/dataprovider/sqlqueries.go +++ b/dataprovider/sqlqueries.go @@ -184,3 +184,11 @@ func getUpdateDBVersionQuery() string { func getCompatVirtualFoldersQuery() string { return fmt.Sprintf(`SELECT id,username,virtual_folders FROM %v`, sqlTableUsers) } + +func getCompatV4FsConfigQuery() string { + return fmt.Sprintf(`SELECT id,username,filesystem FROM %v`, sqlTableUsers) +} + +func updateCompatV4FsConfigQuery() string { + return fmt.Sprintf(`UPDATE %v SET filesystem=%v WHERE id=%v`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1]) +} diff --git a/dataprovider/user.go b/dataprovider/user.go index 04b85987..344f61be 100644 --- a/dataprovider/user.go +++ b/dataprovider/user.go @@ -230,6 +230,19 @@ func (u *User) GetFilesystem(connectionID string) (vfs.Fs, error) { return vfs.NewOsFs(connectionID, u.GetHomeDir(), u.VirtualFolders), nil } +// HideConfidentialData hides user confidential data +func (u *User) HideConfidentialData() { + u.Password = "" + switch u.FsConfig.Provider { + case S3FilesystemProvider: + u.FsConfig.S3Config.AccessSecret.Hide() + case GCSFilesystemProvider: + u.FsConfig.GCSConfig.Credentials.Hide() + case AzureBlobFilesystemProvider: + u.FsConfig.AzBlobConfig.AccountKey.Hide() + } +} + // GetPermissionsForPath returns the permissions for the given path. // The path must be an SFTP path func (u *User) GetPermissionsForPath(p string) []string { @@ -809,6 +822,7 @@ func (u *User) getACopy() User { UploadPartSize: u.FsConfig.AzBlobConfig.UploadPartSize, UploadConcurrency: u.FsConfig.AzBlobConfig.UploadConcurrency, UseEmulator: u.FsConfig.AzBlobConfig.UseEmulator, + AccessTier: u.FsConfig.AzBlobConfig.AccessTier, }, } diff --git a/docker/README.md b/docker/README.md index 01016f5e..09525e3f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -79,15 +79,24 @@ Please take a look [here](../docs/full-configuration.md#environment-variables) t Alternately you can mount your custom configuration file to `/var/lib/sftpgo` or `/var/lib/sftpgo/.config/sftpgo`. +### Loading initial data + +Initial data can be loaded in the following ways: + +- via the `--loaddata-from` flag or the `SFTPGO_LOADDATA_FROM` environment variable +- by providing a dump file to the memory provider + +Please take a look [here](../docs/full-configuration.md) for more details. + ### Running as an arbitrary user The SFTPGo image runs using `1000` as UID/GID by default. If you know the permissions of your data and/or configuration directory are already set appropriately or you have need of running SFTPGo with a specific UID/GID, it is possible to invoke this image with `--user` set to any value (other than `root/0`) in order to achieve the desired access/configuration: ```shell $ ls -lnd data -drwxr-xr-x 2 1100 11000 6 6 nov 09.09 data +drwxr-xr-x 2 1100 1100 6 7 nov 09.09 data $ ls -lnd config -drwxr-xr-x 2 1100 11000 6 6 nov 09.19 config +drwxr-xr-x 2 1100 1100 6 7 nov 09.19 config ``` With the above directory permissions, you can start a SFTPGo instance like this: diff --git a/docs/full-configuration.md b/docs/full-configuration.md index 1404a934..91de40ee 100644 --- a/docs/full-configuration.md +++ b/docs/full-configuration.md @@ -115,7 +115,7 @@ The configuration file contains the following sections: - `max_size`, integer. Maximum number of users to cache. 0 means unlimited. Default: 50. - **"data_provider"**, the configuration for the data provider - `driver`, string. Supported drivers are `sqlite`, `mysql`, `postgresql`, `bolt`, `memory` - - `name`, string. Database name. For driver `sqlite` this can be the database name relative to the config dir or the absolute path to the SQLite database. For driver `memory` this is the (optional) path relative to the config dir or the absolute path to the users dump, obtained using the `dumpdata` REST API, to load. This dump will be loaded at startup and can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows. The `memory` provider will not modify the provided file so quota usage and last login will not be persisted + - `name`, string. Database name. For driver `sqlite` this can be the database name relative to the config dir or the absolute path to the SQLite database. For driver `memory` this is the (optional) path relative to the config dir or the absolute path to the provider dump, obtained using the `dumpdata` REST API, to load. This dump will be loaded at startup and can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows. The `memory` provider will not modify the provided file so quota usage and last login will not be persisted - `host`, string. Database host. Leave empty for drivers `sqlite`, `bolt` and `memory` - `port`, integer. Database port. Leave empty for drivers `sqlite`, `bolt` and `memory` - `username`, string. Database user. Leave empty for drivers `sqlite`, `bolt` and `memory` diff --git a/examples/rest-api-cli/README.md b/examples/rest-api-cli/README.md index 56c479c4..4f100afd 100644 --- a/examples/rest-api-cli/README.md +++ b/examples/rest-api-cli/README.md @@ -58,7 +58,10 @@ Output: "provider": 1, "s3config": { "access_key": "accesskey", - "access_secret": "$aes$6c088ba12b0b261247c8cf331c46d9260b8e58002957d89ad1c0495e3af665cd0227", + "access_secret": { + "payload": "ac46cec75466ba77e47f536436783b729ca5bbbb53252fda0de51f785a6da11ffb03", + "status": "AES-256-GCM" + }, "bucket": "test", "endpoint": "http://127.0.0.1:9000", "key_prefix": "vfolder/", diff --git a/examples/rest-api-cli/sftpgo_api_cli b/examples/rest-api-cli/sftpgo_api_cli index 9c351dc2..55661dcc 100755 --- a/examples/rest-api-cli/sftpgo_api_cli +++ b/examples/rest-api-cli/sftpgo_api_cli @@ -238,23 +238,30 @@ class SFTPGoApiRequests: az_upload_concurrency, az_key_prefix, az_use_emulator, az_access_tier): fs_config = {'provider':0} if fs_provider == 'S3': + secret = {} + if s3_access_secret: + secret.update({"status":"Plain", "payload":s3_access_secret}) s3config = {'bucket':s3_bucket, 'region':s3_region, 'access_key':s3_access_key, 'access_secret': - s3_access_secret, 'endpoint':s3_endpoint, 'storage_class':s3_storage_class, 'key_prefix': + secret, 'endpoint':s3_endpoint, 'storage_class':s3_storage_class, 'key_prefix': s3_key_prefix, 'upload_part_size':s3_upload_part_size, 'upload_concurrency':s3_upload_concurrency} fs_config.update({'provider':1, 's3config':s3config}) elif fs_provider == 'GCS': - gcsconfig = {'bucket':gcs_bucket, 'key_prefix':gcs_key_prefix, 'storage_class':gcs_storage_class} + gcsconfig = {'bucket':gcs_bucket, 'key_prefix':gcs_key_prefix, 'storage_class':gcs_storage_class, + 'credentials':{}} if gcs_automatic_credentials == "automatic": gcsconfig.update({'automatic_credentials':1}) else: gcsconfig.update({'automatic_credentials':0}) if gcs_credentials_file: with open(gcs_credentials_file) as creds: - gcsconfig.update({'credentials':base64.b64encode(creds.read().encode('UTF-8')).decode('UTF-8'), - 'automatic_credentials':0}) + secret = {"status":"Plain", "payload":creds.read()} + gcsconfig.update({'credentials':secret, 'automatic_credentials':0}) fs_config.update({'provider':2, 'gcsconfig':gcsconfig}) elif fs_provider == "AzureBlob": - azureconfig = {'container':az_container, 'account_name':az_account_name, 'account_key':az_account_key, + secret = {} + if az_account_key: + secret.update({"status":"Plain", "payload":az_account_key}) + azureconfig = {'container':az_container, 'account_name':az_account_name, 'account_key':secret, 'sas_url':az_sas_url, 'endpoint':az_endpoint, 'upload_part_size':az_upload_part_size, 'upload_concurrency':az_upload_concurrency, 'key_prefix':az_key_prefix, 'use_emulator': az_use_emulator, 'access_tier':az_access_tier} @@ -609,7 +616,7 @@ def addCommonUserArguments(parser): help='Denied IP/Mask in CIDR notation. For example "192.168.2.0/24" or "2001:db8::/32". Default: %(default)s') parser.add_argument('--denied-patterns', type=str, nargs='*', default=[], help='Denied file patterns case insensitive. ' +'The format is /dir::pattern1,pattern2. For example: "/somedir::*.jpg,*.png" "/otherdir/subdir::a*b?.zip,*.rar". ' + - 'You have to set both denied and allowed patterns to update existing values or none to preserve them.' + + ' You have to set both denied and allowed patterns to update existing values or none to preserve them.' + ' If you only set allowed or denied patterns the missing one is assumed to be an empty list. Default: %(default)s') parser.add_argument('--allowed-patterns', type=str, nargs='*', default=[], help='Allowed file patterns case insensitive. ' +'The format is /dir::pattern1,pattern2. For example: "/somedir::*.jpg,a*b?.png" "/otherdir/subdir::*.zip,*.rar". ' + diff --git a/ftpd/ftpd_test.go b/ftpd/ftpd_test.go index a9769496..9f88f7e0 100644 --- a/ftpd/ftpd_test.go +++ b/ftpd/ftpd_test.go @@ -876,7 +876,10 @@ func TestLoginWithDatabaseCredentials(t *testing.T) { u := getTestUser() u.FsConfig.Provider = dataprovider.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "test" - u.FsConfig.GCSConfig.Credentials = []byte(`{ "type": "service_account" }`) + u.FsConfig.GCSConfig.Credentials = vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: `{ "type": "service_account" }`, + } providerConf := config.GetProviderConf() providerConf.PreferDatabaseCredentials = true @@ -897,9 +900,12 @@ func TestLoginWithDatabaseCredentials(t *testing.T) { user, _, err := httpd.AddUser(u, http.StatusOK) assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.GCSConfig.Credentials.Status) + assert.NotEmpty(t, user.FsConfig.GCSConfig.Credentials.Payload) + assert.Empty(t, user.FsConfig.GCSConfig.Credentials.AdditionalData) + assert.Empty(t, user.FsConfig.GCSConfig.Credentials.Key) - _, err = os.Stat(credentialsFile) - assert.Error(t, err) + assert.NoFileExists(t, credentialsFile) client, err := getFTPClient(user, false) if assert.NoError(t, err) { @@ -922,7 +928,10 @@ func TestLoginInvalidFs(t *testing.T) { u := getTestUser() u.FsConfig.Provider = dataprovider.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "test" - u.FsConfig.GCSConfig.Credentials = []byte("invalid JSON for credentials") + u.FsConfig.GCSConfig.Credentials = vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: "invalid JSON for credentials", + } user, _, err := httpd.AddUser(u, http.StatusOK) assert.NoError(t, err) diff --git a/httpd/api_maintenance.go b/httpd/api_maintenance.go index b218d37b..5ea0e1b8 100644 --- a/httpd/api_maintenance.go +++ b/httpd/api_maintenance.go @@ -96,8 +96,7 @@ func loadData(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - var dump dataprovider.BackupData - err = json.Unmarshal(content, &dump) + dump, err := dataprovider.ParseDumpData(content) if err != nil { sendAPIResponse(w, r, err, fmt.Sprintf("Unable to parse input file: %#v", inputFile), http.StatusBadRequest) return diff --git a/httpd/api_user.go b/httpd/api_user.go index 5c257335..5d786b02 100644 --- a/httpd/api_user.go +++ b/httpd/api_user.go @@ -11,7 +11,7 @@ import ( "github.com/drakkan/sftpgo/common" "github.com/drakkan/sftpgo/dataprovider" - "github.com/drakkan/sftpgo/utils" + "github.com/drakkan/sftpgo/vfs" ) func getUsers(w http.ResponseWriter, r *http.Request) { @@ -67,7 +67,8 @@ func getUserByID(w http.ResponseWriter, r *http.Request) { } user, err := dataprovider.GetUserByID(userID) if err == nil { - render.JSON(w, r, dataprovider.HideUserSensitiveData(&user)) + user.HideConfidentialData() + render.JSON(w, r, user) } else { sendAPIResponse(w, r, err, "", getRespStatus(err)) } @@ -81,11 +82,29 @@ func addUser(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, err, "", http.StatusBadRequest) return } + switch user.FsConfig.Provider { + case dataprovider.S3FilesystemProvider: + if user.FsConfig.S3Config.AccessSecret.IsRedacted() { + sendAPIResponse(w, r, errors.New("invalid access_secret"), "", http.StatusBadRequest) + return + } + case dataprovider.GCSFilesystemProvider: + if user.FsConfig.GCSConfig.Credentials.IsRedacted() { + sendAPIResponse(w, r, errors.New("invalid credentials"), "", http.StatusBadRequest) + return + } + case dataprovider.AzureBlobFilesystemProvider: + if user.FsConfig.AzBlobConfig.AccountKey.IsRedacted() { + sendAPIResponse(w, r, errors.New("invalid account_key"), "", http.StatusBadRequest) + return + } + } err = dataprovider.AddUser(user) if err == nil { user, err = dataprovider.UserExists(user.Username) if err == nil { - render.JSON(w, r, dataprovider.HideUserSensitiveData(&user)) + user.HideConfidentialData() + render.JSON(w, r, user) } else { sendAPIResponse(w, r, err, "", getRespStatus(err)) } @@ -117,15 +136,22 @@ func updateUser(w http.ResponseWriter, r *http.Request) { return } currentPermissions := user.Permissions - currentS3AccessSecret := "" - currentAzAccountKey := "" + var currentS3AccessSecret vfs.Secret + var currentAzAccountKey vfs.Secret + var currentGCSCredentials vfs.Secret if user.FsConfig.Provider == dataprovider.S3FilesystemProvider { currentS3AccessSecret = user.FsConfig.S3Config.AccessSecret } if user.FsConfig.Provider == dataprovider.AzureBlobFilesystemProvider { currentAzAccountKey = user.FsConfig.AzBlobConfig.AccountKey } + if user.FsConfig.Provider == dataprovider.GCSFilesystemProvider { + currentGCSCredentials = user.FsConfig.GCSConfig.Credentials + } user.Permissions = make(map[string][]string) + user.FsConfig.S3Config = vfs.S3FsConfig{} + user.FsConfig.AzBlobConfig = vfs.AzBlobFsConfig{} + user.FsConfig.GCSConfig = vfs.GCSFsConfig{} err = render.DecodeJSON(r.Body, &user) if err != nil { sendAPIResponse(w, r, err, "", http.StatusBadRequest) @@ -135,7 +161,7 @@ func updateUser(w http.ResponseWriter, r *http.Request) { if len(user.Permissions) == 0 { user.Permissions = currentPermissions } - updateEncryptedSecrets(&user, currentS3AccessSecret, currentAzAccountKey) + updateEncryptedSecrets(&user, currentS3AccessSecret, currentAzAccountKey, currentGCSCredentials) if user.ID != userID { sendAPIResponse(w, r, err, "user ID in request body does not match user ID in path parameter", http.StatusBadRequest) @@ -181,18 +207,21 @@ func disconnectUser(username string) { } } -func updateEncryptedSecrets(user *dataprovider.User, currentS3AccessSecret, currentAzAccountKey string) { - // we use the new access secret if different from the old one and not empty +func updateEncryptedSecrets(user *dataprovider.User, currentS3AccessSecret, currentAzAccountKey, currentGCSCredentials vfs.Secret) { + // we use the new access secret if plain or empty, otherwise the old value if user.FsConfig.Provider == dataprovider.S3FilesystemProvider { - if utils.RemoveDecryptionKey(currentS3AccessSecret) == user.FsConfig.S3Config.AccessSecret || - (user.FsConfig.S3Config.AccessSecret == "" && user.FsConfig.S3Config.AccessKey != "") { + if !user.FsConfig.S3Config.AccessSecret.IsPlain() && !user.FsConfig.S3Config.AccessSecret.IsEmpty() { user.FsConfig.S3Config.AccessSecret = currentS3AccessSecret } } if user.FsConfig.Provider == dataprovider.AzureBlobFilesystemProvider { - if utils.RemoveDecryptionKey(currentAzAccountKey) == user.FsConfig.AzBlobConfig.AccountKey || - (user.FsConfig.AzBlobConfig.AccountKey == "" && user.FsConfig.AzBlobConfig.AccountName != "") { + if !user.FsConfig.AzBlobConfig.AccountKey.IsPlain() && !user.FsConfig.AzBlobConfig.AccountKey.IsEmpty() { user.FsConfig.AzBlobConfig.AccountKey = currentAzAccountKey } } + if user.FsConfig.Provider == dataprovider.GCSFilesystemProvider { + if !user.FsConfig.GCSConfig.Credentials.IsPlain() && !user.FsConfig.GCSConfig.Credentials.IsEmpty() { + user.FsConfig.GCSConfig.Credentials = currentGCSCredentials + } + } } diff --git a/httpd/api_utils.go b/httpd/api_utils.go index 79c6f005..37e6e303 100644 --- a/httpd/api_utils.go +++ b/httpd/api_utils.go @@ -707,28 +707,19 @@ func compareAzBlobConfig(expected *dataprovider.User, actual *dataprovider.User) return nil } -func checkEncryptedSecret(expectedAccessSecret, actualAccessSecret string) error { - if len(expectedAccessSecret) > 0 { - vals := strings.Split(expectedAccessSecret, "$") - if strings.HasPrefix(expectedAccessSecret, "$aes$") && len(vals) == 4 { - expectedAccessSecret = utils.RemoveDecryptionKey(expectedAccessSecret) - if expectedAccessSecret != actualAccessSecret { - return fmt.Errorf("secret mismatch, expected: %v", expectedAccessSecret) - } - } else { - // here we check that actualAccessSecret is aes encrypted without the nonce - parts := strings.Split(actualAccessSecret, "$") - if !strings.HasPrefix(actualAccessSecret, "$aes$") || len(parts) != 3 { - return errors.New("invalid secret") - } - if len(parts) == len(vals) { - if expectedAccessSecret != actualAccessSecret { - return errors.New("encrypted secret mismatch") - } - } +func checkEncryptedSecret(expected, actual vfs.Secret) error { + if expected.IsPlain() && actual.IsEncrypted() { + if actual.Payload == "" { + return errors.New("invalid secret payload") + } + if actual.AdditionalData != "" { + return errors.New("invalid secret additional data") + } + if actual.Key != "" { + return errors.New("invalid secret key") } } else { - if expectedAccessSecret != actualAccessSecret { + if expected.Status != actual.Status || expected.Payload != actual.Payload { return errors.New("secret mismatch") } } diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index 78c4c70a..ec213028 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -26,6 +26,7 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/drakkan/sftpgo/common" "github.com/drakkan/sftpgo/config" @@ -190,7 +191,7 @@ func TestMain(m *testing.M) { defer testServer.Close() exitCode := m.Run() - os.Remove(logfilePath) //nolint:errcheck + //os.Remove(logfilePath) //nolint:errcheck os.RemoveAll(backupsPath) //nolint:errcheck os.RemoveAll(credentialsPath) //nolint:errcheck os.Remove(certPath) //nolint:errcheck @@ -438,12 +439,16 @@ func TestAddUserInvalidFsConfig(t *testing.T) { u.FsConfig.S3Config.Bucket = "testbucket" u.FsConfig.S3Config.Region = "eu-west-1" u.FsConfig.S3Config.AccessKey = "access-key" - u.FsConfig.S3Config.AccessSecret = "access-secret" + u.FsConfig.S3Config.AccessSecret.Payload = "access-secret" + u.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusRedacted u.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?a=b" u.FsConfig.S3Config.StorageClass = "Standard" //nolint:goconst u.FsConfig.S3Config.KeyPrefix = "/adir/subdir/" _, _, err = httpd.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) + u.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusPlain + _, _, err = httpd.AddUser(u, http.StatusBadRequest) + assert.NoError(t, err) u.FsConfig.S3Config.KeyPrefix = "" u.FsConfig.S3Config.UploadPartSize = 3 _, _, err = httpd.AddUser(u, http.StatusBadRequest) @@ -463,16 +468,20 @@ func TestAddUserInvalidFsConfig(t *testing.T) { u.FsConfig.GCSConfig.Bucket = "abucket" u.FsConfig.GCSConfig.StorageClass = "Standard" u.FsConfig.GCSConfig.KeyPrefix = "/somedir/subdir/" - u.FsConfig.GCSConfig.Credentials = []byte("test") + u.FsConfig.GCSConfig.Credentials.Payload = "test" //nolint:goconst + u.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusRedacted + _, _, err = httpd.AddUser(u, http.StatusBadRequest) + assert.NoError(t, err) + u.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusPlain _, _, err = httpd.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) u.FsConfig.GCSConfig.KeyPrefix = "somedir/subdir/" //nolint:goconst - u.FsConfig.GCSConfig.Credentials = nil + u.FsConfig.GCSConfig.Credentials = vfs.Secret{} u.FsConfig.GCSConfig.AutomaticCredentials = 0 _, _, err = httpd.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) - - u.FsConfig.GCSConfig.Credentials = invalidBase64{} + u.FsConfig.GCSConfig.Credentials.Payload = "invalid" + u.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusAES256GCM _, _, err = httpd.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) @@ -488,10 +497,14 @@ func TestAddUserInvalidFsConfig(t *testing.T) { u.FsConfig.AzBlobConfig.Container = "container" _, _, err = httpd.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) - u.FsConfig.AzBlobConfig.AccountKey = "key" + u.FsConfig.AzBlobConfig.AccountKey.Payload = "key" + u.FsConfig.AzBlobConfig.AccountKey.Status = vfs.SecretStatusRedacted u.FsConfig.AzBlobConfig.KeyPrefix = "/amedir/subdir/" _, _, err = httpd.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) + u.FsConfig.AzBlobConfig.AccountKey.Status = vfs.SecretStatusPlain + _, _, err = httpd.AddUser(u, http.StatusBadRequest) + assert.NoError(t, err) u.FsConfig.AzBlobConfig.KeyPrefix = "amedir/subdir/" u.FsConfig.AzBlobConfig.UploadPartSize = -1 _, _, err = httpd.AddUser(u, http.StatusBadRequest) @@ -1000,19 +1013,35 @@ func TestUserS3Config(t *testing.T) { user.FsConfig.S3Config.Bucket = "test" //nolint:goconst user.FsConfig.S3Config.Region = "us-east-1" //nolint:goconst user.FsConfig.S3Config.AccessKey = "Server-Access-Key" - user.FsConfig.S3Config.AccessSecret = "Server-Access-Secret" + user.FsConfig.S3Config.AccessSecret.Payload = "Server-Access-Secret" + user.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusPlain user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000" user.FsConfig.S3Config.UploadPartSize = 8 - user, _, err = httpd.UpdateUser(user, http.StatusOK, "") - assert.NoError(t, err) + user, body, err := httpd.UpdateUser(user, http.StatusOK, "") + assert.NoError(t, err, string(body)) + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.S3Config.AccessSecret.Status) + assert.NotEmpty(t, user.FsConfig.S3Config.AccessSecret.Payload) + assert.Empty(t, user.FsConfig.S3Config.AccessSecret.AdditionalData) + assert.Empty(t, user.FsConfig.S3Config.AccessSecret.Key) _, err = httpd.RemoveUser(user, http.StatusOK) assert.NoError(t, err) user.Password = defaultPassword user.ID = 0 - secret, _ := utils.EncryptData("Server-Access-Secret") + secret := vfs.Secret{ + Payload: "Server-Access-Secret", + Status: vfs.SecretStatusAES256GCM, + } user.FsConfig.S3Config.AccessSecret = secret + _, _, err = httpd.AddUser(user, http.StatusOK) + assert.Error(t, err) + user.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusPlain user, _, err = httpd.AddUser(user, http.StatusOK) assert.NoError(t, err) + initialSecretPayload := user.FsConfig.S3Config.AccessSecret.Payload + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.S3Config.AccessSecret.Status) + assert.NotEmpty(t, initialSecretPayload) + assert.Empty(t, user.FsConfig.S3Config.AccessSecret.AdditionalData) + assert.Empty(t, user.FsConfig.S3Config.AccessSecret.Key) user.FsConfig.Provider = dataprovider.S3FilesystemProvider user.FsConfig.S3Config.Bucket = "test-bucket" user.FsConfig.S3Config.Region = "us-east-1" //nolint:goconst @@ -1022,29 +1051,31 @@ func TestUserS3Config(t *testing.T) { user.FsConfig.S3Config.UploadConcurrency = 5 user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) - user.FsConfig.Provider = dataprovider.LocalFilesystemProvider - user.FsConfig.S3Config.Bucket = "" - user.FsConfig.S3Config.Region = "" - user.FsConfig.S3Config.AccessKey = "" - user.FsConfig.S3Config.AccessSecret = "" - user.FsConfig.S3Config.Endpoint = "" - user.FsConfig.S3Config.KeyPrefix = "" - user.FsConfig.S3Config.UploadPartSize = 0 - user.FsConfig.S3Config.UploadConcurrency = 0 - user, _, err = httpd.UpdateUser(user, http.StatusOK, "") - assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.S3Config.AccessSecret.Status) + assert.Equal(t, initialSecretPayload, user.FsConfig.S3Config.AccessSecret.Payload) + assert.Empty(t, user.FsConfig.S3Config.AccessSecret.AdditionalData) + assert.Empty(t, user.FsConfig.S3Config.AccessSecret.Key) // test user without access key and access secret (shared config state) user.FsConfig.Provider = dataprovider.S3FilesystemProvider user.FsConfig.S3Config.Bucket = "testbucket" user.FsConfig.S3Config.Region = "us-east-1" user.FsConfig.S3Config.AccessKey = "" - user.FsConfig.S3Config.AccessSecret = "" + user.FsConfig.S3Config.AccessSecret = vfs.Secret{} user.FsConfig.S3Config.Endpoint = "" user.FsConfig.S3Config.KeyPrefix = "somedir/subdir" user.FsConfig.S3Config.UploadPartSize = 6 user.FsConfig.S3Config.UploadConcurrency = 4 - user, _, err = httpd.UpdateUser(user, http.StatusOK, "") + user, body, err = httpd.UpdateUser(user, http.StatusOK, "") + assert.NoError(t, err, string(body)) + assert.True(t, user.FsConfig.S3Config.AccessSecret.IsEmpty()) + _, err = httpd.RemoveUser(user, http.StatusOK) assert.NoError(t, err) + user.Password = defaultPassword + user.ID = 0 + // shared credential test for add instead of update + user, _, err = httpd.AddUser(user, http.StatusOK) + assert.NoError(t, err) + assert.True(t, user.FsConfig.S3Config.AccessSecret.IsEmpty()) _, err = httpd.RemoveUser(user, http.StatusOK) assert.NoError(t, err) } @@ -1058,36 +1089,69 @@ func TestUserGCSConfig(t *testing.T) { assert.NoError(t, err) user.FsConfig.Provider = dataprovider.GCSFilesystemProvider user.FsConfig.GCSConfig.Bucket = "test" - user.FsConfig.GCSConfig.Credentials = []byte("fake credentials") + user.FsConfig.GCSConfig.Credentials.Payload = "fake credentials" //nolint:goconst + user.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusPlain user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) + credentialFile := filepath.Join(credentialsPath, fmt.Sprintf("%v_gcs_credentials.json", user.Username)) + assert.FileExists(t, credentialFile) + creds, err := ioutil.ReadFile(credentialFile) + assert.NoError(t, err) + secret := &vfs.Secret{} + err = json.Unmarshal(creds, secret) + assert.NoError(t, err) + err = secret.Decrypt() + assert.NoError(t, err) + assert.Equal(t, "fake credentials", secret.Payload) + user.FsConfig.GCSConfig.Credentials.Payload = "fake encrypted credentials" + user.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusAES256GCM + user, _, err = httpd.UpdateUser(user, http.StatusOK, "") + assert.NoError(t, err) + assert.FileExists(t, credentialFile) + creds, err = ioutil.ReadFile(credentialFile) + assert.NoError(t, err) + secret = &vfs.Secret{} + err = json.Unmarshal(creds, secret) + assert.NoError(t, err) + err = secret.Decrypt() + assert.NoError(t, err) + assert.Equal(t, "fake credentials", secret.Payload) _, err = httpd.RemoveUser(user, http.StatusOK) assert.NoError(t, err) user.Password = defaultPassword user.ID = 0 - user.FsConfig.GCSConfig.Credentials = []byte("fake credentials") + user.FsConfig.GCSConfig.Credentials.Payload = "fake credentials" + user.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusAES256GCM + _, _, err = httpd.AddUser(user, http.StatusOK) + assert.Error(t, err) + user.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusPlain user, body, err := httpd.AddUser(user, http.StatusOK) assert.NoError(t, err, string(body)) err = os.RemoveAll(credentialsPath) assert.NoError(t, err) err = os.MkdirAll(credentialsPath, 0700) assert.NoError(t, err) - user.FsConfig.GCSConfig.Credentials = nil + user.FsConfig.GCSConfig.Credentials = vfs.Secret{} user.FsConfig.GCSConfig.AutomaticCredentials = 1 user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) + assert.NoFileExists(t, credentialFile) + user.FsConfig.GCSConfig = vfs.GCSFsConfig{} user.FsConfig.Provider = dataprovider.S3FilesystemProvider user.FsConfig.S3Config.Bucket = "test1" user.FsConfig.S3Config.Region = "us-east-1" user.FsConfig.S3Config.AccessKey = "Server-Access-Key1" - user.FsConfig.S3Config.AccessSecret = "secret" + user.FsConfig.S3Config.AccessSecret.Payload = "secret" + user.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusPlain user.FsConfig.S3Config.Endpoint = "http://localhost:9000" user.FsConfig.S3Config.KeyPrefix = "somedir/subdir" user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) + user.FsConfig.S3Config = vfs.S3FsConfig{} user.FsConfig.Provider = dataprovider.GCSFilesystemProvider user.FsConfig.GCSConfig.Bucket = "test1" - user.FsConfig.GCSConfig.Credentials = []byte("fake credentials") + user.FsConfig.GCSConfig.Credentials.Payload = "fake credentials" + user.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusPlain user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) @@ -1101,42 +1165,248 @@ func TestUserAzureBlobConfig(t *testing.T) { user.FsConfig.Provider = dataprovider.AzureBlobFilesystemProvider user.FsConfig.AzBlobConfig.Container = "test" user.FsConfig.AzBlobConfig.AccountName = "Server-Account-Name" - user.FsConfig.AzBlobConfig.AccountKey = "Server-Account-Key" + user.FsConfig.AzBlobConfig.AccountKey.Payload = "Server-Account-Key" + user.FsConfig.AzBlobConfig.AccountKey.Status = vfs.SecretStatusPlain user.FsConfig.AzBlobConfig.Endpoint = "http://127.0.0.1:9000" user.FsConfig.AzBlobConfig.UploadPartSize = 8 user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) + initialPayload := user.FsConfig.AzBlobConfig.AccountKey.Payload + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.AzBlobConfig.AccountKey.Status) + assert.NotEmpty(t, initialPayload) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.Key) + user.FsConfig.AzBlobConfig.AccountKey.Status = vfs.SecretStatusAES256GCM + user.FsConfig.AzBlobConfig.AccountKey.AdditionalData = "data" + user.FsConfig.AzBlobConfig.AccountKey.Key = "fake key" + user, _, err = httpd.UpdateUser(user, http.StatusOK, "") + assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.AzBlobConfig.AccountKey.Status) + assert.Equal(t, initialPayload, user.FsConfig.AzBlobConfig.AccountKey.Payload) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.Key) + _, err = httpd.RemoveUser(user, http.StatusOK) assert.NoError(t, err) user.Password = defaultPassword user.ID = 0 - secret, _ := utils.EncryptData("Server-Account-Key") + secret := vfs.Secret{ + Payload: "Server-Account-Key", + Status: vfs.SecretStatusAES256GCM, + } user.FsConfig.AzBlobConfig.AccountKey = secret + _, _, err = httpd.AddUser(user, http.StatusOK) + assert.Error(t, err) + user.FsConfig.AzBlobConfig.AccountKey = vfs.Secret{ + Payload: "Server-Account-Key-Test", + Status: vfs.SecretStatusPlain, + } user, _, err = httpd.AddUser(user, http.StatusOK) assert.NoError(t, err) + initialPayload = user.FsConfig.AzBlobConfig.AccountKey.Payload + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.AzBlobConfig.AccountKey.Status) + assert.NotEmpty(t, initialPayload) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.Key) user.FsConfig.Provider = dataprovider.AzureBlobFilesystemProvider user.FsConfig.AzBlobConfig.Container = "test-container" - user.FsConfig.AzBlobConfig.AccountKey = "Server-Account-Key1" user.FsConfig.AzBlobConfig.Endpoint = "http://localhost:9001" user.FsConfig.AzBlobConfig.KeyPrefix = "somedir/subdir" user.FsConfig.AzBlobConfig.UploadConcurrency = 5 user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) - user.FsConfig.Provider = dataprovider.LocalFilesystemProvider - user.FsConfig.AzBlobConfig = vfs.AzBlobFsConfig{} - user, _, err = httpd.UpdateUser(user, http.StatusOK, "") - assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.AzBlobConfig.AccountKey.Status) + assert.NotEmpty(t, initialPayload) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.Key) // test user without access key and access secret (sas) user.FsConfig.Provider = dataprovider.AzureBlobFilesystemProvider - user.FsConfig.AzBlobConfig.SASURL = "https://myaccount.blob.core.windows.net/pictures/profile.jpg?sv=2012-02-12&st=2009-02-09&se=2009-02-10&sr=c&sp=r&si=YWJjZGVmZw%3d%3d&sig=dD80ihBh5jfNpymO5Hg1IdiJIEvHcJpCMiCMnN%2fRnbI%3d" user.FsConfig.AzBlobConfig.KeyPrefix = "somedir/subdir" + user.FsConfig.AzBlobConfig.AccountName = "" + user.FsConfig.AzBlobConfig.AccountKey = vfs.Secret{} user.FsConfig.AzBlobConfig.UploadPartSize = 6 user.FsConfig.AzBlobConfig.UploadConcurrency = 4 user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) + assert.True(t, user.FsConfig.AzBlobConfig.AccountKey.IsEmpty()) _, err = httpd.RemoveUser(user, http.StatusOK) assert.NoError(t, err) + user.Password = defaultPassword + user.ID = 0 + // sas test for add instead of update + user, _, err = httpd.AddUser(user, http.StatusOK) + assert.NoError(t, err) + assert.True(t, user.FsConfig.AzBlobConfig.AccountKey.IsEmpty()) + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) +} + +func TestUserHiddenFields(t *testing.T) { + err := dataprovider.Close() + assert.NoError(t, err) + err = config.LoadConfig(configDir, "") + assert.NoError(t, err) + providerConf := config.GetProviderConf() + providerConf.PreferDatabaseCredentials = true + err = dataprovider.Initialize(providerConf, configDir) + assert.NoError(t, err) + + // sensitive data must be hidden but not deleted from the dataprovider + usernames := []string{"user1", "user2", "user3"} + u1 := getTestUser() + u1.Username = usernames[0] + u1.FsConfig.Provider = dataprovider.S3FilesystemProvider + u1.FsConfig.S3Config.Bucket = "test" + u1.FsConfig.S3Config.Region = "us-east-1" + u1.FsConfig.S3Config.AccessKey = "S3-Access-Key" + u1.FsConfig.S3Config.AccessSecret.Payload = "S3-Access-Secret" + u1.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusPlain + user1, _, err := httpd.AddUser(u1, http.StatusOK) + assert.NoError(t, err) + + u2 := getTestUser() + u2.Username = usernames[1] + u2.FsConfig.Provider = dataprovider.GCSFilesystemProvider + u2.FsConfig.GCSConfig.Bucket = "test" + u2.FsConfig.GCSConfig.Credentials.Payload = "fake credentials" + u2.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusPlain + user2, _, err := httpd.AddUser(u2, http.StatusOK) + assert.NoError(t, err) + + u3 := getTestUser() + u3.Username = usernames[2] + u3.FsConfig.Provider = dataprovider.AzureBlobFilesystemProvider + u3.FsConfig.AzBlobConfig.Container = "test" + u3.FsConfig.AzBlobConfig.AccountName = "Server-Account-Name" + u3.FsConfig.AzBlobConfig.AccountKey.Payload = "Server-Account-Key" + u3.FsConfig.AzBlobConfig.AccountKey.Status = vfs.SecretStatusPlain + user3, _, err := httpd.AddUser(u3, http.StatusOK) + assert.NoError(t, err) + + users, _, err := httpd.GetUsers(0, 0, "", http.StatusOK) + assert.NoError(t, err) + assert.GreaterOrEqual(t, len(users), 3) + for _, username := range usernames { + users, _, err = httpd.GetUsers(0, 0, username, http.StatusOK) + assert.NoError(t, err) + if assert.Len(t, users, 1) { + user := users[0] + assert.Empty(t, user.Password) + } + } + user1, _, err = httpd.GetUserByID(user1.ID, http.StatusOK) + assert.NoError(t, err) + assert.Empty(t, user1.Password) + assert.Empty(t, user1.FsConfig.S3Config.AccessSecret.Key) + assert.Empty(t, user1.FsConfig.S3Config.AccessSecret.AdditionalData) + assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.Status) + assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.Payload) + + user2, _, err = httpd.GetUserByID(user2.ID, http.StatusOK) + assert.NoError(t, err) + assert.Empty(t, user2.Password) + assert.Empty(t, user2.FsConfig.GCSConfig.Credentials.Key) + assert.Empty(t, user2.FsConfig.GCSConfig.Credentials.AdditionalData) + assert.NotEmpty(t, user2.FsConfig.GCSConfig.Credentials.Status) + assert.NotEmpty(t, user2.FsConfig.GCSConfig.Credentials.Payload) + + user3, _, err = httpd.GetUserByID(user3.ID, http.StatusOK) + assert.NoError(t, err) + assert.Empty(t, user3.Password) + assert.Empty(t, user3.FsConfig.AzBlobConfig.AccountKey.Key) + assert.Empty(t, user3.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + assert.NotEmpty(t, user3.FsConfig.AzBlobConfig.AccountKey.Status) + assert.NotEmpty(t, user3.FsConfig.AzBlobConfig.AccountKey.Payload) + + // finally check that we have all the data inside the data provider + user1, err = dataprovider.GetUserByID(user1.ID) + assert.NoError(t, err) + assert.NotEmpty(t, user1.Password) + assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.Key) + assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.AdditionalData) + assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.Status) + assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.Payload) + err = user1.FsConfig.S3Config.AccessSecret.Decrypt() + assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusPlain, user1.FsConfig.S3Config.AccessSecret.Status) + assert.Equal(t, u1.FsConfig.S3Config.AccessSecret.Payload, user1.FsConfig.S3Config.AccessSecret.Payload) + assert.Empty(t, user1.FsConfig.S3Config.AccessSecret.Key) + assert.Empty(t, user1.FsConfig.S3Config.AccessSecret.AdditionalData) + + user2, err = dataprovider.GetUserByID(user2.ID) + assert.NoError(t, err) + assert.NotEmpty(t, user2.Password) + assert.NotEmpty(t, user2.FsConfig.GCSConfig.Credentials.Key) + assert.NotEmpty(t, user2.FsConfig.GCSConfig.Credentials.AdditionalData) + assert.NotEmpty(t, user2.FsConfig.GCSConfig.Credentials.Status) + assert.NotEmpty(t, user2.FsConfig.GCSConfig.Credentials.Payload) + err = user2.FsConfig.GCSConfig.Credentials.Decrypt() + assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusPlain, user2.FsConfig.GCSConfig.Credentials.Status) + assert.Equal(t, u2.FsConfig.GCSConfig.Credentials.Payload, user2.FsConfig.GCSConfig.Credentials.Payload) + assert.Empty(t, user2.FsConfig.GCSConfig.Credentials.Key) + assert.Empty(t, user2.FsConfig.GCSConfig.Credentials.AdditionalData) + + user3, err = dataprovider.GetUserByID(user3.ID) + assert.NoError(t, err) + assert.NotEmpty(t, user3.Password) + assert.NotEmpty(t, user3.FsConfig.AzBlobConfig.AccountKey.Key) + assert.NotEmpty(t, user3.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + assert.NotEmpty(t, user3.FsConfig.AzBlobConfig.AccountKey.Status) + assert.NotEmpty(t, user3.FsConfig.AzBlobConfig.AccountKey.Payload) + err = user3.FsConfig.AzBlobConfig.AccountKey.Decrypt() + assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusPlain, user3.FsConfig.AzBlobConfig.AccountKey.Status) + assert.Equal(t, u3.FsConfig.AzBlobConfig.AccountKey.Payload, user3.FsConfig.AzBlobConfig.AccountKey.Payload) + assert.Empty(t, user3.FsConfig.AzBlobConfig.AccountKey.Key) + assert.Empty(t, user3.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + + _, err = httpd.RemoveUser(user1, http.StatusOK) + assert.NoError(t, err) + _, err = httpd.RemoveUser(user2, http.StatusOK) + assert.NoError(t, err) + _, err = httpd.RemoveUser(user3, http.StatusOK) + assert.NoError(t, err) + + err = dataprovider.Close() + assert.NoError(t, err) + err = config.LoadConfig(configDir, "") + assert.NoError(t, err) + providerConf = config.GetProviderConf() + providerConf.CredentialsPath = credentialsPath + err = os.RemoveAll(credentialsPath) + assert.NoError(t, err) + err = dataprovider.Initialize(providerConf, configDir) + assert.NoError(t, err) +} + +func TestSecretObject(t *testing.T) { + s := vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: "test data", + AdditionalData: "username", + } + require.True(t, s.IsValid()) + err := s.Encrypt() + require.NoError(t, err) + require.Equal(t, vfs.SecretStatusAES256GCM, s.Status) + require.NotEmpty(t, s.Payload) + require.NotEmpty(t, s.Key) + require.True(t, s.IsValid()) + err = s.Decrypt() + require.NoError(t, err) + require.Equal(t, vfs.SecretStatusPlain, s.Status) + require.Equal(t, "test data", s.Payload) + require.Empty(t, s.Key) + + oldFormat := "$aes$5b97e3a3324a2f53e2357483383367c0$0ed3132b584742ab217866219da633266782b69b13e50ebc6ddfb7c4fbf2f2a414c6d5f813" + s, err = vfs.GetSecretFromCompatString(oldFormat) + require.NoError(t, err) + require.True(t, s.IsValid()) + require.Equal(t, vfs.SecretStatusPlain, s.Status) + require.Equal(t, "test data", s.Payload) + require.Empty(t, s.Key) } func TestUpdateUserNoCredentials(t *testing.T) { @@ -2727,7 +2997,8 @@ func TestWebUserS3Mock(t *testing.T) { user.FsConfig.S3Config.Bucket = "test" user.FsConfig.S3Config.Region = "eu-west-1" user.FsConfig.S3Config.AccessKey = "access-key" - user.FsConfig.S3Config.AccessSecret = "access-secret" + user.FsConfig.S3Config.AccessSecret.Payload = "access-secret" + user.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusPlain user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?a=b" user.FsConfig.S3Config.StorageClass = "Standard" user.FsConfig.S3Config.KeyPrefix = "somedir/subdir/" @@ -2753,7 +3024,7 @@ func TestWebUserS3Mock(t *testing.T) { form.Set("s3_bucket", user.FsConfig.S3Config.Bucket) form.Set("s3_region", user.FsConfig.S3Config.Region) form.Set("s3_access_key", user.FsConfig.S3Config.AccessKey) - form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret) + form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret.Payload) form.Set("s3_storage_class", user.FsConfig.S3Config.StorageClass) form.Set("s3_endpoint", user.FsConfig.S3Config.Endpoint) form.Set("s3_key_prefix", user.FsConfig.S3Config.KeyPrefix) @@ -2800,9 +3071,46 @@ func TestWebUserS3Mock(t *testing.T) { assert.Equal(t, updateUser.FsConfig.S3Config.UploadPartSize, user.FsConfig.S3Config.UploadPartSize) assert.Equal(t, updateUser.FsConfig.S3Config.UploadConcurrency, user.FsConfig.S3Config.UploadConcurrency) assert.Equal(t, 2, len(updateUser.Filters.FileExtensions)) - if !strings.HasPrefix(updateUser.FsConfig.S3Config.AccessSecret, "$aes$") { - t.Error("s3 access secret is not encrypted") - } + assert.Equal(t, vfs.SecretStatusAES256GCM, updateUser.FsConfig.S3Config.AccessSecret.Status) + assert.NotEmpty(t, updateUser.FsConfig.S3Config.AccessSecret.Payload) + assert.Empty(t, updateUser.FsConfig.S3Config.AccessSecret.Key) + assert.Empty(t, updateUser.FsConfig.S3Config.AccessSecret.AdditionalData) + // now check that a redacted password is not saved + form.Set("s3_access_secret", "[**redacted**] ") + b, contentType, _ = getMultipartFormData(form, "", "") + req, _ = http.NewRequest(http.MethodPost, webUserPath+"/"+strconv.FormatInt(user.ID, 10), &b) + req.Header.Set("Content-Type", contentType) + rr = executeRequest(req) + checkResponseCode(t, http.StatusSeeOther, rr.Code) + req, _ = http.NewRequest(http.MethodGet, userPath+"?limit=1&offset=0&order=ASC&username="+user.Username, nil) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr.Code) + users = nil + err = render.DecodeJSON(rr.Body, &users) + assert.NoError(t, err) + assert.Equal(t, 1, len(users)) + lastUpdatedUser := users[0] + assert.Equal(t, vfs.SecretStatusAES256GCM, lastUpdatedUser.FsConfig.S3Config.AccessSecret.Status) + assert.Equal(t, updateUser.FsConfig.S3Config.AccessSecret.Payload, lastUpdatedUser.FsConfig.S3Config.AccessSecret.Payload) + assert.Empty(t, lastUpdatedUser.FsConfig.S3Config.AccessSecret.Key) + assert.Empty(t, lastUpdatedUser.FsConfig.S3Config.AccessSecret.AdditionalData) + // now clear credentials + form.Set("s3_access_key", "") + form.Set("s3_access_secret", "") + b, contentType, _ = getMultipartFormData(form, "", "") + req, _ = http.NewRequest(http.MethodPost, webUserPath+"/"+strconv.FormatInt(user.ID, 10), &b) + req.Header.Set("Content-Type", contentType) + rr = executeRequest(req) + checkResponseCode(t, http.StatusSeeOther, rr.Code) + req, _ = http.NewRequest(http.MethodGet, userPath+"?limit=1&offset=0&order=ASC&username="+user.Username, nil) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr.Code) + users = nil + err = render.DecodeJSON(rr.Body, &users) + assert.NoError(t, err) + assert.Equal(t, 1, len(users)) + assert.True(t, users[0].FsConfig.S3Config.AccessSecret.IsEmpty()) + req, _ = http.NewRequest(http.MethodDelete, userPath+"/"+strconv.FormatInt(user.ID, 10), nil) rr = executeRequest(req) checkResponseCode(t, http.StatusOK, rr.Code) @@ -2908,7 +3216,8 @@ func TestWebUserAzureBlobMock(t *testing.T) { user.FsConfig.Provider = dataprovider.AzureBlobFilesystemProvider user.FsConfig.AzBlobConfig.Container = "container" user.FsConfig.AzBlobConfig.AccountName = "aname" - user.FsConfig.AzBlobConfig.AccountKey = "access-skey" + user.FsConfig.AzBlobConfig.AccountKey.Payload = "access-skey" + user.FsConfig.AzBlobConfig.AccountKey.Status = vfs.SecretStatusPlain user.FsConfig.AzBlobConfig.Endpoint = "http://127.0.0.1:9000/path?b=c" user.FsConfig.AzBlobConfig.KeyPrefix = "somedir/subdir/" user.FsConfig.AzBlobConfig.UploadPartSize = 5 @@ -2933,7 +3242,7 @@ func TestWebUserAzureBlobMock(t *testing.T) { form.Set("fs_provider", "3") form.Set("az_container", user.FsConfig.AzBlobConfig.Container) form.Set("az_account_name", user.FsConfig.AzBlobConfig.AccountName) - form.Set("az_account_key", user.FsConfig.AzBlobConfig.AccountKey) + form.Set("az_account_key", user.FsConfig.AzBlobConfig.AccountKey.Payload) form.Set("az_sas_url", user.FsConfig.AzBlobConfig.SASURL) form.Set("az_endpoint", user.FsConfig.AzBlobConfig.Endpoint) form.Set("az_key_prefix", user.FsConfig.AzBlobConfig.KeyPrefix) @@ -2980,9 +3289,29 @@ func TestWebUserAzureBlobMock(t *testing.T) { assert.Equal(t, updateUser.FsConfig.AzBlobConfig.UploadPartSize, user.FsConfig.AzBlobConfig.UploadPartSize) assert.Equal(t, updateUser.FsConfig.AzBlobConfig.UploadConcurrency, user.FsConfig.AzBlobConfig.UploadConcurrency) assert.Equal(t, 2, len(updateUser.Filters.FileExtensions)) - if !strings.HasPrefix(updateUser.FsConfig.AzBlobConfig.AccountKey, "$aes$") { - t.Error("azure account secret is not encrypted") - } + assert.Equal(t, vfs.SecretStatusAES256GCM, updateUser.FsConfig.AzBlobConfig.AccountKey.Status) + assert.NotEmpty(t, updateUser.FsConfig.AzBlobConfig.AccountKey.Payload) + assert.Empty(t, updateUser.FsConfig.AzBlobConfig.AccountKey.Key) + assert.Empty(t, updateUser.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + // now check that a redacted password is not saved + form.Set("az_account_key", "[**redacted**] ") + b, contentType, _ = getMultipartFormData(form, "", "") + req, _ = http.NewRequest(http.MethodPost, webUserPath+"/"+strconv.FormatInt(user.ID, 10), &b) + req.Header.Set("Content-Type", contentType) + rr = executeRequest(req) + checkResponseCode(t, http.StatusSeeOther, rr.Code) + req, _ = http.NewRequest(http.MethodGet, userPath+"?limit=1&offset=0&order=ASC&username="+user.Username, nil) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr.Code) + users = nil + err = render.DecodeJSON(rr.Body, &users) + assert.NoError(t, err) + assert.Equal(t, 1, len(users)) + lastUpdatedUser := users[0] + assert.Equal(t, vfs.SecretStatusAES256GCM, lastUpdatedUser.FsConfig.AzBlobConfig.AccountKey.Status) + assert.Equal(t, updateUser.FsConfig.AzBlobConfig.AccountKey.Payload, lastUpdatedUser.FsConfig.AzBlobConfig.AccountKey.Payload) + assert.Empty(t, lastUpdatedUser.FsConfig.AzBlobConfig.AccountKey.Key) + assert.Empty(t, lastUpdatedUser.FsConfig.AzBlobConfig.AccountKey.AdditionalData) req, _ = http.NewRequest(http.MethodDelete, userPath+"/"+strconv.FormatInt(user.ID, 10), nil) rr = executeRequest(req) checkResponseCode(t, http.StatusOK, rr.Code) @@ -3211,9 +3540,3 @@ func getMultipartFormData(values url.Values, fileFieldName, filePath string) (by err := w.Close() return b, w.FormDataContentType(), err } - -type invalidBase64 []byte - -func (b invalidBase64) MarshalJSON() ([]byte, error) { - return []byte(`not base64`), nil -} diff --git a/httpd/internal_test.go b/httpd/internal_test.go index d50955a9..5a7b93c5 100644 --- a/httpd/internal_test.go +++ b/httpd/internal_test.go @@ -332,24 +332,36 @@ func TestCompareUserFsConfig(t *testing.T) { err = compareUserFsConfig(expected, actual) assert.Error(t, err) expected.FsConfig.S3Config.AccessKey = "" - actual.FsConfig.S3Config.AccessSecret = "access secret" + actual.FsConfig.S3Config.AccessSecret.Payload = "access secret" err = compareUserFsConfig(expected, actual) assert.Error(t, err) secret, _ := utils.EncryptData("access secret") - actual.FsConfig.S3Config.AccessSecret = "" - expected.FsConfig.S3Config.AccessSecret = secret + actual.FsConfig.S3Config.AccessSecret.Payload = "" + expected.FsConfig.S3Config.AccessSecret.Payload = secret err = compareUserFsConfig(expected, actual) assert.Error(t, err) - expected.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(secret) - actual.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(secret) + "a" + expected.FsConfig.S3Config.AccessSecret.Payload = "test" + actual.FsConfig.S3Config.AccessSecret.Payload = "" err = compareUserFsConfig(expected, actual) assert.Error(t, err) - expected.FsConfig.S3Config.AccessSecret = "test" - actual.FsConfig.S3Config.AccessSecret = "" + expected.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusPlain + actual.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusAES256GCM err = compareUserFsConfig(expected, actual) assert.Error(t, err) - expected.FsConfig.S3Config.AccessSecret = "" - actual.FsConfig.S3Config.AccessSecret = "" + actual.FsConfig.S3Config.AccessSecret.Payload = "payload" + actual.FsConfig.S3Config.AccessSecret.AdditionalData = "data" + err = compareUserFsConfig(expected, actual) + assert.Error(t, err) + actual.FsConfig.S3Config.AccessSecret.AdditionalData = "" + actual.FsConfig.S3Config.AccessSecret.Key = "key" + err = compareUserFsConfig(expected, actual) + assert.Error(t, err) + expected.FsConfig.S3Config.AccessSecret.Status = "" + expected.FsConfig.S3Config.AccessSecret.Payload = "" + actual.FsConfig.S3Config.AccessSecret.Status = "" + actual.FsConfig.S3Config.AccessSecret.Payload = "" + actual.FsConfig.S3Config.AccessSecret.AdditionalData = "" + actual.FsConfig.S3Config.AccessSecret.Key = "" expected.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/" err = compareUserFsConfig(expected, actual) assert.Error(t, err) @@ -403,10 +415,10 @@ func TestCompareUserAzureConfig(t *testing.T) { err = compareUserFsConfig(expected, actual) assert.Error(t, err) expected.FsConfig.AzBlobConfig.AccountName = "" - expected.FsConfig.AzBlobConfig.AccountKey = "akey" + expected.FsConfig.AzBlobConfig.AccountKey.Payload = "akey" err = compareUserFsConfig(expected, actual) assert.Error(t, err) - expected.FsConfig.AzBlobConfig.AccountKey = "" + expected.FsConfig.AzBlobConfig.AccountKey.Payload = "" expected.FsConfig.AzBlobConfig.Endpoint = "endpt" err = compareUserFsConfig(expected, actual) assert.Error(t, err) diff --git a/httpd/schema/openapi.yaml b/httpd/schema/openapi.yaml index 5237a4e4..45f80854 100644 --- a/httpd/schema/openapi.yaml +++ b/httpd/schema/openapi.yaml @@ -2,7 +2,7 @@ openapi: 3.0.3 info: title: SFTPGo description: 'SFTPGo REST API' - version: 2.0.3 + version: 2.1.0 servers: - url: /api/v1 @@ -11,6 +11,7 @@ security: paths: /healthz: get: + security: [] servers: - url : / tags: @@ -956,6 +957,22 @@ components: nullable: true description: maximum allowed size, as bytes, for a single file upload. The upload will be aborted if/when the size of the file being sent exceeds this limit. 0 means unlimited. This restriction does not apply for SSH system commands such as `git` and `rsync` description: Additional restrictions + Secret: + type: object + properties: + status: + type: string + enum: + - Plain + - AES-256-GCM + - Redacted + description: Set to "Plain" to add or update an existing secret, set to "Redacted" to preserve the existing value + payload: + type: string + key: + type: string + additional_data: + type: string S3Config: type: object properties: @@ -968,8 +985,7 @@ components: access_key: type: string access_secret: - type: string - description: the access secret is stored encrypted (AES-256-GCM) + $ref: '#/components/schemas/Secret' endpoint: type: string description: optional endpoint @@ -997,9 +1013,7 @@ components: type: string minLength: 1 credentials: - type: string - format: byte - description: Google Cloud Storage JSON credentials base64 encoded. This field must be populated only when adding/updating a user. It will be always omitted, since there are sensitive data, when you search/get users. The credentials will be stored in the configured "credentials_path" + $ref: '#/components/schemas/Secret' automatic_credentials: type: integer nullable: true @@ -1019,7 +1033,7 @@ components: required: - bucket nullable: true - description: Google Cloud Storage configuration details + description: Google Cloud Storage configuration details. The "credentials" field must be populated only when adding/updating a user. It will be always omitted, since there are sensitive data, when you search/get users AzureBlobFsConfig: type: object properties: @@ -1029,8 +1043,7 @@ components: type: string description: Storage Account Name, leave blank to use SAS URL account_key: - type: string - description: Storage Account Key leave blank to use SAS URL. The access key is stored encrypted (AES-256-GCM) + $ref: '#/components/schemas/Secret' sas_url: type: string description: Shared access signature URL, leave blank if using account/key diff --git a/httpd/web.go b/httpd/web.go index 6a0af1f0..b76d9925 100644 --- a/httpd/web.go +++ b/httpd/web.go @@ -39,6 +39,7 @@ const ( page500Body = "The server is unable to fulfill your request." defaultQueryLimit = 500 webDateTimeFormat = "2006-01-02 15:04:05" // YYYY-MM-DD HH:MM:SS + redactedSecret = "[**redacted**]" ) var ( @@ -81,7 +82,6 @@ type connectionsPage struct { type userPage struct { basePage - IsAdd bool User dataprovider.User RootPerms []string Error string @@ -89,6 +89,10 @@ type userPage struct { ValidSSHLoginMethods []string ValidProtocols []string RootDirPerms []string + RedactedSecret string + IsAdd bool + IsS3SecretEnc bool + IsAzSecretEnc bool } type folderPage struct { @@ -210,6 +214,9 @@ func renderAddUserPage(w http.ResponseWriter, user dataprovider.User, error stri ValidSSHLoginMethods: dataprovider.ValidSSHLoginMethods, ValidProtocols: dataprovider.ValidProtocols, RootDirPerms: user.GetPermissionsForPath("/"), + IsS3SecretEnc: user.FsConfig.S3Config.AccessSecret.IsEncrypted(), + IsAzSecretEnc: user.FsConfig.AzBlobConfig.AccountKey.IsEncrypted(), + RedactedSecret: redactedSecret, } renderTemplate(w, templateUser, data) } @@ -224,6 +231,9 @@ func renderUpdateUserPage(w http.ResponseWriter, user dataprovider.User, error s ValidSSHLoginMethods: dataprovider.ValidSSHLoginMethods, ValidProtocols: dataprovider.ValidProtocols, RootDirPerms: user.GetPermissionsForPath("/"), + IsS3SecretEnc: user.FsConfig.S3Config.AccessSecret.IsEncrypted(), + IsAzSecretEnc: user.FsConfig.AzBlobConfig.AccountKey.IsEncrypted(), + RedactedSecret: redactedSecret, } renderTemplate(w, templateUser, data) } @@ -420,6 +430,20 @@ func getFiltersFromUserPostFields(r *http.Request) dataprovider.UserFilters { return filters } +func getSecretFromFormField(r *http.Request, field string) vfs.Secret { + secret := vfs.Secret{ + Payload: r.Form.Get(field), + Status: vfs.SecretStatusPlain, + } + if strings.TrimSpace(secret.Payload) == redactedSecret { + secret.Status = vfs.SecretStatusRedacted + } + if strings.TrimSpace(secret.Payload) == "" { + secret.Status = "" + } + return secret +} + func getFsConfigFromUserPostFields(r *http.Request) (dataprovider.Filesystem, error) { var fs dataprovider.Filesystem provider, err := strconv.Atoi(r.Form.Get("fs_provider")) @@ -431,7 +455,7 @@ func getFsConfigFromUserPostFields(r *http.Request) (dataprovider.Filesystem, er fs.S3Config.Bucket = r.Form.Get("s3_bucket") fs.S3Config.Region = r.Form.Get("s3_region") fs.S3Config.AccessKey = r.Form.Get("s3_access_key") - fs.S3Config.AccessSecret = r.Form.Get("s3_access_secret") + fs.S3Config.AccessSecret = getSecretFromFormField(r, "s3_access_secret") fs.S3Config.Endpoint = r.Form.Get("s3_endpoint") fs.S3Config.StorageClass = r.Form.Get("s3_storage_class") fs.S3Config.KeyPrefix = r.Form.Get("s3_key_prefix") @@ -468,12 +492,15 @@ func getFsConfigFromUserPostFields(r *http.Request) (dataprovider.Filesystem, er } return fs, err } - fs.GCSConfig.Credentials = fileBytes + fs.GCSConfig.Credentials = vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: string(fileBytes), + } fs.GCSConfig.AutomaticCredentials = 0 } else if fs.Provider == dataprovider.AzureBlobFilesystemProvider { fs.AzBlobConfig.Container = r.Form.Get("az_container") fs.AzBlobConfig.AccountName = r.Form.Get("az_account_name") - fs.AzBlobConfig.AccountKey = r.Form.Get("az_account_key") + fs.AzBlobConfig.AccountKey = getSecretFromFormField(r, "az_account_key") fs.AzBlobConfig.SASURL = r.Form.Get("az_sas_url") fs.AzBlobConfig.Endpoint = r.Form.Get("az_endpoint") fs.AzBlobConfig.KeyPrefix = r.Form.Get("az_key_prefix") @@ -655,6 +682,12 @@ func handleWebUpdateUserPost(w http.ResponseWriter, r *http.Request) { if len(updatedUser.Password) == 0 { updatedUser.Password = user.Password } + if !updatedUser.FsConfig.S3Config.AccessSecret.IsPlain() && !updatedUser.FsConfig.S3Config.AccessSecret.IsEmpty() { + updatedUser.FsConfig.S3Config.AccessSecret = user.FsConfig.S3Config.AccessSecret + } + if !updatedUser.FsConfig.AzBlobConfig.AccountKey.IsPlain() && !updatedUser.FsConfig.AzBlobConfig.AccountKey.IsEmpty() { + updatedUser.FsConfig.AzBlobConfig.AccountKey = user.FsConfig.AzBlobConfig.AccountKey + } err = dataprovider.UpdateUser(updatedUser) if err == nil { if len(r.Form.Get("disconnect")) > 0 { diff --git a/service/service.go b/service/service.go index 8e64c608..61580903 100644 --- a/service/service.go +++ b/service/service.go @@ -2,7 +2,6 @@ package service import ( - "encoding/json" "fmt" "io/ioutil" "os" @@ -207,9 +206,7 @@ func (s *Service) loadInitialData() error { if err != nil { return fmt.Errorf("unable to read input file %#v: %v", s.LoadDataFrom, err) } - var dump dataprovider.BackupData - - err = json.Unmarshal(content, &dump) + dump, err := dataprovider.ParseDumpData(content) if err != nil { return fmt.Errorf("unable to parse file to restore %#v: %v", s.LoadDataFrom, err) } diff --git a/sftpd/sftpd_test.go b/sftpd/sftpd_test.go index b18de796..5460e580 100644 --- a/sftpd/sftpd_test.go +++ b/sftpd/sftpd_test.go @@ -1312,7 +1312,10 @@ func TestLoginWithDatabaseCredentials(t *testing.T) { u := getTestUser(usePubKey) u.FsConfig.Provider = dataprovider.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "testbucket" - u.FsConfig.GCSConfig.Credentials = []byte(`{ "type": "service_account" }`) + u.FsConfig.GCSConfig.Credentials = vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: `{ "type": "service_account" }`, + } providerConf := config.GetProviderConf() providerConf.PreferDatabaseCredentials = true @@ -1333,9 +1336,12 @@ func TestLoginWithDatabaseCredentials(t *testing.T) { user, _, err := httpd.AddUser(u, http.StatusOK) assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.GCSConfig.Credentials.Status) + assert.NotEmpty(t, user.FsConfig.GCSConfig.Credentials.Payload) + assert.Empty(t, user.FsConfig.GCSConfig.Credentials.AdditionalData) + assert.Empty(t, user.FsConfig.GCSConfig.Credentials.Key) - _, err = os.Stat(credentialsFile) - assert.Error(t, err) + assert.NoFileExists(t, credentialsFile) client, err := getSftpClient(user, usePubKey) if assert.NoError(t, err) { @@ -1358,7 +1364,10 @@ func TestLoginInvalidFs(t *testing.T) { u := getTestUser(usePubKey) u.FsConfig.Provider = dataprovider.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "test" - u.FsConfig.GCSConfig.Credentials = []byte("invalid JSON for credentials") + u.FsConfig.GCSConfig.Credentials = vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: "invalid JSON for credentials", + } user, _, err := httpd.AddUser(u, http.StatusOK) assert.NoError(t, err) diff --git a/templates/user.html b/templates/user.html index 93449e9f..dccca447 100644 --- a/templates/user.html +++ b/templates/user.html @@ -337,7 +337,7 @@
+ value="{{if .IsS3SecretEnc}}{{.RedactedSecret}}{{else}}{{.User.FsConfig.S3Config.AccessSecret.Payload}}{{end}}" maxlength="1000">
@@ -345,7 +345,7 @@
+ value="{{.User.FsConfig.S3Config.StorageClass}}" maxlength="255">
@@ -448,7 +448,7 @@
+ value="{{if .IsAzSecretEnc}}{{.RedactedSecret}}{{else}}{{.User.FsConfig.AzBlobConfig.AccountKey.Payload}}{{end}}" maxlength="1000">
diff --git a/utils/utils.go b/utils/utils.go index b4e270bf..5d4f4b16 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -193,6 +193,9 @@ func DecryptData(data string) (string, error) { return result, err } nonceSize := gcm.NonceSize() + if len(encrypted) < nonceSize { + return result, errors.New("malformed ciphertext") + } nonce, ciphertext := encrypted[:nonceSize], encrypted[nonceSize:] plaintext, err := gcm.Open(nil, nonce, ciphertext, nil) if err != nil { diff --git a/vfs/azblobfs.go b/vfs/azblobfs.go index 5a8262fb..61a094ee 100644 --- a/vfs/azblobfs.go +++ b/vfs/azblobfs.go @@ -24,7 +24,6 @@ import ( "github.com/drakkan/sftpgo/logger" "github.com/drakkan/sftpgo/metrics" - "github.com/drakkan/sftpgo/utils" "github.com/drakkan/sftpgo/version" ) @@ -61,12 +60,11 @@ func NewAzBlobFs(connectionID, localTempDir string, config AzBlobFsConfig) (Fs, if err := ValidateAzBlobFsConfig(&fs.config); err != nil { return fs, err } - if fs.config.AccountKey != "" { - accountKey, err := utils.DecryptData(fs.config.AccountKey) + if fs.config.AccountKey.IsEncrypted() { + err := fs.config.AccountKey.Decrypt() if err != nil { return fs, err } - fs.config.AccountKey = accountKey } fs.setConfigDefaults() @@ -106,7 +104,7 @@ func NewAzBlobFs(connectionID, localTempDir string, config AzBlobFsConfig) (Fs, return fs, nil } - credential, err := azblob.NewSharedKeyCredential(fs.config.AccountName, fs.config.AccountKey) + credential, err := azblob.NewSharedKeyCredential(fs.config.AccountName, fs.config.AccountKey.Payload) if err != nil { return fs, fmt.Errorf("invalid credentials: %v", err) } diff --git a/vfs/gcsfs.go b/vfs/gcsfs.go index e5d80c53..d09d24b9 100644 --- a/vfs/gcsfs.go +++ b/vfs/gcsfs.go @@ -4,9 +4,11 @@ package vfs import ( "context" + "encoding/json" "errors" "fmt" "io" + "io/ioutil" "mime" "net/http" "os" @@ -60,10 +62,28 @@ func NewGCSFs(connectionID, localTempDir string, config GCSFsConfig) (Fs, error) ctx := context.Background() if fs.config.AutomaticCredentials > 0 { fs.svc, err = storage.NewClient(ctx) - } else if len(fs.config.Credentials) > 0 { - fs.svc, err = storage.NewClient(ctx, option.WithCredentialsJSON(fs.config.Credentials)) + } else if fs.config.Credentials.IsEncrypted() { + err = fs.config.Credentials.Decrypt() + if err != nil { + return fs, err + } + fs.svc, err = storage.NewClient(ctx, option.WithCredentialsJSON([]byte(fs.config.Credentials.Payload))) } else { - fs.svc, err = storage.NewClient(ctx, option.WithCredentialsFile(fs.config.CredentialFile)) + var creds []byte + creds, err = ioutil.ReadFile(fs.config.CredentialFile) + if err != nil { + return fs, err + } + secret := &Secret{} + err = json.Unmarshal(creds, secret) + if err != nil { + return fs, err + } + err = secret.Decrypt() + if err != nil { + return fs, err + } + fs.svc, err = storage.NewClient(ctx, option.WithCredentialsJSON([]byte(secret.Payload))) } return fs, err } diff --git a/vfs/s3fs.go b/vfs/s3fs.go index 6b275306..8bdd309c 100644 --- a/vfs/s3fs.go +++ b/vfs/s3fs.go @@ -60,13 +60,12 @@ func NewS3Fs(connectionID, localTempDir string, config S3FsConfig) (Fs, error) { awsConfig.WithRegion(fs.config.Region) } - if fs.config.AccessSecret != "" { - accessSecret, err := utils.DecryptData(fs.config.AccessSecret) + if fs.config.AccessSecret.IsEncrypted() { + err := fs.config.AccessSecret.Decrypt() if err != nil { return fs, err } - fs.config.AccessSecret = accessSecret - awsConfig.Credentials = credentials.NewStaticCredentials(fs.config.AccessKey, fs.config.AccessSecret, "") + awsConfig.Credentials = credentials.NewStaticCredentials(fs.config.AccessKey, fs.config.AccessSecret.Payload, "") } if fs.config.Endpoint != "" { diff --git a/vfs/secret.go b/vfs/secret.go new file mode 100644 index 00000000..9d16f088 --- /dev/null +++ b/vfs/secret.go @@ -0,0 +1,209 @@ +package vfs + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + + "github.com/drakkan/sftpgo/utils" +) + +// SecretStatus defines the statuses of a Secret object +type SecretStatus = string + +const ( + // SecretStatusPlain means the secret is in plain text and must be encrypted + SecretStatusPlain SecretStatus = "Plain" + // SecretStatusAES256GCM means the secret is encrypted using AES-256-GCM + SecretStatusAES256GCM SecretStatus = "AES-256-GCM" + // SecretStatusRedacted means the secret is redacted + SecretStatusRedacted SecretStatus = "Redacted" +) + +var ( + errWrongSecretStatus = errors.New("wrong secret status") + errMalformedCiphertext = errors.New("malformed ciphertext") + errInvalidSecret = errors.New("invalid secret") + validSecretStatuses = []string{SecretStatusPlain, SecretStatusAES256GCM, SecretStatusRedacted} +) + +// Secret defines the struct used to store confidential data +type Secret struct { + Status SecretStatus `json:"status,omitempty"` + Payload string `json:"payload,omitempty"` + Key string `json:"key,omitempty"` + AdditionalData string `json:"additional_data,omitempty"` +} + +// GetSecretFromCompatString returns a secret from the previous format +func GetSecretFromCompatString(secret string) (Secret, error) { + s := Secret{} + plain, err := utils.DecryptData(secret) + if err != nil { + return s, errMalformedCiphertext + } + s.Status = SecretStatusPlain + s.Payload = plain + return s, nil +} + +// IsEncrypted returns true if the secret is encrypted +// This isn't a pointer receiver because we don't want to pass +// a pointer to html template +func (s *Secret) IsEncrypted() bool { + return s.Status == SecretStatusAES256GCM +} + +// IsPlain returns true if the secret is in plain text +func (s *Secret) IsPlain() bool { + return s.Status == SecretStatusPlain +} + +// IsRedacted returns true if the secret is redacted +func (s *Secret) IsRedacted() bool { + return s.Status == SecretStatusRedacted +} + +// IsEmpty returns true if all fields are empty +func (s *Secret) IsEmpty() bool { + if s.Status != "" { + return false + } + if s.Payload != "" { + return false + } + if s.Key != "" { + return false + } + if s.AdditionalData != "" { + return false + } + return true +} + +// IsValid returns true if the secret is not empty and valid +func (s *Secret) IsValid() bool { + if !s.IsValidInput() { + return false + } + if s.Status == SecretStatusAES256GCM { + if len(s.Key) != 64 { + return false + } + } + return true +} + +// IsValidInput returns true if the secret is a valid user input +func (s *Secret) IsValidInput() bool { + if !utils.IsStringInSlice(s.Status, validSecretStatuses) { + return false + } + if s.Payload == "" { + return false + } + return true +} + +// Hide hides info to decrypt data +func (s *Secret) Hide() { + s.Key = "" + s.AdditionalData = "" +} + +// deriveKey is a weak method of deriving a key but it is still better than using the key as it is. +// We should use a KMS in future +func (s *Secret) deriveKey(key []byte) []byte { + var combined []byte + combined = append(combined, key...) + if s.AdditionalData != "" { + combined = append(combined, []byte(s.AdditionalData)...) + } + combined = append(combined, key...) + hash := sha256.Sum256(combined) + return hash[:] +} + +// Encrypt encrypts a plain text Secret object +func (s *Secret) Encrypt() error { + if s.Payload == "" { + return errInvalidSecret + } + switch s.Status { + case SecretStatusPlain: + key := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, key); err != nil { + return err + } + block, err := aes.NewCipher(s.deriveKey(key)) + if err != nil { + return err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err = io.ReadFull(rand.Reader, nonce); err != nil { + return err + } + var aad []byte + if s.AdditionalData != "" { + aad = []byte(s.AdditionalData) + } + ciphertext := gcm.Seal(nonce, nonce, []byte(s.Payload), aad) + s.Key = hex.EncodeToString(key) + s.Payload = hex.EncodeToString(ciphertext) + s.Status = SecretStatusAES256GCM + return nil + default: + return errWrongSecretStatus + } +} + +// Decrypt decrypts a Secret object +func (s *Secret) Decrypt() error { + switch s.Status { + case SecretStatusAES256GCM: + encrypted, err := hex.DecodeString(s.Payload) + if err != nil { + return err + } + key, err := hex.DecodeString(s.Key) + if err != nil { + return err + } + block, err := aes.NewCipher(s.deriveKey(key)) + if err != nil { + return err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return err + } + nonceSize := gcm.NonceSize() + if len(encrypted) < nonceSize { + return errMalformedCiphertext + } + nonce, ciphertext := encrypted[:nonceSize], encrypted[nonceSize:] + var aad []byte + if s.AdditionalData != "" { + aad = []byte(s.AdditionalData) + } + plaintext, err := gcm.Open(nil, nonce, ciphertext, aad) + if err != nil { + return err + } + s.Status = SecretStatusPlain + s.Payload = string(plaintext) + s.Key = "" + s.AdditionalData = "" + return nil + default: + return errWrongSecretStatus + } +} diff --git a/vfs/vfs.go b/vfs/vfs.go index e60a68cd..1842955b 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -113,7 +113,7 @@ type S3FsConfig struct { KeyPrefix string `json:"key_prefix,omitempty"` Region string `json:"region,omitempty"` AccessKey string `json:"access_key,omitempty"` - AccessSecret string `json:"access_secret,omitempty"` + AccessSecret Secret `json:"access_secret,omitempty"` Endpoint string `json:"endpoint,omitempty"` StorageClass string `json:"storage_class,omitempty"` // The buffer size (in MB) to use for multipart uploads. The minimum allowed part size is 5MB, @@ -137,9 +137,10 @@ type GCSFsConfig struct { // folder. The prefix, if not empty, must not start with "/" and must // end with "/". // If empty the whole bucket contents will be available - KeyPrefix string `json:"key_prefix,omitempty"` - CredentialFile string `json:"-"` - Credentials []byte `json:"credentials,omitempty"` + KeyPrefix string `json:"key_prefix,omitempty"` + CredentialFile string `json:"-"` + Credentials Secret `json:"credentials,omitempty"` + // 0 explicit, 1 automatic AutomaticCredentials int `json:"automatic_credentials,omitempty"` StorageClass string `json:"storage_class,omitempty"` } @@ -151,7 +152,7 @@ type AzBlobFsConfig struct { AccountName string `json:"account_name,omitempty"` // Storage Account Key leave blank to use SAS URL. // The access key is stored encrypted (AES-256-GCM) - AccountKey string `json:"account_key,omitempty"` + AccountKey Secret `json:"account_key,omitempty"` // Optional endpoint. Default is "blob.core.windows.net". // If you use the emulator the endpoint must include the protocol, // for example "http://127.0.0.1:10000" @@ -235,20 +236,33 @@ func IsLocalOsFs(fs Fs) bool { return fs.Name() == osFsName } -// ValidateS3FsConfig returns nil if the specified s3 config is valid, otherwise an error -func ValidateS3FsConfig(config *S3FsConfig) error { - if len(config.Bucket) == 0 { - return errors.New("bucket cannot be empty") - } - if len(config.Region) == 0 { - return errors.New("region cannot be empty") - } - if len(config.AccessKey) == 0 && len(config.AccessSecret) > 0 { +func checkS3Credentials(config *S3FsConfig) error { + if config.AccessKey == "" && !config.AccessSecret.IsEmpty() { return errors.New("access_key cannot be empty with access_secret not empty") } - if len(config.AccessSecret) == 0 && len(config.AccessKey) > 0 { + if config.AccessSecret.IsEmpty() && config.AccessKey != "" { return errors.New("access_secret cannot be empty with access_key not empty") } + if config.AccessSecret.IsEncrypted() && !config.AccessSecret.IsValid() { + return errors.New("invalid encrypted access_secret") + } + if !config.AccessSecret.IsEmpty() && !config.AccessSecret.IsValidInput() { + return errors.New("invalid access_secret") + } + return nil +} + +// ValidateS3FsConfig returns nil if the specified s3 config is valid, otherwise an error +func ValidateS3FsConfig(config *S3FsConfig) error { + if config.Bucket == "" { + return errors.New("bucket cannot be empty") + } + if config.Region == "" { + return errors.New("region cannot be empty") + } + if err := checkS3Credentials(config); err != nil { + return err + } if config.KeyPrefix != "" { if strings.HasPrefix(config.KeyPrefix, "/") { return errors.New("key_prefix cannot start with /") @@ -281,7 +295,10 @@ func ValidateGCSFsConfig(config *GCSFsConfig, credentialsFilePath string) error config.KeyPrefix += "/" } } - if len(config.Credentials) == 0 && config.AutomaticCredentials == 0 { + if config.Credentials.IsEncrypted() && !config.Credentials.IsValid() { + return errors.New("invalid encrypted credentials") + } + if !config.Credentials.IsValidInput() && config.AutomaticCredentials == 0 { fi, err := os.Stat(credentialsFilePath) if err != nil { return fmt.Errorf("invalid credentials %v", err) @@ -302,8 +319,11 @@ func ValidateAzBlobFsConfig(config *AzBlobFsConfig) error { if config.Container == "" { return errors.New("container cannot be empty") } - if config.AccountName == "" || config.AccountKey == "" { - return errors.New("credentials cannot be empty") + if config.AccountName == "" || !config.AccountKey.IsValidInput() { + return errors.New("credentials cannot be empty or invalid") + } + if config.AccountKey.IsEncrypted() && !config.AccountKey.IsValid() { + return errors.New("invalid encrypted account_key") } if config.KeyPrefix != "" { if strings.HasPrefix(config.KeyPrefix, "/") { diff --git a/webdavd/webdavd_test.go b/webdavd/webdavd_test.go index 8a6eca21..6f9838d1 100644 --- a/webdavd/webdavd_test.go +++ b/webdavd/webdavd_test.go @@ -861,7 +861,10 @@ func TestLoginWithDatabaseCredentials(t *testing.T) { u := getTestUser() u.FsConfig.Provider = dataprovider.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "test" - u.FsConfig.GCSConfig.Credentials = []byte(`{ "type": "service_account" }`) + u.FsConfig.GCSConfig.Credentials = vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: `{ "type": "service_account" }`, + } providerConf := config.GetProviderConf() providerConf.PreferDatabaseCredentials = true @@ -882,9 +885,12 @@ func TestLoginWithDatabaseCredentials(t *testing.T) { user, _, err := httpd.AddUser(u, http.StatusOK) assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.GCSConfig.Credentials.Status) + assert.NotEmpty(t, user.FsConfig.GCSConfig.Credentials.Payload) + assert.Empty(t, user.FsConfig.GCSConfig.Credentials.AdditionalData) + assert.Empty(t, user.FsConfig.GCSConfig.Credentials.Key) - _, err = os.Stat(credentialsFile) - assert.Error(t, err) + assert.NoFileExists(t, credentialsFile) client := getWebDavClient(user) @@ -906,7 +912,10 @@ func TestLoginInvalidFs(t *testing.T) { u := getTestUser() u.FsConfig.Provider = dataprovider.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "test" - u.FsConfig.GCSConfig.Credentials = []byte("invalid JSON for credentials") + u.FsConfig.GCSConfig.Credentials = vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: "invalid JSON for credentials", + } user, _, err := httpd.AddUser(u, http.StatusOK) assert.NoError(t, err)