diff --git a/cmd/portable.go b/cmd/portable.go index b9d45449..6b6c8ebb 100644 --- a/cmd/portable.go +++ b/cmd/portable.go @@ -3,7 +3,6 @@ package cmd import ( - "encoding/base64" "fmt" "io/ioutil" "os" @@ -77,7 +76,7 @@ Please take a look at the usage below to customize the serving parameters`, } permissions := make(map[string][]string) permissions["/"] = portablePermissions - portableGCSCredentials := "" + var portableGCSCredentials []byte if fsProvider == dataprovider.GCSFilesystemProvider && len(portableGCSCredentialsFile) > 0 { fi, err := os.Stat(portableGCSCredentialsFile) if err != nil { @@ -93,7 +92,7 @@ Please take a look at the usage below to customize the serving parameters`, if err != nil { fmt.Printf("Unable to read credentials file: %v\n", err) } - portableGCSCredentials = base64.StdEncoding.EncodeToString(creds) + portableGCSCredentials = creds portableGCSAutoCredentials = 0 } if portableFTPDPort >= 0 && len(portableFTPSCert) > 0 && len(portableFTPSKey) > 0 { diff --git a/config/config.go b/config/config.go index 505ddc7d..35f410d3 100644 --- a/config/config.go +++ b/config/config.go @@ -141,7 +141,8 @@ func init() { Parallelism: 2, }, }, - UpdateMode: 0, + UpdateMode: 0, + PreferDatabaseCredentials: false, }, HTTPDConfig: httpd.Conf{ BindPort: 8080, diff --git a/dataprovider/bolt.go b/dataprovider/bolt.go index 55bcd47e..33b0e2dc 100644 --- a/dataprovider/bolt.go +++ b/dataprovider/bolt.go @@ -348,6 +348,7 @@ func (p BoltProvider) updateUser(user User) error { return err } } + user.ID = oldUser.ID user.LastQuotaUpdate = oldUser.LastQuotaUpdate user.UsedQuotaSize = oldUser.UsedQuotaSize user.UsedQuotaFiles = oldUser.UsedQuotaFiles diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go index 2c50e695..bea725ba 100644 --- a/dataprovider/dataprovider.go +++ b/dataprovider/dataprovider.go @@ -249,12 +249,16 @@ type Config struct { // - 4 means WebDAV // you can combine the scopes, for example 6 means FTP and WebDAV CheckPasswordScope int `json:"check_password_scope" mapstructure:"check_password_scope"` - // PasswordHashing defines the configuration for password hashing - PasswordHashing PasswordHashing `json:"password_hashing" mapstructure:"password_hashing"` // Defines how the database will be initialized/updated: // - 0 means automatically // - 1 means manually using the initprovider sub-command UpdateMode int `json:"update_mode" mapstructure:"update_mode"` + // PasswordHashing defines the configuration for password hashing + PasswordHashing PasswordHashing `json:"password_hashing" mapstructure:"password_hashing"` + // PreferDatabaseCredentials indicates whether credential files (currently used for Google + // Cloud Storage) should be stored in the database instead of in the directory specified by + // CredentialsPath. + PreferDatabaseCredentials bool `json:"prefer_database_credentials" mapstructure:"prefer_database_credentials"` } // BackupData defines the structure for the backup/restore files @@ -974,15 +978,14 @@ func saveGCSCredentials(user *User) error { if len(user.FsConfig.GCSConfig.Credentials) == 0 { return nil } - decoded, err := base64.StdEncoding.DecodeString(user.FsConfig.GCSConfig.Credentials) - if err != nil { - return &ValidationError{err: fmt.Sprintf("could not validate GCS credentials: %v", err)} + if config.PreferDatabaseCredentials { + return nil } - err = ioutil.WriteFile(user.getGCSCredentialsFilePath(), decoded, 0600) + err := ioutil.WriteFile(user.getGCSCredentialsFilePath(), user.FsConfig.GCSConfig.Credentials, 0600) if err != nil { return &ValidationError{err: fmt.Sprintf("could not save GCS credentials: %v", err)} } - user.FsConfig.GCSConfig.Credentials = "" + user.FsConfig.GCSConfig.Credentials = nil return nil } @@ -1244,7 +1247,7 @@ func HideUserSensitiveData(user *User) User { if user.FsConfig.Provider == S3FilesystemProvider { user.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(user.FsConfig.S3Config.AccessSecret) } else if user.FsConfig.Provider == GCSFilesystemProvider { - user.FsConfig.GCSConfig.Credentials = "" + user.FsConfig.GCSConfig.Credentials = nil } return *user } @@ -1256,11 +1259,17 @@ func addCredentialsToUser(user *User) error { if user.FsConfig.GCSConfig.AutomaticCredentials > 0 { return nil } + + // Don't read from file if credentials have already been set + if len(user.FsConfig.GCSConfig.Credentials) > 0 { + return nil + } + cred, err := ioutil.ReadFile(user.getGCSCredentialsFilePath()) if err != nil { return err } - user.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString(cred) + user.FsConfig.GCSConfig.Credentials = cred return nil } diff --git a/dataprovider/user.go b/dataprovider/user.go index a65a7149..ee349168 100644 --- a/dataprovider/user.go +++ b/dataprovider/user.go @@ -720,6 +720,7 @@ func (u *User) getACopy() User { GCSConfig: vfs.GCSFsConfig{ Bucket: u.FsConfig.GCSConfig.Bucket, CredentialFile: u.FsConfig.GCSConfig.CredentialFile, + Credentials: u.FsConfig.GCSConfig.Credentials, AutomaticCredentials: u.FsConfig.GCSConfig.AutomaticCredentials, StorageClass: u.FsConfig.GCSConfig.StorageClass, KeyPrefix: u.FsConfig.GCSConfig.KeyPrefix, diff --git a/docs/full-configuration.md b/docs/full-configuration.md index 3240dcb4..09c460b0 100644 --- a/docs/full-configuration.md +++ b/docs/full-configuration.md @@ -137,6 +137,7 @@ The configuration file contains the following sections: - `external_auth_hook`, string. Absolute path to an external program or an HTTP URL to invoke for users authentication. See [External Authentication](./external-auth.md) for more details. Leave empty to disable. - `external_auth_scope`, integer. 0 means all supported authentication scopes (passwords, public keys and keyboard interactive). 1 means passwords only. 2 means public keys only. 4 means key keyboard interactive only. The flags can be combined, for example 6 means public keys and keyboard interactive - `credentials_path`, string. It defines the directory for storing user provided credential files such as Google Cloud Storage credentials. This can be an absolute path or a path relative to the config dir + - `prefer_database_credentials`, boolean. When true, users' Google Cloud Storage credentials will be written to the data provider instead of disk, though pre-existing credentials on disk will be used as a fallback. When false, they will be written to the directory specified by `credentials_path`. - `pre_login_program`, string. Deprecated, please use `pre_login_hook`. - `pre_login_hook`, string. Absolute path to an external program or an HTTP URL to invoke to modify user details just before the login. See [Dynamic user modification](./dynamic-user-mod.md) for more details. Leave empty to disable. - `post_login_hook`, string. Absolute path to an external program or an HTTP URL to invoke to notify a successful or failed login. See [Post-login hook](./post-login-hook.md) for more details. Leave empty to disable. diff --git a/ftpd/ftpd_test.go b/ftpd/ftpd_test.go index 0eb1f0fe..7328ac99 100644 --- a/ftpd/ftpd_test.go +++ b/ftpd/ftpd_test.go @@ -3,7 +3,6 @@ package ftpd_test import ( "crypto/rand" "crypto/tls" - "encoding/base64" "encoding/json" "fmt" "io" @@ -856,21 +855,70 @@ func TestLoginWithIPilters(t *testing.T) { assert.NoError(t, err) } +func TestLoginWithDatabaseCredentials(t *testing.T) { + u := getTestUser() + u.FsConfig.Provider = dataprovider.GCSFilesystemProvider + u.FsConfig.GCSConfig.Bucket = "test" + u.FsConfig.GCSConfig.Credentials = []byte(`{ "type": "service_account" }`) + + providerConf := config.GetProviderConf() + providerConf.PreferDatabaseCredentials = true + credentialsFile := filepath.Join(providerConf.CredentialsPath, fmt.Sprintf("%v_gcs_credentials.json", u.Username)) + if !filepath.IsAbs(credentialsFile) { + credentialsFile = filepath.Join(configDir, credentialsFile) + } + + assert.NoError(t, dataprovider.Close()) + + err := dataprovider.Initialize(providerConf, configDir) + assert.NoError(t, err) + + if _, err = os.Stat(credentialsFile); err == nil { + // remove the credentials file + assert.NoError(t, os.Remove(credentialsFile)) + } + + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + + _, err = os.Stat(credentialsFile) + assert.Error(t, err) + + client, err := getFTPClient(user, false) + if assert.NoError(t, err) { + err = client.Quit() + assert.NoError(t, err) + } + + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + + assert.NoError(t, dataprovider.Close()) + assert.NoError(t, config.LoadConfig(configDir, "")) + providerConf = config.GetProviderConf() + assert.NoError(t, dataprovider.Initialize(providerConf, configDir)) +} + func TestLoginInvalidFs(t *testing.T) { u := getTestUser() u.FsConfig.Provider = dataprovider.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "test" - u.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString([]byte("invalid JSON for credentials")) + u.FsConfig.GCSConfig.Credentials = []byte("invalid JSON for credentials") user, _, err := httpd.AddUser(u, http.StatusOK) assert.NoError(t, err) - // now remove the credentials file so the filesystem creation will fail + providerConf := config.GetProviderConf() credentialsFile := filepath.Join(providerConf.CredentialsPath, fmt.Sprintf("%v_gcs_credentials.json", u.Username)) if !filepath.IsAbs(credentialsFile) { credentialsFile = filepath.Join(configDir, credentialsFile) } + + // now remove the credentials file so the filesystem creation will fail err = os.Remove(credentialsFile) assert.NoError(t, err) + client, err := getFTPClient(user, false) if !assert.Error(t, err) { err = client.Quit() diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index 4f8c7f57..eff43b42 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -3,7 +3,6 @@ package httpd_test import ( "bytes" "crypto/rand" - "encoding/base64" "encoding/json" "fmt" "io" @@ -243,8 +242,12 @@ func TestBasicUserHandling(t *testing.T) { user.UploadBandwidth = 128 user.DownloadBandwidth = 64 user.ExpirationDate = utils.GetTimeAsMsSinceEpoch(time.Now()) + + originalUser := user user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) + assert.Equal(t, originalUser.ID, user.ID) + users, _, err := httpd.GetUsers(0, 0, defaultUsername, http.StatusOK) assert.NoError(t, err) assert.Equal(t, 1, len(users)) @@ -418,15 +421,16 @@ func TestAddUserInvalidFsConfig(t *testing.T) { u.FsConfig.GCSConfig.Bucket = "abucket" u.FsConfig.GCSConfig.StorageClass = "Standard" u.FsConfig.GCSConfig.KeyPrefix = "/somedir/subdir/" - u.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString([]byte("test")) + u.FsConfig.GCSConfig.Credentials = []byte("test") _, _, err = httpd.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) u.FsConfig.GCSConfig.KeyPrefix = "somedir/subdir/" //nolint:goconst - u.FsConfig.GCSConfig.Credentials = "" + u.FsConfig.GCSConfig.Credentials = nil u.FsConfig.GCSConfig.AutomaticCredentials = 0 _, _, err = httpd.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) - u.FsConfig.GCSConfig.Credentials = "no base64 encoded" + + u.FsConfig.GCSConfig.Credentials = invalidBase64{} _, _, err = httpd.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) } @@ -983,21 +987,21 @@ func TestUserGCSConfig(t *testing.T) { assert.NoError(t, err) user.FsConfig.Provider = dataprovider.GCSFilesystemProvider user.FsConfig.GCSConfig.Bucket = "test" - user.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString([]byte("fake credentials")) + user.FsConfig.GCSConfig.Credentials = []byte("fake credentials") user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) _, err = httpd.RemoveUser(user, http.StatusOK) assert.NoError(t, err) user.Password = defaultPassword user.ID = 0 - // the user will be added since the credentials file is found - user, _, err = httpd.AddUser(user, http.StatusOK) - assert.NoError(t, err) + user.FsConfig.GCSConfig.Credentials = []byte("fake credentials") + user, body, err := httpd.AddUser(user, http.StatusOK) + assert.NoError(t, err, string(body)) err = os.RemoveAll(credentialsPath) assert.NoError(t, err) err = os.MkdirAll(credentialsPath, 0700) assert.NoError(t, err) - user.FsConfig.GCSConfig.Credentials = "" + user.FsConfig.GCSConfig.Credentials = nil user.FsConfig.GCSConfig.AutomaticCredentials = 1 user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) @@ -1012,7 +1016,7 @@ func TestUserGCSConfig(t *testing.T) { assert.NoError(t, err) user.FsConfig.Provider = dataprovider.GCSFilesystemProvider user.FsConfig.GCSConfig.Bucket = "test1" - user.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString([]byte("fake credentials")) + user.FsConfig.GCSConfig.Credentials = []byte("fake credentials") user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) @@ -2956,3 +2960,9 @@ func getMultipartFormData(values url.Values, fileFieldName, filePath string) (by err := w.Close() return b, w.FormDataContentType(), err } + +type invalidBase64 []byte + +func (b invalidBase64) MarshalJSON() ([]byte, error) { + return []byte(`not base64`), nil +} diff --git a/httpd/web.go b/httpd/web.go index 87d7cf10..7fc5c44e 100644 --- a/httpd/web.go +++ b/httpd/web.go @@ -1,7 +1,6 @@ package httpd import ( - "encoding/base64" "errors" "fmt" "html/template" @@ -430,7 +429,7 @@ func getFsConfigFromUserPostFields(r *http.Request) (dataprovider.Filesystem, er } return fs, err } - fs.GCSConfig.Credentials = base64.StdEncoding.EncodeToString(fileBytes) + fs.GCSConfig.Credentials = fileBytes fs.GCSConfig.AutomaticCredentials = 0 } return fs, nil diff --git a/service/service_portable.go b/service/service_portable.go index 7005d77f..6085b130 100644 --- a/service/service_portable.go +++ b/service/service_portable.go @@ -7,7 +7,6 @@ import ( "math/rand" "os" "os/signal" - "path/filepath" "strings" "syscall" "time" @@ -48,7 +47,7 @@ func (s *Service) StartPortableMode(sftpdPort, ftpPort, webdavPort int, enabledS dataProviderConf := config.GetProviderConf() dataProviderConf.Driver = dataprovider.MemoryDataProviderName dataProviderConf.Name = "" - dataProviderConf.CredentialsPath = filepath.Join(os.TempDir(), "credentials") + dataProviderConf.PreferDatabaseCredentials = true config.SetProviderConf(dataProviderConf) httpdConf := config.GetHTTPDConfig() httpdConf.BindPort = 0 diff --git a/sftpd/sftpd_test.go b/sftpd/sftpd_test.go index 6d42e4b0..6b4852f9 100644 --- a/sftpd/sftpd_test.go +++ b/sftpd/sftpd_test.go @@ -1308,22 +1308,71 @@ func TestLoginUserExpiration(t *testing.T) { assert.NoError(t, err) } +func TestLoginWithDatabaseCredentials(t *testing.T) { + usePubKey := true + u := getTestUser(usePubKey) + u.FsConfig.Provider = dataprovider.GCSFilesystemProvider + u.FsConfig.GCSConfig.Bucket = "testbucket" + u.FsConfig.GCSConfig.Credentials = []byte(`{ "type": "service_account" }`) + + providerConf := config.GetProviderConf() + providerConf.PreferDatabaseCredentials = true + credentialsFile := filepath.Join(providerConf.CredentialsPath, fmt.Sprintf("%v_gcs_credentials.json", u.Username)) + if !filepath.IsAbs(credentialsFile) { + credentialsFile = filepath.Join(configDir, credentialsFile) + } + + assert.NoError(t, dataprovider.Close()) + + err := dataprovider.Initialize(providerConf, configDir) + assert.NoError(t, err) + + if _, err = os.Stat(credentialsFile); err == nil { + // remove the credentials file + assert.NoError(t, os.Remove(credentialsFile)) + } + + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + + _, err = os.Stat(credentialsFile) + assert.Error(t, err) + + client, err := getSftpClient(user, usePubKey) + if assert.NoError(t, err) { + defer client.Close() + } + + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + + assert.NoError(t, dataprovider.Close()) + assert.NoError(t, config.LoadConfig(configDir, "")) + providerConf = config.GetProviderConf() + assert.NoError(t, dataprovider.Initialize(providerConf, configDir)) +} + func TestLoginInvalidFs(t *testing.T) { usePubKey := true u := getTestUser(usePubKey) u.FsConfig.Provider = dataprovider.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "test" - u.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString([]byte("invalid JSON for credentials")) + u.FsConfig.GCSConfig.Credentials = []byte("invalid JSON for credentials") user, _, err := httpd.AddUser(u, http.StatusOK) assert.NoError(t, err) - // now remove the credentials file so the filesystem creation will fail + providerConf := config.GetProviderConf() credentialsFile := filepath.Join(providerConf.CredentialsPath, fmt.Sprintf("%v_gcs_credentials.json", u.Username)) if !filepath.IsAbs(credentialsFile) { credentialsFile = filepath.Join(configDir, credentialsFile) } + + // now remove the credentials file so the filesystem creation will fail err = os.Remove(credentialsFile) assert.NoError(t, err) + client, err := getSftpClient(user, usePubKey) if !assert.Error(t, err, "login must fail, the user has an invalid filesystem config") { client.Close() diff --git a/sftpgo.json b/sftpgo.json index 9c8e0317..afb2ba45 100644 --- a/sftpgo.json +++ b/sftpgo.json @@ -88,6 +88,7 @@ "external_auth_hook": "", "external_auth_scope": 0, "credentials_path": "credentials", + "prefer_database_credentials": false, "pre_login_hook": "", "post_login_hook": "", "post_login_scope": 0, diff --git a/vfs/gcsfs.go b/vfs/gcsfs.go index 7ce28b46..64e0d403 100644 --- a/vfs/gcsfs.go +++ b/vfs/gcsfs.go @@ -60,6 +60,8 @@ func NewGCSFs(connectionID, localTempDir string, config GCSFsConfig) (Fs, error) ctx := context.Background() if fs.config.AutomaticCredentials > 0 { fs.svc, err = storage.NewClient(ctx) + } else if len(fs.config.Credentials) > 0 { + fs.svc, err = storage.NewClient(ctx, option.WithCredentialsJSON(fs.config.Credentials)) } else { fs.svc, err = storage.NewClient(ctx, option.WithCredentialsFile(fs.config.CredentialFile)) } diff --git a/vfs/vfs.go b/vfs/vfs.go index fbcf7108..119fa2b1 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -121,7 +121,7 @@ type GCSFsConfig struct { // If empty the whole bucket contents will be available KeyPrefix string `json:"key_prefix,omitempty"` CredentialFile string `json:"-"` - Credentials string `json:"credentials,omitempty"` + Credentials []byte `json:"credentials,omitempty"` AutomaticCredentials int `json:"automatic_credentials,omitempty"` StorageClass string `json:"storage_class,omitempty"` } diff --git a/webdavd/webdavd_test.go b/webdavd/webdavd_test.go index 9d9d0857..fae1f900 100644 --- a/webdavd/webdavd_test.go +++ b/webdavd/webdavd_test.go @@ -2,7 +2,6 @@ package webdavd_test import ( "crypto/rand" - "encoding/base64" "encoding/json" "fmt" "io" @@ -789,21 +788,69 @@ func TestClientClose(t *testing.T) { assert.NoError(t, err) } +func TestLoginWithDatabaseCredentials(t *testing.T) { + u := getTestUser() + u.FsConfig.Provider = dataprovider.GCSFilesystemProvider + u.FsConfig.GCSConfig.Bucket = "test" + u.FsConfig.GCSConfig.Credentials = []byte(`{ "type": "service_account" }`) + + providerConf := config.GetProviderConf() + providerConf.PreferDatabaseCredentials = true + credentialsFile := filepath.Join(providerConf.CredentialsPath, fmt.Sprintf("%v_gcs_credentials.json", u.Username)) + if !filepath.IsAbs(credentialsFile) { + credentialsFile = filepath.Join(configDir, credentialsFile) + } + + assert.NoError(t, dataprovider.Close()) + + err := dataprovider.Initialize(providerConf, configDir) + assert.NoError(t, err) + + if _, err = os.Stat(credentialsFile); err == nil { + // remove the credentials file + assert.NoError(t, os.Remove(credentialsFile)) + } + + user, _, err := httpd.AddUser(u, http.StatusOK) + assert.NoError(t, err) + + _, err = os.Stat(credentialsFile) + assert.Error(t, err) + + client := getWebDavClient(user) + + err = client.Connect() + assert.NoError(t, err) + + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + + assert.NoError(t, dataprovider.Close()) + assert.NoError(t, config.LoadConfig(configDir, "")) + providerConf = config.GetProviderConf() + assert.NoError(t, dataprovider.Initialize(providerConf, configDir)) +} + func TestLoginInvalidFs(t *testing.T) { u := getTestUser() u.FsConfig.Provider = dataprovider.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "test" - u.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString([]byte("invalid JSON for credentials")) + u.FsConfig.GCSConfig.Credentials = []byte("invalid JSON for credentials") user, _, err := httpd.AddUser(u, http.StatusOK) assert.NoError(t, err) - // now remove the credentials file so the filesystem creation will fail + providerConf := config.GetProviderConf() credentialsFile := filepath.Join(providerConf.CredentialsPath, fmt.Sprintf("%v_gcs_credentials.json", u.Username)) if !filepath.IsAbs(credentialsFile) { credentialsFile = filepath.Join(configDir, credentialsFile) } + + // now remove the credentials file so the filesystem creation will fail err = os.Remove(credentialsFile) assert.NoError(t, err) + client := getWebDavClient(user) assert.Error(t, checkBasicFunc(client))