add prefer_database_credentials configuration parameter

When true, users' Google Cloud Storage credentials will be written to
the data provider instead of disk.
Pre-existing credentials on disk will be used as a fallback

Fixes #201

cmd/portable.go

@@ -3,7 +3,6 @@
 package cmd
 
 import (
-	"encoding/base64"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -77,7 +76,7 @@ Please take a look at the usage below to customize the serving parameters`,
 			}
 			permissions := make(map[string][]string)
 			permissions["/"] = portablePermissions
-			portableGCSCredentials := ""
+			var portableGCSCredentials []byte
 			if fsProvider == dataprovider.GCSFilesystemProvider && len(portableGCSCredentialsFile) > 0 {
 				fi, err := os.Stat(portableGCSCredentialsFile)
 				if err != nil {
@@ -93,7 +92,7 @@ Please take a look at the usage below to customize the serving parameters`,
 				if err != nil {
 					fmt.Printf("Unable to read credentials file: %v\n", err)
 				}
-				portableGCSCredentials = base64.StdEncoding.EncodeToString(creds)
+				portableGCSCredentials = creds
 				portableGCSAutoCredentials = 0
 			}
 			if portableFTPDPort >= 0 && len(portableFTPSCert) > 0 && len(portableFTPSKey) > 0 {

config/config.go

@@ -141,7 +141,8 @@ func init() {
 					Parallelism: 2,
 				},
 			},
-			UpdateMode: 0,
+			UpdateMode:                0,
+			PreferDatabaseCredentials: false,
 		},
 		HTTPDConfig: httpd.Conf{
 			BindPort:           8080,

dataprovider/bolt.go

@@ -348,6 +348,7 @@ func (p BoltProvider) updateUser(user User) error {
 				return err
 			}
 		}
+		user.ID = oldUser.ID
 		user.LastQuotaUpdate = oldUser.LastQuotaUpdate
 		user.UsedQuotaSize = oldUser.UsedQuotaSize
 		user.UsedQuotaFiles = oldUser.UsedQuotaFiles

dataprovider/dataprovider.go

@@ -249,12 +249,16 @@ type Config struct {
 	// - 4 means WebDAV
 	// you can combine the scopes, for example 6 means FTP and WebDAV
 	CheckPasswordScope int `json:"check_password_scope" mapstructure:"check_password_scope"`
-	// PasswordHashing defines the configuration for password hashing
-	PasswordHashing PasswordHashing `json:"password_hashing" mapstructure:"password_hashing"`
 	// Defines how the database will be initialized/updated:
 	// - 0 means automatically
 	// - 1 means manually using the initprovider sub-command
 	UpdateMode int `json:"update_mode" mapstructure:"update_mode"`
+	// PasswordHashing defines the configuration for password hashing
+	PasswordHashing PasswordHashing `json:"password_hashing" mapstructure:"password_hashing"`
+	// PreferDatabaseCredentials indicates whether credential files (currently used for Google
+	// Cloud Storage) should be stored in the database instead of in the directory specified by
+	// CredentialsPath.
+	PreferDatabaseCredentials bool `json:"prefer_database_credentials" mapstructure:"prefer_database_credentials"`
 }
 
 // BackupData defines the structure for the backup/restore files
@@ -974,15 +978,14 @@ func saveGCSCredentials(user *User) error {
 	if len(user.FsConfig.GCSConfig.Credentials) == 0 {
 		return nil
 	}
-	decoded, err := base64.StdEncoding.DecodeString(user.FsConfig.GCSConfig.Credentials)
-	if err != nil {
-		return &ValidationError{err: fmt.Sprintf("could not validate GCS credentials: %v", err)}
+	if config.PreferDatabaseCredentials {
+		return nil
 	}
-	err = ioutil.WriteFile(user.getGCSCredentialsFilePath(), decoded, 0600)
+	err := ioutil.WriteFile(user.getGCSCredentialsFilePath(), user.FsConfig.GCSConfig.Credentials, 0600)
 	if err != nil {
 		return &ValidationError{err: fmt.Sprintf("could not save GCS credentials: %v", err)}
 	}
-	user.FsConfig.GCSConfig.Credentials = ""
+	user.FsConfig.GCSConfig.Credentials = nil
 	return nil
 }
 
@@ -1244,7 +1247,7 @@ func HideUserSensitiveData(user *User) User {
 	if user.FsConfig.Provider == S3FilesystemProvider {
 		user.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(user.FsConfig.S3Config.AccessSecret)
 	} else if user.FsConfig.Provider == GCSFilesystemProvider {
-		user.FsConfig.GCSConfig.Credentials = ""
+		user.FsConfig.GCSConfig.Credentials = nil
 	}
 	return *user
 }
@@ -1256,11 +1259,17 @@ func addCredentialsToUser(user *User) error {
 	if user.FsConfig.GCSConfig.AutomaticCredentials > 0 {
 		return nil
 	}
+
+	// Don't read from file if credentials have already been set
+	if len(user.FsConfig.GCSConfig.Credentials) > 0 {
+		return nil
+	}
+
 	cred, err := ioutil.ReadFile(user.getGCSCredentialsFilePath())
 	if err != nil {
 		return err
 	}
-	user.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString(cred)
+	user.FsConfig.GCSConfig.Credentials = cred
 	return nil
 }
 

dataprovider/user.go

@@ -720,6 +720,7 @@ func (u *User) getACopy() User {
 		GCSConfig: vfs.GCSFsConfig{
 			Bucket:               u.FsConfig.GCSConfig.Bucket,
 			CredentialFile:       u.FsConfig.GCSConfig.CredentialFile,
+			Credentials:          u.FsConfig.GCSConfig.Credentials,
 			AutomaticCredentials: u.FsConfig.GCSConfig.AutomaticCredentials,
 			StorageClass:         u.FsConfig.GCSConfig.StorageClass,
 			KeyPrefix:            u.FsConfig.GCSConfig.KeyPrefix,

docs/full-configuration.md

@@ -137,6 +137,7 @@ The configuration file contains the following sections:
   - `external_auth_hook`, string. Absolute path to an external program or an HTTP URL to invoke for users authentication. See [External Authentication](./external-auth.md) for more details. Leave empty to disable.
   - `external_auth_scope`, integer. 0 means all supported authentication scopes (passwords, public keys and keyboard interactive). 1 means passwords only. 2 means public keys only. 4 means key keyboard interactive only. The flags can be combined, for example 6 means public keys and keyboard interactive
   - `credentials_path`, string. It defines the directory for storing user provided credential files such as Google Cloud Storage credentials. This can be an absolute path or a path relative to the config dir
+  - `prefer_database_credentials`, boolean. When true, users' Google Cloud Storage credentials will be written to the data provider instead of disk, though pre-existing credentials on disk will be used as a fallback. When false, they will be written to the directory specified by `credentials_path`.
   - `pre_login_program`, string. Deprecated, please use `pre_login_hook`.
   - `pre_login_hook`, string. Absolute path to an external program or an HTTP URL to invoke to modify user details just before the login. See [Dynamic user modification](./dynamic-user-mod.md) for more details. Leave empty to disable.
   - `post_login_hook`, string. Absolute path to an external program or an HTTP URL to invoke to notify a successful or failed login. See [Post-login hook](./post-login-hook.md) for more details. Leave empty to disable.

ftpd/ftpd_test.go

@@ -3,7 +3,6 @@ package ftpd_test
 import (
 	"crypto/rand"
 	"crypto/tls"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -856,21 +855,70 @@ func TestLoginWithIPilters(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestLoginWithDatabaseCredentials(t *testing.T) {
+	u := getTestUser()
+	u.FsConfig.Provider = dataprovider.GCSFilesystemProvider
+	u.FsConfig.GCSConfig.Bucket = "test"
+	u.FsConfig.GCSConfig.Credentials = []byte(`{ "type": "service_account" }`)
+
+	providerConf := config.GetProviderConf()
+	providerConf.PreferDatabaseCredentials = true
+	credentialsFile := filepath.Join(providerConf.CredentialsPath, fmt.Sprintf("%v_gcs_credentials.json", u.Username))
+	if !filepath.IsAbs(credentialsFile) {
+		credentialsFile = filepath.Join(configDir, credentialsFile)
+	}
+
+	assert.NoError(t, dataprovider.Close())
+
+	err := dataprovider.Initialize(providerConf, configDir)
+	assert.NoError(t, err)
+
+	if _, err = os.Stat(credentialsFile); err == nil {
+		// remove the credentials file
+		assert.NoError(t, os.Remove(credentialsFile))
+	}
+
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+
+	_, err = os.Stat(credentialsFile)
+	assert.Error(t, err)
+
+	client, err := getFTPClient(user, false)
+	if assert.NoError(t, err) {
+		err = client.Quit()
+		assert.NoError(t, err)
+	}
+
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+
+	assert.NoError(t, dataprovider.Close())
+	assert.NoError(t, config.LoadConfig(configDir, ""))
+	providerConf = config.GetProviderConf()
+	assert.NoError(t, dataprovider.Initialize(providerConf, configDir))
+}
+
 func TestLoginInvalidFs(t *testing.T) {
 	u := getTestUser()
 	u.FsConfig.Provider = dataprovider.GCSFilesystemProvider
 	u.FsConfig.GCSConfig.Bucket = "test"
-	u.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString([]byte("invalid JSON for credentials"))
+	u.FsConfig.GCSConfig.Credentials = []byte("invalid JSON for credentials")
 	user, _, err := httpd.AddUser(u, http.StatusOK)
 	assert.NoError(t, err)
-	// now remove the credentials file so the filesystem creation will fail
+
 	providerConf := config.GetProviderConf()
 	credentialsFile := filepath.Join(providerConf.CredentialsPath, fmt.Sprintf("%v_gcs_credentials.json", u.Username))
 	if !filepath.IsAbs(credentialsFile) {
 		credentialsFile = filepath.Join(configDir, credentialsFile)
 	}
+
+	// now remove the credentials file so the filesystem creation will fail
 	err = os.Remove(credentialsFile)
 	assert.NoError(t, err)
+
 	client, err := getFTPClient(user, false)
 	if !assert.Error(t, err) {
 		err = client.Quit()

httpd/httpd_test.go

@@ -3,7 +3,6 @@ package httpd_test
 import (
 	"bytes"
 	"crypto/rand"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -243,8 +242,12 @@ func TestBasicUserHandling(t *testing.T) {
 	user.UploadBandwidth = 128
 	user.DownloadBandwidth = 64
 	user.ExpirationDate = utils.GetTimeAsMsSinceEpoch(time.Now())
+
+	originalUser := user
 	user, _, err = httpd.UpdateUser(user, http.StatusOK, "")
 	assert.NoError(t, err)
+	assert.Equal(t, originalUser.ID, user.ID)
+
 	users, _, err := httpd.GetUsers(0, 0, defaultUsername, http.StatusOK)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(users))
@@ -418,15 +421,16 @@ func TestAddUserInvalidFsConfig(t *testing.T) {
 	u.FsConfig.GCSConfig.Bucket = "abucket"
 	u.FsConfig.GCSConfig.StorageClass = "Standard"
 	u.FsConfig.GCSConfig.KeyPrefix = "/somedir/subdir/"
-	u.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString([]byte("test"))
+	u.FsConfig.GCSConfig.Credentials = []byte("test")
 	_, _, err = httpd.AddUser(u, http.StatusBadRequest)
 	assert.NoError(t, err)
 	u.FsConfig.GCSConfig.KeyPrefix = "somedir/subdir/" //nolint:goconst
-	u.FsConfig.GCSConfig.Credentials = ""
+	u.FsConfig.GCSConfig.Credentials = nil
 	u.FsConfig.GCSConfig.AutomaticCredentials = 0
 	_, _, err = httpd.AddUser(u, http.StatusBadRequest)
 	assert.NoError(t, err)
-	u.FsConfig.GCSConfig.Credentials = "no base64 encoded"
+
+	u.FsConfig.GCSConfig.Credentials = invalidBase64{}
 	_, _, err = httpd.AddUser(u, http.StatusBadRequest)
 	assert.NoError(t, err)
 }
@@ -983,21 +987,21 @@ func TestUserGCSConfig(t *testing.T) {
 	assert.NoError(t, err)
 	user.FsConfig.Provider = dataprovider.GCSFilesystemProvider
 	user.FsConfig.GCSConfig.Bucket = "test"
-	user.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString([]byte("fake credentials"))
+	user.FsConfig.GCSConfig.Credentials = []byte("fake credentials")
 	user, _, err = httpd.UpdateUser(user, http.StatusOK, "")
 	assert.NoError(t, err)
 	_, err = httpd.RemoveUser(user, http.StatusOK)
 	assert.NoError(t, err)
 	user.Password = defaultPassword
 	user.ID = 0
-	// the user will be added since the credentials file is found
-	user, _, err = httpd.AddUser(user, http.StatusOK)
-	assert.NoError(t, err)
+	user.FsConfig.GCSConfig.Credentials = []byte("fake credentials")
+	user, body, err := httpd.AddUser(user, http.StatusOK)
+	assert.NoError(t, err, string(body))
 	err = os.RemoveAll(credentialsPath)
 	assert.NoError(t, err)
 	err = os.MkdirAll(credentialsPath, 0700)
 	assert.NoError(t, err)
-	user.FsConfig.GCSConfig.Credentials = ""
+	user.FsConfig.GCSConfig.Credentials = nil
 	user.FsConfig.GCSConfig.AutomaticCredentials = 1
 	user, _, err = httpd.UpdateUser(user, http.StatusOK, "")
 	assert.NoError(t, err)
@@ -1012,7 +1016,7 @@ func TestUserGCSConfig(t *testing.T) {
 	assert.NoError(t, err)
 	user.FsConfig.Provider = dataprovider.GCSFilesystemProvider
 	user.FsConfig.GCSConfig.Bucket = "test1"
-	user.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString([]byte("fake credentials"))
+	user.FsConfig.GCSConfig.Credentials = []byte("fake credentials")
 	user, _, err = httpd.UpdateUser(user, http.StatusOK, "")
 	assert.NoError(t, err)
 
@@ -2956,3 +2960,9 @@ func getMultipartFormData(values url.Values, fileFieldName, filePath string) (by
 	err := w.Close()
 	return b, w.FormDataContentType(), err
 }
+
+type invalidBase64 []byte
+
+func (b invalidBase64) MarshalJSON() ([]byte, error) {
+	return []byte(`not base64`), nil
+}

httpd/web.go

@@ -1,7 +1,6 @@
 package httpd
 
 import (
-	"encoding/base64"
 	"errors"
 	"fmt"
 	"html/template"
@@ -430,7 +429,7 @@ func getFsConfigFromUserPostFields(r *http.Request) (dataprovider.Filesystem, er
 			}
 			return fs, err
 		}
-		fs.GCSConfig.Credentials = base64.StdEncoding.EncodeToString(fileBytes)
+		fs.GCSConfig.Credentials = fileBytes
 		fs.GCSConfig.AutomaticCredentials = 0
 	}
 	return fs, nil

service/service_portable.go

@@ -7,7 +7,6 @@ import (
 	"math/rand"
 	"os"
 	"os/signal"
-	"path/filepath"
 	"strings"
 	"syscall"
 	"time"
@@ -48,7 +47,7 @@ func (s *Service) StartPortableMode(sftpdPort, ftpPort, webdavPort int, enabledS
 	dataProviderConf := config.GetProviderConf()
 	dataProviderConf.Driver = dataprovider.MemoryDataProviderName
 	dataProviderConf.Name = ""
-	dataProviderConf.CredentialsPath = filepath.Join(os.TempDir(), "credentials")
+	dataProviderConf.PreferDatabaseCredentials = true
 	config.SetProviderConf(dataProviderConf)
 	httpdConf := config.GetHTTPDConfig()
 	httpdConf.BindPort = 0

sftpd/sftpd_test.go

@@ -1308,22 +1308,71 @@ func TestLoginUserExpiration(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestLoginWithDatabaseCredentials(t *testing.T) {
+	usePubKey := true
+	u := getTestUser(usePubKey)
+	u.FsConfig.Provider = dataprovider.GCSFilesystemProvider
+	u.FsConfig.GCSConfig.Bucket = "testbucket"
+	u.FsConfig.GCSConfig.Credentials = []byte(`{ "type": "service_account" }`)
+
+	providerConf := config.GetProviderConf()
+	providerConf.PreferDatabaseCredentials = true
+	credentialsFile := filepath.Join(providerConf.CredentialsPath, fmt.Sprintf("%v_gcs_credentials.json", u.Username))
+	if !filepath.IsAbs(credentialsFile) {
+		credentialsFile = filepath.Join(configDir, credentialsFile)
+	}
+
+	assert.NoError(t, dataprovider.Close())
+
+	err := dataprovider.Initialize(providerConf, configDir)
+	assert.NoError(t, err)
+
+	if _, err = os.Stat(credentialsFile); err == nil {
+		// remove the credentials file
+		assert.NoError(t, os.Remove(credentialsFile))
+	}
+
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+
+	_, err = os.Stat(credentialsFile)
+	assert.Error(t, err)
+
+	client, err := getSftpClient(user, usePubKey)
+	if assert.NoError(t, err) {
+		defer client.Close()
+	}
+
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+
+	assert.NoError(t, dataprovider.Close())
+	assert.NoError(t, config.LoadConfig(configDir, ""))
+	providerConf = config.GetProviderConf()
+	assert.NoError(t, dataprovider.Initialize(providerConf, configDir))
+}
+
 func TestLoginInvalidFs(t *testing.T) {
 	usePubKey := true
 	u := getTestUser(usePubKey)
 	u.FsConfig.Provider = dataprovider.GCSFilesystemProvider
 	u.FsConfig.GCSConfig.Bucket = "test"
-	u.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString([]byte("invalid JSON for credentials"))
+	u.FsConfig.GCSConfig.Credentials = []byte("invalid JSON for credentials")
 	user, _, err := httpd.AddUser(u, http.StatusOK)
 	assert.NoError(t, err)
-	// now remove the credentials file so the filesystem creation will fail
+
 	providerConf := config.GetProviderConf()
 	credentialsFile := filepath.Join(providerConf.CredentialsPath, fmt.Sprintf("%v_gcs_credentials.json", u.Username))
 	if !filepath.IsAbs(credentialsFile) {
 		credentialsFile = filepath.Join(configDir, credentialsFile)
 	}
+
+	// now remove the credentials file so the filesystem creation will fail
 	err = os.Remove(credentialsFile)
 	assert.NoError(t, err)
+
 	client, err := getSftpClient(user, usePubKey)
 	if !assert.Error(t, err, "login must fail, the user has an invalid filesystem config") {
 		client.Close()

sftpgo.json

@@ -88,6 +88,7 @@
     "external_auth_hook": "",
     "external_auth_scope": 0,
     "credentials_path": "credentials",
+    "prefer_database_credentials": false,
     "pre_login_hook": "",
     "post_login_hook": "",
     "post_login_scope": 0,

vfs/gcsfs.go

@@ -60,6 +60,8 @@ func NewGCSFs(connectionID, localTempDir string, config GCSFsConfig) (Fs, error)
 	ctx := context.Background()
 	if fs.config.AutomaticCredentials > 0 {
 		fs.svc, err = storage.NewClient(ctx)
+	} else if len(fs.config.Credentials) > 0 {
+		fs.svc, err = storage.NewClient(ctx, option.WithCredentialsJSON(fs.config.Credentials))
 	} else {
 		fs.svc, err = storage.NewClient(ctx, option.WithCredentialsFile(fs.config.CredentialFile))
 	}

vfs/vfs.go

@@ -121,7 +121,7 @@ type GCSFsConfig struct {
 	// If empty the whole bucket contents will be available
 	KeyPrefix            string `json:"key_prefix,omitempty"`
 	CredentialFile       string `json:"-"`
-	Credentials          string `json:"credentials,omitempty"`
+	Credentials          []byte `json:"credentials,omitempty"`
 	AutomaticCredentials int    `json:"automatic_credentials,omitempty"`
 	StorageClass         string `json:"storage_class,omitempty"`
 }

webdavd/webdavd_test.go

@@ -2,7 +2,6 @@ package webdavd_test
 
 import (
 	"crypto/rand"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -789,21 +788,69 @@ func TestClientClose(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestLoginWithDatabaseCredentials(t *testing.T) {
+	u := getTestUser()
+	u.FsConfig.Provider = dataprovider.GCSFilesystemProvider
+	u.FsConfig.GCSConfig.Bucket = "test"
+	u.FsConfig.GCSConfig.Credentials = []byte(`{ "type": "service_account" }`)
+
+	providerConf := config.GetProviderConf()
+	providerConf.PreferDatabaseCredentials = true
+	credentialsFile := filepath.Join(providerConf.CredentialsPath, fmt.Sprintf("%v_gcs_credentials.json", u.Username))
+	if !filepath.IsAbs(credentialsFile) {
+		credentialsFile = filepath.Join(configDir, credentialsFile)
+	}
+
+	assert.NoError(t, dataprovider.Close())
+
+	err := dataprovider.Initialize(providerConf, configDir)
+	assert.NoError(t, err)
+
+	if _, err = os.Stat(credentialsFile); err == nil {
+		// remove the credentials file
+		assert.NoError(t, os.Remove(credentialsFile))
+	}
+
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+
+	_, err = os.Stat(credentialsFile)
+	assert.Error(t, err)
+
+	client := getWebDavClient(user)
+
+	err = client.Connect()
+	assert.NoError(t, err)
+
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+
+	assert.NoError(t, dataprovider.Close())
+	assert.NoError(t, config.LoadConfig(configDir, ""))
+	providerConf = config.GetProviderConf()
+	assert.NoError(t, dataprovider.Initialize(providerConf, configDir))
+}
+
 func TestLoginInvalidFs(t *testing.T) {
 	u := getTestUser()
 	u.FsConfig.Provider = dataprovider.GCSFilesystemProvider
 	u.FsConfig.GCSConfig.Bucket = "test"
-	u.FsConfig.GCSConfig.Credentials = base64.StdEncoding.EncodeToString([]byte("invalid JSON for credentials"))
+	u.FsConfig.GCSConfig.Credentials = []byte("invalid JSON for credentials")
 	user, _, err := httpd.AddUser(u, http.StatusOK)
 	assert.NoError(t, err)
-	// now remove the credentials file so the filesystem creation will fail
+
 	providerConf := config.GetProviderConf()
 	credentialsFile := filepath.Join(providerConf.CredentialsPath, fmt.Sprintf("%v_gcs_credentials.json", u.Username))
 	if !filepath.IsAbs(credentialsFile) {
 		credentialsFile = filepath.Join(configDir, credentialsFile)
 	}
+
+	// now remove the credentials file so the filesystem creation will fail
 	err = os.Remove(credentialsFile)
 	assert.NoError(t, err)
+
 	client := getWebDavClient(user)
 	assert.Error(t, checkBasicFunc(client))