add basic S3-Compatible Object Storage support

we have now an interface for filesystem backeds, this make easy to add
new filesystem backends

.travis.yml

@@ -11,7 +11,7 @@ env:
   - GO111MODULE=on
 
 before_script:
-  - sqlite3 sftpgo.db 'CREATE TABLE "users" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE, "password" varchar(255) NULL, "public_keys" text NULL, "home_dir" varchar(255) NOT NULL, "uid" integer NOT NULL, "gid" integer NOT NULL, "max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "permissions" text NOT NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL, "expiration_date" bigint NOT NULL, "last_login" bigint NOT NULL, "status" integer NOT NULL, "filters" TEXT NULL);'
+  - sqlite3 sftpgo.db 'CREATE TABLE "users" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE, "password" varchar(255) NULL, "public_keys" text NULL, "home_dir" varchar(255) NOT NULL, "uid" integer NOT NULL, "gid" integer NOT NULL, "max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "permissions" text NOT NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL, "expiration_date" bigint NOT NULL, "last_login" bigint NOT NULL, "status" integer NOT NULL, "filters" TEXT NULL, "filesystem" text NULL);'
 
 install:
   - go get -v -t ./...

README.md

@@ -21,6 +21,7 @@ Full featured and highly configurable SFTP server
 - Atomic uploads are configurable.
 - Support for Git repositories over SSH.
 - SCP and rsync are supported.
+- Support for serving S3 Compatible Object Storage over SFTP.
 - Prometheus metrics are exposed.
 - REST API for users management, backup, restore and real time reports of the active connections with possibility of forcibly closing a connection.
 - Web based interface to easily manage users and connections.
@@ -136,7 +137,7 @@ The `sftpgo` configuration file contains the following sections:
     - `idle_timeout`, integer. Time in minutes after which an idle client will be disconnected. 0 menas disabled. Default: 15
     - `max_auth_tries` integer. Maximum number of authentication attempts permitted per connection. If set to a negative number, the number of attempts are unlimited. If set to zero, the number of attempts are limited to 6.
     - `umask`, string. Umask for the new files and directories. This setting has no effect on Windows. Default: "0022"
-    - `banner`, string. Identification string used by the server. Leave empty to use the default banner. Default "SFTPGo_version"
+    - `banner`, string. Identification string used by the server. Leave empty to use the default banner. Default "SFTPGo_<version>"
     - `upload_mode` integer. 0 means standard, the files are uploaded directly to the requested path. 1 means atomic: files are uploaded to a temporary path and renamed to the requested path when the client ends the upload. Atomic mode avoids problems such as a web server that serves partial files when the files are being uploaded. In atomic mode if there is an upload error the temporary file is deleted and so the requested upload path will not contain a partial file. 2 means atomic with resume support: as atomic but if there is an upload error the temporary file is renamed to the requested path and not deleted, this way a client can reconnect and resume the upload.
     - `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions. See the "Custom Actions" paragraph for more details
         - `execute_on`, list of strings. Valid values are `download`, `upload`, `delete`, `rename`, `ssh_cmd`. Leave empty to disable actions.
@@ -150,7 +151,7 @@ The `sftpgo` configuration file contains the following sections:
     - `macs`, list of strings. available MAC (message authentication code) algorithms in preference order. Leave empty to use default values. The supported values can be found here: [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/common.go#L84 "Supported MACs")
     - `login_banner_file`, path to the login banner file. The contents of the specified file, if any, are sent to the remote user before authentication is allowed. It can be a path relative to the config dir or an absolute one. Leave empty to send no login banner
     - `setstat_mode`, integer. 0 means "normal mode": requests for changing permissions, owner/group and access/modification times are executed. 1 means "ignore mode": requests for changing permissions, owner/group and access/modification times are silently ignored.
-    - `enabled_ssh_commands`, list of enabled SSH commands. These SSH commands are enabled by default: `md5sum`, `sha1sum`, `cd`, `pwd`. `*` enables all supported commands. Some commands are implemented directly inside SFTPGo, while for other commands we use system commands that need to be installed and in your system's `PATH`. For system commands we have no direct control on file creation/deletion and so quota check is suboptimal: if quota is enabled, the number of files is checked at the command begin and not while new files are created. The allowed size is calculated as the difference between the max quota and the used one and it is checked against the bytes transferred via SSH. The command is aborted if it uploads more bytes than the remaining allowed size calculated at the command start. Anyway we see the bytes that the remote command send to the local command via SSH, these bytes contain both protocol commands and files and so the size of the files is different from the size trasferred via SSH: for example a command can send compressed files or a protocol command (few bytes) could delete a big file. To mitigate this issue quotas are recalculated at the command end with a full home directory scan, this could be heavy for big directories. If you need system commands and quotas you could consider to disable quota restrictions and periodically update quota usage yourself using the REST API. We support the following SSH commands:
+    - `enabled_ssh_commands`, list of enabled SSH commands. These SSH commands are enabled by default: `md5sum`, `sha1sum`, `cd`, `pwd`. `*` enables all supported commands. Some commands are implemented directly inside SFTPGo, while for other commands we use system commands that need to be installed and in your system's `PATH`. For system commands we have no direct control on file creation/deletion and so we cannot support remote filesystems, such as S3, and quota check is suboptimal: if quota is enabled, the number of files is checked at the command begin and not while new files are created. The allowed size is calculated as the difference between the max quota and the used one and it is checked against the bytes transferred via SSH. The command is aborted if it uploads more bytes than the remaining allowed size calculated at the command start. Anyway we see the bytes that the remote command send to the local command via SSH, these bytes contain both protocol commands and files and so the size of the files is different from the size trasferred via SSH: for example a command can send compressed files or a protocol command (few bytes) could delete a big file. To mitigate this issue quotas are recalculated at the command end with a full home directory scan, this could be heavy for big directories. If you need system commands and quotas you could consider to disable quota restrictions and periodically update quota usage yourself using the REST API. We support the following SSH commands:
       - `scp`, SCP is an experimental feature, we have our own SCP implementation since we can't rely on "scp" system command to proper handle quotas and user's home dir restrictions. The SCP protocol is quite simple but there is no official docs about it, so we need more testing and feedbacks before enabling it by default. We may not handle some borderline cases or have sneaky bugs. Please do accurate tests yourself before enabling SCP and let us known if something does not work as expected for your use cases. SCP between two remote hosts is supported using the `-3` scp option.
       - `md5sum`, `sha1sum`, `sha256sum`, `sha384sum`, `sha512sum`. Useful to check message digests for uploaded files. These commands are implemented inside SFTPGo so they work even if the matching system commands are not available, for example on Windows.
       - `cd`, `pwd`. Some SFTP clients does not support the SFTP SSH_FXP_REALPATH packet type and so they use `cd` and `pwd` SSH commands to get the initial directory. Currently `cd` do nothing and `pwd` always returns the `/` path.
@@ -416,6 +417,25 @@ The `http_notification_url`, if defined, will be called invoked as http POST. Th
 
 The HTTP request has a 15 seconds timeout.
 
+## S3 Compabible Object Storage backends
+
+Each user can be mapped with an S3-Compatible bucket, this way the mapped bucket is exposed over SFTP/SCP.
+SFTPGo uses multipart uploads and parallel downloads for storing and retrieving files from S3 and automatically try to create the mapped bucket if it does not exists.
+
+Some SFTP commands doesn't work over S3:
+
+- `symlink` and `chtimes` will fail
+- `chown`, `chmod` are silently ignored
+- upload resume is not supported
+- upload mode `atomic` is ignored since S3 uploads are already atomic
+
+Other notes:
+
+- `rename` is a two steps operation: server-side copy and then deletion. So it is not atomic as for local filesystem
+- We don't support renaming non empty directories since we should rename all the contents too and this could take long time: think about directories with thousands of files, for each file we should do an AWS API call.
+- For server side encryption you have to configure the mapped bucket to automatically encrypt objects.
+- A local home directory is still required to store temporary files.
+
 ## Portable mode
 
 SFTPGo allows to share a single directory on demand using the `portable` subcommand:
@@ -435,11 +455,18 @@ Flags:
   -C, --advertise-credentials     If the SFTP service is advertised via multicast DNS this flag allows to put username/password inside the advertised TXT record
   -S, --advertise-service         Advertise SFTP service using multicast DNS (default true)
   -d, --directory string          Path to the directory to serve. This can be an absolute path or a path relative to the current directory (default ".")
+  -f, --fs-provider int           0 means local filesystem, 1 S3 compatible
   -h, --help                      help for portable
   -l, --log-file-path string      Leave empty to disable logging
   -p, --password string           Leave empty to use an auto generated value
   -g, --permissions strings       User's permissions. "*" means any permission (default [list,download])
   -k, --public-key strings
+      --s3-access-key string
+      --s3-access-secret string
+      --s3-bucket string
+      --s3-endpoint string
+      --s3-region string
+      --s3-storage-class string
   -s, --sftpd-port int            0 means a random non privileged port
   -c, --ssh-commands strings      SSH commands to enable. "*" means any supported SSH command including scp (default [md5sum,sha1sum,cd,pwd])
   -u, --username string           Leave empty to use an auto generated value
@@ -488,6 +515,13 @@ For each account the following properties can be configured:
 - `download_bandwidth` maximum download bandwidth as KB/s, 0 means unlimited.
 - `allowed_ip`, List of IP/Mask allowed to login. Any IP address not contained in this list cannot login. IP/Mask must be in CIDR notation as defined in RFC 4632 and RFC 4291, for example "192.0.2.0/24" or "2001:db8::/32"
 - `denied_ip`, List of IP/Mask not allowed to login. If an IP address is both allowed and denied then login will be denied
+- `fs_provider`, filesystem to serve via SFTP. Local filesystem and S3 Compatible Object Storage are supported
+- `s3_bucket`, required for S3 filesystem
+- `s3_region`, required for S3 filesystem
+- `s3_access_key`, required for S3 filesystem
+- `s3_access_secret`, required for S3 filesystem. It is stored encrypted (AES-256-GCM)
+- `s3_endpoint`, specifies s3 endpoint (server) different from AWS
+- `s3_storage_class`
 
 These properties are stored inside the data provider.
 

cmd/portable.go

@@ -6,6 +6,7 @@ import (
 	"github.com/drakkan/sftpgo/dataprovider"
 	"github.com/drakkan/sftpgo/service"
 	"github.com/drakkan/sftpgo/sftpd"
+	"github.com/drakkan/sftpgo/vfs"
 	"github.com/spf13/cobra"
 )
 
@@ -20,6 +21,13 @@ var (
 	portablePublicKeys           []string
 	portablePermissions          []string
 	portableSSHCommands          []string
+	portableFsProvider           int
+	portableS3Bucket             string
+	portableS3Region             string
+	portableS3AccessKey          string
+	portableS3AccessSecret       string
+	portableS3Endpoint           string
+	portableS3StorageClass       string
 	portableCmd                  = &cobra.Command{
 		Use:   "portable",
 		Short: "Serve a single directory",
@@ -53,6 +61,17 @@ Please take a look at the usage below to customize the serving parameters`,
 					Permissions: permissions,
 					HomeDir:     portableDir,
 					Status:      1,
+					FsConfig: dataprovider.Filesystem{
+						Provider: portableFsProvider,
+						S3Config: vfs.S3FsConfig{
+							Bucket:       portableS3Bucket,
+							Region:       portableS3Region,
+							AccessKey:    portableS3AccessKey,
+							AccessSecret: portableS3AccessSecret,
+							Endpoint:     portableS3Endpoint,
+							StorageClass: portableS3StorageClass,
+						},
+					},
 				},
 			}
 			if err := service.StartPortableMode(portableSFTPDPort, portableSSHCommands, portableAdvertiseService,
@@ -79,5 +98,12 @@ func init() {
 		"Advertise SFTP service using multicast DNS")
 	portableCmd.Flags().BoolVarP(&portableAdvertiseCredentials, "advertise-credentials", "C", false,
 		"If the SFTP service is advertised via multicast DNS this flag allows to put username/password inside the advertised TXT record")
+	portableCmd.Flags().IntVarP(&portableFsProvider, "fs-provider", "f", 0, "0 means local filesystem, 1 S3 compatible")
+	portableCmd.Flags().StringVar(&portableS3Bucket, "s3-bucket", "", "")
+	portableCmd.Flags().StringVar(&portableS3Region, "s3-region", "", "")
+	portableCmd.Flags().StringVar(&portableS3AccessKey, "s3-access-key", "", "")
+	portableCmd.Flags().StringVar(&portableS3AccessSecret, "s3-access-secret", "", "")
+	portableCmd.Flags().StringVar(&portableS3Endpoint, "s3-endpoint", "", "")
+	portableCmd.Flags().StringVar(&portableS3StorageClass, "s3-storage-class", "", "")
 	rootCmd.AddCommand(portableCmd)
 }

dataprovider/bolt.go

@@ -336,7 +336,7 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str
 		if offset == 0 {
 			user, err := p.userExists(username)
 			if err == nil {
-				users = append(users, getUserNoCredentials(&user))
+				users = append(users, HideUserSensitiveData(&user))
 			}
 		}
 		return users, err
@@ -357,7 +357,7 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str
 				var user User
 				err = json.Unmarshal(v, &user)
 				if err == nil {
-					users = append(users, getUserNoCredentials(&user))
+					users = append(users, HideUserSensitiveData(&user))
 				}
 				if len(users) >= limit {
 					break
@@ -372,7 +372,7 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str
 				var user User
 				err = json.Unmarshal(v, &user)
 				if err == nil {
-					users = append(users, getUserNoCredentials(&user))
+					users = append(users, HideUserSensitiveData(&user))
 				}
 				if len(users) >= limit {
 					break
@@ -388,11 +388,6 @@ func (p BoltProvider) close() error {
 	return p.dbHandle.Close()
 }
 
-func getUserNoCredentials(user *User) User {
-	user.Password = ""
-	return *user
-}
-
 // itob returns an 8-byte big endian representation of v.
 func itob(v int64) []byte {
 	b := make([]byte, 8)

dataprovider/dataprovider.go

@@ -34,6 +34,7 @@ import (
 	"github.com/drakkan/sftpgo/logger"
 	"github.com/drakkan/sftpgo/metrics"
 	"github.com/drakkan/sftpgo/utils"
+	"github.com/drakkan/sftpgo/vfs"
 	unixcrypt "github.com/nathanaelle/password"
 )
 
@@ -479,6 +480,27 @@ func validateFilters(user *User) error {
 	return nil
 }
 
+func validateFilesystemConfig(user *User) error {
+	if user.FsConfig.Provider == 1 {
+		err := vfs.ValidateS3FsConfig(&user.FsConfig.S3Config)
+		if err != nil {
+			return &ValidationError{err: fmt.Sprintf("Could not validate s3config: %v", err)}
+		}
+		vals := strings.Split(user.FsConfig.S3Config.AccessSecret, "$")
+		if !strings.HasPrefix(user.FsConfig.S3Config.AccessSecret, "$aes$") || len(vals) != 4 {
+			accessSecret, err := utils.EncryptData(user.FsConfig.S3Config.AccessSecret)
+			if err != nil {
+				return &ValidationError{err: fmt.Sprintf("Could encrypt s3 access secret: %v", err)}
+			}
+			user.FsConfig.S3Config.AccessSecret = accessSecret
+		}
+		return nil
+	}
+	user.FsConfig.Provider = 0
+	user.FsConfig.S3Config = vfs.S3FsConfig{}
+	return nil
+}
+
 func validateUser(user *User) error {
 	buildUserHomeDir(user)
 	if len(user.Username) == 0 || len(user.HomeDir) == 0 {
@@ -493,6 +515,9 @@ func validateUser(user *User) error {
 	if err := validatePermissions(user); err != nil {
 		return err
 	}
+	if err := validateFilesystemConfig(user); err != nil {
+		return err
+	}
 	if user.Status < 0 || user.Status > 1 {
 		return &ValidationError{err: fmt.Sprintf("invalid user status: %v", user.Status)}
 	}
@@ -645,6 +670,15 @@ func comparePbkdf2PasswordAndHash(password, hashedPassword string) (bool, error)
 	return subtle.ConstantTimeCompare(buf, []byte(expected)) == 1, nil
 }
 
+// HideUserSensitiveData hides user sensitive data
+func HideUserSensitiveData(user *User) User {
+	user.Password = ""
+	if user.FsConfig.Provider == 1 {
+		user.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(user.FsConfig.S3Config.AccessSecret)
+	}
+	return *user
+}
+
 func getSSLMode() string {
 	if config.Driver == PGSQLDataProviderName {
 		if config.SSLMode == 0 {
@@ -772,8 +806,7 @@ func executeAction(operation string, user User) {
 			return
 		}
 	}
-	// hide the hashed password
+	HideUserSensitiveData(&user)
-	user.Password = ""
 	if len(config.Actions.Command) > 0 && filepath.IsAbs(config.Actions.Command) {
 		// we are in a goroutine but if we have to send an HTTP notification we don't want to wait for the
 		// end of the command

dataprovider/memory.go

@@ -244,8 +244,7 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s
 		if offset == 0 {
 			user, err := p.userExistsInternal(username)
 			if err == nil {
-				user.Password = ""
+				users = append(users, HideUserSensitiveData(&user))
-				users = append(users, user)
 			}
 		}
 		return users, err
@@ -258,8 +257,7 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s
 				continue
 			}
 			user := p.dbHandle.users[username]
-			user.Password = ""
+			users = append(users, HideUserSensitiveData(&user))
-			users = append(users, user)
 			if len(users) >= limit {
 				break
 			}
@@ -272,8 +270,7 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s
 			}
 			username := p.dbHandle.usernames[i]
 			user := p.dbHandle.users[username]
-			user.Password = ""
+			users = append(users, HideUserSensitiveData(&user))
-			users = append(users, user)
 			if len(users) >= limit {
 				break
 			}

dataprovider/sqlcommon.go

@@ -162,8 +162,13 @@ func sqlCommonAddUser(user User, dbHandle *sql.DB) error {
 	if err != nil {
 		return err
 	}
+	fsConfig, err := user.GetFsConfigAsJSON()
+	if err != nil {
+		return err
+	}
 	_, err = stmt.Exec(user.Username, user.Password, string(publicKeys), user.HomeDir, user.UID, user.GID, user.MaxSessions, user.QuotaSize,
-		user.QuotaFiles, string(permissions), user.UploadBandwidth, user.DownloadBandwidth, user.Status, user.ExpirationDate, string(filters))
+		user.QuotaFiles, string(permissions), user.UploadBandwidth, user.DownloadBandwidth, user.Status, user.ExpirationDate, string(filters),
+		string(fsConfig))
 	return err
 }
 
@@ -191,9 +196,13 @@ func sqlCommonUpdateUser(user User, dbHandle *sql.DB) error {
 	if err != nil {
 		return err
 	}
+	fsConfig, err := user.GetFsConfigAsJSON()
+	if err != nil {
+		return err
+	}
 	_, err = stmt.Exec(user.Password, string(publicKeys), user.HomeDir, user.UID, user.GID, user.MaxSessions, user.QuotaSize,
 		user.QuotaFiles, string(permissions), user.UploadBandwidth, user.DownloadBandwidth, user.Status, user.ExpirationDate,
-		string(filters), user.ID)
+		string(filters), string(fsConfig), user.ID)
 	return err
 }
 
@@ -253,10 +262,8 @@ func sqlCommonGetUsers(limit int, offset int, order string, username string, dbH
 		defer rows.Close()
 		for rows.Next() {
 			u, err := getUserFromDbRow(nil, rows)
-			// hide password
 			if err == nil {
-				u.Password = ""
+				users = append(users, HideUserSensitiveData(&u))
-				users = append(users, u)
 			} else {
 				break
 			}
@@ -272,16 +279,17 @@ func getUserFromDbRow(row *sql.Row, rows *sql.Rows) (User, error) {
 	var password sql.NullString
 	var publicKey sql.NullString
 	var filters sql.NullString
+	var fsConfig sql.NullString
 	var err error
 	if row != nil {
 		err = row.Scan(&user.ID, &user.Username, &password, &publicKey, &user.HomeDir, &user.UID, &user.GID, &user.MaxSessions,
 			&user.QuotaSize, &user.QuotaFiles, &permissions, &user.UsedQuotaSize, &user.UsedQuotaFiles, &user.LastQuotaUpdate,
-			&user.UploadBandwidth, &user.DownloadBandwidth, &user.ExpirationDate, &user.LastLogin, &user.Status, &filters)
+			&user.UploadBandwidth, &user.DownloadBandwidth, &user.ExpirationDate, &user.LastLogin, &user.Status, &filters, &fsConfig)
 
 	} else {
 		err = rows.Scan(&user.ID, &user.Username, &password, &publicKey, &user.HomeDir, &user.UID, &user.GID, &user.MaxSessions,
 			&user.QuotaSize, &user.QuotaFiles, &permissions, &user.UsedQuotaSize, &user.UsedQuotaFiles, &user.LastQuotaUpdate,
-			&user.UploadBandwidth, &user.DownloadBandwidth, &user.ExpirationDate, &user.LastLogin, &user.Status, &filters)
+			&user.UploadBandwidth, &user.DownloadBandwidth, &user.ExpirationDate, &user.LastLogin, &user.Status, &filters, &fsConfig)
 	}
 	if err != nil {
 		if err == sql.ErrNoRows {
@@ -326,5 +334,16 @@ func getUserFromDbRow(row *sql.Row, rows *sql.Rows) (User, error) {
 			DeniedIP:  []string{},
 		}
 	}
+	if fsConfig.Valid {
+		var fs Filesystem
+		err = json.Unmarshal([]byte(fsConfig.String), &fs)
+		if err == nil {
+			user.FsConfig = fs
+		}
+	} else {
+		user.FsConfig = Filesystem{
+			Provider: 0,
+		}
+	}
 	return user, err
 }

dataprovider/sqlqueries.go

@@ -4,7 +4,7 @@ import "fmt"
 
 const (
 	selectUserFields = "id,username,password,public_keys,home_dir,uid,gid,max_sessions,quota_size,quota_files,permissions,used_quota_size," +
-		"used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,expiration_date,last_login,status,filters"
+		"used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,expiration_date,last_login,status,filters,filesystem"
 )
 
 func getSQLPlaceholders() []string {
@@ -60,19 +60,20 @@ func getQuotaQuery() string {
 
 func getAddUserQuery() string {
 	return fmt.Sprintf(`INSERT INTO %v (username,password,public_keys,home_dir,uid,gid,max_sessions,quota_size,quota_files,permissions,
-		used_quota_size,used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,status,last_login,expiration_date,filters)
+		used_quota_size,used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,status,last_login,expiration_date,filters,
-		VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,0,0,0,%v,%v,%v,0,%v,%v)`, config.UsersTable, sqlPlaceholders[0], sqlPlaceholders[1],
+		filesystem)
+		VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,0,0,0,%v,%v,%v,0,%v,%v,%v)`, config.UsersTable, sqlPlaceholders[0], sqlPlaceholders[1],
 		sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7],
 		sqlPlaceholders[8], sqlPlaceholders[9], sqlPlaceholders[10], sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13],
-		sqlPlaceholders[14])
+		sqlPlaceholders[14], sqlPlaceholders[15])
 }
 
 func getUpdateUserQuery() string {
 	return fmt.Sprintf(`UPDATE %v SET password=%v,public_keys=%v,home_dir=%v,uid=%v,gid=%v,max_sessions=%v,quota_size=%v,
-		quota_files=%v,permissions=%v,upload_bandwidth=%v,download_bandwidth=%v,status=%v,expiration_date=%v,filters=%v WHERE id = %v`,
+		quota_files=%v,permissions=%v,upload_bandwidth=%v,download_bandwidth=%v,status=%v,expiration_date=%v,filters=%v,filesystem=%v
-		config.UsersTable, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4],
+		WHERE id = %v`, config.UsersTable, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3],
-		sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9], sqlPlaceholders[10],
+		sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9],
-		sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13], sqlPlaceholders[14])
+		sqlPlaceholders[10], sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13], sqlPlaceholders[14], sqlPlaceholders[15])
 }
 
 func getDeleteUserQuery() string {

dataprovider/user.go

@@ -7,10 +7,10 @@ import (
 	"path"
 	"path/filepath"
 	"strconv"
-	"strings"
 
 	"github.com/drakkan/sftpgo/logger"
 	"github.com/drakkan/sftpgo/utils"
+	"github.com/drakkan/sftpgo/vfs"
 )
 
 // Available permissions for SFTP users
@@ -53,6 +53,13 @@ type UserFilters struct {
 	DeniedIP []string `json:"denied_ip"`
 }
 
+// Filesystem defines cloud storage filesystem details
+type Filesystem struct {
+	// 0 local filesystem, 1 Amazon S3 compatible
+	Provider int            `json:"provider"`
+	S3Config vfs.S3FsConfig `json:"s3config,omitempty"`
+}
+
 // User defines an SFTP user
 type User struct {
 	// Database unique identifier
@@ -98,6 +105,16 @@ type User struct {
 	LastLogin int64 `json:"last_login"`
 	// Additional restrictions
 	Filters UserFilters `json:"filters"`
+	// Filesystem configuration details
+	FsConfig Filesystem `json:"filesystem"`
+}
+
+// GetFilesystem returns the filesystem for this user
+func (u *User) GetFilesystem(connectionID string) (vfs.Fs, error) {
+	if u.FsConfig.Provider == 1 {
+		return vfs.NewS3Fs(connectionID, u.GetHomeDir(), u.FsConfig.S3Config)
+	}
+	return vfs.NewOsFs(connectionID), nil
 }
 
 // GetPermissionsForPath returns the permissions for the given path.
@@ -211,6 +228,11 @@ func (u *User) GetFiltersAsJSON() ([]byte, error) {
 	return json.Marshal(u.Filters)
 }
 
+// GetFsConfigAsJSON returns the filesystem config as json byte array
+func (u *User) GetFsConfigAsJSON() ([]byte, error) {
+	return json.Marshal(u.FsConfig)
+}
+
 // GetUID returns a validate uid, suitable for use with os.Chown
 func (u *User) GetUID() int {
 	if u.UID <= 0 || u.UID > 65535 {
@@ -237,19 +259,6 @@ func (u *User) HasQuotaRestrictions() bool {
 	return u.QuotaFiles > 0 || u.QuotaSize > 0
 }
 
-// GetRelativePath returns the path for a file relative to the user's home dir.
-// This is the path as seen by SFTP users
-func (u *User) GetRelativePath(path string) string {
-	rel, err := filepath.Rel(u.GetHomeDir(), filepath.Clean(path))
-	if err != nil {
-		return ""
-	}
-	if rel == "." || strings.HasPrefix(rel, "..") {
-		rel = ""
-	}
-	return "/" + filepath.ToSlash(rel)
-}
-
 // GetQuotaSummary returns used quota and limits if defined
 func (u *User) GetQuotaSummary() string {
 	var result string
@@ -319,6 +328,9 @@ func (u *User) GetInfoString() string {
 		t := utils.GetTimeFromMsecSinceEpoch(u.LastLogin)
 		result += fmt.Sprintf("Last login: %v ", t.Format("2006-01-02 15:04:05")) // YYYY-MM-DD HH:MM:SS
 	}
+	if u.FsConfig.Provider == 1 {
+		result += fmt.Sprintf("Storage: S3")
+	}
 	if len(u.PublicKeys) > 0 {
 		result += fmt.Sprintf("Public keys: %v ", len(u.PublicKeys))
 	}
@@ -387,6 +399,17 @@ func (u *User) getACopy() User {
 	copy(filters.AllowedIP, u.Filters.AllowedIP)
 	filters.DeniedIP = make([]string, len(u.Filters.DeniedIP))
 	copy(filters.DeniedIP, u.Filters.DeniedIP)
+	fsConfig := Filesystem{
+		Provider: u.FsConfig.Provider,
+		S3Config: vfs.S3FsConfig{
+			Bucket:       u.FsConfig.S3Config.Bucket,
+			Region:       u.FsConfig.S3Config.Region,
+			AccessKey:    u.FsConfig.S3Config.AccessKey,
+			AccessSecret: u.FsConfig.S3Config.AccessSecret,
+			Endpoint:     u.FsConfig.S3Config.Endpoint,
+			StorageClass: u.FsConfig.S3Config.StorageClass,
+		},
+	}
 
 	return User{
 		ID:                u.ID,
@@ -409,6 +432,7 @@ func (u *User) getACopy() User {
 		ExpirationDate:    u.ExpirationDate,
 		LastLogin:         u.LastLogin,
 		Filters:           filters,
+		FsConfig:          fsConfig,
 	}
 }
 

go.mod

@@ -4,7 +4,9 @@ go 1.13
 
 require (
 	github.com/alexedwards/argon2id v0.0.0-20190612080829-01a59b2b8802
+	github.com/aws/aws-sdk-go v1.28.3
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
+	github.com/eikenb/pipeat v0.0.0-20190316224601-fb1f3a9aa29f
 	github.com/go-chi/chi v4.0.2+incompatible
 	github.com/go-chi/render v1.0.1
 	github.com/go-sql-driver/mysql v1.5.0
@@ -24,3 +26,5 @@ require (
 	golang.org/x/sys v0.0.0-20191220142924-d4481acd189f
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
+
+replace github.com/eikenb/pipeat v0.0.0-20190316224601-fb1f3a9aa29f => github.com/drakkan/pipeat v0.0.0-20200114135659-fac71c64d75d

go.sum

@@ -1,5 +1,4 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -9,6 +8,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
 github.com/alexedwards/argon2id v0.0.0-20190612080829-01a59b2b8802 h1:RwMM1q/QSKYIGbHfOkf843hE8sSUJtf1dMwFPtEDmm0=
 github.com/alexedwards/argon2id v0.0.0-20190612080829-01a59b2b8802/go.mod h1:4dsm7ufQm1Gwl8S2ss57u+2J7KlxIL2QUmFGlGtWogY=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/aws/aws-sdk-go v1.28.3 h1:FnkDp+fz4JHWUW3Ust2Wh89RpdGif077Wjis/sMrGKM=
+github.com/aws/aws-sdk-go v1.28.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -32,6 +33,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/drakkan/pipeat v0.0.0-20200114135659-fac71c64d75d h1:+k0oy9bBY9dXlKHriYg6crXpwIrtM1rCrlUehmc/F3M=
+github.com/drakkan/pipeat v0.0.0-20200114135659-fac71c64d75d/go.mod h1:wNYvIpR5rIhoezOYcpxcXz4HbIEOu7A45EqlQCA+h+w=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -57,10 +60,8 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/grandcat/zeroconf v0.0.0-20190424104450-85eadb44205c h1:svzQzfVE9t7Y1CGULS5PsMWs4/H4Au/ZTJzU/0CKgqc=
@@ -70,12 +71,12 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -84,10 +85,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
 github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
 github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@@ -148,9 +147,7 @@ github.com/rs/zerolog v1.17.2/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -205,7 +202,6 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -214,7 +210,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -244,7 +239,6 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

httpd/api_maintenance.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/drakkan/sftpgo/dataprovider"
 	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/sftpd"
 )
 
 func dumpData(w http.ResponseWriter, r *http.Request) {
@@ -103,10 +104,16 @@ func loadData(w http.ResponseWriter, r *http.Request) {
 		u, err := dataprovider.UserExists(dataProvider, user.Username)
 		if err == nil {
 			user.ID = u.ID
+			user.LastLogin = u.LastLogin
+			user.UsedQuotaSize = u.UsedQuotaSize
+			user.UsedQuotaFiles = u.UsedQuotaFiles
 			err = dataprovider.UpdateUser(dataProvider, user)
 			user.Password = "[redacted]"
 			logger.Debug(logSender, "", "restoring existing user: %+v, dump file: %#v, error: %v", user, inputFile, err)
 		} else {
+			user.LastLogin = 0
+			user.UsedQuotaSize = 0
+			user.UsedQuotaFiles = 0
 			err = dataprovider.AddUser(dataProvider, user)
 			user.Password = "[redacted]"
 			logger.Debug(logSender, "", "adding new user: %+v, dump file: %#v, error: %v", user, inputFile, err)
@@ -115,11 +122,17 @@ func loadData(w http.ResponseWriter, r *http.Request) {
 			sendAPIResponse(w, r, err, "", getRespStatus(err))
 			return
 		}
-		if scanQuota == 1 || (scanQuota == 2 && user.HasQuotaRestrictions()) {
+		if needQuotaScan(scanQuota, &user) {
+			if sftpd.AddQuotaScan(user.Username) {
 				logger.Debug(logSender, "", "starting quota scan for restored user: %#v", user.Username)
-			doQuotaScan(user)
+				go doQuotaScan(user)
+			}
 		}
 	}
 	logger.Debug(logSender, "", "backup restored, users: %v", len(dump.Users))
 	sendAPIResponse(w, r, err, "Data restored", http.StatusOK)
 }
+
+func needQuotaScan(scanQuota int, user *dataprovider.User) bool {
+	return scanQuota == 1 || (scanQuota == 2 && user.HasQuotaRestrictions())
+}

httpd/api_quota.go

@@ -6,7 +6,6 @@ import (
 	"github.com/drakkan/sftpgo/dataprovider"
 	"github.com/drakkan/sftpgo/logger"
 	"github.com/drakkan/sftpgo/sftpd"
-	"github.com/drakkan/sftpgo/utils"
 	"github.com/go-chi/render"
 )
 
@@ -26,26 +25,27 @@ func startQuotaScan(w http.ResponseWriter, r *http.Request) {
 		sendAPIResponse(w, r, err, "", http.StatusNotFound)
 		return
 	}
-	if doQuotaScan(user) {
+	if sftpd.AddQuotaScan(user.Username) {
+		go doQuotaScan(user)
 		sendAPIResponse(w, r, err, "Scan started", http.StatusCreated)
 	} else {
 		sendAPIResponse(w, r, err, "Another scan is already in progress", http.StatusConflict)
 	}
 }
 
-func doQuotaScan(user dataprovider.User) bool {
+func doQuotaScan(user dataprovider.User) error {
-	result := sftpd.AddQuotaScan(user.Username)
+	defer sftpd.RemoveQuotaScan(user.Username)
-	if result {
+	fs, err := user.GetFilesystem("")
-		go func() {
+	if err != nil {
-			numFiles, size, _, err := utils.ScanDirContents(user.HomeDir)
+		logger.Warn(logSender, "", "unable scan quota for user %#v error creating filesystem: %v", user.Username, err)
+		return err
+	}
+	numFiles, size, err := fs.ScanDirContents(user.HomeDir)
 	if err != nil {
 		logger.Warn(logSender, "", "error scanning user home dir %#v: %v", user.HomeDir, err)
 	} else {
-				err := dataprovider.UpdateUserQuota(dataProvider, user, numFiles, size, true)
+		err = dataprovider.UpdateUserQuota(dataProvider, user, numFiles, size, true)
 		logger.Debug(logSender, "", "user home dir scanned, user: %#v, dir: %#v, error: %v", user.Username, user.HomeDir, err)
 	}
-			sftpd.RemoveQuotaScan(user.Username)
+	return err
-		}()
-	}
-	return result
 }

httpd/api_user.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 
 	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/utils"
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
 )
@@ -63,8 +64,7 @@ func getUserByID(w http.ResponseWriter, r *http.Request) {
 	}
 	user, err := dataprovider.GetUserByID(dataProvider, userID)
 	if err == nil {
-		user.Password = ""
+		render.JSON(w, r, dataprovider.HideUserSensitiveData(&user))
-		render.JSON(w, r, user)
 	} else if _, ok := err.(*dataprovider.RecordNotFoundError); ok {
 		sendAPIResponse(w, r, err, "", http.StatusNotFound)
 	} else {
@@ -83,8 +83,7 @@ func addUser(w http.ResponseWriter, r *http.Request) {
 	if err == nil {
 		user, err = dataprovider.UserExists(dataProvider, user.Username)
 		if err == nil {
-			user.Password = ""
+			render.JSON(w, r, dataprovider.HideUserSensitiveData(&user))
-			render.JSON(w, r, user)
 		} else {
 			sendAPIResponse(w, r, err, "", http.StatusInternalServerError)
 		}
@@ -102,6 +101,10 @@ func updateUser(w http.ResponseWriter, r *http.Request) {
 	}
 	user, err := dataprovider.GetUserByID(dataProvider, userID)
 	oldPermissions := user.Permissions
+	oldS3AccessSecret := ""
+	if user.FsConfig.Provider == 1 {
+		oldS3AccessSecret = user.FsConfig.S3Config.AccessSecret
+	}
 	user.Permissions = make(map[string][]string)
 	if _, ok := err.(*dataprovider.RecordNotFoundError); ok {
 		sendAPIResponse(w, r, err, "", http.StatusNotFound)
@@ -119,6 +122,13 @@ func updateUser(w http.ResponseWriter, r *http.Request) {
 	if len(user.Permissions) == 0 {
 		user.Permissions = oldPermissions
 	}
+	// we use the new access secret if different from the old one and not empty
+	if user.FsConfig.Provider == 1 {
+		if utils.RemoveDecryptionKey(oldS3AccessSecret) == user.FsConfig.S3Config.AccessSecret ||
+			len(user.FsConfig.S3Config.AccessSecret) == 0 {
+			user.FsConfig.S3Config.AccessSecret = oldS3AccessSecret
+		}
+	}
 	if user.ID != userID {
 		sendAPIResponse(w, r, err, "user ID in request body does not match user ID in path parameter", http.StatusBadRequest)
 		return

httpd/api_utils.go

@@ -406,10 +406,66 @@ func checkUser(expected *dataprovider.User, actual *dataprovider.User) error {
 	if err := compareUserFilters(expected, actual); err != nil {
 		return err
 	}
+	if err := compareUserFsConfig(expected, actual); err != nil {
+		return err
+	}
 
 	return compareEqualsUserFields(expected, actual)
 }
 
+func compareUserFsConfig(expected *dataprovider.User, actual *dataprovider.User) error {
+	if expected.FsConfig.Provider != actual.FsConfig.Provider {
+		return errors.New("Fs provider mismatch")
+	}
+	if expected.FsConfig.S3Config.Bucket != actual.FsConfig.S3Config.Bucket {
+		return errors.New("S3 bucket mismatch")
+	}
+	if expected.FsConfig.S3Config.Region != actual.FsConfig.S3Config.Region {
+		return errors.New("S3 region mismatch")
+	}
+	if expected.FsConfig.S3Config.AccessKey != actual.FsConfig.S3Config.AccessKey {
+		return errors.New("S3 access key mismatch")
+	}
+	if err := checkS3AccessSecret(expected.FsConfig.S3Config.AccessSecret, actual.FsConfig.S3Config.AccessSecret); err != nil {
+		return err
+	}
+	if expected.FsConfig.S3Config.Endpoint != actual.FsConfig.S3Config.Endpoint {
+		return errors.New("S3 endpoint mismatch")
+	}
+	if expected.FsConfig.S3Config.StorageClass != actual.FsConfig.S3Config.StorageClass {
+		return errors.New("S3 storage class mismatch")
+	}
+	return nil
+}
+
+func checkS3AccessSecret(expectedAccessSecret, actualAccessSecret string) error {
+	if len(expectedAccessSecret) > 0 {
+		vals := strings.Split(expectedAccessSecret, "$")
+		if strings.HasPrefix(expectedAccessSecret, "$aes$") && len(vals) == 4 {
+			expectedAccessSecret = utils.RemoveDecryptionKey(expectedAccessSecret)
+			if expectedAccessSecret != actualAccessSecret {
+				return fmt.Errorf("S3 access secret mismatch, expected: %v", expectedAccessSecret)
+			}
+		} else {
+			// here we check that actualAccessSecret is aes encrypted without the nonce
+			parts := strings.Split(actualAccessSecret, "$")
+			if !strings.HasPrefix(actualAccessSecret, "$aes$") || len(parts) != 3 {
+				return errors.New("Invalid S3 access secret")
+			}
+			if len(parts) == len(vals) {
+				if expectedAccessSecret != actualAccessSecret {
+					return errors.New("S3 encrypted access secret mismatch")
+				}
+			}
+		}
+	} else {
+		if expectedAccessSecret != actualAccessSecret {
+			return errors.New("S3 access secret mismatch")
+		}
+	}
+	return nil
+}
+
 func compareUserFilters(expected *dataprovider.User, actual *dataprovider.User) error {
 	if len(expected.Filters.AllowedIP) != len(actual.Filters.AllowedIP) {
 		return errors.New("AllowedIP mismatch")

httpd/httpd_test.go

@@ -242,6 +242,16 @@ func TestAddUserInvalidFilters(t *testing.T) {
 	}
 }
 
+func TestAddUserInvalidFsConfig(t *testing.T) {
+	u := getTestUser()
+	u.FsConfig.Provider = 1
+	u.FsConfig.S3Config.Bucket = ""
+	_, _, err := httpd.AddUser(u, http.StatusBadRequest)
+	if err != nil {
+		t.Errorf("unexpected error adding user with invalid fs config: %v", err)
+	}
+}
+
 func TestUserPublicKey(t *testing.T) {
 	u := getTestUser()
 	invalidPubKey := "invalid"
@@ -299,6 +309,48 @@ func TestUpdateUser(t *testing.T) {
 	}
 }
 
+func TestUserS3Config(t *testing.T) {
+	user, _, err := httpd.AddUser(getTestUser(), http.StatusOK)
+	if err != nil {
+		t.Errorf("unable to add user: %v", err)
+	}
+	user.FsConfig.Provider = 1
+	user.FsConfig.S3Config.Bucket = "test"
+	user.FsConfig.S3Config.Region = "us-east-1"
+	user.FsConfig.S3Config.AccessKey = "Server-Access-Key"
+	user.FsConfig.S3Config.AccessSecret = "Server-Access-Secret"
+	user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000"
+	user, _, err = httpd.UpdateUser(user, http.StatusOK)
+	if err != nil {
+		t.Errorf("unable to update user: %v", err)
+	}
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	if err != nil {
+		t.Errorf("unable to remove: %v", err)
+	}
+	user.Password = defaultPassword
+	user.ID = 0
+	secret, _ := utils.EncryptData("Server-Access-Secret")
+	user.FsConfig.S3Config.AccessSecret = secret
+	user, _, err = httpd.AddUser(user, http.StatusOK)
+	if err != nil {
+		t.Errorf("unable to add user: %v", err)
+	}
+	user.FsConfig.Provider = 1
+	user.FsConfig.S3Config.Bucket = "test1"
+	user.FsConfig.S3Config.Region = "us-east-1"
+	user.FsConfig.S3Config.AccessKey = "Server-Access-Key1"
+	user.FsConfig.S3Config.Endpoint = "http://localhost:9000"
+	user, _, err = httpd.UpdateUser(user, http.StatusOK)
+	if err != nil {
+		t.Errorf("unable to update user: %v", err)
+	}
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	if err != nil {
+		t.Errorf("unable to remove: %v", err)
+	}
+}
+
 func TestUpdateUserNoCredentials(t *testing.T) {
 	user, _, err := httpd.AddUser(getTestUser(), http.StatusOK)
 	if err != nil {
@@ -1398,6 +1450,91 @@ func TestWebUserUpdateMock(t *testing.T) {
 	checkResponseCode(t, http.StatusOK, rr.Code)
 }
 
+func TestWebUserS3Mock(t *testing.T) {
+	user := getTestUser()
+	userAsJSON := getUserAsJSON(t, user)
+	req, _ := http.NewRequest(http.MethodPost, userPath, bytes.NewBuffer(userAsJSON))
+	rr := executeRequest(req)
+	checkResponseCode(t, http.StatusOK, rr.Code)
+	err := render.DecodeJSON(rr.Body, &user)
+	if err != nil {
+		t.Errorf("Error get user: %v", err)
+	}
+	user.FsConfig.Provider = 1
+	user.FsConfig.S3Config.Bucket = "test"
+	user.FsConfig.S3Config.Region = "eu-west-1"
+	user.FsConfig.S3Config.AccessKey = "access-key"
+	user.FsConfig.S3Config.AccessSecret = "access-secret"
+	user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?a=b"
+	user.FsConfig.S3Config.StorageClass = "Standard"
+	form := make(url.Values)
+	form.Set("username", user.Username)
+	form.Set("home_dir", user.HomeDir)
+	form.Set("uid", "0")
+	form.Set("gid", strconv.FormatInt(int64(user.GID), 10))
+	form.Set("max_sessions", strconv.FormatInt(int64(user.MaxSessions), 10))
+	form.Set("quota_size", strconv.FormatInt(user.QuotaSize, 10))
+	form.Set("quota_files", strconv.FormatInt(int64(user.QuotaFiles), 10))
+	form.Set("upload_bandwidth", "0")
+	form.Set("download_bandwidth", "0")
+	form.Set("permissions", "*")
+	form.Set("sub_dirs_permissions", "")
+	form.Set("status", strconv.Itoa(user.Status))
+	form.Set("expiration_date", "2020-01-01 00:00:00")
+	form.Set("allowed_ip", "")
+	form.Set("denied_ip", "")
+	form.Set("fs_provider", "1")
+	form.Set("s3_bucket", user.FsConfig.S3Config.Bucket)
+	form.Set("s3_region", user.FsConfig.S3Config.Region)
+	form.Set("s3_access_key", user.FsConfig.S3Config.AccessKey)
+	form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret)
+	form.Set("s3_storage_class", user.FsConfig.S3Config.StorageClass)
+	form.Set("s3_endpoint", user.FsConfig.S3Config.Endpoint)
+	req, _ = http.NewRequest(http.MethodPost, webUserPath+"/"+strconv.FormatInt(user.ID, 10), strings.NewReader(form.Encode()))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	rr = executeRequest(req)
+	checkResponseCode(t, http.StatusSeeOther, rr.Code)
+	req, _ = http.NewRequest(http.MethodGet, userPath+"?limit=1&offset=0&order=ASC&username="+user.Username, nil)
+	rr = executeRequest(req)
+	checkResponseCode(t, http.StatusOK, rr.Code)
+	var users []dataprovider.User
+	err = render.DecodeJSON(rr.Body, &users)
+	if err != nil {
+		t.Errorf("Error decoding users: %v", err)
+	}
+	if len(users) != 1 {
+		t.Errorf("1 user is expected")
+	}
+	updateUser := users[0]
+	if updateUser.ExpirationDate != 1577836800000 {
+		t.Errorf("invalid expiration date: %v", updateUser.ExpirationDate)
+	}
+	if updateUser.FsConfig.Provider != user.FsConfig.Provider {
+		t.Error("fs provider mismatch")
+	}
+	if updateUser.FsConfig.S3Config.Bucket != user.FsConfig.S3Config.Bucket {
+		t.Error("s3 bucket mismatch")
+	}
+	if updateUser.FsConfig.S3Config.Region != user.FsConfig.S3Config.Region {
+		t.Error("s3 region mismatch")
+	}
+	if updateUser.FsConfig.S3Config.AccessKey != user.FsConfig.S3Config.AccessKey {
+		t.Error("s3 access key mismatch")
+	}
+	if !strings.HasPrefix(updateUser.FsConfig.S3Config.AccessSecret, "$aes$") {
+		t.Error("s3 access secret is not encrypted")
+	}
+	if updateUser.FsConfig.S3Config.StorageClass != user.FsConfig.S3Config.StorageClass {
+		t.Error("s3 storage class mismatch")
+	}
+	if updateUser.FsConfig.S3Config.Endpoint != user.FsConfig.S3Config.Endpoint {
+		t.Error("s3 endpoint mismatch")
+	}
+	req, _ = http.NewRequest(http.MethodDelete, userPath+"/"+strconv.FormatInt(user.ID, 10), nil)
+	rr = executeRequest(req)
+	checkResponseCode(t, http.StatusOK, rr.Code)
+}
+
 func TestProviderClosedMock(t *testing.T) {
 	if providerDriverName == dataprovider.BoltDataProviderName {
 		t.Skip("skipping test provider errors for bolt provider")

httpd/internal_test.go

@@ -6,9 +6,12 @@ import (
 	"html/template"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"testing"
 
 	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/sftpd"
+	"github.com/drakkan/sftpgo/utils"
 	"github.com/go-chi/chi"
 )
 
@@ -120,6 +123,13 @@ func TestCheckUser(t *testing.T) {
 	if err == nil {
 		t.Errorf("DeniedIP contents are not equal")
 	}
+	expected.Filters.DeniedIP = []string{}
+	actual.Filters.DeniedIP = []string{}
+	actual.FsConfig.Provider = 1
+	err = checkUser(expected, actual)
+	if err == nil {
+		t.Errorf("Fs providers are not equal")
+	}
 }
 
 func TestCompareUserFields(t *testing.T) {
@@ -200,6 +210,72 @@ func TestCompareUserFields(t *testing.T) {
 	}
 }
 
+func TestCompareUserFsConfig(t *testing.T) {
+	expected := &dataprovider.User{}
+	actual := &dataprovider.User{}
+	expected.FsConfig.Provider = 1
+	err := compareUserFsConfig(expected, actual)
+	if err == nil {
+		t.Errorf("Provider does not match")
+	}
+	expected.FsConfig.Provider = 0
+	expected.FsConfig.S3Config.Bucket = "bucket"
+	err = compareUserFsConfig(expected, actual)
+	if err == nil {
+		t.Errorf("S3 bucket does not match")
+	}
+	expected.FsConfig.S3Config.Bucket = ""
+	expected.FsConfig.S3Config.Region = "region"
+	err = compareUserFsConfig(expected, actual)
+	if err == nil {
+		t.Errorf("S3 region does not match")
+	}
+	expected.FsConfig.S3Config.Region = ""
+	expected.FsConfig.S3Config.AccessKey = "access key"
+	err = compareUserFsConfig(expected, actual)
+	if err == nil {
+		t.Errorf("S3 access key does not match")
+	}
+	expected.FsConfig.S3Config.AccessKey = ""
+	actual.FsConfig.S3Config.AccessSecret = "access secret"
+	err = compareUserFsConfig(expected, actual)
+	if err == nil {
+		t.Errorf("S3 access secret does not match")
+	}
+	secret, _ := utils.EncryptData("access secret")
+	actual.FsConfig.S3Config.AccessSecret = ""
+	expected.FsConfig.S3Config.AccessSecret = secret
+	err = compareUserFsConfig(expected, actual)
+	if err == nil {
+		t.Errorf("S3 access secret does not match")
+	}
+	expected.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(secret)
+	actual.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(secret) + "a"
+	err = compareUserFsConfig(expected, actual)
+	if err == nil {
+		t.Errorf("S3 access secret does not match")
+	}
+	expected.FsConfig.S3Config.AccessSecret = "test"
+	actual.FsConfig.S3Config.AccessSecret = ""
+	err = compareUserFsConfig(expected, actual)
+	if err == nil {
+		t.Errorf("S3 access secret does not match")
+	}
+	expected.FsConfig.S3Config.AccessSecret = ""
+	actual.FsConfig.S3Config.AccessSecret = ""
+	expected.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/"
+	err = compareUserFsConfig(expected, actual)
+	if err == nil {
+		t.Errorf("S3 endpoint does not match")
+	}
+	expected.FsConfig.S3Config.Endpoint = ""
+	expected.FsConfig.S3Config.StorageClass = "Standard"
+	err = compareUserFsConfig(expected, actual)
+	if err == nil {
+		t.Errorf("S3 storage class does not match")
+	}
+}
+
 func TestApiCallsWithBadURL(t *testing.T) {
 	oldBaseURL := httpBaseURL
 	SetBaseURL(invalidURL)
@@ -315,3 +391,18 @@ func TestRenderInvalidTemplate(t *testing.T) {
 		}
 	}
 }
+
+func TestQuotaScanInvalidFs(t *testing.T) {
+	user := dataprovider.User{
+		Username: "test",
+		HomeDir:  os.TempDir(),
+		FsConfig: dataprovider.Filesystem{
+			Provider: 1,
+		},
+	}
+	sftpd.AddQuotaScan(user.Username)
+	err := doQuotaScan(user)
+	if err == nil {
+		t.Error("quota scan with bad fs must fail")
+	}
+}

httpd/schema/openapi.yaml

@@ -710,6 +710,50 @@ components:
           nullable: true
           description: clients connecting from these IP/Mask are not allowed. Denied rules are evaluated before allowed ones
           example: [ "172.16.0.0/16" ]
+      description: Additional restrictions
+    S3Config:
+      type: object
+      properties:
+        bucket:
+          type: string
+          minLength: 1
+        region:
+          type: string
+          minLength: 1
+        access_key:
+          type: string
+          minLength: 1
+        access_secret:
+          type: string
+          minLength: 1
+          description: the access secret is stored encrypted (AES-256-GCM)
+        endpoint:
+          type: string
+          description: optional endpoint
+        storage_class:
+          type: string
+      required:
+        - bucket
+        - region
+        - access_key
+        - access_secret
+      nullable: true
+      description: S3 Compatible Object Storage configuration details
+    FilesystemConfig:
+      type: object
+      properties:
+        provider:
+          type: integer
+          enum:
+            - 0
+            - 1
+          description: >
+            Providers:
+              * `0` - local filesystem
+              * `1` - S3 Compatible Object Storage
+        s3config:
+          $ref: '#/components/schemas/S3Config'
+      description: Storage filesystem details
     User:
       type: object
       properties:
@@ -799,8 +843,8 @@ components:
           description: Last user login as unix timestamp in milliseconds
         filters:
           $ref: '#/components/schemas/UserFilters'
-          nullable: true
+        filesystem:
-          description: Additional restrictions
+          $ref: '#/components/schemas/FilesystemConfig'
     Transfer:
       type: object
       properties:

httpd/web.go

@@ -224,6 +224,24 @@ func getFiltersFromUserPostFields(r *http.Request) dataprovider.UserFilters {
 	return filters
 }
 
+func getFsConfigFromUserPostFields(r *http.Request) dataprovider.Filesystem {
+	var fs dataprovider.Filesystem
+	provider, err := strconv.Atoi(r.Form.Get("fs_provider"))
+	if err != nil {
+		provider = 0
+	}
+	fs.Provider = provider
+	if fs.Provider == 1 {
+		fs.S3Config.Bucket = r.Form.Get("s3_bucket")
+		fs.S3Config.Region = r.Form.Get("s3_region")
+		fs.S3Config.AccessKey = r.Form.Get("s3_access_key")
+		fs.S3Config.AccessSecret = r.Form.Get("s3_access_secret")
+		fs.S3Config.Endpoint = r.Form.Get("s3_endpoint")
+		fs.S3Config.StorageClass = r.Form.Get("s3_storage_class")
+	}
+	return fs
+}
+
 func getUserFromPostFields(r *http.Request) (dataprovider.User, error) {
 	var user dataprovider.User
 	err := r.ParseForm()
@@ -289,6 +307,7 @@ func getUserFromPostFields(r *http.Request) (dataprovider.User, error) {
 		Status:            status,
 		ExpirationDate:    expirationDateMillis,
 		Filters:           getFiltersFromUserPostFields(r),
+		FsConfig:          getFsConfigFromUserPostFields(r),
 	}
 	return user, err
 }

scripts/README.md

@@ -44,7 +44,7 @@ Let's see a sample usage for each REST API.
 Command:
 
 ```
-python sftpgo_api_cli.py add-user test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 33 --gid 1000 --max-sessions 2 --quota-size 0 --quota-files 3 --permissions "list" "download" "upload" "delete" "rename" "create_dirs" "overwrite" --subdirs-permissions "/dir1:list,download" "/dir2:*" --upload-bandwidth 100 --download-bandwidth 60 --status 0 --expiration-date 2019-01-01 --allowed-ip "192.168.1.1/32"
+python sftpgo_api_cli.py add-user test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 33 --gid 1000 --max-sessions 2 --quota-size 0 --quota-files 3 --permissions "list" "download" "upload" "delete" "rename" "create_dirs" "overwrite" --subdirs-permissions "/dir1:list,download" "/dir2:*" --upload-bandwidth 100 --download-bandwidth 60 --status 0 --expiration-date 2019-01-01 --allowed-ip "192.168.1.1/32" --fs S3 --s3-bucket test --s3-region eu-west-1 --s3-access-key accesskey --s3-access-secret secret --s3-endpoint "http://127.0.0.1:9000" --s3-storage-class Standard
 ```
 
 Output:
@@ -53,6 +53,17 @@ Output:
 {
   "download_bandwidth": 60,
   "expiration_date": 1546297200000,
+  "filesystem": {
+    "provider": 1,
+    "s3config": {
+      "access_key": "accesskey",
+      "access_secret": "$aes$6c088ba12b0b261247c8cf331c46d9260b8e58002957d89ad1c0495e3af665cd0227",
+      "bucket": "test",
+      "endpoint": "http://127.0.0.1:9000",
+      "region": "eu-west-1",
+      "storage_class": "Standard"
+    }
+  },
   "filters": {
     "allowed_ip": [
       "192.168.1.1/32"
@@ -99,7 +110,7 @@ Output:
 Command:
 
 ```
-python sftpgo_api_cli.py update-user 9576 test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 0 --gid 33 --max-sessions 3 --quota-size 0 --quota-files 4 --permissions "*" --subdirs-permissions "/dir1:list,download,create_symlinks" --upload-bandwidth 90 --download-bandwidth 80 --status 1 --expiration-date "" --allowed-ip "" --denied-ip "192.168.1.0/24"
+python sftpgo_api_cli.py update-user 9576 test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 0 --gid 33 --max-sessions 3 --quota-size 0 --quota-files 4 --permissions "*" --subdirs-permissions "/dir1:list,download,create_symlinks" --upload-bandwidth 90 --download-bandwidth 80 --status 1 --expiration-date "" --allowed-ip "" --denied-ip "192.168.1.0/24" --fs local
 ```
 
 Output:
@@ -126,6 +137,10 @@ Output:
 {
   "download_bandwidth": 80,
   "expiration_date": 0,
+  "filesystem": {
+    "provider": 0,
+    "s3config": {}
+  },
   "filters": {
     "allowed_ip": [],
     "denied_ip": [
@@ -174,6 +189,10 @@ Output:
   {
     "download_bandwidth": 80,
     "expiration_date": 0,
+    "filesystem": {
+      "provider": 0,
+      "s3config": {}
+    },
     "filters": {
       "allowed_ip": [],
       "denied_ip": [

scripts/sftpgo_api_cli.py

@@ -70,9 +70,10 @@ class SFTPGoApiRequests:
 		else:
 			print(r.text)
 
-	def buildUserObject(self, user_id=0, username="", password="", public_keys=[], home_dir="", uid=0,
+	def buildUserObject(self, user_id=0, username="", password="", public_keys=[], home_dir="", uid=0, gid=0,
-					gid=0, max_sessions=0, quota_size=0, quota_files=0, permissions={}, upload_bandwidth=0,
+					max_sessions=0, quota_size=0, quota_files=0, permissions={}, upload_bandwidth=0, download_bandwidth=0,
-					download_bandwidth=0, status=1, expiration_date=0, allowed_ip=[], denied_ip=[]):
+					status=1, expiration_date=0, allowed_ip=[], denied_ip=[], fs_provider='local', s3_bucket='',
+					s3_region='', s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class=''):
 		user = {"id":user_id, "username":username, "uid":uid, "gid":gid,
 			"max_sessions":max_sessions, "quota_size":quota_size, "quota_files":quota_files,
 			"upload_bandwidth":upload_bandwidth, "download_bandwidth":download_bandwidth,
@@ -90,6 +91,8 @@ class SFTPGoApiRequests:
 			user.update({"permissions":permissions})
 		if allowed_ip or denied_ip:
 			user.update({"filters":self.buildFilters(allowed_ip, denied_ip)})
+		user.update({"filesystem":self.buildFsConfig(fs_provider, s3_bucket, s3_region, s3_access_key,
+														s3_access_secret, s3_endpoint, s3_storage_class)})
 		return user
 
 	def buildPermissions(self, root_perms, subdirs_perms):
@@ -113,16 +116,25 @@ class SFTPGoApiRequests:
 		filters = {}
 		if allowed_ip:
 			if len(allowed_ip) == 1 and not allowed_ip[0]:
-				filters.update({"allowed_ip":[]})
+				filters.update({'allowed_ip':[]})
 			else:
-				filters.update({"allowed_ip":allowed_ip})
+				filters.update({'allowed_ip':allowed_ip})
 		if denied_ip:
 			if len(denied_ip) == 1 and not denied_ip[0]:
-				filters.update({"denied_ip":[]})
+				filters.update({'denied_ip':[]})
 			else:
-				filters.update({"denied_ip":denied_ip})
+				filters.update({'denied_ip':denied_ip})
 		return filters
 
+	def buildFsConfig(self, fs_provider, s3_bucket, s3_region, s3_access_key, s3_access_secret, s3_endpoint,
+					s3_storage_class):
+		fs_config = {'provider':0}
+		if fs_provider == 'S3':
+			s3config = {'bucket':s3_bucket, 'region':s3_region, 'access_key':s3_access_key, 'access_secret':
+					s3_access_secret, 'endpoint':s3_endpoint, 'storage_class':s3_storage_class}
+			fs_config.update({'provider':1, 's3config':s3config})
+		return fs_config
+
 	def getUsers(self, limit=100, offset=0, order="ASC", username=""):
 		r = requests.get(self.userPath, params={"limit":limit, "offset":offset, "order":order,
 											"username":username}, auth=self.auth, verify=self.verify)
@@ -132,22 +144,25 @@ class SFTPGoApiRequests:
 		r = requests.get(urlparse.urljoin(self.userPath, "user/" + str(user_id)), auth=self.auth, verify=self.verify)
 		self.printResponse(r)
 
-	def addUser(self, username="", password="", public_keys="", home_dir="", uid=0, gid=0, max_sessions=0,
+	def addUser(self, username="", password="", public_keys="", home_dir="", uid=0, gid=0, max_sessions=0, quota_size=0,
-		quota_size=0, quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1,
+			quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1, expiration_date=0,
-		expiration_date=0, subdirs_permissions=[], allowed_ip=[], denied_ip=[]):
+			subdirs_permissions=[], allowed_ip=[], denied_ip=[], fs_provider='local', s3_bucket='', s3_region='',
+			s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class=''):
 		u = self.buildUserObject(0, username, password, public_keys, home_dir, uid, gid, max_sessions,
 			quota_size, quota_files, self.buildPermissions(perms, subdirs_permissions), upload_bandwidth, download_bandwidth,
-			status, expiration_date, allowed_ip, denied_ip)
+			status, expiration_date, allowed_ip, denied_ip, fs_provider, s3_bucket, s3_region,
+			s3_access_key, s3_access_secret, s3_endpoint, s3_storage_class)
 		r = requests.post(self.userPath, json=u, auth=self.auth, verify=self.verify)
 		self.printResponse(r)
 
-	def updateUser(self, user_id, username="", password="", public_keys="", home_dir="", uid=0, gid=0,
+	def updateUser(self, user_id, username="", password="", public_keys="", home_dir="", uid=0, gid=0, max_sessions=0,
-				max_sessions=0, quota_size=0, quota_files=0, perms=[], upload_bandwidth=0,
+				quota_size=0, quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1,
-				download_bandwidth=0, status=1, expiration_date=0, subdirs_permissions=[],
+				expiration_date=0, subdirs_permissions=[], allowed_ip=[], denied_ip=[], fs_provider='local',
-				allowed_ip=[], denied_ip=[]):
+				s3_bucket='', s3_region='', s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class=''):
 		u = self.buildUserObject(user_id, username, password, public_keys, home_dir, uid, gid, max_sessions,
 			quota_size, quota_files, self.buildPermissions(perms, subdirs_permissions), upload_bandwidth, download_bandwidth,
-			status, expiration_date, allowed_ip, denied_ip)
+			status, expiration_date, allowed_ip, denied_ip, fs_provider, s3_bucket, s3_region, s3_access_key,
+			s3_access_secret, s3_endpoint, s3_storage_class)
 		r = requests.put(urlparse.urljoin(self.userPath, "user/" + str(user_id)), json=u, auth=self.auth, verify=self.verify)
 		self.printResponse(r)
 
@@ -238,7 +253,7 @@ class ConvertUsers:
 			self.convertFromProFTPD()
 		self.saveUsers()
 
-	def isUserValid(self, username, uid, gid):
+	def isUserValid(self, username, uid):
 		if self.usernames and not username in self.usernames:
 			return False
 		if self.min_uid >= 0 and uid < self.min_uid:
@@ -257,7 +272,7 @@ class ConvertUsers:
 			home_dir = user.pw_dir
 			status = 1
 			expiration_date = 0
-			if not self.isUserValid(username, uid, gid):
+			if not self.isUserValid(username, uid):
 				continue
 			if self.force_uid >= 0:
 				uid = self.force_uid
@@ -375,7 +390,7 @@ def addCommonUserArguments(parser):
 	parser.add_argument('username', type=str)
 	parser.add_argument('-P', '--password', type=str, default=None, help='Default: %(default)s')
 	parser.add_argument('-K', '--public-keys', type=str, nargs='+', default=[], help='Default: %(default)s')
-	parser.add_argument('-H', '--home-dir', type=str, default="", help='Default: %(default)s')
+	parser.add_argument('-H', '--home-dir', type=str, default='', help='Default: %(default)s')
 	parser.add_argument('--uid', type=int, default=0, help='Default: %(default)s')
 	parser.add_argument('--gid', type=int, default=0, help='Default: %(default)s')
 	parser.add_argument('-C', '--max-sessions', type=int, default=0,
@@ -401,6 +416,14 @@ def addCommonUserArguments(parser):
 					help='Allowed IP/Mask in CIDR notation. For example "192.168.2.0/24" or "2001:db8::/32". Default: %(default)s')
 	parser.add_argument('-N', '--denied-ip', type=str, nargs='+', default=[],
 					help='Denied IP/Mask in CIDR notation. For example "192.168.2.0/24" or "2001:db8::/32". Default: %(default)s')
+	parser.add_argument('--fs', type=str, default='local', choices=['local', 'S3'],
+					help='Filesystem provider. Default: %(default)s')
+	parser.add_argument('--s3-bucket', type=str, default='', help='Default: %(default)s')
+	parser.add_argument('--s3-region', type=str, default='', help='Default: %(default)s')
+	parser.add_argument('--s3-access-key', type=str, default='', help='Default: %(default)s')
+	parser.add_argument('--s3-access-secret', type=str, default='', help='Default: %(default)s')
+	parser.add_argument('--s3-endpoint', type=str, default='', help='Default: %(default)s')
+	parser.add_argument('--s3-storage-class', type=str, default='', help='Default: %(default)s')
 
 
 if __name__ == '__main__':
@@ -503,12 +526,14 @@ if __name__ == '__main__':
 		api.addUser(args.username, args.password, args.public_keys, args.home_dir, args.uid, args.gid, args.max_sessions,
 				args.quota_size, args.quota_files, args.permissions, args.upload_bandwidth, args.download_bandwidth,
 				args.status, getDatetimeAsMillisSinceEpoch(args.expiration_date), args.subdirs_permissions, args.allowed_ip,
-				args.denied_ip)
+				args.denied_ip, args.fs, args.s3_bucket, args.s3_region, args.s3_access_key, args.s3_access_secret,
+				args.s3_endpoint, args.s3_storage_class)
 	elif args.command == 'update-user':
 		api.updateUser(args.id, args.username, args.password, args.public_keys, args.home_dir, args.uid, args.gid,
 					args.max_sessions, args.quota_size, args.quota_files, args.permissions, args.upload_bandwidth,
 					args.download_bandwidth, args.status, getDatetimeAsMillisSinceEpoch(args.expiration_date),
-					args.subdirs_permissions, args.allowed_ip, args.denied_ip)
+					args.subdirs_permissions, args.allowed_ip, args.denied_ip, args.fs, args.s3_bucket, args.s3_region,
+					args.s3_access_key, args.s3_access_secret, args.s3_endpoint, args.s3_storage_class)
 	elif args.command == 'delete-user':
 		api.deleteUser(args.id)
 	elif args.command == 'get-users':

sftpd/handler.go

@@ -1,19 +1,14 @@
 package sftpd
 
 import (
-	"fmt"
 	"io"
-	"io/ioutil"
 	"net"
 	"os"
 	"path"
-	"path/filepath"
-	"strings"
 	"sync"
 	"time"
 
-	"github.com/drakkan/sftpgo/utils"
+	"github.com/drakkan/sftpgo/vfs"
-	"github.com/rs/xid"
 	"golang.org/x/crypto/ssh"
 
 	"github.com/drakkan/sftpgo/dataprovider"
@@ -40,6 +35,7 @@ type Connection struct {
 	netConn      net.Conn
 	channel      ssh.Channel
 	command      string
+	fs           vfs.Fs
 }
 
 // Log outputs a log entry to the configured logger
@@ -55,26 +51,29 @@ func (c Connection) Fileread(request *sftp.Request) (io.ReaderAt, error) {
 		return nil, sftp.ErrSSHFxPermissionDenied
 	}
 
-	p, err := c.buildPath(request.Filepath)
+	p, err := c.fs.ResolvePath(request.Filepath, c.User.GetHomeDir())
 	if err != nil {
-		return nil, getSFTPErrorFromOSError(err)
+		return nil, vfs.GetSFTPError(c.fs, err)
 	}
 
-	fi, err := os.Stat(p)
+	fi, err := c.fs.Stat(p)
 	if err != nil {
-		return nil, getSFTPErrorFromOSError(err)
+		return nil, vfs.GetSFTPError(c.fs, err)
 	}
 
-	file, err := os.Open(p)
+	file, r, cancelFn, err := c.fs.Open(p)
 	if err != nil {
 		c.Log(logger.LevelWarn, logSender, "could not open file %#v for reading: %v", p, err)
-		return nil, getSFTPErrorFromOSError(err)
+		return nil, vfs.GetSFTPError(c.fs, err)
 	}
 
 	c.Log(logger.LevelDebug, logSender, "fileread requested for path: %#v", p)
 
 	transfer := Transfer{
 		file:           file,
+		readerAt:       r,
+		writerAt:       nil,
+		cancelFn:       cancelFn,
 		path:           p,
 		start:          time.Now(),
 		bytesSent:      0,
@@ -98,18 +97,18 @@ func (c Connection) Fileread(request *sftp.Request) (io.ReaderAt, error) {
 // Filewrite handles the write actions for a file on the system.
 func (c Connection) Filewrite(request *sftp.Request) (io.WriterAt, error) {
 	updateConnectionActivity(c.ID)
-	p, err := c.buildPath(request.Filepath)
+	p, err := c.fs.ResolvePath(request.Filepath, c.User.GetHomeDir())
 	if err != nil {
-		return nil, getSFTPErrorFromOSError(err)
+		return nil, vfs.GetSFTPError(c.fs, err)
 	}
 
 	filePath := p
-	if isAtomicUploadEnabled() {
+	if isAtomicUploadEnabled() && c.fs.IsAtomicUploadSupported() {
-		filePath = getUploadTempFilePath(p)
+		filePath = c.fs.GetAtomicUploadPath(p)
 	}
 
-	stat, statErr := os.Stat(p)
+	stat, statErr := c.fs.Stat(p)
-	if os.IsNotExist(statErr) {
+	if c.fs.IsNotExist(statErr) {
 		if !c.User.HasPerm(dataprovider.PermUpload, path.Dir(request.Filepath)) {
 			return nil, sftp.ErrSSHFxPermissionDenied
 		}
@@ -118,7 +117,7 @@ func (c Connection) Filewrite(request *sftp.Request) (io.WriterAt, error) {
 
 	if statErr != nil {
 		c.Log(logger.LevelError, logSender, "error performing file stat %#v: %v", p, statErr)
-		return nil, getSFTPErrorFromOSError(err)
+		return nil, vfs.GetSFTPError(c.fs, statErr)
 	}
 
 	// This happen if we upload a file that has the same name of an existing directory
@@ -139,9 +138,9 @@ func (c Connection) Filewrite(request *sftp.Request) (io.WriterAt, error) {
 func (c Connection) Filecmd(request *sftp.Request) error {
 	updateConnectionActivity(c.ID)
 
-	p, err := c.buildPath(request.Filepath)
+	p, err := c.fs.ResolvePath(request.Filepath, c.User.GetHomeDir())
 	if err != nil {
-		return getSFTPErrorFromOSError(err)
+		return vfs.GetSFTPError(c.fs, err)
 	}
 	target, err := c.getSFTPCmdTargetPath(request.Target)
 	if err != nil {
@@ -186,7 +185,7 @@ func (c Connection) Filecmd(request *sftp.Request) error {
 	}
 
 	// we return if we remove a file or a dir so source path or target path always exists here
-	utils.SetPathPermissions(fileLocation, c.User.GetUID(), c.User.GetGID())
+	vfs.SetPathPermissions(c.fs, fileLocation, c.User.GetUID(), c.User.GetGID())
 
 	return sftp.ErrSSHFxOk
 }
@@ -195,9 +194,9 @@ func (c Connection) Filecmd(request *sftp.Request) error {
 // a directory as well as perform file/folder stat calls.
 func (c Connection) Filelist(request *sftp.Request) (sftp.ListerAt, error) {
 	updateConnectionActivity(c.ID)
-	p, err := c.buildPath(request.Filepath)
+	p, err := c.fs.ResolvePath(request.Filepath, c.User.GetHomeDir())
 	if err != nil {
-		return nil, getSFTPErrorFromOSError(err)
+		return nil, vfs.GetSFTPError(c.fs, err)
 	}
 
 	switch request.Method {
@@ -208,10 +207,10 @@ func (c Connection) Filelist(request *sftp.Request) (sftp.ListerAt, error) {
 
 		c.Log(logger.LevelDebug, logSender, "requested list file for dir: %#v", p)
 
-		files, err := ioutil.ReadDir(p)
+		files, err := c.fs.ReadDir(p)
 		if err != nil {
 			c.Log(logger.LevelWarn, logSender, "error listing directory: %#v", err)
-			return nil, getSFTPErrorFromOSError(err)
+			return nil, vfs.GetSFTPError(c.fs, err)
 		}
 
 		return listerAt(files), nil
@@ -221,10 +220,10 @@ func (c Connection) Filelist(request *sftp.Request) (sftp.ListerAt, error) {
 		}
 
 		c.Log(logger.LevelDebug, logSender, "requested stat for path: %#v", p)
-		s, err := os.Stat(p)
+		s, err := c.fs.Stat(p)
 		if err != nil {
 			c.Log(logger.LevelWarn, logSender, "error running stat on path: %#v", err)
-			return nil, getSFTPErrorFromOSError(err)
+			return nil, vfs.GetSFTPError(c.fs, err)
 		}
 
 		return listerAt([]os.FileInfo{s}), nil
@@ -239,9 +238,9 @@ func (c Connection) getSFTPCmdTargetPath(requestTarget string) (string, error) {
 	// location for the server. If it is not, return an error
 	if len(requestTarget) > 0 {
 		var err error
-		target, err = c.buildPath(requestTarget)
+		target, err = c.fs.ResolvePath(requestTarget, c.User.GetHomeDir())
 		if err != nil {
-			return target, getSFTPErrorFromOSError(err)
+			return target, vfs.GetSFTPError(c.fs, err)
 		}
 	}
 	return target, nil
@@ -252,7 +251,7 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er
 		return nil
 	}
 	pathForPerms := request.Filepath
-	if fi, err := os.Lstat(filePath); err == nil {
+	if fi, err := c.fs.Lstat(filePath); err == nil {
 		if fi.IsDir() {
 			pathForPerms = path.Dir(request.Filepath)
 		}
@@ -263,9 +262,9 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er
 			return sftp.ErrSSHFxPermissionDenied
 		}
 		fileMode := request.Attributes().FileMode()
-		if err := os.Chmod(filePath, fileMode); err != nil {
+		if err := c.fs.Chmod(filePath, fileMode); err != nil {
 			c.Log(logger.LevelWarn, logSender, "failed to chmod path %#v, mode: %v, err: %v", filePath, fileMode.String(), err)
-			return getSFTPErrorFromOSError(err)
+			return vfs.GetSFTPError(c.fs, err)
 		}
 		logger.CommandLog(chmodLogSender, filePath, "", c.User.Username, fileMode.String(), c.ID, c.protocol, -1, -1, "", "", "")
 		return nil
@@ -275,9 +274,9 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er
 		}
 		uid := int(request.Attributes().UID)
 		gid := int(request.Attributes().GID)
-		if err := os.Chown(filePath, uid, gid); err != nil {
+		if err := c.fs.Chown(filePath, uid, gid); err != nil {
 			c.Log(logger.LevelWarn, logSender, "failed to chown path %#v, uid: %v, gid: %v, err: %v", filePath, uid, gid, err)
-			return getSFTPErrorFromOSError(err)
+			return vfs.GetSFTPError(c.fs, err)
 		}
 		logger.CommandLog(chownLogSender, filePath, "", c.User.Username, "", c.ID, c.protocol, uid, gid, "", "", "")
 		return nil
@@ -290,10 +289,10 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er
 		modificationTime := time.Unix(int64(request.Attributes().Mtime), 0)
 		accessTimeString := accessTime.Format(dateFormat)
 		modificationTimeString := modificationTime.Format(dateFormat)
-		if err := os.Chtimes(filePath, accessTime, modificationTime); err != nil {
+		if err := c.fs.Chtimes(filePath, accessTime, modificationTime); err != nil {
 			c.Log(logger.LevelWarn, logSender, "failed to chtimes for path %#v, access time: %v, modification time: %v, err: %v",
 				filePath, accessTime, modificationTime, err)
-			return getSFTPErrorFromOSError(err)
+			return vfs.GetSFTPError(c.fs, err)
 		}
 		logger.CommandLog(chtimesLogSender, filePath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, accessTimeString,
 			modificationTimeString, "")
@@ -303,16 +302,16 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er
 }
 
 func (c Connection) handleSFTPRename(sourcePath string, targetPath string, request *sftp.Request) error {
-	if c.User.GetRelativePath(sourcePath) == "/" {
+	if c.fs.GetRelativePath(sourcePath, c.User.GetHomeDir()) == "/" {
 		c.Log(logger.LevelWarn, logSender, "renaming root dir is not allowed")
 		return sftp.ErrSSHFxPermissionDenied
 	}
 	if !c.User.HasPerm(dataprovider.PermRename, path.Dir(request.Target)) {
 		return sftp.ErrSSHFxPermissionDenied
 	}
-	if err := os.Rename(sourcePath, targetPath); err != nil {
+	if err := c.fs.Rename(sourcePath, targetPath); err != nil {
 		c.Log(logger.LevelWarn, logSender, "failed to rename file, source: %#v target: %#v: %v", sourcePath, targetPath, err)
-		return getSFTPErrorFromOSError(err)
+		return vfs.GetSFTPError(c.fs, err)
 	}
 	logger.CommandLog(renameLogSender, sourcePath, targetPath, c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
 	go executeAction(operationRename, c.User.Username, sourcePath, targetPath, "", 0)
@@ -320,7 +319,7 @@ func (c Connection) handleSFTPRename(sourcePath string, targetPath string, reque
 }
 
 func (c Connection) handleSFTPRmdir(dirPath string, request *sftp.Request) error {
-	if c.User.GetRelativePath(dirPath) == "/" {
+	if c.fs.GetRelativePath(dirPath, c.User.GetHomeDir()) == "/" {
 		c.Log(logger.LevelWarn, logSender, "removing root dir is not allowed")
 		return sftp.ErrSSHFxPermissionDenied
 	}
@@ -330,18 +329,18 @@ func (c Connection) handleSFTPRmdir(dirPath string, request *sftp.Request) error
 
 	var fi os.FileInfo
 	var err error
-	if fi, err = os.Lstat(dirPath); err != nil {
+	if fi, err = c.fs.Lstat(dirPath); err != nil {
 		c.Log(logger.LevelWarn, logSender, "failed to remove a dir %#v: stat error: %v", dirPath, err)
-		return getSFTPErrorFromOSError(err)
+		return vfs.GetSFTPError(c.fs, err)
 	}
 	if !fi.IsDir() || fi.Mode()&os.ModeSymlink == os.ModeSymlink {
 		c.Log(logger.LevelDebug, logSender, "cannot remove %#v is not a directory", dirPath)
 		return sftp.ErrSSHFxFailure
 	}
 
-	if err = os.Remove(dirPath); err != nil {
+	if err = c.fs.Remove(dirPath, true); err != nil {
 		c.Log(logger.LevelWarn, logSender, "failed to remove directory %#v: %v", dirPath, err)
-		return getSFTPErrorFromOSError(err)
+		return vfs.GetSFTPError(c.fs, err)
 	}
 
 	logger.CommandLog(rmdirLogSender, dirPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
@@ -349,16 +348,16 @@ func (c Connection) handleSFTPRmdir(dirPath string, request *sftp.Request) error
 }
 
 func (c Connection) handleSFTPSymlink(sourcePath string, targetPath string, request *sftp.Request) error {
-	if c.User.GetRelativePath(sourcePath) == "/" {
+	if c.fs.GetRelativePath(sourcePath, c.User.GetHomeDir()) == "/" {
 		c.Log(logger.LevelWarn, logSender, "symlinking root dir is not allowed")
 		return sftp.ErrSSHFxPermissionDenied
 	}
 	if !c.User.HasPerm(dataprovider.PermCreateSymlinks, path.Dir(request.Target)) {
 		return sftp.ErrSSHFxPermissionDenied
 	}
-	if err := os.Symlink(sourcePath, targetPath); err != nil {
+	if err := c.fs.Symlink(sourcePath, targetPath); err != nil {
 		c.Log(logger.LevelWarn, logSender, "failed to create symlink %#v -> %#v: %v", sourcePath, targetPath, err)
-		return getSFTPErrorFromOSError(err)
+		return vfs.GetSFTPError(c.fs, err)
 	}
 
 	logger.CommandLog(symlinkLogSender, sourcePath, targetPath, c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
@@ -369,11 +368,11 @@ func (c Connection) handleSFTPMkdir(dirPath string, request *sftp.Request) error
 	if !c.User.HasPerm(dataprovider.PermCreateDirs, path.Dir(request.Filepath)) {
 		return sftp.ErrSSHFxPermissionDenied
 	}
-	if err := os.Mkdir(dirPath, 0777); err != nil {
+	if err := c.fs.Mkdir(dirPath); err != nil {
 		c.Log(logger.LevelWarn, logSender, "error creating missing dir: %#v error: %v", dirPath, err)
-		return getSFTPErrorFromOSError(err)
+		return vfs.GetSFTPError(c.fs, err)
 	}
-	utils.SetPathPermissions(dirPath, c.User.GetUID(), c.User.GetGID())
+	vfs.SetPathPermissions(c.fs, dirPath, c.User.GetUID(), c.User.GetGID())
 
 	logger.CommandLog(mkdirLogSender, dirPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
 	return nil
@@ -387,18 +386,18 @@ func (c Connection) handleSFTPRemove(filePath string, request *sftp.Request) err
 	var size int64
 	var fi os.FileInfo
 	var err error
-	if fi, err = os.Lstat(filePath); err != nil {
+	if fi, err = c.fs.Lstat(filePath); err != nil {
 		c.Log(logger.LevelWarn, logSender, "failed to remove a file %#v: stat error: %v", filePath, err)
-		return getSFTPErrorFromOSError(err)
+		return vfs.GetSFTPError(c.fs, err)
 	}
 	if fi.IsDir() && fi.Mode()&os.ModeSymlink != os.ModeSymlink {
 		c.Log(logger.LevelDebug, logSender, "cannot remove %#v is not a file/symlink", filePath)
 		return sftp.ErrSSHFxFailure
 	}
 	size = fi.Size()
-	if err := os.Remove(filePath); err != nil {
+	if err := c.fs.Remove(filePath, false); err != nil {
 		c.Log(logger.LevelWarn, logSender, "failed to remove a file/symlink %#v: %v", filePath, err)
-		return getSFTPErrorFromOSError(err)
+		return vfs.GetSFTPError(c.fs, err)
 	}
 
 	logger.CommandLog(removeLogSender, filePath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
@@ -416,16 +415,19 @@ func (c Connection) handleSFTPUploadToNewFile(requestPath, filePath string) (io.
 		return nil, sftp.ErrSSHFxFailure
 	}
 
-	file, err := os.Create(filePath)
+	file, w, cancelFn, err := c.fs.Create(filePath, 0)
 	if err != nil {
 		c.Log(logger.LevelWarn, logSender, "error creating file %#v: %v", requestPath, err)
-		return nil, getSFTPErrorFromOSError(err)
+		return nil, vfs.GetSFTPError(c.fs, err)
 	}
 
-	utils.SetPathPermissions(filePath, c.User.GetUID(), c.User.GetGID())
+	vfs.SetPathPermissions(c.fs, filePath, c.User.GetUID(), c.User.GetGID())
 
 	transfer := Transfer{
 		file:           file,
+		writerAt:       w,
+		readerAt:       nil,
+		cancelFn:       cancelFn,
 		path:           requestPath,
 		start:          time.Now(),
 		bytesSent:      0,
@@ -456,19 +458,25 @@ func (c Connection) handleSFTPUploadToExistingFile(pflags sftp.FileOpenFlags, re
 	minWriteOffset := int64(0)
 	osFlags := getOSOpenFlags(pflags)
 
-	if isAtomicUploadEnabled() {
+	if pflags.Append && osFlags&os.O_TRUNC == 0 && !c.fs.IsUploadResumeSupported() {
-		err = os.Rename(requestPath, filePath)
+		c.Log(logger.LevelInfo, logSender, "upload resume requested for path: %#v but not supported in fs implementation",
+			requestPath)
+		return nil, sftp.ErrSSHFxOpUnsupported
+	}
+
+	if isAtomicUploadEnabled() && c.fs.IsAtomicUploadSupported() {
+		err = c.fs.Rename(requestPath, filePath)
 		if err != nil {
 			c.Log(logger.LevelWarn, logSender, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %v",
 				requestPath, filePath, err)
-			return nil, getSFTPErrorFromOSError(err)
+			return nil, vfs.GetSFTPError(c.fs, err)
 		}
 	}
-	// we use 0666 so the umask is applied
+
-	file, err := os.OpenFile(filePath, osFlags, 0666)
+	file, w, cancelFn, err := c.fs.Create(filePath, osFlags)
 	if err != nil {
 		c.Log(logger.LevelWarn, logSender, "error opening existing file, flags: %v, source: %#v, err: %v", pflags, filePath, err)
-		return nil, getSFTPErrorFromOSError(err)
+		return nil, vfs.GetSFTPError(c.fs, err)
 	}
 
 	if pflags.Append && osFlags&os.O_TRUNC == 0 {
@@ -478,10 +486,13 @@ func (c Connection) handleSFTPUploadToExistingFile(pflags sftp.FileOpenFlags, re
 		dataprovider.UpdateUserQuota(dataProvider, c.User, 0, -fileSize, false)
 	}
 
-	utils.SetPathPermissions(filePath, c.User.GetUID(), c.User.GetGID())
+	vfs.SetPathPermissions(c.fs, filePath, c.User.GetUID(), c.User.GetGID())
 
 	transfer := Transfer{
 		file:           file,
+		writerAt:       w,
+		readerAt:       nil,
+		cancelFn:       cancelFn,
 		path:           requestPath,
 		start:          time.Now(),
 		bytesSent:      0,
@@ -522,103 +533,6 @@ func (c Connection) hasSpace(checkFiles bool) bool {
 	return true
 }
 
-// Normalizes a file/directory we get from the SFTP request to ensure the user is not able to escape
-// from their data directory. After normalization if the file/directory is still within their home
-// path it is returned. If they managed to "escape" an error will be returned.
-func (c Connection) buildPath(rawPath string) (string, error) {
-	r := filepath.Clean(filepath.Join(c.User.HomeDir, rawPath))
-	p, err := filepath.EvalSymlinks(r)
-	if err != nil && !os.IsNotExist(err) {
-		return "", err
-	} else if os.IsNotExist(err) {
-		// The requested path doesn't exist, so at this point we need to iterate up the
-		// path chain until we hit a directory that _does_ exist and can be validated.
-		_, err = c.findFirstExistingDir(r)
-		if err != nil {
-			c.Log(logger.LevelWarn, logSender, "error resolving not existent path: %#v", err)
-		}
-		return r, err
-	}
-
-	err = c.isSubDir(p)
-	if err != nil {
-		c.Log(logger.LevelWarn, logSender, "Invalid path resolution, dir: %#v outside user home: %#v err: %v", p, c.User.HomeDir, err)
-	}
-	return r, err
-}
-
-// iterate up the path chain until we hit a directory that does exist and can be validated.
-// all nonexistent directories will be returned
-func (c Connection) findNonexistentDirs(path string) ([]string, error) {
-	results := []string{}
-	cleanPath := filepath.Clean(path)
-	parent := filepath.Dir(cleanPath)
-	_, err := os.Stat(parent)
-
-	for os.IsNotExist(err) {
-		results = append(results, parent)
-		parent = filepath.Dir(parent)
-		_, err = os.Stat(parent)
-	}
-	if err != nil {
-		return results, err
-	}
-	p, err := filepath.EvalSymlinks(parent)
-	if err != nil {
-		return results, err
-	}
-	err = c.isSubDir(p)
-	if err != nil {
-		c.Log(logger.LevelWarn, logSender, "Error finding non existing dir: %v", err)
-	}
-	return results, err
-}
-
-// iterate up the path chain until we hit a directory that does exist and can be validated.
-func (c Connection) findFirstExistingDir(path string) (string, error) {
-	results, err := c.findNonexistentDirs(path)
-	if err != nil {
-		c.Log(logger.LevelWarn, logSender, "unable to find non existent dirs: %v", err)
-		return "", err
-	}
-	var parent string
-	if len(results) > 0 {
-		lastMissingDir := results[len(results)-1]
-		parent = filepath.Dir(lastMissingDir)
-	} else {
-		parent = c.User.GetHomeDir()
-	}
-	p, err := filepath.EvalSymlinks(parent)
-	if err != nil {
-		return "", err
-	}
-	fileInfo, err := os.Stat(p)
-	if err != nil {
-		return "", err
-	}
-	if !fileInfo.IsDir() {
-		return "", fmt.Errorf("resolved path is not a dir: %#v", p)
-	}
-	err = c.isSubDir(p)
-	return p, err
-}
-
-// checks if sub is a subpath of the user home dir.
-// EvalSymlink must be used on sub before calling this method
-func (c Connection) isSubDir(sub string) error {
-	// home dir must exist and it is already a validated absolute path
-	parent, err := filepath.EvalSymlinks(c.User.HomeDir)
-	if err != nil {
-		c.Log(logger.LevelWarn, logSender, "invalid home dir %#v: %v", c.User.HomeDir, err)
-		return err
-	}
-	if !strings.HasPrefix(sub, parent) {
-		c.Log(logger.LevelWarn, logSender, "path %#v is not inside: %#v ", sub, parent)
-		return fmt.Errorf("path %#v is not inside: %#v", sub, parent)
-	}
-	return nil
-}
-
 func (c Connection) close() error {
 	if c.channel != nil {
 		err := c.channel.Close()
@@ -649,20 +563,3 @@ func getOSOpenFlags(requestFlags sftp.FileOpenFlags) (flags int) {
 	}
 	return osFlags
 }
-
-func getUploadTempFilePath(path string) string {
-	dir := filepath.Dir(path)
-	guid := xid.New().String()
-	return filepath.Join(dir, ".sftpgo-upload."+guid+"."+filepath.Base(path))
-}
-
-func getSFTPErrorFromOSError(err error) error {
-	if os.IsNotExist(err) {
-		return sftp.ErrSSHFxNoSuchFile
-	} else if os.IsPermission(err) {
-		return sftp.ErrSSHFxPermissionDenied
-	} else if err != nil {
-		return sftp.ErrSSHFxFailure
-	}
-	return nil
-}

sftpd/internal_test.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"sync"
@@ -16,6 +17,8 @@ import (
 
 	"github.com/drakkan/sftpgo/dataprovider"
 	"github.com/drakkan/sftpgo/utils"
+	"github.com/drakkan/sftpgo/vfs"
+	"github.com/eikenb/pipeat"
 	"github.com/pkg/sftp"
 )
 
@@ -60,6 +63,61 @@ func (c *MockChannel) Stderr() io.ReadWriter {
 	return c.StdErrBuffer
 }
 
+// MockOsFs mockable OsFs
+type MockOsFs struct {
+	vfs.OsFs
+	err                     error
+	statErr                 error
+	isAtomicUploadSupported bool
+}
+
+// Name returns the name for the Fs implementation
+func (fs MockOsFs) Name() string {
+	return "mockOsFs"
+}
+
+// IsUploadResumeSupported returns true if upload resume is supported
+func (MockOsFs) IsUploadResumeSupported() bool {
+	return false
+}
+
+// IsAtomicUploadSupported returns true if atomic upload is supported
+func (fs MockOsFs) IsAtomicUploadSupported() bool {
+	return fs.isAtomicUploadSupported
+}
+
+// Stat returns a FileInfo describing the named file
+func (fs MockOsFs) Stat(name string) (os.FileInfo, error) {
+	if fs.statErr != nil {
+		return nil, fs.statErr
+	}
+	return os.Stat(name)
+}
+
+// Remove removes the named file or (empty) directory.
+func (fs MockOsFs) Remove(name string, isDir bool) error {
+	if fs.err != nil {
+		return fs.err
+	}
+	return os.Remove(name)
+}
+
+// Rename renames (moves) source to target
+func (fs MockOsFs) Rename(source, target string) error {
+	if fs.err != nil {
+		return fs.err
+	}
+	return os.Rename(source, target)
+}
+
+func newMockOsFs(err, statErr error, atomicUpload bool) vfs.Fs {
+	return &MockOsFs{
+		err:                     err,
+		statErr:                 statErr,
+		isAtomicUploadSupported: atomicUpload,
+	}
+}
+
 func TestWrongActions(t *testing.T) {
 	actionsCopy := actions
 	badCommand := "/bad/command"
@@ -218,13 +276,134 @@ func TestReadWriteErrors(t *testing.T) {
 	if err == nil {
 		t.Error("upoload must fail the expected size does not match")
 	}
+	r, _, _ := pipeat.Pipe()
+	transfer = Transfer{
+		readerAt:      r,
+		writerAt:      nil,
+		start:         time.Now(),
+		bytesSent:     0,
+		bytesReceived: 0,
+		user: dataprovider.User{
+			Username: "testuser",
+		},
+		connectionID:  "",
+		transferType:  transferDownload,
+		lastActivity:  time.Now(),
+		isNewFile:     false,
+		protocol:      protocolSFTP,
+		transferError: nil,
+		isFinished:    false,
+		lock:          new(sync.Mutex),
+	}
+	transfer.closeIO()
+	_, err = transfer.ReadAt(buf, 0)
+	if err == nil {
+		t.Error("reading from a closed pipe must fail")
+	}
+	r, w, _ := pipeat.Pipe()
+	transfer = Transfer{
+		readerAt:      nil,
+		writerAt:      w,
+		start:         time.Now(),
+		bytesSent:     0,
+		bytesReceived: 0,
+		user: dataprovider.User{
+			Username: "testuser",
+		},
+		connectionID:  "",
+		transferType:  transferDownload,
+		lastActivity:  time.Now(),
+		isNewFile:     false,
+		protocol:      protocolSFTP,
+		transferError: nil,
+		isFinished:    false,
+		lock:          new(sync.Mutex),
+	}
+	r.Close()
+	transfer.closeIO()
+	_, err = transfer.WriteAt([]byte("test"), 0)
+	if err == nil {
+		t.Error("writing to closed pipe must fail")
+	}
+	os.Remove(testfile)
+}
+
+func TestTransferCancelFn(t *testing.T) {
+	testfile := "testfile"
+	file, _ := os.Create(testfile)
+	isCancelled := false
+	cancelFn := func() {
+		isCancelled = true
+	}
+	transfer := Transfer{
+		file:          file,
+		cancelFn:      cancelFn,
+		path:          file.Name(),
+		start:         time.Now(),
+		bytesSent:     0,
+		bytesReceived: 0,
+		user: dataprovider.User{
+			Username: "testuser",
+		},
+		connectionID:   "",
+		transferType:   transferDownload,
+		lastActivity:   time.Now(),
+		isNewFile:      false,
+		protocol:       protocolSFTP,
+		transferError:  nil,
+		isFinished:     false,
+		minWriteOffset: 0,
+		expectedSize:   10,
+		lock:           new(sync.Mutex),
+	}
+	transfer.TransferError(errors.New("fake error, this will trigger cancelFn"))
+	transfer.Close()
+	if !isCancelled {
+		t.Error("cancelFn not called")
+	}
+	os.Remove(testfile)
+}
+
+func TestMockFsErrors(t *testing.T) {
+	errFake := errors.New("fake error")
+	fs := newMockOsFs(errFake, errFake, false)
+	u := dataprovider.User{}
+	u.Username = "test"
+	u.Permissions = make(map[string][]string)
+	u.Permissions["/"] = []string{dataprovider.PermAny}
+	u.HomeDir = os.TempDir()
+	c := Connection{
+		fs:   fs,
+		User: u,
+	}
+	testfile := filepath.Join(u.HomeDir, "testfile")
+	request := sftp.NewRequest("Remove", testfile)
+	ioutil.WriteFile(testfile, []byte("test"), 0666)
+	err := c.handleSFTPRemove(testfile, request)
+	if err != sftp.ErrSSHFxFailure {
+		t.Errorf("unexpected error: %v", err)
+	}
+	_, err = c.Filewrite(request)
+	if err != sftp.ErrSSHFxFailure {
+		t.Errorf("unexpected error: %v", err)
+	}
+	var flags sftp.FileOpenFlags
+	flags.Write = true
+	flags.Trunc = false
+	flags.Append = true
+	_, err = c.handleSFTPUploadToExistingFile(flags, testfile, testfile, 0)
+	if err != sftp.ErrSSHFxOpUnsupported {
+		t.Errorf("unexpected error: %v", err)
+	}
 	os.Remove(testfile)
 }
 
 func TestUploadFiles(t *testing.T) {
 	oldUploadMode := uploadMode
 	uploadMode = uploadModeAtomic
-	c := Connection{}
+	c := Connection{
+		fs: vfs.NewOsFs("123"),
+	}
 	var flags sftp.FileOpenFlags
 	flags.Write = true
 	flags.Trunc = true
@@ -255,10 +434,13 @@ func TestWithInvalidHome(t *testing.T) {
 	if err == nil {
 		t.Errorf("login a user with an invalid home_dir must fail")
 	}
+	fs, _ := u.GetFilesystem("123")
 	c := Connection{
 		User: u,
+		fs:   fs,
 	}
-	err = c.isSubDir("dir_rel_path")
+	u.HomeDir = os.TempDir()
+	_, err = c.fs.ResolvePath("../upper_path", u.GetHomeDir())
 	if err == nil {
 		t.Errorf("tested path is not a home subdir")
 	}
@@ -266,12 +448,18 @@ func TestWithInvalidHome(t *testing.T) {
 
 func TestSFTPCmdTargetPath(t *testing.T) {
 	u := dataprovider.User{}
-	u.HomeDir = "home_rel_path"
+	if runtime.GOOS == "windows" {
+		u.HomeDir = "C:\\invalid_home"
+	} else {
+		u.HomeDir = "/invalid_home"
+	}
 	u.Username = "test"
 	u.Permissions = make(map[string][]string)
 	u.Permissions["/"] = []string{dataprovider.PermAny}
+	fs, _ := u.GetFilesystem("123")
 	connection := Connection{
 		User: u,
+		fs:   fs,
 	}
 	_, err := connection.getSFTPCmdTargetPath("invalid_path")
 	if err != sftp.ErrSSHFxNoSuchFile {
@@ -281,16 +469,17 @@ func TestSFTPCmdTargetPath(t *testing.T) {
 
 func TestGetSFTPErrorFromOSError(t *testing.T) {
 	err := os.ErrNotExist
-	err = getSFTPErrorFromOSError(err)
+	fs := vfs.NewOsFs("")
+	err = vfs.GetSFTPError(fs, err)
 	if err != sftp.ErrSSHFxNoSuchFile {
 		t.Errorf("unexpected error: %v", err)
 	}
 	err = os.ErrPermission
-	err = getSFTPErrorFromOSError(err)
+	err = vfs.GetSFTPError(fs, err)
 	if err != sftp.ErrSSHFxPermissionDenied {
 		t.Errorf("unexpected error: %v", err)
 	}
-	err = getSFTPErrorFromOSError(nil)
+	err = vfs.GetSFTPError(fs, nil)
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
@@ -418,10 +607,12 @@ func TestSSHCommandErrors(t *testing.T) {
 	user := dataprovider.User{}
 	user.Permissions = make(map[string][]string)
 	user.Permissions["/"] = []string{dataprovider.PermAny}
+	fs, _ := user.GetFilesystem("123")
 	connection := Connection{
 		channel: &mockSSHChannel,
 		netConn: client,
 		User:    user,
+		fs:      fs,
 	}
 	cmd := sshCommand{
 		command:    "md5sum",
@@ -499,6 +690,45 @@ func TestSSHCommandErrors(t *testing.T) {
 	}
 }
 
+func TestSSHCommandsRemoteFs(t *testing.T) {
+	buf := make([]byte, 65535)
+	stdErrBuf := make([]byte, 65535)
+	mockSSHChannel := MockChannel{
+		Buffer:       bytes.NewBuffer(buf),
+		StdErrBuffer: bytes.NewBuffer(stdErrBuf),
+	}
+	server, client := net.Pipe()
+	defer server.Close()
+	defer client.Close()
+	user := dataprovider.User{}
+	user.FsConfig = dataprovider.Filesystem{
+		Provider: 1}
+	fs, _ := user.GetFilesystem("123")
+	connection := Connection{
+		channel: &mockSSHChannel,
+		netConn: client,
+		User:    user,
+		fs:      fs,
+	}
+	cmd := sshCommand{
+		command:    "md5sum",
+		connection: connection,
+		args:       []string{},
+	}
+	err := cmd.handleHashCommands()
+	if err == nil {
+		t.Error("command must fail for a non local filesystem")
+	}
+	command, err := cmd.getSystemCommand()
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	err = cmd.executeSystemCommand(command)
+	if err == nil {
+		t.Error("command must fail for a non local filesystem")
+	}
+}
+
 func TestSSHCommandQuotaScan(t *testing.T) {
 	buf := make([]byte, 65535)
 	stdErrBuf := make([]byte, 65535)
@@ -513,14 +743,17 @@ func TestSSHCommandQuotaScan(t *testing.T) {
 	defer client.Close()
 	permissions := make(map[string][]string)
 	permissions["/"] = []string{dataprovider.PermAny}
-	connection := Connection{
+	user := dataprovider.User{
-		channel: &mockSSHChannel,
-		netConn: client,
-		User: dataprovider.User{
 		Permissions: permissions,
 		QuotaFiles:  1,
 		HomeDir:     "invalid_path",
-		},
+	}
+	fs, _ := user.GetFilesystem("123")
+	connection := Connection{
+		channel: &mockSSHChannel,
+		netConn: client,
+		User:    user,
+		fs:      fs,
 	}
 	cmd := sshCommand{
 		command:    "git-receive-pack",
@@ -536,11 +769,14 @@ func TestSSHCommandQuotaScan(t *testing.T) {
 func TestRsyncOptions(t *testing.T) {
 	permissions := make(map[string][]string)
 	permissions["/"] = []string{dataprovider.PermAny}
-	conn := Connection{
+	user := dataprovider.User{
-		User: dataprovider.User{
 		Permissions: permissions,
 		HomeDir:     os.TempDir(),
-		},
+	}
+	fs, _ := user.GetFilesystem("123")
+	conn := Connection{
+		User: user,
+		fs:   fs,
 	}
 	sshCmd := sshCommand{
 		command:    "rsync",
@@ -556,11 +792,11 @@ func TestRsyncOptions(t *testing.T) {
 	}
 	permissions["/"] = []string{dataprovider.PermDownload, dataprovider.PermUpload, dataprovider.PermCreateDirs,
 		dataprovider.PermListItems, dataprovider.PermOverwrite, dataprovider.PermDelete, dataprovider.PermRename}
+	user.Permissions = permissions
+	fs, _ = user.GetFilesystem("123")
 	conn = Connection{
-		User: dataprovider.User{
+		User: user,
-			Permissions: permissions,
+		fs:   fs,
-			HomeDir:     os.TempDir(),
-		},
 	}
 	sshCmd = sshCommand{
 		command:    "rsync",
@@ -592,13 +828,16 @@ func TestSystemCommandErrors(t *testing.T) {
 	defer client.Close()
 	permissions := make(map[string][]string)
 	permissions["/"] = []string{dataprovider.PermAny}
+	user := dataprovider.User{
+		Permissions: permissions,
+		HomeDir:     os.TempDir(),
+	}
+	fs, _ := user.GetFilesystem("123")
 	connection := Connection{
 		channel: &mockSSHChannel,
 		netConn: client,
-		User: dataprovider.User{
+		User:    user,
-			Permissions: permissions,
+		fs:      fs,
-			HomeDir:     os.TempDir(),
-		},
 	}
 	sshCmd := sshCommand{
 		command:    "ls",
@@ -934,6 +1173,55 @@ func TestSCPCommandHandleErrors(t *testing.T) {
 	}
 }
 
+func TestSCPErrorsMockFs(t *testing.T) {
+	errFake := errors.New("fake error")
+	fs := newMockOsFs(errFake, errFake, false)
+	u := dataprovider.User{}
+	u.Username = "test"
+	u.Permissions = make(map[string][]string)
+	u.Permissions["/"] = []string{dataprovider.PermAny}
+	u.HomeDir = os.TempDir()
+	buf := make([]byte, 65535)
+	stdErrBuf := make([]byte, 65535)
+	mockSSHChannel := MockChannel{
+		Buffer:       bytes.NewBuffer(buf),
+		StdErrBuffer: bytes.NewBuffer(stdErrBuf),
+	}
+	server, client := net.Pipe()
+	defer server.Close()
+	defer client.Close()
+	connection := Connection{
+		channel: &mockSSHChannel,
+		netConn: client,
+		fs:      fs,
+		User:    u,
+	}
+	scpCommand := scpCommand{
+		sshCommand: sshCommand{
+			command:    "scp",
+			connection: connection,
+			args:       []string{"-r", "-t", "/tmp"},
+		},
+	}
+	err := scpCommand.handleUpload("test", 0)
+	if err != errFake {
+		t.Errorf("unexpected error: %v", err)
+	}
+	testfile := filepath.Join(u.HomeDir, "testfile")
+	ioutil.WriteFile(testfile, []byte("test"), 0666)
+	stat, _ := os.Stat(u.HomeDir)
+	err = scpCommand.handleRecursiveDownload(u.HomeDir, stat)
+	if err != errFake {
+		t.Errorf("unexpected error: %v", err)
+	}
+	scpCommand.sshCommand.connection.fs = newMockOsFs(errFake, nil, true)
+	err = scpCommand.handleUpload(filepath.Base(testfile), 0)
+	if err != errFake {
+		t.Errorf("unexpected error: %v", err)
+	}
+	os.Remove(testfile)
+}
+
 func TestSCPRecursiveDownloadErrors(t *testing.T) {
 	buf := make([]byte, 65535)
 	stdErrBuf := make([]byte, 65535)
@@ -951,6 +1239,7 @@ func TestSCPRecursiveDownloadErrors(t *testing.T) {
 	connection := Connection{
 		channel: &mockSSHChannel,
 		netConn: client,
+		fs:      vfs.NewOsFs("123"),
 	}
 	scpCommand := scpCommand{
 		sshCommand: sshCommand{
@@ -1033,9 +1322,11 @@ func TestSCPCreateDirs(t *testing.T) {
 		ReadError:    nil,
 		WriteError:   nil,
 	}
+	fs, _ := u.GetFilesystem("123")
 	connection := Connection{
 		User:    u,
 		channel: &mockSSHChannel,
+		fs:      fs,
 	}
 	scpCommand := scpCommand{
 		sshCommand: sshCommand{

sftpd/scp.go

@@ -3,7 +3,6 @@ package sftpd
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math"
 	"os"
 	"path"
@@ -16,6 +15,7 @@ import (
 	"github.com/drakkan/sftpgo/dataprovider"
 	"github.com/drakkan/sftpgo/logger"
 	"github.com/drakkan/sftpgo/utils"
+	"github.com/drakkan/sftpgo/vfs"
 )
 
 var (
@@ -116,7 +116,7 @@ func (c *scpCommand) handleRecursiveUpload() error {
 
 func (c *scpCommand) handleCreateDir(dirPath string) error {
 	updateConnectionActivity(c.connection.ID)
-	p, err := c.connection.buildPath(dirPath)
+	p, err := c.connection.fs.ResolvePath(dirPath, c.connection.User.GetHomeDir())
 	if err != nil {
 		c.connection.Log(logger.LevelWarn, logSenderSCP, "error creating dir: %#v, invalid file path, err: %v", dirPath, err)
 		c.sendErrorMessage(err.Error())
@@ -189,17 +189,20 @@ func (c *scpCommand) handleUploadFile(requestPath, filePath string, sizeToRead i
 		return err
 	}
 
-	file, err := os.Create(filePath)
+	file, w, cancelFn, err := c.connection.fs.Create(filePath, 0)
 	if err != nil {
 		c.connection.Log(logger.LevelError, logSenderSCP, "error creating file %#v: %v", requestPath, err)
 		c.sendErrorMessage(err.Error())
 		return err
 	}
 
-	utils.SetPathPermissions(filePath, c.connection.User.GetUID(), c.connection.User.GetGID())
+	vfs.SetPathPermissions(c.connection.fs, filePath, c.connection.User.GetUID(), c.connection.User.GetGID())
 
 	transfer := Transfer{
 		file:           file,
+		readerAt:       nil,
+		writerAt:       w,
+		cancelFn:       cancelFn,
 		path:           requestPath,
 		start:          time.Now(),
 		bytesSent:      0,
@@ -225,18 +228,18 @@ func (c *scpCommand) handleUpload(uploadFilePath string, sizeToRead int64) error
 
 	updateConnectionActivity(c.connection.ID)
 
-	p, err := c.connection.buildPath(uploadFilePath)
+	p, err := c.connection.fs.ResolvePath(uploadFilePath, c.connection.User.GetHomeDir())
 	if err != nil {
 		c.connection.Log(logger.LevelWarn, logSenderSCP, "error uploading file: %#v, err: %v", uploadFilePath, err)
 		c.sendErrorMessage(err.Error())
 		return err
 	}
 	filePath := p
-	if isAtomicUploadEnabled() {
+	if isAtomicUploadEnabled() && c.connection.fs.IsAtomicUploadSupported() {
-		filePath = getUploadTempFilePath(p)
+		filePath = c.connection.fs.GetAtomicUploadPath(p)
 	}
-	stat, statErr := os.Stat(p)
+	stat, statErr := c.connection.fs.Stat(p)
-	if os.IsNotExist(statErr) {
+	if c.connection.fs.IsNotExist(statErr) {
 		if !c.connection.User.HasPerm(dataprovider.PermUpload, path.Dir(uploadFilePath)) {
 			err := fmt.Errorf("Permission denied")
 			c.connection.Log(logger.LevelWarn, logSenderSCP, "cannot upload file: %#v, permission denied", uploadFilePath)
@@ -248,8 +251,8 @@ func (c *scpCommand) handleUpload(uploadFilePath string, sizeToRead int64) error
 
 	if statErr != nil {
 		c.connection.Log(logger.LevelError, logSenderSCP, "error performing file stat %#v: %v", p, statErr)
-		c.sendErrorMessage(err.Error())
+		c.sendErrorMessage(statErr.Error())
-		return err
+		return statErr
 	}
 
 	if stat.IsDir() {
@@ -266,8 +269,8 @@ func (c *scpCommand) handleUpload(uploadFilePath string, sizeToRead int64) error
 		return err
 	}
 
-	if isAtomicUploadEnabled() {
+	if isAtomicUploadEnabled() && c.connection.fs.IsAtomicUploadSupported() {
-		err = os.Rename(p, filePath)
+		err = c.connection.fs.Rename(p, filePath)
 		if err != nil {
 			c.connection.Log(logger.LevelError, logSenderSCP, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %v",
 				p, filePath, err)
@@ -315,14 +318,14 @@ func (c *scpCommand) handleRecursiveDownload(dirPath string, stat os.FileInfo) e
 		if err != nil {
 			return err
 		}
-		files, err := ioutil.ReadDir(dirPath)
+		files, err := c.connection.fs.ReadDir(dirPath)
 		if err != nil {
 			c.sendErrorMessage(err.Error())
 			return err
 		}
 		var dirs []string
 		for _, file := range files {
-			filePath := c.connection.User.GetRelativePath(filepath.Join(dirPath, file.Name()))
+			filePath := c.connection.fs.GetRelativePath(c.connection.fs.Join(dirPath, file.Name()), c.connection.User.GetHomeDir())
 			if file.Mode().IsRegular() || file.Mode()&os.ModeSymlink == os.ModeSymlink {
 				err = c.handleDownload(filePath)
 				if err != nil {
@@ -419,7 +422,7 @@ func (c *scpCommand) handleDownload(filePath string) error {
 
 	updateConnectionActivity(c.connection.ID)
 
-	p, err := c.connection.buildPath(filePath)
+	p, err := c.connection.fs.ResolvePath(filePath, c.connection.User.GetHomeDir())
 	if err != nil {
 		err := fmt.Errorf("Invalid file path")
 		c.connection.Log(logger.LevelWarn, logSenderSCP, "error downloading file: %#v, invalid file path", filePath)
@@ -428,7 +431,7 @@ func (c *scpCommand) handleDownload(filePath string) error {
 	}
 
 	var stat os.FileInfo
-	if stat, err = os.Stat(p); os.IsNotExist(err) {
+	if stat, err = c.connection.fs.Stat(p); err != nil {
 		c.connection.Log(logger.LevelWarn, logSenderSCP, "error downloading file: %#v, err: %v", p, err)
 		c.sendErrorMessage(err.Error())
 		return err
@@ -452,7 +455,7 @@ func (c *scpCommand) handleDownload(filePath string) error {
 		return err
 	}
 
-	file, err := os.Open(p)
+	file, r, cancelFn, err := c.connection.fs.Open(p)
 	if err != nil {
 		c.connection.Log(logger.LevelError, logSenderSCP, "could not open file %#v for reading: %v", p, err)
 		c.sendErrorMessage(err.Error())
@@ -461,6 +464,9 @@ func (c *scpCommand) handleDownload(filePath string) error {
 
 	transfer := Transfer{
 		file:           file,
+		readerAt:       r,
+		writerAt:       nil,
+		cancelFn:       cancelFn,
 		path:           p,
 		start:          time.Now(),
 		bytesSent:      0,
@@ -608,12 +614,12 @@ func (c *scpCommand) getNextUploadProtocolMessage() (string, error) {
 
 func (c *scpCommand) createDir(dirPath string) error {
 	var err error
-	if err = os.Mkdir(dirPath, 0777); err != nil {
+	if err = c.connection.fs.Mkdir(dirPath); err != nil {
 		c.connection.Log(logger.LevelError, logSenderSCP, "error creating dir: %v", dirPath)
 		c.sendErrorMessage(err.Error())
 		return err
 	}
-	utils.SetPathPermissions(dirPath, c.connection.User.GetUID(), c.connection.User.GetGID())
+	vfs.SetPathPermissions(c.connection.fs, dirPath, c.connection.User.GetUID(), c.connection.User.GetGID())
 	return err
 }
 
@@ -668,8 +674,8 @@ func (c *scpCommand) getFileUploadDestPath(scpDestPath, fileName string) string
 			// but if scpDestPath is an existing directory then we put the uploaded file
 			// inside that directory this is as scp command works, for example:
 			// scp fileName.txt user@127.0.0.1:/existing_dir
-			if p, err := c.connection.buildPath(scpDestPath); err == nil {
+			if p, err := c.connection.fs.ResolvePath(scpDestPath, c.connection.User.GetHomeDir()); err == nil {
-				if stat, err := os.Stat(p); err == nil {
+				if stat, err := c.connection.fs.Stat(p); err == nil {
 					if stat.IsDir() {
 						return path.Join(scpDestPath, fileName)
 					}

sftpd/server.go

@@ -266,6 +266,14 @@ func (c Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.Server
 	loginType = sconn.Permissions.Extensions["login_type"]
 	connectionID := hex.EncodeToString(sconn.SessionID())
 
+	fs, err := user.GetFilesystem(connectionID)
+
+	if err != nil {
+		logger.Warn(logSender, "", "could create filesystem for user %#v err: %v", user.Username, err)
+		conn.Close()
+		return
+	}
+
 	connection := Connection{
 		ID:            connectionID,
 		User:          user,
@@ -275,7 +283,11 @@ func (c Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.Server
 		lastActivity:  time.Now(),
 		netConn:       conn,
 		channel:       nil,
+		fs:            fs,
 	}
+
+	connection.fs.CheckRootPath(user.GetHomeDir(), user.Username, user.GetUID(), user.GetGID())
+
 	connection.Log(logger.LevelInfo, logSender, "User id: %d, logged in with: %#v, username: %#v, home_dir: %#v remote addr: %#v",
 		user.ID, loginType, user.Username, user.HomeDir, remoteAddr.String())
 	dataprovider.UpdateLastLogin(dataProvider, user)
@@ -368,14 +380,6 @@ func loginUser(user dataprovider.User, loginType string, remoteAddr string) (*ss
 		logger.Debug(logSender, "", "cannot login user %#v, remote address is not allowed: %v", user.Username, remoteAddr)
 		return nil, fmt.Errorf("Login is not allowed from this address: %v", remoteAddr)
 	}
-	if _, err := os.Stat(user.HomeDir); os.IsNotExist(err) {
-		err := os.MkdirAll(user.HomeDir, 0777)
-		logger.Debug(logSender, "", "home directory %#v for user %#v does not exist, try to create, mkdir error: %v",
-			user.HomeDir, user.Username, err)
-		if err == nil {
-			utils.SetPathPermissions(user.HomeDir, user.GetUID(), user.GetGID())
-		}
-	}
 
 	json, err := json.Marshal(user)
 	if err != nil {

sftpd/sftpd.go

@@ -307,7 +307,7 @@ func GetConnectionsStats() []ConnectionStatus {
 					StartTime:     utils.GetTimeAsMsSinceEpoch(t.start),
 					Size:          size,
 					LastActivity:  utils.GetTimeAsMsSinceEpoch(t.lastActivity),
-					Path:          c.User.GetRelativePath(t.path),
+					Path:          c.fs.GetRelativePath(t.path, c.User.GetHomeDir()),
 				}
 				conn.Transfers = append(conn.Transfers, connTransfer)
 			}

sftpd/sftpd_test.go

@@ -35,6 +35,7 @@ import (
 	"github.com/drakkan/sftpgo/logger"
 	"github.com/drakkan/sftpgo/sftpd"
 	"github.com/drakkan/sftpgo/utils"
+	"github.com/drakkan/sftpgo/vfs"
 	"github.com/pkg/sftp"
 	"github.com/rs/zerolog"
 )
@@ -1017,6 +1018,45 @@ func TestLoginUserExpiration(t *testing.T) {
 	os.RemoveAll(user.GetHomeDir())
 }
 
+func TestLoginInvalidFs(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("this test is not available on Windows")
+	}
+	config.LoadConfig(configDir, "")
+	providerConf := config.GetProviderConf()
+	if providerConf.Driver != dataprovider.SQLiteDataProviderName {
+		t.Skip("this test require sqlite provider")
+	}
+	dbPath := providerConf.Name
+	if !filepath.IsAbs(dbPath) {
+		dbPath = filepath.Join(configDir, dbPath)
+	}
+	usePubKey := true
+	u := getTestUser(usePubKey)
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	if err != nil {
+		t.Errorf("unable to add user: %v", err)
+	}
+	// we update the database using sqlite3 CLI since we cannot add an user with an invalid config
+	time.Sleep(150 * time.Millisecond)
+	updateUserQuery := fmt.Sprintf("UPDATE users SET filesystem='{\"provider\":1}' WHERE id=%v", user.ID)
+	cmd := exec.Command("sqlite3", dbPath, updateUserQuery)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Errorf("unexpected error: %v, cmd out: %v", err, string(out))
+	}
+	time.Sleep(200 * time.Millisecond)
+	_, err = getSftpClient(user, usePubKey)
+	if err == nil {
+		t.Error("login must fail, the user has an invalid filesystem config")
+	}
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	if err != nil {
+		t.Errorf("unable to remove user: %v", err)
+	}
+	os.RemoveAll(user.GetHomeDir())
+}
+
 func TestLoginWithIPFilters(t *testing.T) {
 	usePubKey := true
 	u := getTestUser(usePubKey)
@@ -2864,52 +2904,53 @@ func TestRootDirCommands(t *testing.T) {
 func TestRelativePaths(t *testing.T) {
 	user := getTestUser(true)
 	path := filepath.Join(user.HomeDir, "/")
-	rel := user.GetRelativePath(path)
+	fs := vfs.NewOsFs("")
+	rel := fs.GetRelativePath(path, user.GetHomeDir())
 	if rel != "/" {
 		t.Errorf("Unexpected relative path: %v", rel)
 	}
 	path = filepath.Join(user.HomeDir, "//")
-	rel = user.GetRelativePath(path)
+	rel = fs.GetRelativePath(path, user.GetHomeDir())
 	if rel != "/" {
 		t.Errorf("Unexpected relative path: %v", rel)
 	}
 	path = filepath.Join(user.HomeDir, "../..")
-	rel = user.GetRelativePath(path)
+	rel = fs.GetRelativePath(path, user.GetHomeDir())
 	if rel != "/" {
 		t.Errorf("Unexpected relative path: %v", rel)
 	}
 	path = filepath.Join(user.HomeDir, "../../../../../")
-	rel = user.GetRelativePath(path)
+	rel = fs.GetRelativePath(path, user.GetHomeDir())
 	if rel != "/" {
 		t.Errorf("Unexpected relative path: %v", rel)
 	}
 	path = filepath.Join(user.HomeDir, "/..")
-	rel = user.GetRelativePath(path)
+	rel = fs.GetRelativePath(path, user.GetHomeDir())
 	if rel != "/" {
 		t.Errorf("Unexpected relative path: %v", rel)
 	}
 	path = filepath.Join(user.HomeDir, "/../../../..")
-	rel = user.GetRelativePath(path)
+	rel = fs.GetRelativePath(path, user.GetHomeDir())
 	if rel != "/" {
 		t.Errorf("Unexpected relative path: %v", rel)
 	}
 	path = filepath.Join(user.HomeDir, "")
-	rel = user.GetRelativePath(path)
+	rel = fs.GetRelativePath(path, user.GetHomeDir())
 	if rel != "/" {
 		t.Errorf("Unexpected relative path: %v", rel)
 	}
 	path = filepath.Join(user.HomeDir, ".")
-	rel = user.GetRelativePath(path)
+	rel = fs.GetRelativePath(path, user.GetHomeDir())
 	if rel != "/" {
 		t.Errorf("Unexpected relative path: %v", rel)
 	}
 	path = filepath.Join(user.HomeDir, "somedir")
-	rel = user.GetRelativePath(path)
+	rel = fs.GetRelativePath(path, user.GetHomeDir())
 	if rel != "/somedir" {
 		t.Errorf("Unexpected relative path: %v", rel)
 	}
 	path = filepath.Join(user.HomeDir, "/somedir/subdir")
-	rel = user.GetRelativePath(path)
+	rel = fs.GetRelativePath(path, user.GetHomeDir())
 	if rel != "/somedir/subdir" {
 		t.Errorf("Unexpected relative path: %v", rel)
 	}

sftpd/ssh_cmd.go

@@ -21,12 +21,14 @@ import (
 	"github.com/drakkan/sftpgo/logger"
 	"github.com/drakkan/sftpgo/metrics"
 	"github.com/drakkan/sftpgo/utils"
+	"github.com/drakkan/sftpgo/vfs"
 	"golang.org/x/crypto/ssh"
 )
 
 var (
 	errQuotaExceeded     = errors.New("denying write due to space limit")
 	errPermissionDenied  = errors.New("Permission denied. You don't have the permissions to execute this command")
+	errUnsupportedConfig = errors.New("command unsupported for this configuration")
 )
 
 type sshCommand struct {
@@ -101,6 +103,9 @@ func (c *sshCommand) handle() error {
 }
 
 func (c *sshCommand) handleHashCommands() error {
+	if !vfs.IsLocalOsFs(c.connection.fs) {
+		return c.sendErrorResponse(errUnsupportedConfig)
+	}
 	var h hash.Hash
 	if c.command == "md5sum" {
 		h = md5.New()
@@ -125,14 +130,14 @@ func (c *sshCommand) handleHashCommands() error {
 		response = fmt.Sprintf("%x  -\n", h.Sum(nil))
 	} else {
 		sshPath := c.getDestPath()
-		path, err := c.connection.buildPath(sshPath)
+		fsPath, err := c.connection.fs.ResolvePath(sshPath, c.connection.User.GetHomeDir())
 		if err != nil {
 			return c.sendErrorResponse(err)
 		}
 		if !c.connection.User.HasPerm(dataprovider.PermListItems, sshPath) {
 			return c.sendErrorResponse(errPermissionDenied)
 		}
-		hash, err := computeHashForFile(h, path)
+		hash, err := computeHashForFile(h, fsPath)
 		if err != nil {
 			return c.sendErrorResponse(err)
 		}
@@ -144,6 +149,9 @@ func (c *sshCommand) handleHashCommands() error {
 }
 
 func (c *sshCommand) executeSystemCommand(command systemCommand) error {
+	if !vfs.IsLocalOsFs(c.connection.fs) {
+		return c.sendErrorResponse(errUnsupportedConfig)
+	}
 	if c.connection.User.QuotaFiles > 0 && c.connection.User.UsedQuotaFiles > c.connection.User.QuotaFiles {
 		return c.sendErrorResponse(errQuotaExceeded)
 	}
@@ -288,7 +296,7 @@ func (c *sshCommand) getSystemCommand() (systemCommand, error) {
 	if len(c.args) > 0 {
 		var err error
 		sshPath := c.getDestPath()
-		path, err = c.connection.buildPath(sshPath)
+		path, err = c.connection.fs.ResolvePath(sshPath, c.connection.User.GetHomeDir())
 		if err != nil {
 			return command, err
 		}
@@ -331,7 +339,7 @@ func (c *sshCommand) rescanHomeDir() error {
 	var numFiles int
 	var size int64
 	if AddQuotaScan(c.connection.User.Username) {
-		numFiles, size, _, err = utils.ScanDirContents(c.connection.User.HomeDir)
+		numFiles, size, err = c.connection.fs.ScanDirContents(c.connection.User.HomeDir)
 		if err != nil {
 			c.connection.Log(logger.LevelWarn, logSenderSSH, "error scanning user home dir %#v: %v", c.connection.User.HomeDir, err)
 		} else {
@@ -389,7 +397,7 @@ func (c *sshCommand) sendExitStatus(err error) {
 	if err == nil && c.command != "scp" {
 		realPath := c.getDestPath()
 		if len(realPath) > 0 {
-			p, err := c.connection.buildPath(realPath)
+			p, err := c.connection.fs.ResolvePath(realPath, c.connection.User.GetHomeDir())
 			if err == nil {
 				realPath = p
 			}

sftpd/transfer.go

@@ -11,6 +11,7 @@ import (
 	"github.com/drakkan/sftpgo/dataprovider"
 	"github.com/drakkan/sftpgo/logger"
 	"github.com/drakkan/sftpgo/metrics"
+	"github.com/eikenb/pipeat"
 )
 
 const (
@@ -26,6 +27,9 @@ var (
 // It implements the io Reader and Writer interface to handle files downloads and uploads
 type Transfer struct {
 	file           *os.File
+	writerAt       *pipeat.PipeWriterAt
+	readerAt       *pipeat.PipeReaderAt
+	cancelFn       func()
 	path           string
 	start          time.Time
 	bytesSent      int64
@@ -52,6 +56,9 @@ func (t *Transfer) TransferError(err error) {
 		return
 	}
 	t.transferError = err
+	if t.cancelFn != nil {
+		t.cancelFn()
+	}
 	elapsed := time.Since(t.start).Nanoseconds() / 1000000
 	logger.Warn(logSender, t.connectionID, "Unexpected error for transfer, path: %#v, error: \"%v\" bytes sent: %v, "+
 		"bytes received: %v transfer running since %v ms", t.path, t.transferError, t.bytesSent, t.bytesReceived, elapsed)
@@ -61,7 +68,13 @@ func (t *Transfer) TransferError(err error) {
 // It handles download bandwidth throttling too
 func (t *Transfer) ReadAt(p []byte, off int64) (n int, err error) {
 	t.lastActivity = time.Now()
-	readed, e := t.file.ReadAt(p, off)
+	var readed int
+	var e error
+	if t.readerAt != nil {
+		readed, e = t.readerAt.ReadAt(p, off)
+	} else {
+		readed, e = t.file.ReadAt(p, off)
+	}
 	t.lock.Lock()
 	t.bytesSent += int64(readed)
 	t.lock.Unlock()
@@ -82,7 +95,13 @@ func (t *Transfer) WriteAt(p []byte, off int64) (n int, err error) {
 		t.TransferError(err)
 		return 0, err
 	}
-	written, e := t.file.WriteAt(p, off)
+	var written int
+	var e error
+	if t.writerAt != nil {
+		written, e = t.writerAt.WriteAt(p, off)
+	} else {
+		written, e = t.file.WriteAt(p, off)
+	}
 	t.lock.Lock()
 	t.bytesReceived += int64(written)
 	t.lock.Unlock()
@@ -105,14 +124,14 @@ func (t *Transfer) Close() error {
 	if t.isFinished {
 		return errTransferClosed
 	}
-	err := t.file.Close()
+	err := t.closeIO()
 	t.isFinished = true
 	numFiles := 0
 	if t.isNewFile {
 		numFiles = 1
 	}
 	t.checkDownloadSize()
-	if t.transferType == transferUpload && t.file.Name() != t.path {
+	if t.transferType == transferUpload && t.file != nil && t.file.Name() != t.path {
 		if t.transferError == nil || uploadMode == uploadModeAtomicWithResume {
 			err = os.Rename(t.file.Name(), t.path)
 			logger.Debug(logSender, t.connectionID, "atomic upload completed, rename: %#v -> %#v, error: %v",
@@ -150,6 +169,18 @@ func (t *Transfer) Close() error {
 	return err
 }
 
+func (t *Transfer) closeIO() error {
+	var err error
+	if t.writerAt != nil {
+		err = t.writerAt.Close()
+	} else if t.readerAt != nil {
+		err = t.readerAt.Close()
+	} else {
+		err = t.file.Close()
+	}
+	return err
+}
+
 func (t *Transfer) checkDownloadSize() {
 	if t.transferType == transferDownload && t.transferError == nil && t.bytesSent < t.expectedSize {
 		t.transferError = fmt.Errorf("incomplete download: %v/%v bytes transferred", t.bytesSent, t.expectedSize)

sql/mysql/20200116.sql

@@ -0,0 +1,6 @@
+BEGIN;
+--
+-- Add field filesystem to user
+--
+ALTER TABLE `users` ADD COLUMN `filesystem` longtext NULL;
+COMMIT;

sql/pgsql/20200116.sql

@@ -0,0 +1,6 @@
+BEGIN;
+--
+-- Add field filesystem to user
+--
+ALTER TABLE "users" ADD COLUMN "filesystem" text NULL;
+COMMIT;

sql/sqlite/20200116.sql

@@ -0,0 +1,9 @@
+BEGIN;
+--
+-- Add field filesystem to user
+--
+CREATE TABLE "new__users" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE, "password" varchar(255) NULL, "public_keys" text NULL, "home_dir" varchar(255) NOT NULL, "uid" integer NOT NULL, "gid" integer NOT NULL, "max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "permissions" text NOT NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL, "expiration_date" bigint NOT NULL, "last_login" bigint NOT NULL, "status" integer NOT NULL, "filters" text NULL, "filesystem" text NULL);
+INSERT INTO "new__users" ("id", "username", "password", "public_keys", "home_dir", "uid", "gid", "max_sessions", "quota_size", "quota_files", "permissions", "used_quota_size", "used_quota_files", "last_quota_update", "upload_bandwidth", "download_bandwidth", "expiration_date", "last_login", "status", "filters", "filesystem") SELECT "id", "username", "password", "public_keys", "home_dir", "uid", "gid", "max_sessions", "quota_size", "quota_files", "permissions", "used_quota_size", "used_quota_files", "last_quota_update", "upload_bandwidth", "download_bandwidth", "expiration_date", "last_login", "status", "filters", NULL FROM "users";
+DROP TABLE "users";
+ALTER TABLE "new__users" RENAME TO "users";
+COMMIT;

templates/user.html

@@ -191,6 +191,58 @@
         </div>
     </div>
 
+    <div class="form-group row">
+        <label for="idFilesystem" class="col-sm-2 col-form-label">Storage</label>
+        <div class="col-sm-10">
+            <select class="form-control" id="idFilesystem" name="fs_provider">
+                <option value="0" {{if eq .User.FsConfig.Provider 0 }}selected{{end}}>local</option>
+                <option value="1" {{if eq .User.FsConfig.Provider 1 }}selected{{end}}>S3</option>
+            </select>
+        </div>
+    </div>
+
+    <div class="form-group row">
+        <label for="idS3Bucket" class="col-sm-2 col-form-label">S3 Bucket</label>
+        <div class="col-sm-3">
+            <input type="text" class="form-control" id="idS3Bucket" name="s3_bucket" placeholder=""
+                value="{{.User.FsConfig.S3Config.Bucket}}" maxlength="255">
+        </div>
+        <div class="col-sm-2"></div>
+        <label for="idS3Region" class="col-sm-2 col-form-label">S3 Region</label>
+        <div class="col-sm-3">
+            <input type="text" class="form-control" id="idS3Region" name="s3_region" placeholder=""
+                value="{{.User.FsConfig.S3Config.Region}}" maxlength="255">
+        </div>
+    </div>
+
+    <div class="form-group row">
+        <label for="idS3AccessKey" class="col-sm-2 col-form-label">S3 Access Key</label>
+        <div class="col-sm-3">
+            <input type="text" class="form-control" id="idS3AccessKey" name="s3_access_key" placeholder=""
+                value="{{.User.FsConfig.S3Config.AccessKey}}" maxlength="255">
+        </div>
+        <div class="col-sm-2"></div>
+        <label for="idS3AccessSecret" class="col-sm-2 col-form-label">S3 Access Secret</label>
+        <div class="col-sm-3">
+            <input type="text" class="form-control" id="idS3AccessSecret" name="s3_access_secret" placeholder=""
+                value="{{.User.FsConfig.S3Config.AccessSecret}}" maxlength="1000">
+        </div>
+    </div>
+
+    <div class="form-group row">
+        <label for="idS3StorageClass" class="col-sm-2 col-form-label">S3 Storage Class</label>
+        <div class="col-sm-3">
+            <input type="text" class="form-control" id="idS3StorageClass" name="s3_storage_class" placeholder=""
+                value="{{.User.FsConfig.S3Config.StorageClass}}" maxlength="1000">
+        </div>
+        <div class="col-sm-2"></div>
+        <label for="idS3Endpoint" class="col-sm-2 col-form-label">S3 Endpoint</label>
+        <div class="col-sm-3">
+            <input type="text" class="form-control" id="idS3Endpoint" name="s3_endpoint" placeholder=""
+                value="{{.User.FsConfig.S3Config.Endpoint}}" maxlength="255">
+        </div>
+    </div>
+
     <input type="hidden" name="expiration_date" id="hidden_start_datetime" value="">
     <button type="submit" class="btn btn-primary float-right mt-3 mb-5 px-5 px-3">Submit</button>
 </form>

utils/utils.go

@@ -2,15 +2,16 @@
 package utils
 
 import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
 	"fmt"
+	"io"
 	"net"
-	"os"
-	"path/filepath"
-	"runtime"
 	"strings"
 	"time"
-
-	"github.com/drakkan/sftpgo/logger"
 )
 
 const logSender = "utils"
@@ -46,49 +47,6 @@ func GetTimeFromMsecSinceEpoch(msec int64) time.Time {
 	return time.Unix(0, msec*1000000)
 }
 
-// ScanDirContents returns the number of files contained in a directory, their size and a slice with the file paths
-func ScanDirContents(path string) (int, int64, []string, error) {
-	var numFiles int
-	var size int64
-	var fileList []string
-	var err error
-	numFiles = 0
-	size = 0
-	isDir, err := isDirectory(path)
-	if err == nil && isDir {
-		err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			if info != nil && info.Mode().IsRegular() {
-				size += info.Size()
-				numFiles++
-				fileList = append(fileList, path)
-			}
-			return err
-		})
-	}
-
-	return numFiles, size, fileList, err
-}
-
-func isDirectory(path string) (bool, error) {
-	fileInfo, err := os.Stat(path)
-	if err != nil {
-		return false, err
-	}
-	return fileInfo.IsDir(), err
-}
-
-// SetPathPermissions call os.Chown on unix, it does nothing on windows
-func SetPathPermissions(path string, uid int, gid int) {
-	if runtime.GOOS != "windows" {
-		if err := os.Chown(path, uid, gid); err != nil {
-			logger.Warn(logSender, "", "error chowning path %v: %v", path, err)
-		}
-	}
-}
-
 // GetAppVersion returns VersionInfo struct
 func GetAppVersion() VersionInfo {
 	return versionInfo
@@ -144,3 +102,74 @@ func GetIPFromRemoteAddress(remoteAddress string) string {
 	}
 	return remoteAddress
 }
+
+// NilIfEmpty returns nil if the input string is empty
+func NilIfEmpty(s string) *string {
+	if len(s) == 0 {
+		return nil
+	}
+	return &s
+}
+
+// EncryptData encrypts data using the given key
+func EncryptData(data string) (string, error) {
+	var result string
+	key := make([]byte, 16)
+	if _, err := io.ReadFull(rand.Reader, key); err != nil {
+		return result, err
+	}
+	keyHex := hex.EncodeToString(key)
+	block, err := aes.NewCipher([]byte(keyHex))
+	if err != nil {
+		return result, err
+	}
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return result, err
+	}
+	nonce := make([]byte, gcm.NonceSize())
+	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
+		return result, err
+	}
+	ciphertext := gcm.Seal(nonce, nonce, []byte(data), nil)
+	result = fmt.Sprintf("$aes$%s$%x", keyHex, ciphertext)
+	return result, err
+}
+
+// RemoveDecryptionKey returns encrypted data without the decryption key
+func RemoveDecryptionKey(encryptData string) string {
+	vals := strings.Split(encryptData, "$")
+	if len(vals) == 4 {
+		return fmt.Sprintf("$%v$%v", vals[1], vals[3])
+	}
+	return encryptData
+}
+
+// DecryptData decrypts data encrypted using EncryptData
+func DecryptData(data string) (string, error) {
+	var result string
+	vals := strings.Split(data, "$")
+	if len(vals) != 4 {
+		return "", errors.New("data to decrypt is not in the correct format")
+	}
+	key := vals[2]
+	encrypted, err := hex.DecodeString(vals[3])
+	if err != nil {
+		return result, err
+	}
+	block, err := aes.NewCipher([]byte(key))
+	if err != nil {
+		return result, err
+	}
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return result, err
+	}
+	nonceSize := gcm.NonceSize()
+	nonce, ciphertext := encrypted[:nonceSize], encrypted[nonceSize:]
+	plaintext, err := gcm.Open(nil, nonce, ciphertext, nil)
+	if err != nil {
+		return result, err
+	}
+	return string(plaintext), nil
+}

vfs/osfs.go

@@ -0,0 +1,288 @@
+package vfs
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/eikenb/pipeat"
+	"github.com/rs/xid"
+)
+
+const (
+	// osFsName is the name for the local Fs implementation
+	osFsName = "osfs"
+)
+
+// OsFs is a Fs implementation that uses functions provided by the os package.
+type OsFs struct {
+	name         string
+	connectionID string
+}
+
+// NewOsFs returns an OsFs object that allows to interact with local Os filesystem
+func NewOsFs(connectionID string) Fs {
+	return &OsFs{
+		name:         osFsName,
+		connectionID: connectionID}
+}
+
+// Name returns the name for the Fs implementation
+func (fs OsFs) Name() string {
+	return fs.name
+}
+
+// ConnectionID returns the SSH connection ID associated to this Fs implementation
+func (fs OsFs) ConnectionID() string {
+	return fs.connectionID
+}
+
+// Stat returns a FileInfo describing the named file
+func (OsFs) Stat(name string) (os.FileInfo, error) {
+	return os.Stat(name)
+}
+
+// Lstat returns a FileInfo describing the named file
+func (OsFs) Lstat(name string) (os.FileInfo, error) {
+	return os.Lstat(name)
+}
+
+// Open opens the named file for reading
+func (OsFs) Open(name string) (*os.File, *pipeat.PipeReaderAt, func(), error) {
+	f, err := os.Open(name)
+	return f, nil, nil, err
+}
+
+// Create creates or opens the named file for writing
+func (OsFs) Create(name string, flag int) (*os.File, *pipeat.PipeWriterAt, func(), error) {
+	var err error
+	var f *os.File
+	if flag == 0 {
+		f, err = os.Create(name)
+	} else {
+		f, err = os.OpenFile(name, flag, 0666)
+
+	}
+	return f, nil, nil, err
+}
+
+// Rename renames (moves) source to target
+func (OsFs) Rename(source, target string) error {
+	return os.Rename(source, target)
+}
+
+// Remove removes the named file or (empty) directory.
+func (OsFs) Remove(name string, isDir bool) error {
+	return os.Remove(name)
+}
+
+// Mkdir creates a new directory with the specified name and default permissions
+func (OsFs) Mkdir(name string) error {
+	return os.Mkdir(name, 0777)
+}
+
+// Symlink creates source as a symbolic link to target.
+func (OsFs) Symlink(source, target string) error {
+	return os.Symlink(source, target)
+}
+
+// Chown changes the numeric uid and gid of the named file.
+func (OsFs) Chown(name string, uid int, gid int) error {
+	return os.Chown(name, uid, gid)
+}
+
+// Chmod changes the mode of the named file to mode
+func (OsFs) Chmod(name string, mode os.FileMode) error {
+	return os.Chmod(name, mode)
+}
+
+// Chtimes changes the access and modification times of the named file
+func (OsFs) Chtimes(name string, atime, mtime time.Time) error {
+	return os.Chtimes(name, atime, mtime)
+}
+
+// ReadDir reads the directory named by dirname and returns
+// a list of directory entries.
+func (OsFs) ReadDir(dirname string) ([]os.FileInfo, error) {
+	return ioutil.ReadDir(dirname)
+}
+
+// IsUploadResumeSupported returns true if upload resume is supported
+func (OsFs) IsUploadResumeSupported() bool {
+	return true
+}
+
+// IsAtomicUploadSupported returns true if atomic upload is supported
+func (OsFs) IsAtomicUploadSupported() bool {
+	return true
+}
+
+// IsNotExist returns a boolean indicating whether the error is known to
+// report that a file or directory does not exist
+func (OsFs) IsNotExist(err error) bool {
+	return os.IsNotExist(err)
+}
+
+// IsPermission returns a boolean indicating whether the error is known to
+// report that permission is denied.
+func (OsFs) IsPermission(err error) bool {
+	return os.IsPermission(err)
+}
+
+// CheckRootPath creates the specified root directory if it does not exists
+func (fs OsFs) CheckRootPath(rootPath, username string, uid int, gid int) bool {
+	var err error
+	if _, err = fs.Stat(rootPath); fs.IsNotExist(err) {
+		err = os.MkdirAll(rootPath, 0777)
+		fsLog(fs, logger.LevelDebug, "root directory %#v for user %#v does not exist, try to create, mkdir error: %v",
+			rootPath, username, err)
+		if err == nil {
+			SetPathPermissions(fs, rootPath, uid, gid)
+		}
+	}
+	return (err == nil)
+}
+
+// ScanDirContents returns the number of files contained in a directory and
+// their size
+func (fs OsFs) ScanDirContents(dirPath string) (int, int64, error) {
+	numFiles := 0
+	size := int64(0)
+	isDir, err := IsDirectory(fs, dirPath)
+	if err == nil && isDir {
+		err = filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info != nil && info.Mode().IsRegular() {
+				size += info.Size()
+				numFiles++
+			}
+			return err
+		})
+	}
+	return numFiles, size, err
+}
+
+// GetAtomicUploadPath returns the path to use for an atomic upload
+func (OsFs) GetAtomicUploadPath(name string) string {
+	dir := filepath.Dir(name)
+	guid := xid.New().String()
+	return filepath.Join(dir, ".sftpgo-upload."+guid+"."+filepath.Base(name))
+}
+
+// GetRelativePath returns the path for a file relative to the user's home dir.
+// This is the path as seen by SFTP users
+func (OsFs) GetRelativePath(name, rootPath string) string {
+	rel, err := filepath.Rel(rootPath, filepath.Clean(name))
+	if err != nil {
+		return ""
+	}
+	if rel == "." || strings.HasPrefix(rel, "..") {
+		rel = ""
+	}
+	return "/" + filepath.ToSlash(rel)
+}
+
+// Join joins any number of path elements into a single path
+func (OsFs) Join(elem ...string) string {
+	return filepath.Join(elem...)
+}
+
+// ResolvePath returns the matching filesystem path for the specified sftp path
+func (fs OsFs) ResolvePath(sftpPath, rootPath string) (string, error) {
+	if !filepath.IsAbs(rootPath) {
+		return "", fmt.Errorf("Invalid root path: %v", rootPath)
+	}
+	r := filepath.Clean(filepath.Join(rootPath, sftpPath))
+	p, err := filepath.EvalSymlinks(r)
+	if err != nil && !os.IsNotExist(err) {
+		return "", err
+	} else if os.IsNotExist(err) {
+		// The requested path doesn't exist, so at this point we need to iterate up the
+		// path chain until we hit a directory that _does_ exist and can be validated.
+		_, err = fs.findFirstExistingDir(r, rootPath)
+		if err != nil {
+			fsLog(fs, logger.LevelWarn, "error resolving not existent path: %#v", err)
+		}
+		return r, err
+	}
+
+	err = fs.isSubDir(p, rootPath)
+	if err != nil {
+		fsLog(fs, logger.LevelWarn, "Invalid path resolution, dir: %#v outside user home: %#v err: %v", p, rootPath, err)
+	}
+	return r, err
+}
+
+func (fs *OsFs) findNonexistentDirs(path, rootPath string) ([]string, error) {
+	results := []string{}
+	cleanPath := filepath.Clean(path)
+	parent := filepath.Dir(cleanPath)
+	_, err := os.Stat(parent)
+
+	for os.IsNotExist(err) {
+		results = append(results, parent)
+		parent = filepath.Dir(parent)
+		_, err = os.Stat(parent)
+	}
+	if err != nil {
+		return results, err
+	}
+	p, err := filepath.EvalSymlinks(parent)
+	if err != nil {
+		return results, err
+	}
+	err = fs.isSubDir(p, rootPath)
+	if err != nil {
+		fsLog(fs, logger.LevelWarn, "error finding non existing dir: %v", err)
+	}
+	return results, err
+}
+
+func (fs *OsFs) findFirstExistingDir(path, rootPath string) (string, error) {
+	results, err := fs.findNonexistentDirs(path, rootPath)
+	if err != nil {
+		fsLog(fs, logger.LevelWarn, "unable to find non existent dirs: %v", err)
+		return "", err
+	}
+	var parent string
+	if len(results) > 0 {
+		lastMissingDir := results[len(results)-1]
+		parent = filepath.Dir(lastMissingDir)
+	} else {
+		parent = rootPath
+	}
+	p, err := filepath.EvalSymlinks(parent)
+	if err != nil {
+		return "", err
+	}
+	fileInfo, err := os.Stat(p)
+	if err != nil {
+		return "", err
+	}
+	if !fileInfo.IsDir() {
+		return "", fmt.Errorf("resolved path is not a dir: %#v", p)
+	}
+	err = fs.isSubDir(p, rootPath)
+	return p, err
+}
+
+func (fs *OsFs) isSubDir(sub, rootPath string) error {
+	// rootPath must exist and it is already a validated absolute path
+	parent, err := filepath.EvalSymlinks(rootPath)
+	if err != nil {
+		fsLog(fs, logger.LevelWarn, "invalid home dir %#v: %v", rootPath, err)
+		return err
+	}
+	if !strings.HasPrefix(sub, parent) {
+		err = fmt.Errorf("path %#v is not inside: %#v", sub, parent)
+		fsLog(fs, logger.LevelWarn, "error: %v ", err)
+		return err
+	}
+	return nil
+}

vfs/s3fileinfo.go

@@ -0,0 +1,60 @@
+package vfs
+
+import (
+	"os"
+	"time"
+)
+
+// S3FileInfo implements os.FileInfo for a file in S3.
+type S3FileInfo struct {
+	name        string
+	sizeInBytes int64
+	modTime     time.Time
+	mode        os.FileMode
+	sys         interface{}
+}
+
+// NewS3FileInfo creates file info.
+func NewS3FileInfo(name string, isDirectory bool, sizeInBytes int64, modTime time.Time) S3FileInfo {
+	mode := os.FileMode(0644)
+	if isDirectory {
+		mode = os.FileMode(0755) | os.ModeDir
+	}
+
+	return S3FileInfo{
+		name:        name,
+		sizeInBytes: sizeInBytes,
+		modTime:     modTime,
+		mode:        mode,
+	}
+}
+
+// Name provides the base name of the file.
+func (fi S3FileInfo) Name() string {
+	return fi.name
+}
+
+// Size provides the length in bytes for a file.
+func (fi S3FileInfo) Size() int64 {
+	return fi.sizeInBytes
+}
+
+// Mode provides the file mode bits
+func (fi S3FileInfo) Mode() os.FileMode {
+	return fi.mode
+}
+
+// ModTime provides the last modification time.
+func (fi S3FileInfo) ModTime() time.Time {
+	return fi.modTime
+}
+
+// IsDir provides the abbreviation for Mode().IsDir()
+func (fi S3FileInfo) IsDir() bool {
+	return fi.mode&os.ModeDir != 0
+}
+
+// Sys provides the underlying data source (can return nil)
+func (fi S3FileInfo) Sys() interface{} {
+	return fi.getFileInfoSys()
+}

vfs/s3fileinfo_unix.go

@@ -0,0 +1,28 @@
+// +build !windows
+
+package vfs
+
+import "syscall"
+
+import "os"
+
+var (
+	defaultUID, defaultGID int
+)
+
+func init() {
+	defaultUID = os.Getuid()
+	defaultGID = os.Getuid()
+	if defaultUID < 0 {
+		defaultUID = 65534
+	}
+	if defaultGID < 0 {
+		defaultGID = 65534
+	}
+}
+
+func (fi S3FileInfo) getFileInfoSys() interface{} {
+	return &syscall.Stat_t{
+		Uid: uint32(defaultUID),
+		Gid: uint32(defaultGID)}
+}

vfs/s3fileinfo_windows.go

@@ -0,0 +1,7 @@
+package vfs
+
+import "syscall"
+
+func (fi S3FileInfo) getFileInfoSys() interface{} {
+	return syscall.Win32FileAttributeData{}
+}

vfs/s3fs.go

@@ -0,0 +1,491 @@
+package vfs
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/utils"
+	"github.com/eikenb/pipeat"
+)
+
+// S3FsConfig defines the configuration for S3fs
+type S3FsConfig struct {
+	Bucket       string `json:"bucket,omitempty"`
+	Region       string `json:"region,omitempty"`
+	AccessKey    string `json:"access_key,omitempty"`
+	AccessSecret string `json:"access_secret,omitempty"`
+	Endpoint     string `json:"endpoint,omitempty"`
+	StorageClass string `json:"storage_class,omitempty"`
+}
+
+// S3Fs is a Fs implementation for Amazon S3 compatible object storage.
+type S3Fs struct {
+	connectionID   string
+	localTempDir   string
+	config         S3FsConfig
+	svc            *s3.S3
+	ctxTimeout     time.Duration
+	ctxLongTimeout time.Duration
+}
+
+// NewS3Fs returns an S3Fs object that allows to interact with an s3 compatible
+// object storage
+func NewS3Fs(connectionID, localTempDir string, config S3FsConfig) (Fs, error) {
+	fs := S3Fs{
+		connectionID:   connectionID,
+		localTempDir:   localTempDir,
+		config:         config,
+		ctxTimeout:     30 * time.Second,
+		ctxLongTimeout: 300 * time.Second,
+	}
+	if err := ValidateS3FsConfig(&fs.config); err != nil {
+		return fs, err
+	}
+	accessSecret, err := utils.DecryptData(fs.config.AccessSecret)
+	if err != nil {
+		return fs, err
+	}
+	fs.config.AccessSecret = accessSecret
+	awsConfig := &aws.Config{
+		Region:      aws.String(fs.config.Region),
+		Credentials: credentials.NewStaticCredentials(fs.config.AccessKey, fs.config.AccessSecret, ""),
+	}
+	//config.WithLogLevel(aws.LogDebugWithHTTPBody)
+	if len(fs.config.Endpoint) > 0 {
+		awsConfig.Endpoint = aws.String(fs.config.Endpoint)
+		awsConfig.S3ForcePathStyle = aws.Bool(true)
+	}
+	sess, err := session.NewSession(awsConfig)
+	if err != nil {
+		return fs, err
+	}
+	fs.svc = s3.New(sess)
+	return fs, nil
+}
+
+// Name returns the name for the Fs implementation
+func (fs S3Fs) Name() string {
+	return fmt.Sprintf("S3Fs bucket: %#v", fs.config.Bucket)
+}
+
+// ConnectionID returns the SSH connection ID associated to this Fs implementation
+func (fs S3Fs) ConnectionID() string {
+	return fs.connectionID
+}
+
+// Stat returns a FileInfo describing the named file
+func (fs S3Fs) Stat(name string) (os.FileInfo, error) {
+	var result S3FileInfo
+	if name == "/" || name == "." {
+		err := fs.checkIfBucketExists()
+		if err != nil {
+			return result, err
+		}
+		return NewS3FileInfo(name, true, 0, time.Time{}), nil
+	}
+	prefix := path.Dir(name)
+	if prefix == "/" || prefix == "." {
+		prefix = ""
+	} else {
+		prefix = strings.TrimPrefix(prefix, "/")
+		if !strings.HasSuffix(prefix, "/") {
+			prefix += "/"
+		}
+	}
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+	err := fs.svc.ListObjectsV2PagesWithContext(ctx, &s3.ListObjectsV2Input{
+		Bucket:    aws.String(fs.config.Bucket),
+		Prefix:    aws.String(prefix),
+		Delimiter: aws.String("/"),
+	}, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+		for _, p := range page.CommonPrefixes {
+			if fs.isEqual(p.Prefix, name) {
+				result = NewS3FileInfo(name, true, 0, time.Time{})
+				return false
+			}
+		}
+		for _, fileObject := range page.Contents {
+			if fs.isEqual(fileObject.Key, name) {
+				objectSize := *fileObject.Size
+				objectModTime := *fileObject.LastModified
+				isDir := strings.HasSuffix(*fileObject.Key, "/")
+				result = NewS3FileInfo(name, isDir, objectSize, objectModTime)
+				return false
+			}
+		}
+		return true
+	})
+	if err == nil && len(result.Name()) == 0 {
+		err = errors.New("404 no such file or directory")
+	}
+	return result, err
+}
+
+// Lstat returns a FileInfo describing the named file
+func (fs S3Fs) Lstat(name string) (os.FileInfo, error) {
+	return fs.Stat(name)
+}
+
+// Open opens the named file for reading
+func (fs S3Fs) Open(name string) (*os.File, *pipeat.PipeReaderAt, func(), error) {
+	r, w, err := pipeat.AsyncWriterPipeInDir(fs.localTempDir)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	ctx, cancelFn := context.WithCancel(context.Background())
+	downloader := s3manager.NewDownloaderWithClient(fs.svc)
+	go func() {
+		defer cancelFn()
+		key := name
+		n, err := downloader.DownloadWithContext(ctx, w, &s3.GetObjectInput{
+			Bucket: aws.String(fs.config.Bucket),
+			Key:    aws.String(key),
+		})
+		fsLog(fs, logger.LevelDebug, "download completed, path: %#v size: %v, err: %v", name, n, err)
+		w.CloseWithError(err)
+	}()
+	return nil, r, cancelFn, nil
+}
+
+// Create creates or opens the named file for writing
+func (fs S3Fs) Create(name string, flag int) (*os.File, *pipeat.PipeWriterAt, func(), error) {
+	r, w, err := pipeat.PipeInDir(fs.localTempDir)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	ctx, cancelFn := context.WithCancel(context.Background())
+	uploader := s3manager.NewUploaderWithClient(fs.svc)
+	go func() {
+		defer cancelFn()
+		key := name
+		response, err := uploader.UploadWithContext(ctx, &s3manager.UploadInput{
+			Bucket:       aws.String(fs.config.Bucket),
+			Key:          aws.String(key),
+			Body:         r,
+			StorageClass: utils.NilIfEmpty(fs.config.StorageClass),
+		})
+		fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, response: %v, err: %v", name, response, err)
+		r.CloseWithError(err)
+	}()
+	return nil, w, cancelFn, nil
+}
+
+// Rename renames (moves) source to target.
+// We don't support renaming non empty directories since we should
+// rename all the contents too and this could take long time: think
+// about directories with thousands of files, for each file we should
+// execute a CopyObject call.
+func (fs S3Fs) Rename(source, target string) error {
+	if source == target {
+		return nil
+	}
+	fi, err := fs.Stat(source)
+	if err != nil {
+		return err
+	}
+	copySource := fs.Join(fs.config.Bucket, source)
+	if fi.IsDir() {
+		contents, err := fs.ReadDir(source)
+		if err != nil {
+			return err
+		}
+		if len(contents) > 0 {
+			return fmt.Errorf("Cannot rename non empty directory: %#v", source)
+		}
+		if !strings.HasSuffix(copySource, "/") {
+			copySource += "/"
+		}
+		if !strings.HasSuffix(target, "/") {
+			target += "/"
+		}
+	}
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+	_, err = fs.svc.CopyObjectWithContext(ctx, &s3.CopyObjectInput{
+		Bucket:     aws.String(fs.config.Bucket),
+		CopySource: aws.String(copySource),
+		Key:        aws.String(target),
+	})
+	if err != nil {
+		return err
+	}
+	return fs.Remove(source, fi.IsDir())
+}
+
+// Remove removes the named file or (empty) directory.
+func (fs S3Fs) Remove(name string, isDir bool) error {
+	if isDir {
+		contents, err := fs.ReadDir(name)
+		if err != nil {
+			return err
+		}
+		if len(contents) > 0 {
+			return fmt.Errorf("Cannot remove non empty directory: %#v", name)
+		}
+		if !strings.HasSuffix(name, "/") {
+			name += "/"
+		}
+	}
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+	_, err := fs.svc.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
+		Bucket: aws.String(fs.config.Bucket),
+		Key:    aws.String(name),
+	})
+	return err
+}
+
+// Mkdir creates a new directory with the specified name and default permissions
+func (fs S3Fs) Mkdir(name string) error {
+	_, err := fs.Stat(name)
+	if !fs.IsNotExist(err) {
+		return err
+	}
+	if !strings.HasSuffix(name, "/") {
+		name += "/"
+	}
+	_, w, _, err := fs.Create(name, 0)
+	if err != nil {
+		return err
+	}
+	return w.Close()
+}
+
+// Symlink creates source as a symbolic link to target.
+func (S3Fs) Symlink(source, target string) error {
+	return errors.New("403 symlinks are not supported")
+}
+
+// Chown changes the numeric uid and gid of the named file.
+// Silently ignored.
+func (S3Fs) Chown(name string, uid int, gid int) error {
+	return nil
+}
+
+// Chmod changes the mode of the named file to mode.
+// Silently ignored.
+func (S3Fs) Chmod(name string, mode os.FileMode) error {
+	return nil
+}
+
+// Chtimes changes the access and modification times of the named file.
+// Silently ignored.
+func (S3Fs) Chtimes(name string, atime, mtime time.Time) error {
+	return errors.New("403 chtimes is not supported")
+}
+
+// ReadDir reads the directory named by dirname and returns
+// a list of directory entries.
+func (fs S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) {
+	var result []os.FileInfo
+	// dirname deve essere già cleaned
+	prefix := ""
+	if dirname != "/" && dirname != "." {
+		prefix = strings.TrimPrefix(dirname, "/")
+		if !strings.HasSuffix(prefix, "/") {
+			prefix += "/"
+		}
+	}
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+	err := fs.svc.ListObjectsV2PagesWithContext(ctx, &s3.ListObjectsV2Input{
+		Bucket:    aws.String(fs.config.Bucket),
+		Prefix:    aws.String(prefix),
+		Delimiter: aws.String("/"),
+	}, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+		for _, p := range page.CommonPrefixes {
+			name, isDir := fs.resolve(p.Prefix, prefix)
+			result = append(result, NewS3FileInfo(name, isDir, 0, time.Time{}))
+		}
+		for _, fileObject := range page.Contents {
+			objectSize := *fileObject.Size
+			objectModTime := *fileObject.LastModified
+			name, isDir := fs.resolve(fileObject.Key, prefix)
+			if len(name) == 0 {
+				continue
+			}
+			result = append(result, NewS3FileInfo(name, isDir, objectSize, objectModTime))
+		}
+		return true
+	})
+	return result, err
+}
+
+// IsUploadResumeSupported returns true if upload resume is supported.
+// SFTP Resume is not supported on S3
+func (S3Fs) IsUploadResumeSupported() bool {
+	return false
+}
+
+// IsAtomicUploadSupported returns true if atomic upload is supported.
+// S3 uploads are already atomic, we don't need to upload to a temporary
+// file
+func (S3Fs) IsAtomicUploadSupported() bool {
+	return false
+}
+
+// IsNotExist returns a boolean indicating whether the error is known to
+// report that a file or directory does not exist
+func (S3Fs) IsNotExist(err error) bool {
+	if err == nil {
+		return false
+	}
+	if aerr, ok := err.(awserr.Error); ok {
+		if aerr.Code() == s3.ErrCodeNoSuchKey {
+			return true
+		}
+		if aerr.Code() == s3.ErrCodeNoSuchBucket {
+			return true
+		}
+	}
+	if multierr, ok := err.(s3manager.MultiUploadFailure); ok {
+		if multierr.Code() == s3.ErrCodeNoSuchKey {
+			return true
+		}
+		if multierr.Code() == s3.ErrCodeNoSuchBucket {
+			return true
+		}
+	}
+	return strings.Contains(err.Error(), "404")
+}
+
+// IsPermission returns a boolean indicating whether the error is known to
+// report that permission is denied.
+func (S3Fs) IsPermission(err error) bool {
+	if err == nil {
+		return false
+	}
+	return strings.Contains(err.Error(), "403")
+}
+
+// CheckRootPath creates the specified root directory if it does not exists
+func (fs S3Fs) CheckRootPath(rootPath, username string, uid int, gid int) bool {
+	// we need a local directory for temporary files
+	osFs := NewOsFs(fs.ConnectionID())
+	osFs.CheckRootPath(fs.localTempDir, username, uid, gid)
+	err := fs.checkIfBucketExists()
+	if err == nil {
+		return true
+	}
+	if !fs.IsNotExist(err) {
+		return false
+	}
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+	input := &s3.CreateBucketInput{
+		Bucket: aws.String(fs.config.Bucket),
+	}
+	_, err = fs.svc.CreateBucketWithContext(ctx, input)
+	fsLog(fs, logger.LevelDebug, "bucket %#v for user %#v does not exists, try to create, error: %v",
+		fs.config.Bucket, username, err)
+	return err == nil
+}
+
+// ScanDirContents returns the number of files contained in the bucket,
+// and their size
+func (fs S3Fs) ScanDirContents(dirPath string) (int, int64, error) {
+	numFiles := 0
+	size := int64(0)
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout))
+	defer cancelFn()
+	err := fs.svc.ListObjectsV2PagesWithContext(ctx, &s3.ListObjectsV2Input{
+		Bucket: aws.String(fs.config.Bucket),
+		Prefix: aws.String(""),
+	}, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+		for _, fileObject := range page.Contents {
+			numFiles++
+			size += *fileObject.Size
+		}
+		return true
+	})
+
+	return numFiles, size, err
+}
+
+// GetAtomicUploadPath returns the path to use for an atomic upload.
+// S3 uploads are already atomic, we never call this method for S3
+func (S3Fs) GetAtomicUploadPath(name string) string {
+	return ""
+}
+
+// GetRelativePath returns the path for a file relative to the user's home dir.
+// This is the path as seen by SFTP users
+func (S3Fs) GetRelativePath(name, rootPath string) string {
+	rel := name
+	if name == "." {
+		rel = ""
+	}
+	if !strings.HasPrefix(rel, "/") {
+		return "/" + rel
+	}
+	return rel
+}
+
+// Join joins any number of path elements into a single path
+func (S3Fs) Join(elem ...string) string {
+	return path.Join(elem...)
+}
+
+// ResolvePath returns the matching filesystem path for the specified sftp path
+func (fs S3Fs) ResolvePath(sftpPath, rootPath string) (string, error) {
+	return sftpPath, nil
+}
+
+func (fs *S3Fs) resolve(name *string, prefix string) (string, bool) {
+	result := strings.TrimPrefix(*name, prefix)
+	isDir := strings.HasSuffix(result, "/")
+	if isDir {
+		result = strings.TrimSuffix(result, "/")
+	}
+	if strings.Contains(result, "/") {
+		i := strings.Index(result, "/")
+		isDir = true
+		result = result[:i]
+	}
+	return result, isDir
+}
+
+func (fs *S3Fs) isEqual(s3Key *string, sftpName string) bool {
+	if *s3Key == sftpName {
+		return true
+	}
+	if "/"+*s3Key == sftpName {
+		return true
+	}
+	if "/"+*s3Key == sftpName+"/" {
+		return true
+	}
+	return false
+}
+
+func (fs *S3Fs) checkIfBucketExists() error {
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+	_, err := fs.svc.HeadBucketWithContext(ctx, &s3.HeadBucketInput{
+		Bucket: aws.String(fs.config.Bucket),
+	})
+	return err
+}
+
+func (fs *S3Fs) getObjectDetails(key string) (*s3.HeadObjectOutput, error) {
+	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
+	defer cancelFn()
+	input := &s3.HeadObjectInput{
+		Bucket: aws.String(fs.config.Bucket),
+		Key:    aws.String(key),
+	}
+	return fs.svc.HeadObjectWithContext(ctx, input)
+}

vfs/vfs.go

@@ -0,0 +1,100 @@
+package vfs
+
+import (
+	"errors"
+	"os"
+	"runtime"
+	"time"
+
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/eikenb/pipeat"
+	"github.com/pkg/sftp"
+)
+
+// Fs defines the interface for filesystems backends
+type Fs interface {
+	Name() string
+	ConnectionID() string
+	Stat(name string) (os.FileInfo, error)
+	Lstat(name string) (os.FileInfo, error)
+	Open(name string) (*os.File, *pipeat.PipeReaderAt, func(), error)
+	Create(name string, flag int) (*os.File, *pipeat.PipeWriterAt, func(), error)
+	Rename(source, target string) error
+	Remove(name string, isDir bool) error
+	Mkdir(name string) error
+	Symlink(source, target string) error
+	Chown(name string, uid int, gid int) error
+	Chmod(name string, mode os.FileMode) error
+	Chtimes(name string, atime, mtime time.Time) error
+	ReadDir(dirname string) ([]os.FileInfo, error)
+	IsUploadResumeSupported() bool
+	IsAtomicUploadSupported() bool
+	CheckRootPath(rootPath, username string, uid int, gid int) bool
+	ResolvePath(sftpPath, rootPath string) (string, error)
+	IsNotExist(err error) bool
+	IsPermission(err error) bool
+	ScanDirContents(dirPath string) (int, int64, error)
+	GetAtomicUploadPath(name string) string
+	GetRelativePath(name, rootPath string) string
+	Join(elem ...string) string
+}
+
+// IsDirectory checks if a path exists and is a directory
+func IsDirectory(fs Fs, path string) (bool, error) {
+	fileInfo, err := fs.Stat(path)
+	if err != nil {
+		return false, err
+	}
+	return fileInfo.IsDir(), err
+}
+
+// GetSFTPError returns an sftp error from a filesystem error
+func GetSFTPError(fs Fs, err error) error {
+	if fs.IsNotExist(err) {
+		return sftp.ErrSSHFxNoSuchFile
+	} else if fs.IsPermission(err) {
+		return sftp.ErrSSHFxPermissionDenied
+	} else if err != nil {
+		return sftp.ErrSSHFxFailure
+	}
+	return nil
+}
+
+// IsLocalOsFs returns true if fs is the local filesystem implementation
+func IsLocalOsFs(fs Fs) bool {
+	return fs.Name() == osFsName
+}
+
+// ValidateS3FsConfig returns nil if the specified s3 config is valid, otherwise an error
+func ValidateS3FsConfig(config *S3FsConfig) error {
+	if len(config.Bucket) == 0 {
+		return errors.New("bucket cannot be empty")
+	}
+	if len(config.Region) == 0 {
+		return errors.New("region cannot be empty")
+	}
+	if len(config.AccessKey) == 0 {
+		return errors.New("access_key cannot be empty")
+	}
+	if len(config.AccessSecret) == 0 {
+		return errors.New("access_secret cannot be empty")
+	}
+	return nil
+}
+
+// SetPathPermissions calls fs.Chown.
+// It does nothing for local filesystem on windows
+func SetPathPermissions(fs Fs, path string, uid int, gid int) {
+	if IsLocalOsFs(fs) {
+		if runtime.GOOS == "windows" {
+			return
+		}
+	}
+	if err := fs.Chown(path, uid, gid); err != nil {
+		fsLog(fs, logger.LevelWarn, "error chowning path %v: %v", path, err)
+	}
+}
+
+func fsLog(fs Fs, level logger.LogLevel, format string, v ...interface{}) {
+	logger.Log(level, fs.Name(), fs.ConnectionID(), format, v...)
+}