From a4834f4a83ea2762d141e0bf015f0b63f39f1adf Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sun, 19 Jan 2020 07:41:05 +0100 Subject: [PATCH] add basic S3-Compatible Object Storage support we have now an interface for filesystem backeds, this make easy to add new filesystem backends --- .travis.yml | 2 +- README.md | 58 ++++- cmd/portable.go | 26 ++ dataprovider/bolt.go | 11 +- dataprovider/dataprovider.go | 37 ++- dataprovider/memory.go | 9 +- dataprovider/sqlcommon.go | 33 ++- dataprovider/sqlqueries.go | 17 +- dataprovider/user.go | 52 +++- go.mod | 4 + go.sum | 18 +- httpd/api_maintenance.go | 19 +- httpd/api_quota.go | 32 +-- httpd/api_user.go | 18 +- httpd/api_utils.go | 56 ++++ httpd/httpd_test.go | 137 ++++++++++ httpd/internal_test.go | 91 +++++++ httpd/schema/openapi.yaml | 48 +++- httpd/web.go | 19 ++ scripts/README.md | 23 +- scripts/sftpgo_api_cli.py | 67 +++-- sftpd/handler.go | 255 ++++++------------ sftpd/internal_test.go | 337 ++++++++++++++++++++++-- sftpd/scp.go | 50 ++-- sftpd/server.go | 20 +- sftpd/sftpd.go | 2 +- sftpd/sftpd_test.go | 61 ++++- sftpd/ssh_cmd.go | 22 +- sftpd/transfer.go | 39 ++- sql/mysql/20200116.sql | 6 + sql/pgsql/20200116.sql | 6 + sql/sqlite/20200116.sql | 9 + templates/user.html | 52 ++++ utils/utils.go | 125 +++++---- vfs/osfs.go | 288 ++++++++++++++++++++ vfs/s3fileinfo.go | 60 +++++ vfs/s3fileinfo_unix.go | 28 ++ vfs/s3fileinfo_windows.go | 7 + vfs/s3fs.go | 491 +++++++++++++++++++++++++++++++++++ vfs/vfs.go | 100 +++++++ 40 files changed, 2315 insertions(+), 420 deletions(-) create mode 100644 sql/mysql/20200116.sql create mode 100644 sql/pgsql/20200116.sql create mode 100644 sql/sqlite/20200116.sql create mode 100644 vfs/osfs.go create mode 100644 vfs/s3fileinfo.go create mode 100644 vfs/s3fileinfo_unix.go create mode 100644 vfs/s3fileinfo_windows.go create mode 100644 vfs/s3fs.go create mode 100644 vfs/vfs.go diff --git a/.travis.yml b/.travis.yml index 41698ff2..431df0e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ env: - GO111MODULE=on before_script: - - sqlite3 sftpgo.db 'CREATE TABLE "users" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE, "password" varchar(255) NULL, "public_keys" text NULL, "home_dir" varchar(255) NOT NULL, "uid" integer NOT NULL, "gid" integer NOT NULL, "max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "permissions" text NOT NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL, "expiration_date" bigint NOT NULL, "last_login" bigint NOT NULL, "status" integer NOT NULL, "filters" TEXT NULL);' + - sqlite3 sftpgo.db 'CREATE TABLE "users" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE, "password" varchar(255) NULL, "public_keys" text NULL, "home_dir" varchar(255) NOT NULL, "uid" integer NOT NULL, "gid" integer NOT NULL, "max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "permissions" text NOT NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL, "expiration_date" bigint NOT NULL, "last_login" bigint NOT NULL, "status" integer NOT NULL, "filters" TEXT NULL, "filesystem" text NULL);' install: - go get -v -t ./... diff --git a/README.md b/README.md index d72e1350..62d1dc54 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Full featured and highly configurable SFTP server - Atomic uploads are configurable. - Support for Git repositories over SSH. - SCP and rsync are supported. +- Support for serving S3 Compatible Object Storage over SFTP. - Prometheus metrics are exposed. - REST API for users management, backup, restore and real time reports of the active connections with possibility of forcibly closing a connection. - Web based interface to easily manage users and connections. @@ -136,7 +137,7 @@ The `sftpgo` configuration file contains the following sections: - `idle_timeout`, integer. Time in minutes after which an idle client will be disconnected. 0 menas disabled. Default: 15 - `max_auth_tries` integer. Maximum number of authentication attempts permitted per connection. If set to a negative number, the number of attempts are unlimited. If set to zero, the number of attempts are limited to 6. - `umask`, string. Umask for the new files and directories. This setting has no effect on Windows. Default: "0022" - - `banner`, string. Identification string used by the server. Leave empty to use the default banner. Default "SFTPGo_version" + - `banner`, string. Identification string used by the server. Leave empty to use the default banner. Default "SFTPGo_" - `upload_mode` integer. 0 means standard, the files are uploaded directly to the requested path. 1 means atomic: files are uploaded to a temporary path and renamed to the requested path when the client ends the upload. Atomic mode avoids problems such as a web server that serves partial files when the files are being uploaded. In atomic mode if there is an upload error the temporary file is deleted and so the requested upload path will not contain a partial file. 2 means atomic with resume support: as atomic but if there is an upload error the temporary file is renamed to the requested path and not deleted, this way a client can reconnect and resume the upload. - `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions. See the "Custom Actions" paragraph for more details - `execute_on`, list of strings. Valid values are `download`, `upload`, `delete`, `rename`, `ssh_cmd`. Leave empty to disable actions. @@ -150,7 +151,7 @@ The `sftpgo` configuration file contains the following sections: - `macs`, list of strings. available MAC (message authentication code) algorithms in preference order. Leave empty to use default values. The supported values can be found here: [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/common.go#L84 "Supported MACs") - `login_banner_file`, path to the login banner file. The contents of the specified file, if any, are sent to the remote user before authentication is allowed. It can be a path relative to the config dir or an absolute one. Leave empty to send no login banner - `setstat_mode`, integer. 0 means "normal mode": requests for changing permissions, owner/group and access/modification times are executed. 1 means "ignore mode": requests for changing permissions, owner/group and access/modification times are silently ignored. - - `enabled_ssh_commands`, list of enabled SSH commands. These SSH commands are enabled by default: `md5sum`, `sha1sum`, `cd`, `pwd`. `*` enables all supported commands. Some commands are implemented directly inside SFTPGo, while for other commands we use system commands that need to be installed and in your system's `PATH`. For system commands we have no direct control on file creation/deletion and so quota check is suboptimal: if quota is enabled, the number of files is checked at the command begin and not while new files are created. The allowed size is calculated as the difference between the max quota and the used one and it is checked against the bytes transferred via SSH. The command is aborted if it uploads more bytes than the remaining allowed size calculated at the command start. Anyway we see the bytes that the remote command send to the local command via SSH, these bytes contain both protocol commands and files and so the size of the files is different from the size trasferred via SSH: for example a command can send compressed files or a protocol command (few bytes) could delete a big file. To mitigate this issue quotas are recalculated at the command end with a full home directory scan, this could be heavy for big directories. If you need system commands and quotas you could consider to disable quota restrictions and periodically update quota usage yourself using the REST API. We support the following SSH commands: + - `enabled_ssh_commands`, list of enabled SSH commands. These SSH commands are enabled by default: `md5sum`, `sha1sum`, `cd`, `pwd`. `*` enables all supported commands. Some commands are implemented directly inside SFTPGo, while for other commands we use system commands that need to be installed and in your system's `PATH`. For system commands we have no direct control on file creation/deletion and so we cannot support remote filesystems, such as S3, and quota check is suboptimal: if quota is enabled, the number of files is checked at the command begin and not while new files are created. The allowed size is calculated as the difference between the max quota and the used one and it is checked against the bytes transferred via SSH. The command is aborted if it uploads more bytes than the remaining allowed size calculated at the command start. Anyway we see the bytes that the remote command send to the local command via SSH, these bytes contain both protocol commands and files and so the size of the files is different from the size trasferred via SSH: for example a command can send compressed files or a protocol command (few bytes) could delete a big file. To mitigate this issue quotas are recalculated at the command end with a full home directory scan, this could be heavy for big directories. If you need system commands and quotas you could consider to disable quota restrictions and periodically update quota usage yourself using the REST API. We support the following SSH commands: - `scp`, SCP is an experimental feature, we have our own SCP implementation since we can't rely on "scp" system command to proper handle quotas and user's home dir restrictions. The SCP protocol is quite simple but there is no official docs about it, so we need more testing and feedbacks before enabling it by default. We may not handle some borderline cases or have sneaky bugs. Please do accurate tests yourself before enabling SCP and let us known if something does not work as expected for your use cases. SCP between two remote hosts is supported using the `-3` scp option. - `md5sum`, `sha1sum`, `sha256sum`, `sha384sum`, `sha512sum`. Useful to check message digests for uploaded files. These commands are implemented inside SFTPGo so they work even if the matching system commands are not available, for example on Windows. - `cd`, `pwd`. Some SFTP clients does not support the SFTP SSH_FXP_REALPATH packet type and so they use `cd` and `pwd` SSH commands to get the initial directory. Currently `cd` do nothing and `pwd` always returns the `/` path. @@ -416,6 +417,25 @@ The `http_notification_url`, if defined, will be called invoked as http POST. Th The HTTP request has a 15 seconds timeout. +## S3 Compabible Object Storage backends + +Each user can be mapped with an S3-Compatible bucket, this way the mapped bucket is exposed over SFTP/SCP. +SFTPGo uses multipart uploads and parallel downloads for storing and retrieving files from S3 and automatically try to create the mapped bucket if it does not exists. + +Some SFTP commands doesn't work over S3: + +- `symlink` and `chtimes` will fail +- `chown`, `chmod` are silently ignored +- upload resume is not supported +- upload mode `atomic` is ignored since S3 uploads are already atomic + +Other notes: + +- `rename` is a two steps operation: server-side copy and then deletion. So it is not atomic as for local filesystem +- We don't support renaming non empty directories since we should rename all the contents too and this could take long time: think about directories with thousands of files, for each file we should do an AWS API call. +- For server side encryption you have to configure the mapped bucket to automatically encrypt objects. +- A local home directory is still required to store temporary files. + ## Portable mode SFTPGo allows to share a single directory on demand using the `portable` subcommand: @@ -432,17 +452,24 @@ Usage: sftpgo portable [flags] Flags: - -C, --advertise-credentials If the SFTP service is advertised via multicast DNS this flag allows to put username/password inside the advertised TXT record - -S, --advertise-service Advertise SFTP service using multicast DNS (default true) - -d, --directory string Path to the directory to serve. This can be an absolute path or a path relative to the current directory (default ".") - -h, --help help for portable - -l, --log-file-path string Leave empty to disable logging - -p, --password string Leave empty to use an auto generated value - -g, --permissions strings User's permissions. "*" means any permission (default [list,download]) + -C, --advertise-credentials If the SFTP service is advertised via multicast DNS this flag allows to put username/password inside the advertised TXT record + -S, --advertise-service Advertise SFTP service using multicast DNS (default true) + -d, --directory string Path to the directory to serve. This can be an absolute path or a path relative to the current directory (default ".") + -f, --fs-provider int 0 means local filesystem, 1 S3 compatible + -h, --help help for portable + -l, --log-file-path string Leave empty to disable logging + -p, --password string Leave empty to use an auto generated value + -g, --permissions strings User's permissions. "*" means any permission (default [list,download]) -k, --public-key strings - -s, --sftpd-port int 0 means a random non privileged port - -c, --ssh-commands strings SSH commands to enable. "*" means any supported SSH command including scp (default [md5sum,sha1sum,cd,pwd]) - -u, --username string Leave empty to use an auto generated value + --s3-access-key string + --s3-access-secret string + --s3-bucket string + --s3-endpoint string + --s3-region string + --s3-storage-class string + -s, --sftpd-port int 0 means a random non privileged port + -c, --ssh-commands strings SSH commands to enable. "*" means any supported SSH command including scp (default [md5sum,sha1sum,cd,pwd]) + -u, --username string Leave empty to use an auto generated value ``` In portable mode SFTPGo can advertise the SFTP service and, optionally, the credentials via multicast DNS, so there is a standard way to discover the service and to automatically connect to it. @@ -488,6 +515,13 @@ For each account the following properties can be configured: - `download_bandwidth` maximum download bandwidth as KB/s, 0 means unlimited. - `allowed_ip`, List of IP/Mask allowed to login. Any IP address not contained in this list cannot login. IP/Mask must be in CIDR notation as defined in RFC 4632 and RFC 4291, for example "192.0.2.0/24" or "2001:db8::/32" - `denied_ip`, List of IP/Mask not allowed to login. If an IP address is both allowed and denied then login will be denied +- `fs_provider`, filesystem to serve via SFTP. Local filesystem and S3 Compatible Object Storage are supported +- `s3_bucket`, required for S3 filesystem +- `s3_region`, required for S3 filesystem +- `s3_access_key`, required for S3 filesystem +- `s3_access_secret`, required for S3 filesystem. It is stored encrypted (AES-256-GCM) +- `s3_endpoint`, specifies s3 endpoint (server) different from AWS +- `s3_storage_class` These properties are stored inside the data provider. diff --git a/cmd/portable.go b/cmd/portable.go index 3beacfc6..bd5194db 100644 --- a/cmd/portable.go +++ b/cmd/portable.go @@ -6,6 +6,7 @@ import ( "github.com/drakkan/sftpgo/dataprovider" "github.com/drakkan/sftpgo/service" "github.com/drakkan/sftpgo/sftpd" + "github.com/drakkan/sftpgo/vfs" "github.com/spf13/cobra" ) @@ -20,6 +21,13 @@ var ( portablePublicKeys []string portablePermissions []string portableSSHCommands []string + portableFsProvider int + portableS3Bucket string + portableS3Region string + portableS3AccessKey string + portableS3AccessSecret string + portableS3Endpoint string + portableS3StorageClass string portableCmd = &cobra.Command{ Use: "portable", Short: "Serve a single directory", @@ -53,6 +61,17 @@ Please take a look at the usage below to customize the serving parameters`, Permissions: permissions, HomeDir: portableDir, Status: 1, + FsConfig: dataprovider.Filesystem{ + Provider: portableFsProvider, + S3Config: vfs.S3FsConfig{ + Bucket: portableS3Bucket, + Region: portableS3Region, + AccessKey: portableS3AccessKey, + AccessSecret: portableS3AccessSecret, + Endpoint: portableS3Endpoint, + StorageClass: portableS3StorageClass, + }, + }, }, } if err := service.StartPortableMode(portableSFTPDPort, portableSSHCommands, portableAdvertiseService, @@ -79,5 +98,12 @@ func init() { "Advertise SFTP service using multicast DNS") portableCmd.Flags().BoolVarP(&portableAdvertiseCredentials, "advertise-credentials", "C", false, "If the SFTP service is advertised via multicast DNS this flag allows to put username/password inside the advertised TXT record") + portableCmd.Flags().IntVarP(&portableFsProvider, "fs-provider", "f", 0, "0 means local filesystem, 1 S3 compatible") + portableCmd.Flags().StringVar(&portableS3Bucket, "s3-bucket", "", "") + portableCmd.Flags().StringVar(&portableS3Region, "s3-region", "", "") + portableCmd.Flags().StringVar(&portableS3AccessKey, "s3-access-key", "", "") + portableCmd.Flags().StringVar(&portableS3AccessSecret, "s3-access-secret", "", "") + portableCmd.Flags().StringVar(&portableS3Endpoint, "s3-endpoint", "", "") + portableCmd.Flags().StringVar(&portableS3StorageClass, "s3-storage-class", "", "") rootCmd.AddCommand(portableCmd) } diff --git a/dataprovider/bolt.go b/dataprovider/bolt.go index 8f924de5..57acbdeb 100644 --- a/dataprovider/bolt.go +++ b/dataprovider/bolt.go @@ -336,7 +336,7 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str if offset == 0 { user, err := p.userExists(username) if err == nil { - users = append(users, getUserNoCredentials(&user)) + users = append(users, HideUserSensitiveData(&user)) } } return users, err @@ -357,7 +357,7 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str var user User err = json.Unmarshal(v, &user) if err == nil { - users = append(users, getUserNoCredentials(&user)) + users = append(users, HideUserSensitiveData(&user)) } if len(users) >= limit { break @@ -372,7 +372,7 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str var user User err = json.Unmarshal(v, &user) if err == nil { - users = append(users, getUserNoCredentials(&user)) + users = append(users, HideUserSensitiveData(&user)) } if len(users) >= limit { break @@ -388,11 +388,6 @@ func (p BoltProvider) close() error { return p.dbHandle.Close() } -func getUserNoCredentials(user *User) User { - user.Password = "" - return *user -} - // itob returns an 8-byte big endian representation of v. func itob(v int64) []byte { b := make([]byte, 8) diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go index c2637013..6587e44e 100644 --- a/dataprovider/dataprovider.go +++ b/dataprovider/dataprovider.go @@ -34,6 +34,7 @@ import ( "github.com/drakkan/sftpgo/logger" "github.com/drakkan/sftpgo/metrics" "github.com/drakkan/sftpgo/utils" + "github.com/drakkan/sftpgo/vfs" unixcrypt "github.com/nathanaelle/password" ) @@ -479,6 +480,27 @@ func validateFilters(user *User) error { return nil } +func validateFilesystemConfig(user *User) error { + if user.FsConfig.Provider == 1 { + err := vfs.ValidateS3FsConfig(&user.FsConfig.S3Config) + if err != nil { + return &ValidationError{err: fmt.Sprintf("Could not validate s3config: %v", err)} + } + vals := strings.Split(user.FsConfig.S3Config.AccessSecret, "$") + if !strings.HasPrefix(user.FsConfig.S3Config.AccessSecret, "$aes$") || len(vals) != 4 { + accessSecret, err := utils.EncryptData(user.FsConfig.S3Config.AccessSecret) + if err != nil { + return &ValidationError{err: fmt.Sprintf("Could encrypt s3 access secret: %v", err)} + } + user.FsConfig.S3Config.AccessSecret = accessSecret + } + return nil + } + user.FsConfig.Provider = 0 + user.FsConfig.S3Config = vfs.S3FsConfig{} + return nil +} + func validateUser(user *User) error { buildUserHomeDir(user) if len(user.Username) == 0 || len(user.HomeDir) == 0 { @@ -493,6 +515,9 @@ func validateUser(user *User) error { if err := validatePermissions(user); err != nil { return err } + if err := validateFilesystemConfig(user); err != nil { + return err + } if user.Status < 0 || user.Status > 1 { return &ValidationError{err: fmt.Sprintf("invalid user status: %v", user.Status)} } @@ -645,6 +670,15 @@ func comparePbkdf2PasswordAndHash(password, hashedPassword string) (bool, error) return subtle.ConstantTimeCompare(buf, []byte(expected)) == 1, nil } +// HideUserSensitiveData hides user sensitive data +func HideUserSensitiveData(user *User) User { + user.Password = "" + if user.FsConfig.Provider == 1 { + user.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(user.FsConfig.S3Config.AccessSecret) + } + return *user +} + func getSSLMode() string { if config.Driver == PGSQLDataProviderName { if config.SSLMode == 0 { @@ -772,8 +806,7 @@ func executeAction(operation string, user User) { return } } - // hide the hashed password - user.Password = "" + HideUserSensitiveData(&user) if len(config.Actions.Command) > 0 && filepath.IsAbs(config.Actions.Command) { // we are in a goroutine but if we have to send an HTTP notification we don't want to wait for the // end of the command diff --git a/dataprovider/memory.go b/dataprovider/memory.go index d03a1e88..e07ed722 100644 --- a/dataprovider/memory.go +++ b/dataprovider/memory.go @@ -244,8 +244,7 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s if offset == 0 { user, err := p.userExistsInternal(username) if err == nil { - user.Password = "" - users = append(users, user) + users = append(users, HideUserSensitiveData(&user)) } } return users, err @@ -258,8 +257,7 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s continue } user := p.dbHandle.users[username] - user.Password = "" - users = append(users, user) + users = append(users, HideUserSensitiveData(&user)) if len(users) >= limit { break } @@ -272,8 +270,7 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s } username := p.dbHandle.usernames[i] user := p.dbHandle.users[username] - user.Password = "" - users = append(users, user) + users = append(users, HideUserSensitiveData(&user)) if len(users) >= limit { break } diff --git a/dataprovider/sqlcommon.go b/dataprovider/sqlcommon.go index 7b39e7d3..a9538dde 100644 --- a/dataprovider/sqlcommon.go +++ b/dataprovider/sqlcommon.go @@ -162,8 +162,13 @@ func sqlCommonAddUser(user User, dbHandle *sql.DB) error { if err != nil { return err } + fsConfig, err := user.GetFsConfigAsJSON() + if err != nil { + return err + } _, err = stmt.Exec(user.Username, user.Password, string(publicKeys), user.HomeDir, user.UID, user.GID, user.MaxSessions, user.QuotaSize, - user.QuotaFiles, string(permissions), user.UploadBandwidth, user.DownloadBandwidth, user.Status, user.ExpirationDate, string(filters)) + user.QuotaFiles, string(permissions), user.UploadBandwidth, user.DownloadBandwidth, user.Status, user.ExpirationDate, string(filters), + string(fsConfig)) return err } @@ -191,9 +196,13 @@ func sqlCommonUpdateUser(user User, dbHandle *sql.DB) error { if err != nil { return err } + fsConfig, err := user.GetFsConfigAsJSON() + if err != nil { + return err + } _, err = stmt.Exec(user.Password, string(publicKeys), user.HomeDir, user.UID, user.GID, user.MaxSessions, user.QuotaSize, user.QuotaFiles, string(permissions), user.UploadBandwidth, user.DownloadBandwidth, user.Status, user.ExpirationDate, - string(filters), user.ID) + string(filters), string(fsConfig), user.ID) return err } @@ -253,10 +262,8 @@ func sqlCommonGetUsers(limit int, offset int, order string, username string, dbH defer rows.Close() for rows.Next() { u, err := getUserFromDbRow(nil, rows) - // hide password if err == nil { - u.Password = "" - users = append(users, u) + users = append(users, HideUserSensitiveData(&u)) } else { break } @@ -272,16 +279,17 @@ func getUserFromDbRow(row *sql.Row, rows *sql.Rows) (User, error) { var password sql.NullString var publicKey sql.NullString var filters sql.NullString + var fsConfig sql.NullString var err error if row != nil { err = row.Scan(&user.ID, &user.Username, &password, &publicKey, &user.HomeDir, &user.UID, &user.GID, &user.MaxSessions, &user.QuotaSize, &user.QuotaFiles, &permissions, &user.UsedQuotaSize, &user.UsedQuotaFiles, &user.LastQuotaUpdate, - &user.UploadBandwidth, &user.DownloadBandwidth, &user.ExpirationDate, &user.LastLogin, &user.Status, &filters) + &user.UploadBandwidth, &user.DownloadBandwidth, &user.ExpirationDate, &user.LastLogin, &user.Status, &filters, &fsConfig) } else { err = rows.Scan(&user.ID, &user.Username, &password, &publicKey, &user.HomeDir, &user.UID, &user.GID, &user.MaxSessions, &user.QuotaSize, &user.QuotaFiles, &permissions, &user.UsedQuotaSize, &user.UsedQuotaFiles, &user.LastQuotaUpdate, - &user.UploadBandwidth, &user.DownloadBandwidth, &user.ExpirationDate, &user.LastLogin, &user.Status, &filters) + &user.UploadBandwidth, &user.DownloadBandwidth, &user.ExpirationDate, &user.LastLogin, &user.Status, &filters, &fsConfig) } if err != nil { if err == sql.ErrNoRows { @@ -326,5 +334,16 @@ func getUserFromDbRow(row *sql.Row, rows *sql.Rows) (User, error) { DeniedIP: []string{}, } } + if fsConfig.Valid { + var fs Filesystem + err = json.Unmarshal([]byte(fsConfig.String), &fs) + if err == nil { + user.FsConfig = fs + } + } else { + user.FsConfig = Filesystem{ + Provider: 0, + } + } return user, err } diff --git a/dataprovider/sqlqueries.go b/dataprovider/sqlqueries.go index a94e40cf..84593261 100644 --- a/dataprovider/sqlqueries.go +++ b/dataprovider/sqlqueries.go @@ -4,7 +4,7 @@ import "fmt" const ( selectUserFields = "id,username,password,public_keys,home_dir,uid,gid,max_sessions,quota_size,quota_files,permissions,used_quota_size," + - "used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,expiration_date,last_login,status,filters" + "used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,expiration_date,last_login,status,filters,filesystem" ) func getSQLPlaceholders() []string { @@ -60,19 +60,20 @@ func getQuotaQuery() string { func getAddUserQuery() string { return fmt.Sprintf(`INSERT INTO %v (username,password,public_keys,home_dir,uid,gid,max_sessions,quota_size,quota_files,permissions, - used_quota_size,used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,status,last_login,expiration_date,filters) - VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,0,0,0,%v,%v,%v,0,%v,%v)`, config.UsersTable, sqlPlaceholders[0], sqlPlaceholders[1], + used_quota_size,used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,status,last_login,expiration_date,filters, + filesystem) + VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,0,0,0,%v,%v,%v,0,%v,%v,%v)`, config.UsersTable, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9], sqlPlaceholders[10], sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13], - sqlPlaceholders[14]) + sqlPlaceholders[14], sqlPlaceholders[15]) } func getUpdateUserQuery() string { return fmt.Sprintf(`UPDATE %v SET password=%v,public_keys=%v,home_dir=%v,uid=%v,gid=%v,max_sessions=%v,quota_size=%v, - quota_files=%v,permissions=%v,upload_bandwidth=%v,download_bandwidth=%v,status=%v,expiration_date=%v,filters=%v WHERE id = %v`, - config.UsersTable, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4], - sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9], sqlPlaceholders[10], - sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13], sqlPlaceholders[14]) + quota_files=%v,permissions=%v,upload_bandwidth=%v,download_bandwidth=%v,status=%v,expiration_date=%v,filters=%v,filesystem=%v + WHERE id = %v`, config.UsersTable, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], + sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9], + sqlPlaceholders[10], sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13], sqlPlaceholders[14], sqlPlaceholders[15]) } func getDeleteUserQuery() string { diff --git a/dataprovider/user.go b/dataprovider/user.go index 7ab0a481..699db11b 100644 --- a/dataprovider/user.go +++ b/dataprovider/user.go @@ -7,10 +7,10 @@ import ( "path" "path/filepath" "strconv" - "strings" "github.com/drakkan/sftpgo/logger" "github.com/drakkan/sftpgo/utils" + "github.com/drakkan/sftpgo/vfs" ) // Available permissions for SFTP users @@ -53,6 +53,13 @@ type UserFilters struct { DeniedIP []string `json:"denied_ip"` } +// Filesystem defines cloud storage filesystem details +type Filesystem struct { + // 0 local filesystem, 1 Amazon S3 compatible + Provider int `json:"provider"` + S3Config vfs.S3FsConfig `json:"s3config,omitempty"` +} + // User defines an SFTP user type User struct { // Database unique identifier @@ -98,6 +105,16 @@ type User struct { LastLogin int64 `json:"last_login"` // Additional restrictions Filters UserFilters `json:"filters"` + // Filesystem configuration details + FsConfig Filesystem `json:"filesystem"` +} + +// GetFilesystem returns the filesystem for this user +func (u *User) GetFilesystem(connectionID string) (vfs.Fs, error) { + if u.FsConfig.Provider == 1 { + return vfs.NewS3Fs(connectionID, u.GetHomeDir(), u.FsConfig.S3Config) + } + return vfs.NewOsFs(connectionID), nil } // GetPermissionsForPath returns the permissions for the given path. @@ -211,6 +228,11 @@ func (u *User) GetFiltersAsJSON() ([]byte, error) { return json.Marshal(u.Filters) } +// GetFsConfigAsJSON returns the filesystem config as json byte array +func (u *User) GetFsConfigAsJSON() ([]byte, error) { + return json.Marshal(u.FsConfig) +} + // GetUID returns a validate uid, suitable for use with os.Chown func (u *User) GetUID() int { if u.UID <= 0 || u.UID > 65535 { @@ -237,19 +259,6 @@ func (u *User) HasQuotaRestrictions() bool { return u.QuotaFiles > 0 || u.QuotaSize > 0 } -// GetRelativePath returns the path for a file relative to the user's home dir. -// This is the path as seen by SFTP users -func (u *User) GetRelativePath(path string) string { - rel, err := filepath.Rel(u.GetHomeDir(), filepath.Clean(path)) - if err != nil { - return "" - } - if rel == "." || strings.HasPrefix(rel, "..") { - rel = "" - } - return "/" + filepath.ToSlash(rel) -} - // GetQuotaSummary returns used quota and limits if defined func (u *User) GetQuotaSummary() string { var result string @@ -319,6 +328,9 @@ func (u *User) GetInfoString() string { t := utils.GetTimeFromMsecSinceEpoch(u.LastLogin) result += fmt.Sprintf("Last login: %v ", t.Format("2006-01-02 15:04:05")) // YYYY-MM-DD HH:MM:SS } + if u.FsConfig.Provider == 1 { + result += fmt.Sprintf("Storage: S3") + } if len(u.PublicKeys) > 0 { result += fmt.Sprintf("Public keys: %v ", len(u.PublicKeys)) } @@ -387,6 +399,17 @@ func (u *User) getACopy() User { copy(filters.AllowedIP, u.Filters.AllowedIP) filters.DeniedIP = make([]string, len(u.Filters.DeniedIP)) copy(filters.DeniedIP, u.Filters.DeniedIP) + fsConfig := Filesystem{ + Provider: u.FsConfig.Provider, + S3Config: vfs.S3FsConfig{ + Bucket: u.FsConfig.S3Config.Bucket, + Region: u.FsConfig.S3Config.Region, + AccessKey: u.FsConfig.S3Config.AccessKey, + AccessSecret: u.FsConfig.S3Config.AccessSecret, + Endpoint: u.FsConfig.S3Config.Endpoint, + StorageClass: u.FsConfig.S3Config.StorageClass, + }, + } return User{ ID: u.ID, @@ -409,6 +432,7 @@ func (u *User) getACopy() User { ExpirationDate: u.ExpirationDate, LastLogin: u.LastLogin, Filters: filters, + FsConfig: fsConfig, } } diff --git a/go.mod b/go.mod index 372b6cdf..328d03cc 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,9 @@ go 1.13 require ( github.com/alexedwards/argon2id v0.0.0-20190612080829-01a59b2b8802 + github.com/aws/aws-sdk-go v1.28.3 github.com/cenkalti/backoff v2.2.1+incompatible // indirect + github.com/eikenb/pipeat v0.0.0-20190316224601-fb1f3a9aa29f github.com/go-chi/chi v4.0.2+incompatible github.com/go-chi/render v1.0.1 github.com/go-sql-driver/mysql v1.5.0 @@ -24,3 +26,5 @@ require ( golang.org/x/sys v0.0.0-20191220142924-d4481acd189f gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) + +replace github.com/eikenb/pipeat v0.0.0-20190316224601-fb1f3a9aa29f => github.com/drakkan/pipeat v0.0.0-20200114135659-fac71c64d75d diff --git a/go.sum b/go.sum index d4db4fe1..c213057d 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,4 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -9,6 +8,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alexedwards/argon2id v0.0.0-20190612080829-01a59b2b8802 h1:RwMM1q/QSKYIGbHfOkf843hE8sSUJtf1dMwFPtEDmm0= github.com/alexedwards/argon2id v0.0.0-20190612080829-01a59b2b8802/go.mod h1:4dsm7ufQm1Gwl8S2ss57u+2J7KlxIL2QUmFGlGtWogY= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aws/aws-sdk-go v1.28.3 h1:FnkDp+fz4JHWUW3Ust2Wh89RpdGif077Wjis/sMrGKM= +github.com/aws/aws-sdk-go v1.28.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -32,6 +33,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/drakkan/pipeat v0.0.0-20200114135659-fac71c64d75d h1:+k0oy9bBY9dXlKHriYg6crXpwIrtM1rCrlUehmc/F3M= +github.com/drakkan/pipeat v0.0.0-20200114135659-fac71c64d75d/go.mod h1:wNYvIpR5rIhoezOYcpxcXz4HbIEOu7A45EqlQCA+h+w= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -57,10 +60,8 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grandcat/zeroconf v0.0.0-20190424104450-85eadb44205c h1:svzQzfVE9t7Y1CGULS5PsMWs4/H4Au/ZTJzU/0CKgqc= @@ -70,12 +71,12 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -84,10 +85,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -148,9 +147,7 @@ github.com/rs/zerolog v1.17.2/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -205,7 +202,6 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -214,7 +210,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -244,7 +239,6 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/httpd/api_maintenance.go b/httpd/api_maintenance.go index 17f04147..3dcc304e 100644 --- a/httpd/api_maintenance.go +++ b/httpd/api_maintenance.go @@ -13,6 +13,7 @@ import ( "github.com/drakkan/sftpgo/dataprovider" "github.com/drakkan/sftpgo/logger" + "github.com/drakkan/sftpgo/sftpd" ) func dumpData(w http.ResponseWriter, r *http.Request) { @@ -103,10 +104,16 @@ func loadData(w http.ResponseWriter, r *http.Request) { u, err := dataprovider.UserExists(dataProvider, user.Username) if err == nil { user.ID = u.ID + user.LastLogin = u.LastLogin + user.UsedQuotaSize = u.UsedQuotaSize + user.UsedQuotaFiles = u.UsedQuotaFiles err = dataprovider.UpdateUser(dataProvider, user) user.Password = "[redacted]" logger.Debug(logSender, "", "restoring existing user: %+v, dump file: %#v, error: %v", user, inputFile, err) } else { + user.LastLogin = 0 + user.UsedQuotaSize = 0 + user.UsedQuotaFiles = 0 err = dataprovider.AddUser(dataProvider, user) user.Password = "[redacted]" logger.Debug(logSender, "", "adding new user: %+v, dump file: %#v, error: %v", user, inputFile, err) @@ -115,11 +122,17 @@ func loadData(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - if scanQuota == 1 || (scanQuota == 2 && user.HasQuotaRestrictions()) { - logger.Debug(logSender, "", "starting quota scan for restored user: %#v", user.Username) - doQuotaScan(user) + if needQuotaScan(scanQuota, &user) { + if sftpd.AddQuotaScan(user.Username) { + logger.Debug(logSender, "", "starting quota scan for restored user: %#v", user.Username) + go doQuotaScan(user) + } } } logger.Debug(logSender, "", "backup restored, users: %v", len(dump.Users)) sendAPIResponse(w, r, err, "Data restored", http.StatusOK) } + +func needQuotaScan(scanQuota int, user *dataprovider.User) bool { + return scanQuota == 1 || (scanQuota == 2 && user.HasQuotaRestrictions()) +} diff --git a/httpd/api_quota.go b/httpd/api_quota.go index 5278a43e..f6468c4e 100644 --- a/httpd/api_quota.go +++ b/httpd/api_quota.go @@ -6,7 +6,6 @@ import ( "github.com/drakkan/sftpgo/dataprovider" "github.com/drakkan/sftpgo/logger" "github.com/drakkan/sftpgo/sftpd" - "github.com/drakkan/sftpgo/utils" "github.com/go-chi/render" ) @@ -26,26 +25,27 @@ func startQuotaScan(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, err, "", http.StatusNotFound) return } - if doQuotaScan(user) { + if sftpd.AddQuotaScan(user.Username) { + go doQuotaScan(user) sendAPIResponse(w, r, err, "Scan started", http.StatusCreated) } else { sendAPIResponse(w, r, err, "Another scan is already in progress", http.StatusConflict) } } -func doQuotaScan(user dataprovider.User) bool { - result := sftpd.AddQuotaScan(user.Username) - if result { - go func() { - numFiles, size, _, err := utils.ScanDirContents(user.HomeDir) - if err != nil { - logger.Warn(logSender, "", "error scanning user home dir %#v: %v", user.HomeDir, err) - } else { - err := dataprovider.UpdateUserQuota(dataProvider, user, numFiles, size, true) - logger.Debug(logSender, "", "user home dir scanned, user: %#v, dir: %#v, error: %v", user.Username, user.HomeDir, err) - } - sftpd.RemoveQuotaScan(user.Username) - }() +func doQuotaScan(user dataprovider.User) error { + defer sftpd.RemoveQuotaScan(user.Username) + fs, err := user.GetFilesystem("") + if err != nil { + logger.Warn(logSender, "", "unable scan quota for user %#v error creating filesystem: %v", user.Username, err) + return err } - return result + numFiles, size, err := fs.ScanDirContents(user.HomeDir) + if err != nil { + logger.Warn(logSender, "", "error scanning user home dir %#v: %v", user.HomeDir, err) + } else { + err = dataprovider.UpdateUserQuota(dataProvider, user, numFiles, size, true) + logger.Debug(logSender, "", "user home dir scanned, user: %#v, dir: %#v, error: %v", user.Username, user.HomeDir, err) + } + return err } diff --git a/httpd/api_user.go b/httpd/api_user.go index dbb92b87..97d13abc 100644 --- a/httpd/api_user.go +++ b/httpd/api_user.go @@ -6,6 +6,7 @@ import ( "strconv" "github.com/drakkan/sftpgo/dataprovider" + "github.com/drakkan/sftpgo/utils" "github.com/go-chi/chi" "github.com/go-chi/render" ) @@ -63,8 +64,7 @@ func getUserByID(w http.ResponseWriter, r *http.Request) { } user, err := dataprovider.GetUserByID(dataProvider, userID) if err == nil { - user.Password = "" - render.JSON(w, r, user) + render.JSON(w, r, dataprovider.HideUserSensitiveData(&user)) } else if _, ok := err.(*dataprovider.RecordNotFoundError); ok { sendAPIResponse(w, r, err, "", http.StatusNotFound) } else { @@ -83,8 +83,7 @@ func addUser(w http.ResponseWriter, r *http.Request) { if err == nil { user, err = dataprovider.UserExists(dataProvider, user.Username) if err == nil { - user.Password = "" - render.JSON(w, r, user) + render.JSON(w, r, dataprovider.HideUserSensitiveData(&user)) } else { sendAPIResponse(w, r, err, "", http.StatusInternalServerError) } @@ -102,6 +101,10 @@ func updateUser(w http.ResponseWriter, r *http.Request) { } user, err := dataprovider.GetUserByID(dataProvider, userID) oldPermissions := user.Permissions + oldS3AccessSecret := "" + if user.FsConfig.Provider == 1 { + oldS3AccessSecret = user.FsConfig.S3Config.AccessSecret + } user.Permissions = make(map[string][]string) if _, ok := err.(*dataprovider.RecordNotFoundError); ok { sendAPIResponse(w, r, err, "", http.StatusNotFound) @@ -119,6 +122,13 @@ func updateUser(w http.ResponseWriter, r *http.Request) { if len(user.Permissions) == 0 { user.Permissions = oldPermissions } + // we use the new access secret if different from the old one and not empty + if user.FsConfig.Provider == 1 { + if utils.RemoveDecryptionKey(oldS3AccessSecret) == user.FsConfig.S3Config.AccessSecret || + len(user.FsConfig.S3Config.AccessSecret) == 0 { + user.FsConfig.S3Config.AccessSecret = oldS3AccessSecret + } + } if user.ID != userID { sendAPIResponse(w, r, err, "user ID in request body does not match user ID in path parameter", http.StatusBadRequest) return diff --git a/httpd/api_utils.go b/httpd/api_utils.go index 71d9c0ce..f12107b6 100644 --- a/httpd/api_utils.go +++ b/httpd/api_utils.go @@ -406,10 +406,66 @@ func checkUser(expected *dataprovider.User, actual *dataprovider.User) error { if err := compareUserFilters(expected, actual); err != nil { return err } + if err := compareUserFsConfig(expected, actual); err != nil { + return err + } return compareEqualsUserFields(expected, actual) } +func compareUserFsConfig(expected *dataprovider.User, actual *dataprovider.User) error { + if expected.FsConfig.Provider != actual.FsConfig.Provider { + return errors.New("Fs provider mismatch") + } + if expected.FsConfig.S3Config.Bucket != actual.FsConfig.S3Config.Bucket { + return errors.New("S3 bucket mismatch") + } + if expected.FsConfig.S3Config.Region != actual.FsConfig.S3Config.Region { + return errors.New("S3 region mismatch") + } + if expected.FsConfig.S3Config.AccessKey != actual.FsConfig.S3Config.AccessKey { + return errors.New("S3 access key mismatch") + } + if err := checkS3AccessSecret(expected.FsConfig.S3Config.AccessSecret, actual.FsConfig.S3Config.AccessSecret); err != nil { + return err + } + if expected.FsConfig.S3Config.Endpoint != actual.FsConfig.S3Config.Endpoint { + return errors.New("S3 endpoint mismatch") + } + if expected.FsConfig.S3Config.StorageClass != actual.FsConfig.S3Config.StorageClass { + return errors.New("S3 storage class mismatch") + } + return nil +} + +func checkS3AccessSecret(expectedAccessSecret, actualAccessSecret string) error { + if len(expectedAccessSecret) > 0 { + vals := strings.Split(expectedAccessSecret, "$") + if strings.HasPrefix(expectedAccessSecret, "$aes$") && len(vals) == 4 { + expectedAccessSecret = utils.RemoveDecryptionKey(expectedAccessSecret) + if expectedAccessSecret != actualAccessSecret { + return fmt.Errorf("S3 access secret mismatch, expected: %v", expectedAccessSecret) + } + } else { + // here we check that actualAccessSecret is aes encrypted without the nonce + parts := strings.Split(actualAccessSecret, "$") + if !strings.HasPrefix(actualAccessSecret, "$aes$") || len(parts) != 3 { + return errors.New("Invalid S3 access secret") + } + if len(parts) == len(vals) { + if expectedAccessSecret != actualAccessSecret { + return errors.New("S3 encrypted access secret mismatch") + } + } + } + } else { + if expectedAccessSecret != actualAccessSecret { + return errors.New("S3 access secret mismatch") + } + } + return nil +} + func compareUserFilters(expected *dataprovider.User, actual *dataprovider.User) error { if len(expected.Filters.AllowedIP) != len(actual.Filters.AllowedIP) { return errors.New("AllowedIP mismatch") diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index 72bbfe5f..43928f5e 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -242,6 +242,16 @@ func TestAddUserInvalidFilters(t *testing.T) { } } +func TestAddUserInvalidFsConfig(t *testing.T) { + u := getTestUser() + u.FsConfig.Provider = 1 + u.FsConfig.S3Config.Bucket = "" + _, _, err := httpd.AddUser(u, http.StatusBadRequest) + if err != nil { + t.Errorf("unexpected error adding user with invalid fs config: %v", err) + } +} + func TestUserPublicKey(t *testing.T) { u := getTestUser() invalidPubKey := "invalid" @@ -299,6 +309,48 @@ func TestUpdateUser(t *testing.T) { } } +func TestUserS3Config(t *testing.T) { + user, _, err := httpd.AddUser(getTestUser(), http.StatusOK) + if err != nil { + t.Errorf("unable to add user: %v", err) + } + user.FsConfig.Provider = 1 + user.FsConfig.S3Config.Bucket = "test" + user.FsConfig.S3Config.Region = "us-east-1" + user.FsConfig.S3Config.AccessKey = "Server-Access-Key" + user.FsConfig.S3Config.AccessSecret = "Server-Access-Secret" + user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000" + user, _, err = httpd.UpdateUser(user, http.StatusOK) + if err != nil { + t.Errorf("unable to update user: %v", err) + } + _, err = httpd.RemoveUser(user, http.StatusOK) + if err != nil { + t.Errorf("unable to remove: %v", err) + } + user.Password = defaultPassword + user.ID = 0 + secret, _ := utils.EncryptData("Server-Access-Secret") + user.FsConfig.S3Config.AccessSecret = secret + user, _, err = httpd.AddUser(user, http.StatusOK) + if err != nil { + t.Errorf("unable to add user: %v", err) + } + user.FsConfig.Provider = 1 + user.FsConfig.S3Config.Bucket = "test1" + user.FsConfig.S3Config.Region = "us-east-1" + user.FsConfig.S3Config.AccessKey = "Server-Access-Key1" + user.FsConfig.S3Config.Endpoint = "http://localhost:9000" + user, _, err = httpd.UpdateUser(user, http.StatusOK) + if err != nil { + t.Errorf("unable to update user: %v", err) + } + _, err = httpd.RemoveUser(user, http.StatusOK) + if err != nil { + t.Errorf("unable to remove: %v", err) + } +} + func TestUpdateUserNoCredentials(t *testing.T) { user, _, err := httpd.AddUser(getTestUser(), http.StatusOK) if err != nil { @@ -1398,6 +1450,91 @@ func TestWebUserUpdateMock(t *testing.T) { checkResponseCode(t, http.StatusOK, rr.Code) } +func TestWebUserS3Mock(t *testing.T) { + user := getTestUser() + userAsJSON := getUserAsJSON(t, user) + req, _ := http.NewRequest(http.MethodPost, userPath, bytes.NewBuffer(userAsJSON)) + rr := executeRequest(req) + checkResponseCode(t, http.StatusOK, rr.Code) + err := render.DecodeJSON(rr.Body, &user) + if err != nil { + t.Errorf("Error get user: %v", err) + } + user.FsConfig.Provider = 1 + user.FsConfig.S3Config.Bucket = "test" + user.FsConfig.S3Config.Region = "eu-west-1" + user.FsConfig.S3Config.AccessKey = "access-key" + user.FsConfig.S3Config.AccessSecret = "access-secret" + user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?a=b" + user.FsConfig.S3Config.StorageClass = "Standard" + form := make(url.Values) + form.Set("username", user.Username) + form.Set("home_dir", user.HomeDir) + form.Set("uid", "0") + form.Set("gid", strconv.FormatInt(int64(user.GID), 10)) + form.Set("max_sessions", strconv.FormatInt(int64(user.MaxSessions), 10)) + form.Set("quota_size", strconv.FormatInt(user.QuotaSize, 10)) + form.Set("quota_files", strconv.FormatInt(int64(user.QuotaFiles), 10)) + form.Set("upload_bandwidth", "0") + form.Set("download_bandwidth", "0") + form.Set("permissions", "*") + form.Set("sub_dirs_permissions", "") + form.Set("status", strconv.Itoa(user.Status)) + form.Set("expiration_date", "2020-01-01 00:00:00") + form.Set("allowed_ip", "") + form.Set("denied_ip", "") + form.Set("fs_provider", "1") + form.Set("s3_bucket", user.FsConfig.S3Config.Bucket) + form.Set("s3_region", user.FsConfig.S3Config.Region) + form.Set("s3_access_key", user.FsConfig.S3Config.AccessKey) + form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret) + form.Set("s3_storage_class", user.FsConfig.S3Config.StorageClass) + form.Set("s3_endpoint", user.FsConfig.S3Config.Endpoint) + req, _ = http.NewRequest(http.MethodPost, webUserPath+"/"+strconv.FormatInt(user.ID, 10), strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + rr = executeRequest(req) + checkResponseCode(t, http.StatusSeeOther, rr.Code) + req, _ = http.NewRequest(http.MethodGet, userPath+"?limit=1&offset=0&order=ASC&username="+user.Username, nil) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr.Code) + var users []dataprovider.User + err = render.DecodeJSON(rr.Body, &users) + if err != nil { + t.Errorf("Error decoding users: %v", err) + } + if len(users) != 1 { + t.Errorf("1 user is expected") + } + updateUser := users[0] + if updateUser.ExpirationDate != 1577836800000 { + t.Errorf("invalid expiration date: %v", updateUser.ExpirationDate) + } + if updateUser.FsConfig.Provider != user.FsConfig.Provider { + t.Error("fs provider mismatch") + } + if updateUser.FsConfig.S3Config.Bucket != user.FsConfig.S3Config.Bucket { + t.Error("s3 bucket mismatch") + } + if updateUser.FsConfig.S3Config.Region != user.FsConfig.S3Config.Region { + t.Error("s3 region mismatch") + } + if updateUser.FsConfig.S3Config.AccessKey != user.FsConfig.S3Config.AccessKey { + t.Error("s3 access key mismatch") + } + if !strings.HasPrefix(updateUser.FsConfig.S3Config.AccessSecret, "$aes$") { + t.Error("s3 access secret is not encrypted") + } + if updateUser.FsConfig.S3Config.StorageClass != user.FsConfig.S3Config.StorageClass { + t.Error("s3 storage class mismatch") + } + if updateUser.FsConfig.S3Config.Endpoint != user.FsConfig.S3Config.Endpoint { + t.Error("s3 endpoint mismatch") + } + req, _ = http.NewRequest(http.MethodDelete, userPath+"/"+strconv.FormatInt(user.ID, 10), nil) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr.Code) +} + func TestProviderClosedMock(t *testing.T) { if providerDriverName == dataprovider.BoltDataProviderName { t.Skip("skipping test provider errors for bolt provider") diff --git a/httpd/internal_test.go b/httpd/internal_test.go index b8e5b798..65643127 100644 --- a/httpd/internal_test.go +++ b/httpd/internal_test.go @@ -6,9 +6,12 @@ import ( "html/template" "net/http" "net/http/httptest" + "os" "testing" "github.com/drakkan/sftpgo/dataprovider" + "github.com/drakkan/sftpgo/sftpd" + "github.com/drakkan/sftpgo/utils" "github.com/go-chi/chi" ) @@ -120,6 +123,13 @@ func TestCheckUser(t *testing.T) { if err == nil { t.Errorf("DeniedIP contents are not equal") } + expected.Filters.DeniedIP = []string{} + actual.Filters.DeniedIP = []string{} + actual.FsConfig.Provider = 1 + err = checkUser(expected, actual) + if err == nil { + t.Errorf("Fs providers are not equal") + } } func TestCompareUserFields(t *testing.T) { @@ -200,6 +210,72 @@ func TestCompareUserFields(t *testing.T) { } } +func TestCompareUserFsConfig(t *testing.T) { + expected := &dataprovider.User{} + actual := &dataprovider.User{} + expected.FsConfig.Provider = 1 + err := compareUserFsConfig(expected, actual) + if err == nil { + t.Errorf("Provider does not match") + } + expected.FsConfig.Provider = 0 + expected.FsConfig.S3Config.Bucket = "bucket" + err = compareUserFsConfig(expected, actual) + if err == nil { + t.Errorf("S3 bucket does not match") + } + expected.FsConfig.S3Config.Bucket = "" + expected.FsConfig.S3Config.Region = "region" + err = compareUserFsConfig(expected, actual) + if err == nil { + t.Errorf("S3 region does not match") + } + expected.FsConfig.S3Config.Region = "" + expected.FsConfig.S3Config.AccessKey = "access key" + err = compareUserFsConfig(expected, actual) + if err == nil { + t.Errorf("S3 access key does not match") + } + expected.FsConfig.S3Config.AccessKey = "" + actual.FsConfig.S3Config.AccessSecret = "access secret" + err = compareUserFsConfig(expected, actual) + if err == nil { + t.Errorf("S3 access secret does not match") + } + secret, _ := utils.EncryptData("access secret") + actual.FsConfig.S3Config.AccessSecret = "" + expected.FsConfig.S3Config.AccessSecret = secret + err = compareUserFsConfig(expected, actual) + if err == nil { + t.Errorf("S3 access secret does not match") + } + expected.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(secret) + actual.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(secret) + "a" + err = compareUserFsConfig(expected, actual) + if err == nil { + t.Errorf("S3 access secret does not match") + } + expected.FsConfig.S3Config.AccessSecret = "test" + actual.FsConfig.S3Config.AccessSecret = "" + err = compareUserFsConfig(expected, actual) + if err == nil { + t.Errorf("S3 access secret does not match") + } + expected.FsConfig.S3Config.AccessSecret = "" + actual.FsConfig.S3Config.AccessSecret = "" + expected.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/" + err = compareUserFsConfig(expected, actual) + if err == nil { + t.Errorf("S3 endpoint does not match") + } + expected.FsConfig.S3Config.Endpoint = "" + expected.FsConfig.S3Config.StorageClass = "Standard" + err = compareUserFsConfig(expected, actual) + if err == nil { + t.Errorf("S3 storage class does not match") + } +} + func TestApiCallsWithBadURL(t *testing.T) { oldBaseURL := httpBaseURL SetBaseURL(invalidURL) @@ -315,3 +391,18 @@ func TestRenderInvalidTemplate(t *testing.T) { } } } + +func TestQuotaScanInvalidFs(t *testing.T) { + user := dataprovider.User{ + Username: "test", + HomeDir: os.TempDir(), + FsConfig: dataprovider.Filesystem{ + Provider: 1, + }, + } + sftpd.AddQuotaScan(user.Username) + err := doQuotaScan(user) + if err == nil { + t.Error("quota scan with bad fs must fail") + } +} diff --git a/httpd/schema/openapi.yaml b/httpd/schema/openapi.yaml index 324a19e4..f7337291 100644 --- a/httpd/schema/openapi.yaml +++ b/httpd/schema/openapi.yaml @@ -710,6 +710,50 @@ components: nullable: true description: clients connecting from these IP/Mask are not allowed. Denied rules are evaluated before allowed ones example: [ "172.16.0.0/16" ] + description: Additional restrictions + S3Config: + type: object + properties: + bucket: + type: string + minLength: 1 + region: + type: string + minLength: 1 + access_key: + type: string + minLength: 1 + access_secret: + type: string + minLength: 1 + description: the access secret is stored encrypted (AES-256-GCM) + endpoint: + type: string + description: optional endpoint + storage_class: + type: string + required: + - bucket + - region + - access_key + - access_secret + nullable: true + description: S3 Compatible Object Storage configuration details + FilesystemConfig: + type: object + properties: + provider: + type: integer + enum: + - 0 + - 1 + description: > + Providers: + * `0` - local filesystem + * `1` - S3 Compatible Object Storage + s3config: + $ref: '#/components/schemas/S3Config' + description: Storage filesystem details User: type: object properties: @@ -799,8 +843,8 @@ components: description: Last user login as unix timestamp in milliseconds filters: $ref: '#/components/schemas/UserFilters' - nullable: true - description: Additional restrictions + filesystem: + $ref: '#/components/schemas/FilesystemConfig' Transfer: type: object properties: diff --git a/httpd/web.go b/httpd/web.go index c3d7682f..9e5500a1 100644 --- a/httpd/web.go +++ b/httpd/web.go @@ -224,6 +224,24 @@ func getFiltersFromUserPostFields(r *http.Request) dataprovider.UserFilters { return filters } +func getFsConfigFromUserPostFields(r *http.Request) dataprovider.Filesystem { + var fs dataprovider.Filesystem + provider, err := strconv.Atoi(r.Form.Get("fs_provider")) + if err != nil { + provider = 0 + } + fs.Provider = provider + if fs.Provider == 1 { + fs.S3Config.Bucket = r.Form.Get("s3_bucket") + fs.S3Config.Region = r.Form.Get("s3_region") + fs.S3Config.AccessKey = r.Form.Get("s3_access_key") + fs.S3Config.AccessSecret = r.Form.Get("s3_access_secret") + fs.S3Config.Endpoint = r.Form.Get("s3_endpoint") + fs.S3Config.StorageClass = r.Form.Get("s3_storage_class") + } + return fs +} + func getUserFromPostFields(r *http.Request) (dataprovider.User, error) { var user dataprovider.User err := r.ParseForm() @@ -289,6 +307,7 @@ func getUserFromPostFields(r *http.Request) (dataprovider.User, error) { Status: status, ExpirationDate: expirationDateMillis, Filters: getFiltersFromUserPostFields(r), + FsConfig: getFsConfigFromUserPostFields(r), } return user, err } diff --git a/scripts/README.md b/scripts/README.md index 8340f529..78b03b22 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -44,7 +44,7 @@ Let's see a sample usage for each REST API. Command: ``` -python sftpgo_api_cli.py add-user test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 33 --gid 1000 --max-sessions 2 --quota-size 0 --quota-files 3 --permissions "list" "download" "upload" "delete" "rename" "create_dirs" "overwrite" --subdirs-permissions "/dir1:list,download" "/dir2:*" --upload-bandwidth 100 --download-bandwidth 60 --status 0 --expiration-date 2019-01-01 --allowed-ip "192.168.1.1/32" +python sftpgo_api_cli.py add-user test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 33 --gid 1000 --max-sessions 2 --quota-size 0 --quota-files 3 --permissions "list" "download" "upload" "delete" "rename" "create_dirs" "overwrite" --subdirs-permissions "/dir1:list,download" "/dir2:*" --upload-bandwidth 100 --download-bandwidth 60 --status 0 --expiration-date 2019-01-01 --allowed-ip "192.168.1.1/32" --fs S3 --s3-bucket test --s3-region eu-west-1 --s3-access-key accesskey --s3-access-secret secret --s3-endpoint "http://127.0.0.1:9000" --s3-storage-class Standard ``` Output: @@ -53,6 +53,17 @@ Output: { "download_bandwidth": 60, "expiration_date": 1546297200000, + "filesystem": { + "provider": 1, + "s3config": { + "access_key": "accesskey", + "access_secret": "$aes$6c088ba12b0b261247c8cf331c46d9260b8e58002957d89ad1c0495e3af665cd0227", + "bucket": "test", + "endpoint": "http://127.0.0.1:9000", + "region": "eu-west-1", + "storage_class": "Standard" + } + }, "filters": { "allowed_ip": [ "192.168.1.1/32" @@ -99,7 +110,7 @@ Output: Command: ``` -python sftpgo_api_cli.py update-user 9576 test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 0 --gid 33 --max-sessions 3 --quota-size 0 --quota-files 4 --permissions "*" --subdirs-permissions "/dir1:list,download,create_symlinks" --upload-bandwidth 90 --download-bandwidth 80 --status 1 --expiration-date "" --allowed-ip "" --denied-ip "192.168.1.0/24" +python sftpgo_api_cli.py update-user 9576 test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 0 --gid 33 --max-sessions 3 --quota-size 0 --quota-files 4 --permissions "*" --subdirs-permissions "/dir1:list,download,create_symlinks" --upload-bandwidth 90 --download-bandwidth 80 --status 1 --expiration-date "" --allowed-ip "" --denied-ip "192.168.1.0/24" --fs local ``` Output: @@ -126,6 +137,10 @@ Output: { "download_bandwidth": 80, "expiration_date": 0, + "filesystem": { + "provider": 0, + "s3config": {} + }, "filters": { "allowed_ip": [], "denied_ip": [ @@ -174,6 +189,10 @@ Output: { "download_bandwidth": 80, "expiration_date": 0, + "filesystem": { + "provider": 0, + "s3config": {} + }, "filters": { "allowed_ip": [], "denied_ip": [ diff --git a/scripts/sftpgo_api_cli.py b/scripts/sftpgo_api_cli.py index b9bbe7c5..eda6a1a4 100755 --- a/scripts/sftpgo_api_cli.py +++ b/scripts/sftpgo_api_cli.py @@ -70,9 +70,10 @@ class SFTPGoApiRequests: else: print(r.text) - def buildUserObject(self, user_id=0, username="", password="", public_keys=[], home_dir="", uid=0, - gid=0, max_sessions=0, quota_size=0, quota_files=0, permissions={}, upload_bandwidth=0, - download_bandwidth=0, status=1, expiration_date=0, allowed_ip=[], denied_ip=[]): + def buildUserObject(self, user_id=0, username="", password="", public_keys=[], home_dir="", uid=0, gid=0, + max_sessions=0, quota_size=0, quota_files=0, permissions={}, upload_bandwidth=0, download_bandwidth=0, + status=1, expiration_date=0, allowed_ip=[], denied_ip=[], fs_provider='local', s3_bucket='', + s3_region='', s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class=''): user = {"id":user_id, "username":username, "uid":uid, "gid":gid, "max_sessions":max_sessions, "quota_size":quota_size, "quota_files":quota_files, "upload_bandwidth":upload_bandwidth, "download_bandwidth":download_bandwidth, @@ -90,6 +91,8 @@ class SFTPGoApiRequests: user.update({"permissions":permissions}) if allowed_ip or denied_ip: user.update({"filters":self.buildFilters(allowed_ip, denied_ip)}) + user.update({"filesystem":self.buildFsConfig(fs_provider, s3_bucket, s3_region, s3_access_key, + s3_access_secret, s3_endpoint, s3_storage_class)}) return user def buildPermissions(self, root_perms, subdirs_perms): @@ -113,16 +116,25 @@ class SFTPGoApiRequests: filters = {} if allowed_ip: if len(allowed_ip) == 1 and not allowed_ip[0]: - filters.update({"allowed_ip":[]}) + filters.update({'allowed_ip':[]}) else: - filters.update({"allowed_ip":allowed_ip}) + filters.update({'allowed_ip':allowed_ip}) if denied_ip: if len(denied_ip) == 1 and not denied_ip[0]: - filters.update({"denied_ip":[]}) + filters.update({'denied_ip':[]}) else: - filters.update({"denied_ip":denied_ip}) + filters.update({'denied_ip':denied_ip}) return filters + def buildFsConfig(self, fs_provider, s3_bucket, s3_region, s3_access_key, s3_access_secret, s3_endpoint, + s3_storage_class): + fs_config = {'provider':0} + if fs_provider == 'S3': + s3config = {'bucket':s3_bucket, 'region':s3_region, 'access_key':s3_access_key, 'access_secret': + s3_access_secret, 'endpoint':s3_endpoint, 'storage_class':s3_storage_class} + fs_config.update({'provider':1, 's3config':s3config}) + return fs_config + def getUsers(self, limit=100, offset=0, order="ASC", username=""): r = requests.get(self.userPath, params={"limit":limit, "offset":offset, "order":order, "username":username}, auth=self.auth, verify=self.verify) @@ -132,22 +144,25 @@ class SFTPGoApiRequests: r = requests.get(urlparse.urljoin(self.userPath, "user/" + str(user_id)), auth=self.auth, verify=self.verify) self.printResponse(r) - def addUser(self, username="", password="", public_keys="", home_dir="", uid=0, gid=0, max_sessions=0, - quota_size=0, quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1, - expiration_date=0, subdirs_permissions=[], allowed_ip=[], denied_ip=[]): + def addUser(self, username="", password="", public_keys="", home_dir="", uid=0, gid=0, max_sessions=0, quota_size=0, + quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1, expiration_date=0, + subdirs_permissions=[], allowed_ip=[], denied_ip=[], fs_provider='local', s3_bucket='', s3_region='', + s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class=''): u = self.buildUserObject(0, username, password, public_keys, home_dir, uid, gid, max_sessions, quota_size, quota_files, self.buildPermissions(perms, subdirs_permissions), upload_bandwidth, download_bandwidth, - status, expiration_date, allowed_ip, denied_ip) + status, expiration_date, allowed_ip, denied_ip, fs_provider, s3_bucket, s3_region, + s3_access_key, s3_access_secret, s3_endpoint, s3_storage_class) r = requests.post(self.userPath, json=u, auth=self.auth, verify=self.verify) self.printResponse(r) - def updateUser(self, user_id, username="", password="", public_keys="", home_dir="", uid=0, gid=0, - max_sessions=0, quota_size=0, quota_files=0, perms=[], upload_bandwidth=0, - download_bandwidth=0, status=1, expiration_date=0, subdirs_permissions=[], - allowed_ip=[], denied_ip=[]): + def updateUser(self, user_id, username="", password="", public_keys="", home_dir="", uid=0, gid=0, max_sessions=0, + quota_size=0, quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1, + expiration_date=0, subdirs_permissions=[], allowed_ip=[], denied_ip=[], fs_provider='local', + s3_bucket='', s3_region='', s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class=''): u = self.buildUserObject(user_id, username, password, public_keys, home_dir, uid, gid, max_sessions, quota_size, quota_files, self.buildPermissions(perms, subdirs_permissions), upload_bandwidth, download_bandwidth, - status, expiration_date, allowed_ip, denied_ip) + status, expiration_date, allowed_ip, denied_ip, fs_provider, s3_bucket, s3_region, s3_access_key, + s3_access_secret, s3_endpoint, s3_storage_class) r = requests.put(urlparse.urljoin(self.userPath, "user/" + str(user_id)), json=u, auth=self.auth, verify=self.verify) self.printResponse(r) @@ -238,7 +253,7 @@ class ConvertUsers: self.convertFromProFTPD() self.saveUsers() - def isUserValid(self, username, uid, gid): + def isUserValid(self, username, uid): if self.usernames and not username in self.usernames: return False if self.min_uid >= 0 and uid < self.min_uid: @@ -257,7 +272,7 @@ class ConvertUsers: home_dir = user.pw_dir status = 1 expiration_date = 0 - if not self.isUserValid(username, uid, gid): + if not self.isUserValid(username, uid): continue if self.force_uid >= 0: uid = self.force_uid @@ -375,7 +390,7 @@ def addCommonUserArguments(parser): parser.add_argument('username', type=str) parser.add_argument('-P', '--password', type=str, default=None, help='Default: %(default)s') parser.add_argument('-K', '--public-keys', type=str, nargs='+', default=[], help='Default: %(default)s') - parser.add_argument('-H', '--home-dir', type=str, default="", help='Default: %(default)s') + parser.add_argument('-H', '--home-dir', type=str, default='', help='Default: %(default)s') parser.add_argument('--uid', type=int, default=0, help='Default: %(default)s') parser.add_argument('--gid', type=int, default=0, help='Default: %(default)s') parser.add_argument('-C', '--max-sessions', type=int, default=0, @@ -401,6 +416,14 @@ def addCommonUserArguments(parser): help='Allowed IP/Mask in CIDR notation. For example "192.168.2.0/24" or "2001:db8::/32". Default: %(default)s') parser.add_argument('-N', '--denied-ip', type=str, nargs='+', default=[], help='Denied IP/Mask in CIDR notation. For example "192.168.2.0/24" or "2001:db8::/32". Default: %(default)s') + parser.add_argument('--fs', type=str, default='local', choices=['local', 'S3'], + help='Filesystem provider. Default: %(default)s') + parser.add_argument('--s3-bucket', type=str, default='', help='Default: %(default)s') + parser.add_argument('--s3-region', type=str, default='', help='Default: %(default)s') + parser.add_argument('--s3-access-key', type=str, default='', help='Default: %(default)s') + parser.add_argument('--s3-access-secret', type=str, default='', help='Default: %(default)s') + parser.add_argument('--s3-endpoint', type=str, default='', help='Default: %(default)s') + parser.add_argument('--s3-storage-class', type=str, default='', help='Default: %(default)s') if __name__ == '__main__': @@ -503,12 +526,14 @@ if __name__ == '__main__': api.addUser(args.username, args.password, args.public_keys, args.home_dir, args.uid, args.gid, args.max_sessions, args.quota_size, args.quota_files, args.permissions, args.upload_bandwidth, args.download_bandwidth, args.status, getDatetimeAsMillisSinceEpoch(args.expiration_date), args.subdirs_permissions, args.allowed_ip, - args.denied_ip) + args.denied_ip, args.fs, args.s3_bucket, args.s3_region, args.s3_access_key, args.s3_access_secret, + args.s3_endpoint, args.s3_storage_class) elif args.command == 'update-user': api.updateUser(args.id, args.username, args.password, args.public_keys, args.home_dir, args.uid, args.gid, args.max_sessions, args.quota_size, args.quota_files, args.permissions, args.upload_bandwidth, args.download_bandwidth, args.status, getDatetimeAsMillisSinceEpoch(args.expiration_date), - args.subdirs_permissions, args.allowed_ip, args.denied_ip) + args.subdirs_permissions, args.allowed_ip, args.denied_ip, args.fs, args.s3_bucket, args.s3_region, + args.s3_access_key, args.s3_access_secret, args.s3_endpoint, args.s3_storage_class) elif args.command == 'delete-user': api.deleteUser(args.id) elif args.command == 'get-users': diff --git a/sftpd/handler.go b/sftpd/handler.go index 07dc245d..18f64dc7 100644 --- a/sftpd/handler.go +++ b/sftpd/handler.go @@ -1,19 +1,14 @@ package sftpd import ( - "fmt" "io" - "io/ioutil" "net" "os" "path" - "path/filepath" - "strings" "sync" "time" - "github.com/drakkan/sftpgo/utils" - "github.com/rs/xid" + "github.com/drakkan/sftpgo/vfs" "golang.org/x/crypto/ssh" "github.com/drakkan/sftpgo/dataprovider" @@ -40,6 +35,7 @@ type Connection struct { netConn net.Conn channel ssh.Channel command string + fs vfs.Fs } // Log outputs a log entry to the configured logger @@ -55,26 +51,29 @@ func (c Connection) Fileread(request *sftp.Request) (io.ReaderAt, error) { return nil, sftp.ErrSSHFxPermissionDenied } - p, err := c.buildPath(request.Filepath) + p, err := c.fs.ResolvePath(request.Filepath, c.User.GetHomeDir()) if err != nil { - return nil, getSFTPErrorFromOSError(err) + return nil, vfs.GetSFTPError(c.fs, err) } - fi, err := os.Stat(p) + fi, err := c.fs.Stat(p) if err != nil { - return nil, getSFTPErrorFromOSError(err) + return nil, vfs.GetSFTPError(c.fs, err) } - file, err := os.Open(p) + file, r, cancelFn, err := c.fs.Open(p) if err != nil { c.Log(logger.LevelWarn, logSender, "could not open file %#v for reading: %v", p, err) - return nil, getSFTPErrorFromOSError(err) + return nil, vfs.GetSFTPError(c.fs, err) } c.Log(logger.LevelDebug, logSender, "fileread requested for path: %#v", p) transfer := Transfer{ file: file, + readerAt: r, + writerAt: nil, + cancelFn: cancelFn, path: p, start: time.Now(), bytesSent: 0, @@ -98,18 +97,18 @@ func (c Connection) Fileread(request *sftp.Request) (io.ReaderAt, error) { // Filewrite handles the write actions for a file on the system. func (c Connection) Filewrite(request *sftp.Request) (io.WriterAt, error) { updateConnectionActivity(c.ID) - p, err := c.buildPath(request.Filepath) + p, err := c.fs.ResolvePath(request.Filepath, c.User.GetHomeDir()) if err != nil { - return nil, getSFTPErrorFromOSError(err) + return nil, vfs.GetSFTPError(c.fs, err) } filePath := p - if isAtomicUploadEnabled() { - filePath = getUploadTempFilePath(p) + if isAtomicUploadEnabled() && c.fs.IsAtomicUploadSupported() { + filePath = c.fs.GetAtomicUploadPath(p) } - stat, statErr := os.Stat(p) - if os.IsNotExist(statErr) { + stat, statErr := c.fs.Stat(p) + if c.fs.IsNotExist(statErr) { if !c.User.HasPerm(dataprovider.PermUpload, path.Dir(request.Filepath)) { return nil, sftp.ErrSSHFxPermissionDenied } @@ -118,7 +117,7 @@ func (c Connection) Filewrite(request *sftp.Request) (io.WriterAt, error) { if statErr != nil { c.Log(logger.LevelError, logSender, "error performing file stat %#v: %v", p, statErr) - return nil, getSFTPErrorFromOSError(err) + return nil, vfs.GetSFTPError(c.fs, statErr) } // This happen if we upload a file that has the same name of an existing directory @@ -139,9 +138,9 @@ func (c Connection) Filewrite(request *sftp.Request) (io.WriterAt, error) { func (c Connection) Filecmd(request *sftp.Request) error { updateConnectionActivity(c.ID) - p, err := c.buildPath(request.Filepath) + p, err := c.fs.ResolvePath(request.Filepath, c.User.GetHomeDir()) if err != nil { - return getSFTPErrorFromOSError(err) + return vfs.GetSFTPError(c.fs, err) } target, err := c.getSFTPCmdTargetPath(request.Target) if err != nil { @@ -186,7 +185,7 @@ func (c Connection) Filecmd(request *sftp.Request) error { } // we return if we remove a file or a dir so source path or target path always exists here - utils.SetPathPermissions(fileLocation, c.User.GetUID(), c.User.GetGID()) + vfs.SetPathPermissions(c.fs, fileLocation, c.User.GetUID(), c.User.GetGID()) return sftp.ErrSSHFxOk } @@ -195,9 +194,9 @@ func (c Connection) Filecmd(request *sftp.Request) error { // a directory as well as perform file/folder stat calls. func (c Connection) Filelist(request *sftp.Request) (sftp.ListerAt, error) { updateConnectionActivity(c.ID) - p, err := c.buildPath(request.Filepath) + p, err := c.fs.ResolvePath(request.Filepath, c.User.GetHomeDir()) if err != nil { - return nil, getSFTPErrorFromOSError(err) + return nil, vfs.GetSFTPError(c.fs, err) } switch request.Method { @@ -208,10 +207,10 @@ func (c Connection) Filelist(request *sftp.Request) (sftp.ListerAt, error) { c.Log(logger.LevelDebug, logSender, "requested list file for dir: %#v", p) - files, err := ioutil.ReadDir(p) + files, err := c.fs.ReadDir(p) if err != nil { c.Log(logger.LevelWarn, logSender, "error listing directory: %#v", err) - return nil, getSFTPErrorFromOSError(err) + return nil, vfs.GetSFTPError(c.fs, err) } return listerAt(files), nil @@ -221,10 +220,10 @@ func (c Connection) Filelist(request *sftp.Request) (sftp.ListerAt, error) { } c.Log(logger.LevelDebug, logSender, "requested stat for path: %#v", p) - s, err := os.Stat(p) + s, err := c.fs.Stat(p) if err != nil { c.Log(logger.LevelWarn, logSender, "error running stat on path: %#v", err) - return nil, getSFTPErrorFromOSError(err) + return nil, vfs.GetSFTPError(c.fs, err) } return listerAt([]os.FileInfo{s}), nil @@ -239,9 +238,9 @@ func (c Connection) getSFTPCmdTargetPath(requestTarget string) (string, error) { // location for the server. If it is not, return an error if len(requestTarget) > 0 { var err error - target, err = c.buildPath(requestTarget) + target, err = c.fs.ResolvePath(requestTarget, c.User.GetHomeDir()) if err != nil { - return target, getSFTPErrorFromOSError(err) + return target, vfs.GetSFTPError(c.fs, err) } } return target, nil @@ -252,7 +251,7 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er return nil } pathForPerms := request.Filepath - if fi, err := os.Lstat(filePath); err == nil { + if fi, err := c.fs.Lstat(filePath); err == nil { if fi.IsDir() { pathForPerms = path.Dir(request.Filepath) } @@ -263,9 +262,9 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er return sftp.ErrSSHFxPermissionDenied } fileMode := request.Attributes().FileMode() - if err := os.Chmod(filePath, fileMode); err != nil { + if err := c.fs.Chmod(filePath, fileMode); err != nil { c.Log(logger.LevelWarn, logSender, "failed to chmod path %#v, mode: %v, err: %v", filePath, fileMode.String(), err) - return getSFTPErrorFromOSError(err) + return vfs.GetSFTPError(c.fs, err) } logger.CommandLog(chmodLogSender, filePath, "", c.User.Username, fileMode.String(), c.ID, c.protocol, -1, -1, "", "", "") return nil @@ -275,9 +274,9 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er } uid := int(request.Attributes().UID) gid := int(request.Attributes().GID) - if err := os.Chown(filePath, uid, gid); err != nil { + if err := c.fs.Chown(filePath, uid, gid); err != nil { c.Log(logger.LevelWarn, logSender, "failed to chown path %#v, uid: %v, gid: %v, err: %v", filePath, uid, gid, err) - return getSFTPErrorFromOSError(err) + return vfs.GetSFTPError(c.fs, err) } logger.CommandLog(chownLogSender, filePath, "", c.User.Username, "", c.ID, c.protocol, uid, gid, "", "", "") return nil @@ -290,10 +289,10 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er modificationTime := time.Unix(int64(request.Attributes().Mtime), 0) accessTimeString := accessTime.Format(dateFormat) modificationTimeString := modificationTime.Format(dateFormat) - if err := os.Chtimes(filePath, accessTime, modificationTime); err != nil { + if err := c.fs.Chtimes(filePath, accessTime, modificationTime); err != nil { c.Log(logger.LevelWarn, logSender, "failed to chtimes for path %#v, access time: %v, modification time: %v, err: %v", filePath, accessTime, modificationTime, err) - return getSFTPErrorFromOSError(err) + return vfs.GetSFTPError(c.fs, err) } logger.CommandLog(chtimesLogSender, filePath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, accessTimeString, modificationTimeString, "") @@ -303,16 +302,16 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er } func (c Connection) handleSFTPRename(sourcePath string, targetPath string, request *sftp.Request) error { - if c.User.GetRelativePath(sourcePath) == "/" { + if c.fs.GetRelativePath(sourcePath, c.User.GetHomeDir()) == "/" { c.Log(logger.LevelWarn, logSender, "renaming root dir is not allowed") return sftp.ErrSSHFxPermissionDenied } if !c.User.HasPerm(dataprovider.PermRename, path.Dir(request.Target)) { return sftp.ErrSSHFxPermissionDenied } - if err := os.Rename(sourcePath, targetPath); err != nil { + if err := c.fs.Rename(sourcePath, targetPath); err != nil { c.Log(logger.LevelWarn, logSender, "failed to rename file, source: %#v target: %#v: %v", sourcePath, targetPath, err) - return getSFTPErrorFromOSError(err) + return vfs.GetSFTPError(c.fs, err) } logger.CommandLog(renameLogSender, sourcePath, targetPath, c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "") go executeAction(operationRename, c.User.Username, sourcePath, targetPath, "", 0) @@ -320,7 +319,7 @@ func (c Connection) handleSFTPRename(sourcePath string, targetPath string, reque } func (c Connection) handleSFTPRmdir(dirPath string, request *sftp.Request) error { - if c.User.GetRelativePath(dirPath) == "/" { + if c.fs.GetRelativePath(dirPath, c.User.GetHomeDir()) == "/" { c.Log(logger.LevelWarn, logSender, "removing root dir is not allowed") return sftp.ErrSSHFxPermissionDenied } @@ -330,18 +329,18 @@ func (c Connection) handleSFTPRmdir(dirPath string, request *sftp.Request) error var fi os.FileInfo var err error - if fi, err = os.Lstat(dirPath); err != nil { + if fi, err = c.fs.Lstat(dirPath); err != nil { c.Log(logger.LevelWarn, logSender, "failed to remove a dir %#v: stat error: %v", dirPath, err) - return getSFTPErrorFromOSError(err) + return vfs.GetSFTPError(c.fs, err) } if !fi.IsDir() || fi.Mode()&os.ModeSymlink == os.ModeSymlink { c.Log(logger.LevelDebug, logSender, "cannot remove %#v is not a directory", dirPath) return sftp.ErrSSHFxFailure } - if err = os.Remove(dirPath); err != nil { + if err = c.fs.Remove(dirPath, true); err != nil { c.Log(logger.LevelWarn, logSender, "failed to remove directory %#v: %v", dirPath, err) - return getSFTPErrorFromOSError(err) + return vfs.GetSFTPError(c.fs, err) } logger.CommandLog(rmdirLogSender, dirPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "") @@ -349,16 +348,16 @@ func (c Connection) handleSFTPRmdir(dirPath string, request *sftp.Request) error } func (c Connection) handleSFTPSymlink(sourcePath string, targetPath string, request *sftp.Request) error { - if c.User.GetRelativePath(sourcePath) == "/" { + if c.fs.GetRelativePath(sourcePath, c.User.GetHomeDir()) == "/" { c.Log(logger.LevelWarn, logSender, "symlinking root dir is not allowed") return sftp.ErrSSHFxPermissionDenied } if !c.User.HasPerm(dataprovider.PermCreateSymlinks, path.Dir(request.Target)) { return sftp.ErrSSHFxPermissionDenied } - if err := os.Symlink(sourcePath, targetPath); err != nil { + if err := c.fs.Symlink(sourcePath, targetPath); err != nil { c.Log(logger.LevelWarn, logSender, "failed to create symlink %#v -> %#v: %v", sourcePath, targetPath, err) - return getSFTPErrorFromOSError(err) + return vfs.GetSFTPError(c.fs, err) } logger.CommandLog(symlinkLogSender, sourcePath, targetPath, c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "") @@ -369,11 +368,11 @@ func (c Connection) handleSFTPMkdir(dirPath string, request *sftp.Request) error if !c.User.HasPerm(dataprovider.PermCreateDirs, path.Dir(request.Filepath)) { return sftp.ErrSSHFxPermissionDenied } - if err := os.Mkdir(dirPath, 0777); err != nil { + if err := c.fs.Mkdir(dirPath); err != nil { c.Log(logger.LevelWarn, logSender, "error creating missing dir: %#v error: %v", dirPath, err) - return getSFTPErrorFromOSError(err) + return vfs.GetSFTPError(c.fs, err) } - utils.SetPathPermissions(dirPath, c.User.GetUID(), c.User.GetGID()) + vfs.SetPathPermissions(c.fs, dirPath, c.User.GetUID(), c.User.GetGID()) logger.CommandLog(mkdirLogSender, dirPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "") return nil @@ -387,18 +386,18 @@ func (c Connection) handleSFTPRemove(filePath string, request *sftp.Request) err var size int64 var fi os.FileInfo var err error - if fi, err = os.Lstat(filePath); err != nil { + if fi, err = c.fs.Lstat(filePath); err != nil { c.Log(logger.LevelWarn, logSender, "failed to remove a file %#v: stat error: %v", filePath, err) - return getSFTPErrorFromOSError(err) + return vfs.GetSFTPError(c.fs, err) } if fi.IsDir() && fi.Mode()&os.ModeSymlink != os.ModeSymlink { c.Log(logger.LevelDebug, logSender, "cannot remove %#v is not a file/symlink", filePath) return sftp.ErrSSHFxFailure } size = fi.Size() - if err := os.Remove(filePath); err != nil { + if err := c.fs.Remove(filePath, false); err != nil { c.Log(logger.LevelWarn, logSender, "failed to remove a file/symlink %#v: %v", filePath, err) - return getSFTPErrorFromOSError(err) + return vfs.GetSFTPError(c.fs, err) } logger.CommandLog(removeLogSender, filePath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "") @@ -416,16 +415,19 @@ func (c Connection) handleSFTPUploadToNewFile(requestPath, filePath string) (io. return nil, sftp.ErrSSHFxFailure } - file, err := os.Create(filePath) + file, w, cancelFn, err := c.fs.Create(filePath, 0) if err != nil { c.Log(logger.LevelWarn, logSender, "error creating file %#v: %v", requestPath, err) - return nil, getSFTPErrorFromOSError(err) + return nil, vfs.GetSFTPError(c.fs, err) } - utils.SetPathPermissions(filePath, c.User.GetUID(), c.User.GetGID()) + vfs.SetPathPermissions(c.fs, filePath, c.User.GetUID(), c.User.GetGID()) transfer := Transfer{ file: file, + writerAt: w, + readerAt: nil, + cancelFn: cancelFn, path: requestPath, start: time.Now(), bytesSent: 0, @@ -456,19 +458,25 @@ func (c Connection) handleSFTPUploadToExistingFile(pflags sftp.FileOpenFlags, re minWriteOffset := int64(0) osFlags := getOSOpenFlags(pflags) - if isAtomicUploadEnabled() { - err = os.Rename(requestPath, filePath) + if pflags.Append && osFlags&os.O_TRUNC == 0 && !c.fs.IsUploadResumeSupported() { + c.Log(logger.LevelInfo, logSender, "upload resume requested for path: %#v but not supported in fs implementation", + requestPath) + return nil, sftp.ErrSSHFxOpUnsupported + } + + if isAtomicUploadEnabled() && c.fs.IsAtomicUploadSupported() { + err = c.fs.Rename(requestPath, filePath) if err != nil { c.Log(logger.LevelWarn, logSender, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %v", requestPath, filePath, err) - return nil, getSFTPErrorFromOSError(err) + return nil, vfs.GetSFTPError(c.fs, err) } } - // we use 0666 so the umask is applied - file, err := os.OpenFile(filePath, osFlags, 0666) + + file, w, cancelFn, err := c.fs.Create(filePath, osFlags) if err != nil { c.Log(logger.LevelWarn, logSender, "error opening existing file, flags: %v, source: %#v, err: %v", pflags, filePath, err) - return nil, getSFTPErrorFromOSError(err) + return nil, vfs.GetSFTPError(c.fs, err) } if pflags.Append && osFlags&os.O_TRUNC == 0 { @@ -478,10 +486,13 @@ func (c Connection) handleSFTPUploadToExistingFile(pflags sftp.FileOpenFlags, re dataprovider.UpdateUserQuota(dataProvider, c.User, 0, -fileSize, false) } - utils.SetPathPermissions(filePath, c.User.GetUID(), c.User.GetGID()) + vfs.SetPathPermissions(c.fs, filePath, c.User.GetUID(), c.User.GetGID()) transfer := Transfer{ file: file, + writerAt: w, + readerAt: nil, + cancelFn: cancelFn, path: requestPath, start: time.Now(), bytesSent: 0, @@ -522,103 +533,6 @@ func (c Connection) hasSpace(checkFiles bool) bool { return true } -// Normalizes a file/directory we get from the SFTP request to ensure the user is not able to escape -// from their data directory. After normalization if the file/directory is still within their home -// path it is returned. If they managed to "escape" an error will be returned. -func (c Connection) buildPath(rawPath string) (string, error) { - r := filepath.Clean(filepath.Join(c.User.HomeDir, rawPath)) - p, err := filepath.EvalSymlinks(r) - if err != nil && !os.IsNotExist(err) { - return "", err - } else if os.IsNotExist(err) { - // The requested path doesn't exist, so at this point we need to iterate up the - // path chain until we hit a directory that _does_ exist and can be validated. - _, err = c.findFirstExistingDir(r) - if err != nil { - c.Log(logger.LevelWarn, logSender, "error resolving not existent path: %#v", err) - } - return r, err - } - - err = c.isSubDir(p) - if err != nil { - c.Log(logger.LevelWarn, logSender, "Invalid path resolution, dir: %#v outside user home: %#v err: %v", p, c.User.HomeDir, err) - } - return r, err -} - -// iterate up the path chain until we hit a directory that does exist and can be validated. -// all nonexistent directories will be returned -func (c Connection) findNonexistentDirs(path string) ([]string, error) { - results := []string{} - cleanPath := filepath.Clean(path) - parent := filepath.Dir(cleanPath) - _, err := os.Stat(parent) - - for os.IsNotExist(err) { - results = append(results, parent) - parent = filepath.Dir(parent) - _, err = os.Stat(parent) - } - if err != nil { - return results, err - } - p, err := filepath.EvalSymlinks(parent) - if err != nil { - return results, err - } - err = c.isSubDir(p) - if err != nil { - c.Log(logger.LevelWarn, logSender, "Error finding non existing dir: %v", err) - } - return results, err -} - -// iterate up the path chain until we hit a directory that does exist and can be validated. -func (c Connection) findFirstExistingDir(path string) (string, error) { - results, err := c.findNonexistentDirs(path) - if err != nil { - c.Log(logger.LevelWarn, logSender, "unable to find non existent dirs: %v", err) - return "", err - } - var parent string - if len(results) > 0 { - lastMissingDir := results[len(results)-1] - parent = filepath.Dir(lastMissingDir) - } else { - parent = c.User.GetHomeDir() - } - p, err := filepath.EvalSymlinks(parent) - if err != nil { - return "", err - } - fileInfo, err := os.Stat(p) - if err != nil { - return "", err - } - if !fileInfo.IsDir() { - return "", fmt.Errorf("resolved path is not a dir: %#v", p) - } - err = c.isSubDir(p) - return p, err -} - -// checks if sub is a subpath of the user home dir. -// EvalSymlink must be used on sub before calling this method -func (c Connection) isSubDir(sub string) error { - // home dir must exist and it is already a validated absolute path - parent, err := filepath.EvalSymlinks(c.User.HomeDir) - if err != nil { - c.Log(logger.LevelWarn, logSender, "invalid home dir %#v: %v", c.User.HomeDir, err) - return err - } - if !strings.HasPrefix(sub, parent) { - c.Log(logger.LevelWarn, logSender, "path %#v is not inside: %#v ", sub, parent) - return fmt.Errorf("path %#v is not inside: %#v", sub, parent) - } - return nil -} - func (c Connection) close() error { if c.channel != nil { err := c.channel.Close() @@ -649,20 +563,3 @@ func getOSOpenFlags(requestFlags sftp.FileOpenFlags) (flags int) { } return osFlags } - -func getUploadTempFilePath(path string) string { - dir := filepath.Dir(path) - guid := xid.New().String() - return filepath.Join(dir, ".sftpgo-upload."+guid+"."+filepath.Base(path)) -} - -func getSFTPErrorFromOSError(err error) error { - if os.IsNotExist(err) { - return sftp.ErrSSHFxNoSuchFile - } else if os.IsPermission(err) { - return sftp.ErrSSHFxPermissionDenied - } else if err != nil { - return sftp.ErrSSHFxFailure - } - return nil -} diff --git a/sftpd/internal_test.go b/sftpd/internal_test.go index 7209880d..db045898 100644 --- a/sftpd/internal_test.go +++ b/sftpd/internal_test.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "net" "os" + "path/filepath" "runtime" "strings" "sync" @@ -16,6 +17,8 @@ import ( "github.com/drakkan/sftpgo/dataprovider" "github.com/drakkan/sftpgo/utils" + "github.com/drakkan/sftpgo/vfs" + "github.com/eikenb/pipeat" "github.com/pkg/sftp" ) @@ -60,6 +63,61 @@ func (c *MockChannel) Stderr() io.ReadWriter { return c.StdErrBuffer } +// MockOsFs mockable OsFs +type MockOsFs struct { + vfs.OsFs + err error + statErr error + isAtomicUploadSupported bool +} + +// Name returns the name for the Fs implementation +func (fs MockOsFs) Name() string { + return "mockOsFs" +} + +// IsUploadResumeSupported returns true if upload resume is supported +func (MockOsFs) IsUploadResumeSupported() bool { + return false +} + +// IsAtomicUploadSupported returns true if atomic upload is supported +func (fs MockOsFs) IsAtomicUploadSupported() bool { + return fs.isAtomicUploadSupported +} + +// Stat returns a FileInfo describing the named file +func (fs MockOsFs) Stat(name string) (os.FileInfo, error) { + if fs.statErr != nil { + return nil, fs.statErr + } + return os.Stat(name) +} + +// Remove removes the named file or (empty) directory. +func (fs MockOsFs) Remove(name string, isDir bool) error { + if fs.err != nil { + return fs.err + } + return os.Remove(name) +} + +// Rename renames (moves) source to target +func (fs MockOsFs) Rename(source, target string) error { + if fs.err != nil { + return fs.err + } + return os.Rename(source, target) +} + +func newMockOsFs(err, statErr error, atomicUpload bool) vfs.Fs { + return &MockOsFs{ + err: err, + statErr: statErr, + isAtomicUploadSupported: atomicUpload, + } +} + func TestWrongActions(t *testing.T) { actionsCopy := actions badCommand := "/bad/command" @@ -218,13 +276,134 @@ func TestReadWriteErrors(t *testing.T) { if err == nil { t.Error("upoload must fail the expected size does not match") } + r, _, _ := pipeat.Pipe() + transfer = Transfer{ + readerAt: r, + writerAt: nil, + start: time.Now(), + bytesSent: 0, + bytesReceived: 0, + user: dataprovider.User{ + Username: "testuser", + }, + connectionID: "", + transferType: transferDownload, + lastActivity: time.Now(), + isNewFile: false, + protocol: protocolSFTP, + transferError: nil, + isFinished: false, + lock: new(sync.Mutex), + } + transfer.closeIO() + _, err = transfer.ReadAt(buf, 0) + if err == nil { + t.Error("reading from a closed pipe must fail") + } + r, w, _ := pipeat.Pipe() + transfer = Transfer{ + readerAt: nil, + writerAt: w, + start: time.Now(), + bytesSent: 0, + bytesReceived: 0, + user: dataprovider.User{ + Username: "testuser", + }, + connectionID: "", + transferType: transferDownload, + lastActivity: time.Now(), + isNewFile: false, + protocol: protocolSFTP, + transferError: nil, + isFinished: false, + lock: new(sync.Mutex), + } + r.Close() + transfer.closeIO() + _, err = transfer.WriteAt([]byte("test"), 0) + if err == nil { + t.Error("writing to closed pipe must fail") + } + os.Remove(testfile) +} + +func TestTransferCancelFn(t *testing.T) { + testfile := "testfile" + file, _ := os.Create(testfile) + isCancelled := false + cancelFn := func() { + isCancelled = true + } + transfer := Transfer{ + file: file, + cancelFn: cancelFn, + path: file.Name(), + start: time.Now(), + bytesSent: 0, + bytesReceived: 0, + user: dataprovider.User{ + Username: "testuser", + }, + connectionID: "", + transferType: transferDownload, + lastActivity: time.Now(), + isNewFile: false, + protocol: protocolSFTP, + transferError: nil, + isFinished: false, + minWriteOffset: 0, + expectedSize: 10, + lock: new(sync.Mutex), + } + transfer.TransferError(errors.New("fake error, this will trigger cancelFn")) + transfer.Close() + if !isCancelled { + t.Error("cancelFn not called") + } + os.Remove(testfile) +} + +func TestMockFsErrors(t *testing.T) { + errFake := errors.New("fake error") + fs := newMockOsFs(errFake, errFake, false) + u := dataprovider.User{} + u.Username = "test" + u.Permissions = make(map[string][]string) + u.Permissions["/"] = []string{dataprovider.PermAny} + u.HomeDir = os.TempDir() + c := Connection{ + fs: fs, + User: u, + } + testfile := filepath.Join(u.HomeDir, "testfile") + request := sftp.NewRequest("Remove", testfile) + ioutil.WriteFile(testfile, []byte("test"), 0666) + err := c.handleSFTPRemove(testfile, request) + if err != sftp.ErrSSHFxFailure { + t.Errorf("unexpected error: %v", err) + } + _, err = c.Filewrite(request) + if err != sftp.ErrSSHFxFailure { + t.Errorf("unexpected error: %v", err) + } + var flags sftp.FileOpenFlags + flags.Write = true + flags.Trunc = false + flags.Append = true + _, err = c.handleSFTPUploadToExistingFile(flags, testfile, testfile, 0) + if err != sftp.ErrSSHFxOpUnsupported { + t.Errorf("unexpected error: %v", err) + } os.Remove(testfile) } func TestUploadFiles(t *testing.T) { oldUploadMode := uploadMode uploadMode = uploadModeAtomic - c := Connection{} + c := Connection{ + fs: vfs.NewOsFs("123"), + } var flags sftp.FileOpenFlags flags.Write = true flags.Trunc = true @@ -255,10 +434,13 @@ func TestWithInvalidHome(t *testing.T) { if err == nil { t.Errorf("login a user with an invalid home_dir must fail") } + fs, _ := u.GetFilesystem("123") c := Connection{ User: u, + fs: fs, } - err = c.isSubDir("dir_rel_path") + u.HomeDir = os.TempDir() + _, err = c.fs.ResolvePath("../upper_path", u.GetHomeDir()) if err == nil { t.Errorf("tested path is not a home subdir") } @@ -266,12 +448,18 @@ func TestWithInvalidHome(t *testing.T) { func TestSFTPCmdTargetPath(t *testing.T) { u := dataprovider.User{} - u.HomeDir = "home_rel_path" + if runtime.GOOS == "windows" { + u.HomeDir = "C:\\invalid_home" + } else { + u.HomeDir = "/invalid_home" + } u.Username = "test" u.Permissions = make(map[string][]string) u.Permissions["/"] = []string{dataprovider.PermAny} + fs, _ := u.GetFilesystem("123") connection := Connection{ User: u, + fs: fs, } _, err := connection.getSFTPCmdTargetPath("invalid_path") if err != sftp.ErrSSHFxNoSuchFile { @@ -281,16 +469,17 @@ func TestSFTPCmdTargetPath(t *testing.T) { func TestGetSFTPErrorFromOSError(t *testing.T) { err := os.ErrNotExist - err = getSFTPErrorFromOSError(err) + fs := vfs.NewOsFs("") + err = vfs.GetSFTPError(fs, err) if err != sftp.ErrSSHFxNoSuchFile { t.Errorf("unexpected error: %v", err) } err = os.ErrPermission - err = getSFTPErrorFromOSError(err) + err = vfs.GetSFTPError(fs, err) if err != sftp.ErrSSHFxPermissionDenied { t.Errorf("unexpected error: %v", err) } - err = getSFTPErrorFromOSError(nil) + err = vfs.GetSFTPError(fs, nil) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -418,10 +607,12 @@ func TestSSHCommandErrors(t *testing.T) { user := dataprovider.User{} user.Permissions = make(map[string][]string) user.Permissions["/"] = []string{dataprovider.PermAny} + fs, _ := user.GetFilesystem("123") connection := Connection{ channel: &mockSSHChannel, netConn: client, User: user, + fs: fs, } cmd := sshCommand{ command: "md5sum", @@ -499,6 +690,45 @@ func TestSSHCommandErrors(t *testing.T) { } } +func TestSSHCommandsRemoteFs(t *testing.T) { + buf := make([]byte, 65535) + stdErrBuf := make([]byte, 65535) + mockSSHChannel := MockChannel{ + Buffer: bytes.NewBuffer(buf), + StdErrBuffer: bytes.NewBuffer(stdErrBuf), + } + server, client := net.Pipe() + defer server.Close() + defer client.Close() + user := dataprovider.User{} + user.FsConfig = dataprovider.Filesystem{ + Provider: 1} + fs, _ := user.GetFilesystem("123") + connection := Connection{ + channel: &mockSSHChannel, + netConn: client, + User: user, + fs: fs, + } + cmd := sshCommand{ + command: "md5sum", + connection: connection, + args: []string{}, + } + err := cmd.handleHashCommands() + if err == nil { + t.Error("command must fail for a non local filesystem") + } + command, err := cmd.getSystemCommand() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + err = cmd.executeSystemCommand(command) + if err == nil { + t.Error("command must fail for a non local filesystem") + } +} + func TestSSHCommandQuotaScan(t *testing.T) { buf := make([]byte, 65535) stdErrBuf := make([]byte, 65535) @@ -513,14 +743,17 @@ func TestSSHCommandQuotaScan(t *testing.T) { defer client.Close() permissions := make(map[string][]string) permissions["/"] = []string{dataprovider.PermAny} + user := dataprovider.User{ + Permissions: permissions, + QuotaFiles: 1, + HomeDir: "invalid_path", + } + fs, _ := user.GetFilesystem("123") connection := Connection{ channel: &mockSSHChannel, netConn: client, - User: dataprovider.User{ - Permissions: permissions, - QuotaFiles: 1, - HomeDir: "invalid_path", - }, + User: user, + fs: fs, } cmd := sshCommand{ command: "git-receive-pack", @@ -536,11 +769,14 @@ func TestSSHCommandQuotaScan(t *testing.T) { func TestRsyncOptions(t *testing.T) { permissions := make(map[string][]string) permissions["/"] = []string{dataprovider.PermAny} + user := dataprovider.User{ + Permissions: permissions, + HomeDir: os.TempDir(), + } + fs, _ := user.GetFilesystem("123") conn := Connection{ - User: dataprovider.User{ - Permissions: permissions, - HomeDir: os.TempDir(), - }, + User: user, + fs: fs, } sshCmd := sshCommand{ command: "rsync", @@ -556,11 +792,11 @@ func TestRsyncOptions(t *testing.T) { } permissions["/"] = []string{dataprovider.PermDownload, dataprovider.PermUpload, dataprovider.PermCreateDirs, dataprovider.PermListItems, dataprovider.PermOverwrite, dataprovider.PermDelete, dataprovider.PermRename} + user.Permissions = permissions + fs, _ = user.GetFilesystem("123") conn = Connection{ - User: dataprovider.User{ - Permissions: permissions, - HomeDir: os.TempDir(), - }, + User: user, + fs: fs, } sshCmd = sshCommand{ command: "rsync", @@ -592,13 +828,16 @@ func TestSystemCommandErrors(t *testing.T) { defer client.Close() permissions := make(map[string][]string) permissions["/"] = []string{dataprovider.PermAny} + user := dataprovider.User{ + Permissions: permissions, + HomeDir: os.TempDir(), + } + fs, _ := user.GetFilesystem("123") connection := Connection{ channel: &mockSSHChannel, netConn: client, - User: dataprovider.User{ - Permissions: permissions, - HomeDir: os.TempDir(), - }, + User: user, + fs: fs, } sshCmd := sshCommand{ command: "ls", @@ -934,6 +1173,55 @@ func TestSCPCommandHandleErrors(t *testing.T) { } } +func TestSCPErrorsMockFs(t *testing.T) { + errFake := errors.New("fake error") + fs := newMockOsFs(errFake, errFake, false) + u := dataprovider.User{} + u.Username = "test" + u.Permissions = make(map[string][]string) + u.Permissions["/"] = []string{dataprovider.PermAny} + u.HomeDir = os.TempDir() + buf := make([]byte, 65535) + stdErrBuf := make([]byte, 65535) + mockSSHChannel := MockChannel{ + Buffer: bytes.NewBuffer(buf), + StdErrBuffer: bytes.NewBuffer(stdErrBuf), + } + server, client := net.Pipe() + defer server.Close() + defer client.Close() + connection := Connection{ + channel: &mockSSHChannel, + netConn: client, + fs: fs, + User: u, + } + scpCommand := scpCommand{ + sshCommand: sshCommand{ + command: "scp", + connection: connection, + args: []string{"-r", "-t", "/tmp"}, + }, + } + err := scpCommand.handleUpload("test", 0) + if err != errFake { + t.Errorf("unexpected error: %v", err) + } + testfile := filepath.Join(u.HomeDir, "testfile") + ioutil.WriteFile(testfile, []byte("test"), 0666) + stat, _ := os.Stat(u.HomeDir) + err = scpCommand.handleRecursiveDownload(u.HomeDir, stat) + if err != errFake { + t.Errorf("unexpected error: %v", err) + } + scpCommand.sshCommand.connection.fs = newMockOsFs(errFake, nil, true) + err = scpCommand.handleUpload(filepath.Base(testfile), 0) + if err != errFake { + t.Errorf("unexpected error: %v", err) + } + os.Remove(testfile) +} + func TestSCPRecursiveDownloadErrors(t *testing.T) { buf := make([]byte, 65535) stdErrBuf := make([]byte, 65535) @@ -951,6 +1239,7 @@ func TestSCPRecursiveDownloadErrors(t *testing.T) { connection := Connection{ channel: &mockSSHChannel, netConn: client, + fs: vfs.NewOsFs("123"), } scpCommand := scpCommand{ sshCommand: sshCommand{ @@ -1033,9 +1322,11 @@ func TestSCPCreateDirs(t *testing.T) { ReadError: nil, WriteError: nil, } + fs, _ := u.GetFilesystem("123") connection := Connection{ User: u, channel: &mockSSHChannel, + fs: fs, } scpCommand := scpCommand{ sshCommand: sshCommand{ diff --git a/sftpd/scp.go b/sftpd/scp.go index dc8161bc..cde6eabc 100644 --- a/sftpd/scp.go +++ b/sftpd/scp.go @@ -3,7 +3,6 @@ package sftpd import ( "fmt" "io" - "io/ioutil" "math" "os" "path" @@ -16,6 +15,7 @@ import ( "github.com/drakkan/sftpgo/dataprovider" "github.com/drakkan/sftpgo/logger" "github.com/drakkan/sftpgo/utils" + "github.com/drakkan/sftpgo/vfs" ) var ( @@ -116,7 +116,7 @@ func (c *scpCommand) handleRecursiveUpload() error { func (c *scpCommand) handleCreateDir(dirPath string) error { updateConnectionActivity(c.connection.ID) - p, err := c.connection.buildPath(dirPath) + p, err := c.connection.fs.ResolvePath(dirPath, c.connection.User.GetHomeDir()) if err != nil { c.connection.Log(logger.LevelWarn, logSenderSCP, "error creating dir: %#v, invalid file path, err: %v", dirPath, err) c.sendErrorMessage(err.Error()) @@ -189,17 +189,20 @@ func (c *scpCommand) handleUploadFile(requestPath, filePath string, sizeToRead i return err } - file, err := os.Create(filePath) + file, w, cancelFn, err := c.connection.fs.Create(filePath, 0) if err != nil { c.connection.Log(logger.LevelError, logSenderSCP, "error creating file %#v: %v", requestPath, err) c.sendErrorMessage(err.Error()) return err } - utils.SetPathPermissions(filePath, c.connection.User.GetUID(), c.connection.User.GetGID()) + vfs.SetPathPermissions(c.connection.fs, filePath, c.connection.User.GetUID(), c.connection.User.GetGID()) transfer := Transfer{ file: file, + readerAt: nil, + writerAt: w, + cancelFn: cancelFn, path: requestPath, start: time.Now(), bytesSent: 0, @@ -225,18 +228,18 @@ func (c *scpCommand) handleUpload(uploadFilePath string, sizeToRead int64) error updateConnectionActivity(c.connection.ID) - p, err := c.connection.buildPath(uploadFilePath) + p, err := c.connection.fs.ResolvePath(uploadFilePath, c.connection.User.GetHomeDir()) if err != nil { c.connection.Log(logger.LevelWarn, logSenderSCP, "error uploading file: %#v, err: %v", uploadFilePath, err) c.sendErrorMessage(err.Error()) return err } filePath := p - if isAtomicUploadEnabled() { - filePath = getUploadTempFilePath(p) + if isAtomicUploadEnabled() && c.connection.fs.IsAtomicUploadSupported() { + filePath = c.connection.fs.GetAtomicUploadPath(p) } - stat, statErr := os.Stat(p) - if os.IsNotExist(statErr) { + stat, statErr := c.connection.fs.Stat(p) + if c.connection.fs.IsNotExist(statErr) { if !c.connection.User.HasPerm(dataprovider.PermUpload, path.Dir(uploadFilePath)) { err := fmt.Errorf("Permission denied") c.connection.Log(logger.LevelWarn, logSenderSCP, "cannot upload file: %#v, permission denied", uploadFilePath) @@ -248,8 +251,8 @@ func (c *scpCommand) handleUpload(uploadFilePath string, sizeToRead int64) error if statErr != nil { c.connection.Log(logger.LevelError, logSenderSCP, "error performing file stat %#v: %v", p, statErr) - c.sendErrorMessage(err.Error()) - return err + c.sendErrorMessage(statErr.Error()) + return statErr } if stat.IsDir() { @@ -266,8 +269,8 @@ func (c *scpCommand) handleUpload(uploadFilePath string, sizeToRead int64) error return err } - if isAtomicUploadEnabled() { - err = os.Rename(p, filePath) + if isAtomicUploadEnabled() && c.connection.fs.IsAtomicUploadSupported() { + err = c.connection.fs.Rename(p, filePath) if err != nil { c.connection.Log(logger.LevelError, logSenderSCP, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %v", p, filePath, err) @@ -315,14 +318,14 @@ func (c *scpCommand) handleRecursiveDownload(dirPath string, stat os.FileInfo) e if err != nil { return err } - files, err := ioutil.ReadDir(dirPath) + files, err := c.connection.fs.ReadDir(dirPath) if err != nil { c.sendErrorMessage(err.Error()) return err } var dirs []string for _, file := range files { - filePath := c.connection.User.GetRelativePath(filepath.Join(dirPath, file.Name())) + filePath := c.connection.fs.GetRelativePath(c.connection.fs.Join(dirPath, file.Name()), c.connection.User.GetHomeDir()) if file.Mode().IsRegular() || file.Mode()&os.ModeSymlink == os.ModeSymlink { err = c.handleDownload(filePath) if err != nil { @@ -419,7 +422,7 @@ func (c *scpCommand) handleDownload(filePath string) error { updateConnectionActivity(c.connection.ID) - p, err := c.connection.buildPath(filePath) + p, err := c.connection.fs.ResolvePath(filePath, c.connection.User.GetHomeDir()) if err != nil { err := fmt.Errorf("Invalid file path") c.connection.Log(logger.LevelWarn, logSenderSCP, "error downloading file: %#v, invalid file path", filePath) @@ -428,7 +431,7 @@ func (c *scpCommand) handleDownload(filePath string) error { } var stat os.FileInfo - if stat, err = os.Stat(p); os.IsNotExist(err) { + if stat, err = c.connection.fs.Stat(p); err != nil { c.connection.Log(logger.LevelWarn, logSenderSCP, "error downloading file: %#v, err: %v", p, err) c.sendErrorMessage(err.Error()) return err @@ -452,7 +455,7 @@ func (c *scpCommand) handleDownload(filePath string) error { return err } - file, err := os.Open(p) + file, r, cancelFn, err := c.connection.fs.Open(p) if err != nil { c.connection.Log(logger.LevelError, logSenderSCP, "could not open file %#v for reading: %v", p, err) c.sendErrorMessage(err.Error()) @@ -461,6 +464,9 @@ func (c *scpCommand) handleDownload(filePath string) error { transfer := Transfer{ file: file, + readerAt: r, + writerAt: nil, + cancelFn: cancelFn, path: p, start: time.Now(), bytesSent: 0, @@ -608,12 +614,12 @@ func (c *scpCommand) getNextUploadProtocolMessage() (string, error) { func (c *scpCommand) createDir(dirPath string) error { var err error - if err = os.Mkdir(dirPath, 0777); err != nil { + if err = c.connection.fs.Mkdir(dirPath); err != nil { c.connection.Log(logger.LevelError, logSenderSCP, "error creating dir: %v", dirPath) c.sendErrorMessage(err.Error()) return err } - utils.SetPathPermissions(dirPath, c.connection.User.GetUID(), c.connection.User.GetGID()) + vfs.SetPathPermissions(c.connection.fs, dirPath, c.connection.User.GetUID(), c.connection.User.GetGID()) return err } @@ -668,8 +674,8 @@ func (c *scpCommand) getFileUploadDestPath(scpDestPath, fileName string) string // but if scpDestPath is an existing directory then we put the uploaded file // inside that directory this is as scp command works, for example: // scp fileName.txt user@127.0.0.1:/existing_dir - if p, err := c.connection.buildPath(scpDestPath); err == nil { - if stat, err := os.Stat(p); err == nil { + if p, err := c.connection.fs.ResolvePath(scpDestPath, c.connection.User.GetHomeDir()); err == nil { + if stat, err := c.connection.fs.Stat(p); err == nil { if stat.IsDir() { return path.Join(scpDestPath, fileName) } diff --git a/sftpd/server.go b/sftpd/server.go index 37185872..80fb85f8 100644 --- a/sftpd/server.go +++ b/sftpd/server.go @@ -266,6 +266,14 @@ func (c Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.Server loginType = sconn.Permissions.Extensions["login_type"] connectionID := hex.EncodeToString(sconn.SessionID()) + fs, err := user.GetFilesystem(connectionID) + + if err != nil { + logger.Warn(logSender, "", "could create filesystem for user %#v err: %v", user.Username, err) + conn.Close() + return + } + connection := Connection{ ID: connectionID, User: user, @@ -275,7 +283,11 @@ func (c Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.Server lastActivity: time.Now(), netConn: conn, channel: nil, + fs: fs, } + + connection.fs.CheckRootPath(user.GetHomeDir(), user.Username, user.GetUID(), user.GetGID()) + connection.Log(logger.LevelInfo, logSender, "User id: %d, logged in with: %#v, username: %#v, home_dir: %#v remote addr: %#v", user.ID, loginType, user.Username, user.HomeDir, remoteAddr.String()) dataprovider.UpdateLastLogin(dataProvider, user) @@ -368,14 +380,6 @@ func loginUser(user dataprovider.User, loginType string, remoteAddr string) (*ss logger.Debug(logSender, "", "cannot login user %#v, remote address is not allowed: %v", user.Username, remoteAddr) return nil, fmt.Errorf("Login is not allowed from this address: %v", remoteAddr) } - if _, err := os.Stat(user.HomeDir); os.IsNotExist(err) { - err := os.MkdirAll(user.HomeDir, 0777) - logger.Debug(logSender, "", "home directory %#v for user %#v does not exist, try to create, mkdir error: %v", - user.HomeDir, user.Username, err) - if err == nil { - utils.SetPathPermissions(user.HomeDir, user.GetUID(), user.GetGID()) - } - } json, err := json.Marshal(user) if err != nil { diff --git a/sftpd/sftpd.go b/sftpd/sftpd.go index 8fd04652..5b2ebc49 100644 --- a/sftpd/sftpd.go +++ b/sftpd/sftpd.go @@ -307,7 +307,7 @@ func GetConnectionsStats() []ConnectionStatus { StartTime: utils.GetTimeAsMsSinceEpoch(t.start), Size: size, LastActivity: utils.GetTimeAsMsSinceEpoch(t.lastActivity), - Path: c.User.GetRelativePath(t.path), + Path: c.fs.GetRelativePath(t.path, c.User.GetHomeDir()), } conn.Transfers = append(conn.Transfers, connTransfer) } diff --git a/sftpd/sftpd_test.go b/sftpd/sftpd_test.go index 02278491..8d04c993 100644 --- a/sftpd/sftpd_test.go +++ b/sftpd/sftpd_test.go @@ -35,6 +35,7 @@ import ( "github.com/drakkan/sftpgo/logger" "github.com/drakkan/sftpgo/sftpd" "github.com/drakkan/sftpgo/utils" + "github.com/drakkan/sftpgo/vfs" "github.com/pkg/sftp" "github.com/rs/zerolog" ) @@ -1017,6 +1018,45 @@ func TestLoginUserExpiration(t *testing.T) { os.RemoveAll(user.GetHomeDir()) } +func TestLoginInvalidFs(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("this test is not available on Windows") + } + config.LoadConfig(configDir, "") + providerConf := config.GetProviderConf() + if providerConf.Driver != dataprovider.SQLiteDataProviderName { + t.Skip("this test require sqlite provider") + } + dbPath := providerConf.Name + if !filepath.IsAbs(dbPath) { + dbPath = filepath.Join(configDir, dbPath) + } + usePubKey := true + u := getTestUser(usePubKey) + user, _, err := httpd.AddUser(u, http.StatusOK) + if err != nil { + t.Errorf("unable to add user: %v", err) + } + // we update the database using sqlite3 CLI since we cannot add an user with an invalid config + time.Sleep(150 * time.Millisecond) + updateUserQuery := fmt.Sprintf("UPDATE users SET filesystem='{\"provider\":1}' WHERE id=%v", user.ID) + cmd := exec.Command("sqlite3", dbPath, updateUserQuery) + out, err := cmd.CombinedOutput() + if err != nil { + t.Errorf("unexpected error: %v, cmd out: %v", err, string(out)) + } + time.Sleep(200 * time.Millisecond) + _, err = getSftpClient(user, usePubKey) + if err == nil { + t.Error("login must fail, the user has an invalid filesystem config") + } + _, err = httpd.RemoveUser(user, http.StatusOK) + if err != nil { + t.Errorf("unable to remove user: %v", err) + } + os.RemoveAll(user.GetHomeDir()) +} + func TestLoginWithIPFilters(t *testing.T) { usePubKey := true u := getTestUser(usePubKey) @@ -2864,52 +2904,53 @@ func TestRootDirCommands(t *testing.T) { func TestRelativePaths(t *testing.T) { user := getTestUser(true) path := filepath.Join(user.HomeDir, "/") - rel := user.GetRelativePath(path) + fs := vfs.NewOsFs("") + rel := fs.GetRelativePath(path, user.GetHomeDir()) if rel != "/" { t.Errorf("Unexpected relative path: %v", rel) } path = filepath.Join(user.HomeDir, "//") - rel = user.GetRelativePath(path) + rel = fs.GetRelativePath(path, user.GetHomeDir()) if rel != "/" { t.Errorf("Unexpected relative path: %v", rel) } path = filepath.Join(user.HomeDir, "../..") - rel = user.GetRelativePath(path) + rel = fs.GetRelativePath(path, user.GetHomeDir()) if rel != "/" { t.Errorf("Unexpected relative path: %v", rel) } path = filepath.Join(user.HomeDir, "../../../../../") - rel = user.GetRelativePath(path) + rel = fs.GetRelativePath(path, user.GetHomeDir()) if rel != "/" { t.Errorf("Unexpected relative path: %v", rel) } path = filepath.Join(user.HomeDir, "/..") - rel = user.GetRelativePath(path) + rel = fs.GetRelativePath(path, user.GetHomeDir()) if rel != "/" { t.Errorf("Unexpected relative path: %v", rel) } path = filepath.Join(user.HomeDir, "/../../../..") - rel = user.GetRelativePath(path) + rel = fs.GetRelativePath(path, user.GetHomeDir()) if rel != "/" { t.Errorf("Unexpected relative path: %v", rel) } path = filepath.Join(user.HomeDir, "") - rel = user.GetRelativePath(path) + rel = fs.GetRelativePath(path, user.GetHomeDir()) if rel != "/" { t.Errorf("Unexpected relative path: %v", rel) } path = filepath.Join(user.HomeDir, ".") - rel = user.GetRelativePath(path) + rel = fs.GetRelativePath(path, user.GetHomeDir()) if rel != "/" { t.Errorf("Unexpected relative path: %v", rel) } path = filepath.Join(user.HomeDir, "somedir") - rel = user.GetRelativePath(path) + rel = fs.GetRelativePath(path, user.GetHomeDir()) if rel != "/somedir" { t.Errorf("Unexpected relative path: %v", rel) } path = filepath.Join(user.HomeDir, "/somedir/subdir") - rel = user.GetRelativePath(path) + rel = fs.GetRelativePath(path, user.GetHomeDir()) if rel != "/somedir/subdir" { t.Errorf("Unexpected relative path: %v", rel) } diff --git a/sftpd/ssh_cmd.go b/sftpd/ssh_cmd.go index 5edf0770..985d3280 100644 --- a/sftpd/ssh_cmd.go +++ b/sftpd/ssh_cmd.go @@ -21,12 +21,14 @@ import ( "github.com/drakkan/sftpgo/logger" "github.com/drakkan/sftpgo/metrics" "github.com/drakkan/sftpgo/utils" + "github.com/drakkan/sftpgo/vfs" "golang.org/x/crypto/ssh" ) var ( - errQuotaExceeded = errors.New("denying write due to space limit") - errPermissionDenied = errors.New("Permission denied. You don't have the permissions to execute this command") + errQuotaExceeded = errors.New("denying write due to space limit") + errPermissionDenied = errors.New("Permission denied. You don't have the permissions to execute this command") + errUnsupportedConfig = errors.New("command unsupported for this configuration") ) type sshCommand struct { @@ -101,6 +103,9 @@ func (c *sshCommand) handle() error { } func (c *sshCommand) handleHashCommands() error { + if !vfs.IsLocalOsFs(c.connection.fs) { + return c.sendErrorResponse(errUnsupportedConfig) + } var h hash.Hash if c.command == "md5sum" { h = md5.New() @@ -125,14 +130,14 @@ func (c *sshCommand) handleHashCommands() error { response = fmt.Sprintf("%x -\n", h.Sum(nil)) } else { sshPath := c.getDestPath() - path, err := c.connection.buildPath(sshPath) + fsPath, err := c.connection.fs.ResolvePath(sshPath, c.connection.User.GetHomeDir()) if err != nil { return c.sendErrorResponse(err) } if !c.connection.User.HasPerm(dataprovider.PermListItems, sshPath) { return c.sendErrorResponse(errPermissionDenied) } - hash, err := computeHashForFile(h, path) + hash, err := computeHashForFile(h, fsPath) if err != nil { return c.sendErrorResponse(err) } @@ -144,6 +149,9 @@ func (c *sshCommand) handleHashCommands() error { } func (c *sshCommand) executeSystemCommand(command systemCommand) error { + if !vfs.IsLocalOsFs(c.connection.fs) { + return c.sendErrorResponse(errUnsupportedConfig) + } if c.connection.User.QuotaFiles > 0 && c.connection.User.UsedQuotaFiles > c.connection.User.QuotaFiles { return c.sendErrorResponse(errQuotaExceeded) } @@ -288,7 +296,7 @@ func (c *sshCommand) getSystemCommand() (systemCommand, error) { if len(c.args) > 0 { var err error sshPath := c.getDestPath() - path, err = c.connection.buildPath(sshPath) + path, err = c.connection.fs.ResolvePath(sshPath, c.connection.User.GetHomeDir()) if err != nil { return command, err } @@ -331,7 +339,7 @@ func (c *sshCommand) rescanHomeDir() error { var numFiles int var size int64 if AddQuotaScan(c.connection.User.Username) { - numFiles, size, _, err = utils.ScanDirContents(c.connection.User.HomeDir) + numFiles, size, err = c.connection.fs.ScanDirContents(c.connection.User.HomeDir) if err != nil { c.connection.Log(logger.LevelWarn, logSenderSSH, "error scanning user home dir %#v: %v", c.connection.User.HomeDir, err) } else { @@ -389,7 +397,7 @@ func (c *sshCommand) sendExitStatus(err error) { if err == nil && c.command != "scp" { realPath := c.getDestPath() if len(realPath) > 0 { - p, err := c.connection.buildPath(realPath) + p, err := c.connection.fs.ResolvePath(realPath, c.connection.User.GetHomeDir()) if err == nil { realPath = p } diff --git a/sftpd/transfer.go b/sftpd/transfer.go index 40b4833e..7ec2fa55 100644 --- a/sftpd/transfer.go +++ b/sftpd/transfer.go @@ -11,6 +11,7 @@ import ( "github.com/drakkan/sftpgo/dataprovider" "github.com/drakkan/sftpgo/logger" "github.com/drakkan/sftpgo/metrics" + "github.com/eikenb/pipeat" ) const ( @@ -26,6 +27,9 @@ var ( // It implements the io Reader and Writer interface to handle files downloads and uploads type Transfer struct { file *os.File + writerAt *pipeat.PipeWriterAt + readerAt *pipeat.PipeReaderAt + cancelFn func() path string start time.Time bytesSent int64 @@ -52,6 +56,9 @@ func (t *Transfer) TransferError(err error) { return } t.transferError = err + if t.cancelFn != nil { + t.cancelFn() + } elapsed := time.Since(t.start).Nanoseconds() / 1000000 logger.Warn(logSender, t.connectionID, "Unexpected error for transfer, path: %#v, error: \"%v\" bytes sent: %v, "+ "bytes received: %v transfer running since %v ms", t.path, t.transferError, t.bytesSent, t.bytesReceived, elapsed) @@ -61,7 +68,13 @@ func (t *Transfer) TransferError(err error) { // It handles download bandwidth throttling too func (t *Transfer) ReadAt(p []byte, off int64) (n int, err error) { t.lastActivity = time.Now() - readed, e := t.file.ReadAt(p, off) + var readed int + var e error + if t.readerAt != nil { + readed, e = t.readerAt.ReadAt(p, off) + } else { + readed, e = t.file.ReadAt(p, off) + } t.lock.Lock() t.bytesSent += int64(readed) t.lock.Unlock() @@ -82,7 +95,13 @@ func (t *Transfer) WriteAt(p []byte, off int64) (n int, err error) { t.TransferError(err) return 0, err } - written, e := t.file.WriteAt(p, off) + var written int + var e error + if t.writerAt != nil { + written, e = t.writerAt.WriteAt(p, off) + } else { + written, e = t.file.WriteAt(p, off) + } t.lock.Lock() t.bytesReceived += int64(written) t.lock.Unlock() @@ -105,14 +124,14 @@ func (t *Transfer) Close() error { if t.isFinished { return errTransferClosed } - err := t.file.Close() + err := t.closeIO() t.isFinished = true numFiles := 0 if t.isNewFile { numFiles = 1 } t.checkDownloadSize() - if t.transferType == transferUpload && t.file.Name() != t.path { + if t.transferType == transferUpload && t.file != nil && t.file.Name() != t.path { if t.transferError == nil || uploadMode == uploadModeAtomicWithResume { err = os.Rename(t.file.Name(), t.path) logger.Debug(logSender, t.connectionID, "atomic upload completed, rename: %#v -> %#v, error: %v", @@ -150,6 +169,18 @@ func (t *Transfer) Close() error { return err } +func (t *Transfer) closeIO() error { + var err error + if t.writerAt != nil { + err = t.writerAt.Close() + } else if t.readerAt != nil { + err = t.readerAt.Close() + } else { + err = t.file.Close() + } + return err +} + func (t *Transfer) checkDownloadSize() { if t.transferType == transferDownload && t.transferError == nil && t.bytesSent < t.expectedSize { t.transferError = fmt.Errorf("incomplete download: %v/%v bytes transferred", t.bytesSent, t.expectedSize) diff --git a/sql/mysql/20200116.sql b/sql/mysql/20200116.sql new file mode 100644 index 00000000..04b45c4e --- /dev/null +++ b/sql/mysql/20200116.sql @@ -0,0 +1,6 @@ +BEGIN; +-- +-- Add field filesystem to user +-- +ALTER TABLE `users` ADD COLUMN `filesystem` longtext NULL; +COMMIT; \ No newline at end of file diff --git a/sql/pgsql/20200116.sql b/sql/pgsql/20200116.sql new file mode 100644 index 00000000..d7f2b8ee --- /dev/null +++ b/sql/pgsql/20200116.sql @@ -0,0 +1,6 @@ +BEGIN; +-- +-- Add field filesystem to user +-- +ALTER TABLE "users" ADD COLUMN "filesystem" text NULL; +COMMIT; \ No newline at end of file diff --git a/sql/sqlite/20200116.sql b/sql/sqlite/20200116.sql new file mode 100644 index 00000000..6ce17c89 --- /dev/null +++ b/sql/sqlite/20200116.sql @@ -0,0 +1,9 @@ +BEGIN; +-- +-- Add field filesystem to user +-- +CREATE TABLE "new__users" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE, "password" varchar(255) NULL, "public_keys" text NULL, "home_dir" varchar(255) NOT NULL, "uid" integer NOT NULL, "gid" integer NOT NULL, "max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "permissions" text NOT NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL, "expiration_date" bigint NOT NULL, "last_login" bigint NOT NULL, "status" integer NOT NULL, "filters" text NULL, "filesystem" text NULL); +INSERT INTO "new__users" ("id", "username", "password", "public_keys", "home_dir", "uid", "gid", "max_sessions", "quota_size", "quota_files", "permissions", "used_quota_size", "used_quota_files", "last_quota_update", "upload_bandwidth", "download_bandwidth", "expiration_date", "last_login", "status", "filters", "filesystem") SELECT "id", "username", "password", "public_keys", "home_dir", "uid", "gid", "max_sessions", "quota_size", "quota_files", "permissions", "used_quota_size", "used_quota_files", "last_quota_update", "upload_bandwidth", "download_bandwidth", "expiration_date", "last_login", "status", "filters", NULL FROM "users"; +DROP TABLE "users"; +ALTER TABLE "new__users" RENAME TO "users"; +COMMIT; \ No newline at end of file diff --git a/templates/user.html b/templates/user.html index b9ddfc65..581ad58c 100644 --- a/templates/user.html +++ b/templates/user.html @@ -191,6 +191,58 @@ +
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+
+ diff --git a/utils/utils.go b/utils/utils.go index 299e5168..5928729a 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -2,15 +2,16 @@ package utils import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/hex" + "errors" "fmt" + "io" "net" - "os" - "path/filepath" - "runtime" "strings" "time" - - "github.com/drakkan/sftpgo/logger" ) const logSender = "utils" @@ -46,49 +47,6 @@ func GetTimeFromMsecSinceEpoch(msec int64) time.Time { return time.Unix(0, msec*1000000) } -// ScanDirContents returns the number of files contained in a directory, their size and a slice with the file paths -func ScanDirContents(path string) (int, int64, []string, error) { - var numFiles int - var size int64 - var fileList []string - var err error - numFiles = 0 - size = 0 - isDir, err := isDirectory(path) - if err == nil && isDir { - err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info != nil && info.Mode().IsRegular() { - size += info.Size() - numFiles++ - fileList = append(fileList, path) - } - return err - }) - } - - return numFiles, size, fileList, err -} - -func isDirectory(path string) (bool, error) { - fileInfo, err := os.Stat(path) - if err != nil { - return false, err - } - return fileInfo.IsDir(), err -} - -// SetPathPermissions call os.Chown on unix, it does nothing on windows -func SetPathPermissions(path string, uid int, gid int) { - if runtime.GOOS != "windows" { - if err := os.Chown(path, uid, gid); err != nil { - logger.Warn(logSender, "", "error chowning path %v: %v", path, err) - } - } -} - // GetAppVersion returns VersionInfo struct func GetAppVersion() VersionInfo { return versionInfo @@ -144,3 +102,74 @@ func GetIPFromRemoteAddress(remoteAddress string) string { } return remoteAddress } + +// NilIfEmpty returns nil if the input string is empty +func NilIfEmpty(s string) *string { + if len(s) == 0 { + return nil + } + return &s +} + +// EncryptData encrypts data using the given key +func EncryptData(data string) (string, error) { + var result string + key := make([]byte, 16) + if _, err := io.ReadFull(rand.Reader, key); err != nil { + return result, err + } + keyHex := hex.EncodeToString(key) + block, err := aes.NewCipher([]byte(keyHex)) + if err != nil { + return result, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return result, err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err = io.ReadFull(rand.Reader, nonce); err != nil { + return result, err + } + ciphertext := gcm.Seal(nonce, nonce, []byte(data), nil) + result = fmt.Sprintf("$aes$%s$%x", keyHex, ciphertext) + return result, err +} + +// RemoveDecryptionKey returns encrypted data without the decryption key +func RemoveDecryptionKey(encryptData string) string { + vals := strings.Split(encryptData, "$") + if len(vals) == 4 { + return fmt.Sprintf("$%v$%v", vals[1], vals[3]) + } + return encryptData +} + +// DecryptData decrypts data encrypted using EncryptData +func DecryptData(data string) (string, error) { + var result string + vals := strings.Split(data, "$") + if len(vals) != 4 { + return "", errors.New("data to decrypt is not in the correct format") + } + key := vals[2] + encrypted, err := hex.DecodeString(vals[3]) + if err != nil { + return result, err + } + block, err := aes.NewCipher([]byte(key)) + if err != nil { + return result, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return result, err + } + nonceSize := gcm.NonceSize() + nonce, ciphertext := encrypted[:nonceSize], encrypted[nonceSize:] + plaintext, err := gcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return result, err + } + return string(plaintext), nil +} diff --git a/vfs/osfs.go b/vfs/osfs.go new file mode 100644 index 00000000..e97987a9 --- /dev/null +++ b/vfs/osfs.go @@ -0,0 +1,288 @@ +package vfs + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" + + "github.com/drakkan/sftpgo/logger" + "github.com/eikenb/pipeat" + "github.com/rs/xid" +) + +const ( + // osFsName is the name for the local Fs implementation + osFsName = "osfs" +) + +// OsFs is a Fs implementation that uses functions provided by the os package. +type OsFs struct { + name string + connectionID string +} + +// NewOsFs returns an OsFs object that allows to interact with local Os filesystem +func NewOsFs(connectionID string) Fs { + return &OsFs{ + name: osFsName, + connectionID: connectionID} +} + +// Name returns the name for the Fs implementation +func (fs OsFs) Name() string { + return fs.name +} + +// ConnectionID returns the SSH connection ID associated to this Fs implementation +func (fs OsFs) ConnectionID() string { + return fs.connectionID +} + +// Stat returns a FileInfo describing the named file +func (OsFs) Stat(name string) (os.FileInfo, error) { + return os.Stat(name) +} + +// Lstat returns a FileInfo describing the named file +func (OsFs) Lstat(name string) (os.FileInfo, error) { + return os.Lstat(name) +} + +// Open opens the named file for reading +func (OsFs) Open(name string) (*os.File, *pipeat.PipeReaderAt, func(), error) { + f, err := os.Open(name) + return f, nil, nil, err +} + +// Create creates or opens the named file for writing +func (OsFs) Create(name string, flag int) (*os.File, *pipeat.PipeWriterAt, func(), error) { + var err error + var f *os.File + if flag == 0 { + f, err = os.Create(name) + } else { + f, err = os.OpenFile(name, flag, 0666) + + } + return f, nil, nil, err +} + +// Rename renames (moves) source to target +func (OsFs) Rename(source, target string) error { + return os.Rename(source, target) +} + +// Remove removes the named file or (empty) directory. +func (OsFs) Remove(name string, isDir bool) error { + return os.Remove(name) +} + +// Mkdir creates a new directory with the specified name and default permissions +func (OsFs) Mkdir(name string) error { + return os.Mkdir(name, 0777) +} + +// Symlink creates source as a symbolic link to target. +func (OsFs) Symlink(source, target string) error { + return os.Symlink(source, target) +} + +// Chown changes the numeric uid and gid of the named file. +func (OsFs) Chown(name string, uid int, gid int) error { + return os.Chown(name, uid, gid) +} + +// Chmod changes the mode of the named file to mode +func (OsFs) Chmod(name string, mode os.FileMode) error { + return os.Chmod(name, mode) +} + +// Chtimes changes the access and modification times of the named file +func (OsFs) Chtimes(name string, atime, mtime time.Time) error { + return os.Chtimes(name, atime, mtime) +} + +// ReadDir reads the directory named by dirname and returns +// a list of directory entries. +func (OsFs) ReadDir(dirname string) ([]os.FileInfo, error) { + return ioutil.ReadDir(dirname) +} + +// IsUploadResumeSupported returns true if upload resume is supported +func (OsFs) IsUploadResumeSupported() bool { + return true +} + +// IsAtomicUploadSupported returns true if atomic upload is supported +func (OsFs) IsAtomicUploadSupported() bool { + return true +} + +// IsNotExist returns a boolean indicating whether the error is known to +// report that a file or directory does not exist +func (OsFs) IsNotExist(err error) bool { + return os.IsNotExist(err) +} + +// IsPermission returns a boolean indicating whether the error is known to +// report that permission is denied. +func (OsFs) IsPermission(err error) bool { + return os.IsPermission(err) +} + +// CheckRootPath creates the specified root directory if it does not exists +func (fs OsFs) CheckRootPath(rootPath, username string, uid int, gid int) bool { + var err error + if _, err = fs.Stat(rootPath); fs.IsNotExist(err) { + err = os.MkdirAll(rootPath, 0777) + fsLog(fs, logger.LevelDebug, "root directory %#v for user %#v does not exist, try to create, mkdir error: %v", + rootPath, username, err) + if err == nil { + SetPathPermissions(fs, rootPath, uid, gid) + } + } + return (err == nil) +} + +// ScanDirContents returns the number of files contained in a directory and +// their size +func (fs OsFs) ScanDirContents(dirPath string) (int, int64, error) { + numFiles := 0 + size := int64(0) + isDir, err := IsDirectory(fs, dirPath) + if err == nil && isDir { + err = filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info != nil && info.Mode().IsRegular() { + size += info.Size() + numFiles++ + } + return err + }) + } + return numFiles, size, err +} + +// GetAtomicUploadPath returns the path to use for an atomic upload +func (OsFs) GetAtomicUploadPath(name string) string { + dir := filepath.Dir(name) + guid := xid.New().String() + return filepath.Join(dir, ".sftpgo-upload."+guid+"."+filepath.Base(name)) +} + +// GetRelativePath returns the path for a file relative to the user's home dir. +// This is the path as seen by SFTP users +func (OsFs) GetRelativePath(name, rootPath string) string { + rel, err := filepath.Rel(rootPath, filepath.Clean(name)) + if err != nil { + return "" + } + if rel == "." || strings.HasPrefix(rel, "..") { + rel = "" + } + return "/" + filepath.ToSlash(rel) +} + +// Join joins any number of path elements into a single path +func (OsFs) Join(elem ...string) string { + return filepath.Join(elem...) +} + +// ResolvePath returns the matching filesystem path for the specified sftp path +func (fs OsFs) ResolvePath(sftpPath, rootPath string) (string, error) { + if !filepath.IsAbs(rootPath) { + return "", fmt.Errorf("Invalid root path: %v", rootPath) + } + r := filepath.Clean(filepath.Join(rootPath, sftpPath)) + p, err := filepath.EvalSymlinks(r) + if err != nil && !os.IsNotExist(err) { + return "", err + } else if os.IsNotExist(err) { + // The requested path doesn't exist, so at this point we need to iterate up the + // path chain until we hit a directory that _does_ exist and can be validated. + _, err = fs.findFirstExistingDir(r, rootPath) + if err != nil { + fsLog(fs, logger.LevelWarn, "error resolving not existent path: %#v", err) + } + return r, err + } + + err = fs.isSubDir(p, rootPath) + if err != nil { + fsLog(fs, logger.LevelWarn, "Invalid path resolution, dir: %#v outside user home: %#v err: %v", p, rootPath, err) + } + return r, err +} + +func (fs *OsFs) findNonexistentDirs(path, rootPath string) ([]string, error) { + results := []string{} + cleanPath := filepath.Clean(path) + parent := filepath.Dir(cleanPath) + _, err := os.Stat(parent) + + for os.IsNotExist(err) { + results = append(results, parent) + parent = filepath.Dir(parent) + _, err = os.Stat(parent) + } + if err != nil { + return results, err + } + p, err := filepath.EvalSymlinks(parent) + if err != nil { + return results, err + } + err = fs.isSubDir(p, rootPath) + if err != nil { + fsLog(fs, logger.LevelWarn, "error finding non existing dir: %v", err) + } + return results, err +} + +func (fs *OsFs) findFirstExistingDir(path, rootPath string) (string, error) { + results, err := fs.findNonexistentDirs(path, rootPath) + if err != nil { + fsLog(fs, logger.LevelWarn, "unable to find non existent dirs: %v", err) + return "", err + } + var parent string + if len(results) > 0 { + lastMissingDir := results[len(results)-1] + parent = filepath.Dir(lastMissingDir) + } else { + parent = rootPath + } + p, err := filepath.EvalSymlinks(parent) + if err != nil { + return "", err + } + fileInfo, err := os.Stat(p) + if err != nil { + return "", err + } + if !fileInfo.IsDir() { + return "", fmt.Errorf("resolved path is not a dir: %#v", p) + } + err = fs.isSubDir(p, rootPath) + return p, err +} + +func (fs *OsFs) isSubDir(sub, rootPath string) error { + // rootPath must exist and it is already a validated absolute path + parent, err := filepath.EvalSymlinks(rootPath) + if err != nil { + fsLog(fs, logger.LevelWarn, "invalid home dir %#v: %v", rootPath, err) + return err + } + if !strings.HasPrefix(sub, parent) { + err = fmt.Errorf("path %#v is not inside: %#v", sub, parent) + fsLog(fs, logger.LevelWarn, "error: %v ", err) + return err + } + return nil +} diff --git a/vfs/s3fileinfo.go b/vfs/s3fileinfo.go new file mode 100644 index 00000000..2cdab62c --- /dev/null +++ b/vfs/s3fileinfo.go @@ -0,0 +1,60 @@ +package vfs + +import ( + "os" + "time" +) + +// S3FileInfo implements os.FileInfo for a file in S3. +type S3FileInfo struct { + name string + sizeInBytes int64 + modTime time.Time + mode os.FileMode + sys interface{} +} + +// NewS3FileInfo creates file info. +func NewS3FileInfo(name string, isDirectory bool, sizeInBytes int64, modTime time.Time) S3FileInfo { + mode := os.FileMode(0644) + if isDirectory { + mode = os.FileMode(0755) | os.ModeDir + } + + return S3FileInfo{ + name: name, + sizeInBytes: sizeInBytes, + modTime: modTime, + mode: mode, + } +} + +// Name provides the base name of the file. +func (fi S3FileInfo) Name() string { + return fi.name +} + +// Size provides the length in bytes for a file. +func (fi S3FileInfo) Size() int64 { + return fi.sizeInBytes +} + +// Mode provides the file mode bits +func (fi S3FileInfo) Mode() os.FileMode { + return fi.mode +} + +// ModTime provides the last modification time. +func (fi S3FileInfo) ModTime() time.Time { + return fi.modTime +} + +// IsDir provides the abbreviation for Mode().IsDir() +func (fi S3FileInfo) IsDir() bool { + return fi.mode&os.ModeDir != 0 +} + +// Sys provides the underlying data source (can return nil) +func (fi S3FileInfo) Sys() interface{} { + return fi.getFileInfoSys() +} diff --git a/vfs/s3fileinfo_unix.go b/vfs/s3fileinfo_unix.go new file mode 100644 index 00000000..375efe0b --- /dev/null +++ b/vfs/s3fileinfo_unix.go @@ -0,0 +1,28 @@ +// +build !windows + +package vfs + +import "syscall" + +import "os" + +var ( + defaultUID, defaultGID int +) + +func init() { + defaultUID = os.Getuid() + defaultGID = os.Getuid() + if defaultUID < 0 { + defaultUID = 65534 + } + if defaultGID < 0 { + defaultGID = 65534 + } +} + +func (fi S3FileInfo) getFileInfoSys() interface{} { + return &syscall.Stat_t{ + Uid: uint32(defaultUID), + Gid: uint32(defaultGID)} +} diff --git a/vfs/s3fileinfo_windows.go b/vfs/s3fileinfo_windows.go new file mode 100644 index 00000000..3707a4ba --- /dev/null +++ b/vfs/s3fileinfo_windows.go @@ -0,0 +1,7 @@ +package vfs + +import "syscall" + +func (fi S3FileInfo) getFileInfoSys() interface{} { + return syscall.Win32FileAttributeData{} +} diff --git a/vfs/s3fs.go b/vfs/s3fs.go new file mode 100644 index 00000000..4dba9719 --- /dev/null +++ b/vfs/s3fs.go @@ -0,0 +1,491 @@ +package vfs + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/drakkan/sftpgo/logger" + "github.com/drakkan/sftpgo/utils" + "github.com/eikenb/pipeat" +) + +// S3FsConfig defines the configuration for S3fs +type S3FsConfig struct { + Bucket string `json:"bucket,omitempty"` + Region string `json:"region,omitempty"` + AccessKey string `json:"access_key,omitempty"` + AccessSecret string `json:"access_secret,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + StorageClass string `json:"storage_class,omitempty"` +} + +// S3Fs is a Fs implementation for Amazon S3 compatible object storage. +type S3Fs struct { + connectionID string + localTempDir string + config S3FsConfig + svc *s3.S3 + ctxTimeout time.Duration + ctxLongTimeout time.Duration +} + +// NewS3Fs returns an S3Fs object that allows to interact with an s3 compatible +// object storage +func NewS3Fs(connectionID, localTempDir string, config S3FsConfig) (Fs, error) { + fs := S3Fs{ + connectionID: connectionID, + localTempDir: localTempDir, + config: config, + ctxTimeout: 30 * time.Second, + ctxLongTimeout: 300 * time.Second, + } + if err := ValidateS3FsConfig(&fs.config); err != nil { + return fs, err + } + accessSecret, err := utils.DecryptData(fs.config.AccessSecret) + if err != nil { + return fs, err + } + fs.config.AccessSecret = accessSecret + awsConfig := &aws.Config{ + Region: aws.String(fs.config.Region), + Credentials: credentials.NewStaticCredentials(fs.config.AccessKey, fs.config.AccessSecret, ""), + } + //config.WithLogLevel(aws.LogDebugWithHTTPBody) + if len(fs.config.Endpoint) > 0 { + awsConfig.Endpoint = aws.String(fs.config.Endpoint) + awsConfig.S3ForcePathStyle = aws.Bool(true) + } + sess, err := session.NewSession(awsConfig) + if err != nil { + return fs, err + } + fs.svc = s3.New(sess) + return fs, nil +} + +// Name returns the name for the Fs implementation +func (fs S3Fs) Name() string { + return fmt.Sprintf("S3Fs bucket: %#v", fs.config.Bucket) +} + +// ConnectionID returns the SSH connection ID associated to this Fs implementation +func (fs S3Fs) ConnectionID() string { + return fs.connectionID +} + +// Stat returns a FileInfo describing the named file +func (fs S3Fs) Stat(name string) (os.FileInfo, error) { + var result S3FileInfo + if name == "/" || name == "." { + err := fs.checkIfBucketExists() + if err != nil { + return result, err + } + return NewS3FileInfo(name, true, 0, time.Time{}), nil + } + prefix := path.Dir(name) + if prefix == "/" || prefix == "." { + prefix = "" + } else { + prefix = strings.TrimPrefix(prefix, "/") + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + } + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) + defer cancelFn() + err := fs.svc.ListObjectsV2PagesWithContext(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(fs.config.Bucket), + Prefix: aws.String(prefix), + Delimiter: aws.String("/"), + }, func(page *s3.ListObjectsV2Output, lastPage bool) bool { + for _, p := range page.CommonPrefixes { + if fs.isEqual(p.Prefix, name) { + result = NewS3FileInfo(name, true, 0, time.Time{}) + return false + } + } + for _, fileObject := range page.Contents { + if fs.isEqual(fileObject.Key, name) { + objectSize := *fileObject.Size + objectModTime := *fileObject.LastModified + isDir := strings.HasSuffix(*fileObject.Key, "/") + result = NewS3FileInfo(name, isDir, objectSize, objectModTime) + return false + } + } + return true + }) + if err == nil && len(result.Name()) == 0 { + err = errors.New("404 no such file or directory") + } + return result, err +} + +// Lstat returns a FileInfo describing the named file +func (fs S3Fs) Lstat(name string) (os.FileInfo, error) { + return fs.Stat(name) +} + +// Open opens the named file for reading +func (fs S3Fs) Open(name string) (*os.File, *pipeat.PipeReaderAt, func(), error) { + r, w, err := pipeat.AsyncWriterPipeInDir(fs.localTempDir) + if err != nil { + return nil, nil, nil, err + } + ctx, cancelFn := context.WithCancel(context.Background()) + downloader := s3manager.NewDownloaderWithClient(fs.svc) + go func() { + defer cancelFn() + key := name + n, err := downloader.DownloadWithContext(ctx, w, &s3.GetObjectInput{ + Bucket: aws.String(fs.config.Bucket), + Key: aws.String(key), + }) + fsLog(fs, logger.LevelDebug, "download completed, path: %#v size: %v, err: %v", name, n, err) + w.CloseWithError(err) + }() + return nil, r, cancelFn, nil +} + +// Create creates or opens the named file for writing +func (fs S3Fs) Create(name string, flag int) (*os.File, *pipeat.PipeWriterAt, func(), error) { + r, w, err := pipeat.PipeInDir(fs.localTempDir) + if err != nil { + return nil, nil, nil, err + } + ctx, cancelFn := context.WithCancel(context.Background()) + uploader := s3manager.NewUploaderWithClient(fs.svc) + go func() { + defer cancelFn() + key := name + response, err := uploader.UploadWithContext(ctx, &s3manager.UploadInput{ + Bucket: aws.String(fs.config.Bucket), + Key: aws.String(key), + Body: r, + StorageClass: utils.NilIfEmpty(fs.config.StorageClass), + }) + fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, response: %v, err: %v", name, response, err) + r.CloseWithError(err) + }() + return nil, w, cancelFn, nil +} + +// Rename renames (moves) source to target. +// We don't support renaming non empty directories since we should +// rename all the contents too and this could take long time: think +// about directories with thousands of files, for each file we should +// execute a CopyObject call. +func (fs S3Fs) Rename(source, target string) error { + if source == target { + return nil + } + fi, err := fs.Stat(source) + if err != nil { + return err + } + copySource := fs.Join(fs.config.Bucket, source) + if fi.IsDir() { + contents, err := fs.ReadDir(source) + if err != nil { + return err + } + if len(contents) > 0 { + return fmt.Errorf("Cannot rename non empty directory: %#v", source) + } + if !strings.HasSuffix(copySource, "/") { + copySource += "/" + } + if !strings.HasSuffix(target, "/") { + target += "/" + } + } + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) + defer cancelFn() + _, err = fs.svc.CopyObjectWithContext(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(fs.config.Bucket), + CopySource: aws.String(copySource), + Key: aws.String(target), + }) + if err != nil { + return err + } + return fs.Remove(source, fi.IsDir()) +} + +// Remove removes the named file or (empty) directory. +func (fs S3Fs) Remove(name string, isDir bool) error { + if isDir { + contents, err := fs.ReadDir(name) + if err != nil { + return err + } + if len(contents) > 0 { + return fmt.Errorf("Cannot remove non empty directory: %#v", name) + } + if !strings.HasSuffix(name, "/") { + name += "/" + } + } + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) + defer cancelFn() + _, err := fs.svc.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(fs.config.Bucket), + Key: aws.String(name), + }) + return err +} + +// Mkdir creates a new directory with the specified name and default permissions +func (fs S3Fs) Mkdir(name string) error { + _, err := fs.Stat(name) + if !fs.IsNotExist(err) { + return err + } + if !strings.HasSuffix(name, "/") { + name += "/" + } + _, w, _, err := fs.Create(name, 0) + if err != nil { + return err + } + return w.Close() +} + +// Symlink creates source as a symbolic link to target. +func (S3Fs) Symlink(source, target string) error { + return errors.New("403 symlinks are not supported") +} + +// Chown changes the numeric uid and gid of the named file. +// Silently ignored. +func (S3Fs) Chown(name string, uid int, gid int) error { + return nil +} + +// Chmod changes the mode of the named file to mode. +// Silently ignored. +func (S3Fs) Chmod(name string, mode os.FileMode) error { + return nil +} + +// Chtimes changes the access and modification times of the named file. +// Silently ignored. +func (S3Fs) Chtimes(name string, atime, mtime time.Time) error { + return errors.New("403 chtimes is not supported") +} + +// ReadDir reads the directory named by dirname and returns +// a list of directory entries. +func (fs S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) { + var result []os.FileInfo + // dirname deve essere già cleaned + prefix := "" + if dirname != "/" && dirname != "." { + prefix = strings.TrimPrefix(dirname, "/") + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + } + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) + defer cancelFn() + err := fs.svc.ListObjectsV2PagesWithContext(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(fs.config.Bucket), + Prefix: aws.String(prefix), + Delimiter: aws.String("/"), + }, func(page *s3.ListObjectsV2Output, lastPage bool) bool { + for _, p := range page.CommonPrefixes { + name, isDir := fs.resolve(p.Prefix, prefix) + result = append(result, NewS3FileInfo(name, isDir, 0, time.Time{})) + } + for _, fileObject := range page.Contents { + objectSize := *fileObject.Size + objectModTime := *fileObject.LastModified + name, isDir := fs.resolve(fileObject.Key, prefix) + if len(name) == 0 { + continue + } + result = append(result, NewS3FileInfo(name, isDir, objectSize, objectModTime)) + } + return true + }) + return result, err +} + +// IsUploadResumeSupported returns true if upload resume is supported. +// SFTP Resume is not supported on S3 +func (S3Fs) IsUploadResumeSupported() bool { + return false +} + +// IsAtomicUploadSupported returns true if atomic upload is supported. +// S3 uploads are already atomic, we don't need to upload to a temporary +// file +func (S3Fs) IsAtomicUploadSupported() bool { + return false +} + +// IsNotExist returns a boolean indicating whether the error is known to +// report that a file or directory does not exist +func (S3Fs) IsNotExist(err error) bool { + if err == nil { + return false + } + if aerr, ok := err.(awserr.Error); ok { + if aerr.Code() == s3.ErrCodeNoSuchKey { + return true + } + if aerr.Code() == s3.ErrCodeNoSuchBucket { + return true + } + } + if multierr, ok := err.(s3manager.MultiUploadFailure); ok { + if multierr.Code() == s3.ErrCodeNoSuchKey { + return true + } + if multierr.Code() == s3.ErrCodeNoSuchBucket { + return true + } + } + return strings.Contains(err.Error(), "404") +} + +// IsPermission returns a boolean indicating whether the error is known to +// report that permission is denied. +func (S3Fs) IsPermission(err error) bool { + if err == nil { + return false + } + return strings.Contains(err.Error(), "403") +} + +// CheckRootPath creates the specified root directory if it does not exists +func (fs S3Fs) CheckRootPath(rootPath, username string, uid int, gid int) bool { + // we need a local directory for temporary files + osFs := NewOsFs(fs.ConnectionID()) + osFs.CheckRootPath(fs.localTempDir, username, uid, gid) + err := fs.checkIfBucketExists() + if err == nil { + return true + } + if !fs.IsNotExist(err) { + return false + } + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) + defer cancelFn() + input := &s3.CreateBucketInput{ + Bucket: aws.String(fs.config.Bucket), + } + _, err = fs.svc.CreateBucketWithContext(ctx, input) + fsLog(fs, logger.LevelDebug, "bucket %#v for user %#v does not exists, try to create, error: %v", + fs.config.Bucket, username, err) + return err == nil +} + +// ScanDirContents returns the number of files contained in the bucket, +// and their size +func (fs S3Fs) ScanDirContents(dirPath string) (int, int64, error) { + numFiles := 0 + size := int64(0) + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout)) + defer cancelFn() + err := fs.svc.ListObjectsV2PagesWithContext(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(fs.config.Bucket), + Prefix: aws.String(""), + }, func(page *s3.ListObjectsV2Output, lastPage bool) bool { + for _, fileObject := range page.Contents { + numFiles++ + size += *fileObject.Size + } + return true + }) + + return numFiles, size, err +} + +// GetAtomicUploadPath returns the path to use for an atomic upload. +// S3 uploads are already atomic, we never call this method for S3 +func (S3Fs) GetAtomicUploadPath(name string) string { + return "" +} + +// GetRelativePath returns the path for a file relative to the user's home dir. +// This is the path as seen by SFTP users +func (S3Fs) GetRelativePath(name, rootPath string) string { + rel := name + if name == "." { + rel = "" + } + if !strings.HasPrefix(rel, "/") { + return "/" + rel + } + return rel +} + +// Join joins any number of path elements into a single path +func (S3Fs) Join(elem ...string) string { + return path.Join(elem...) +} + +// ResolvePath returns the matching filesystem path for the specified sftp path +func (fs S3Fs) ResolvePath(sftpPath, rootPath string) (string, error) { + return sftpPath, nil +} + +func (fs *S3Fs) resolve(name *string, prefix string) (string, bool) { + result := strings.TrimPrefix(*name, prefix) + isDir := strings.HasSuffix(result, "/") + if isDir { + result = strings.TrimSuffix(result, "/") + } + if strings.Contains(result, "/") { + i := strings.Index(result, "/") + isDir = true + result = result[:i] + } + return result, isDir +} + +func (fs *S3Fs) isEqual(s3Key *string, sftpName string) bool { + if *s3Key == sftpName { + return true + } + if "/"+*s3Key == sftpName { + return true + } + if "/"+*s3Key == sftpName+"/" { + return true + } + return false +} + +func (fs *S3Fs) checkIfBucketExists() error { + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) + defer cancelFn() + _, err := fs.svc.HeadBucketWithContext(ctx, &s3.HeadBucketInput{ + Bucket: aws.String(fs.config.Bucket), + }) + return err +} + +func (fs *S3Fs) getObjectDetails(key string) (*s3.HeadObjectOutput, error) { + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) + defer cancelFn() + input := &s3.HeadObjectInput{ + Bucket: aws.String(fs.config.Bucket), + Key: aws.String(key), + } + return fs.svc.HeadObjectWithContext(ctx, input) +} diff --git a/vfs/vfs.go b/vfs/vfs.go new file mode 100644 index 00000000..1b97e8dc --- /dev/null +++ b/vfs/vfs.go @@ -0,0 +1,100 @@ +package vfs + +import ( + "errors" + "os" + "runtime" + "time" + + "github.com/drakkan/sftpgo/logger" + "github.com/eikenb/pipeat" + "github.com/pkg/sftp" +) + +// Fs defines the interface for filesystems backends +type Fs interface { + Name() string + ConnectionID() string + Stat(name string) (os.FileInfo, error) + Lstat(name string) (os.FileInfo, error) + Open(name string) (*os.File, *pipeat.PipeReaderAt, func(), error) + Create(name string, flag int) (*os.File, *pipeat.PipeWriterAt, func(), error) + Rename(source, target string) error + Remove(name string, isDir bool) error + Mkdir(name string) error + Symlink(source, target string) error + Chown(name string, uid int, gid int) error + Chmod(name string, mode os.FileMode) error + Chtimes(name string, atime, mtime time.Time) error + ReadDir(dirname string) ([]os.FileInfo, error) + IsUploadResumeSupported() bool + IsAtomicUploadSupported() bool + CheckRootPath(rootPath, username string, uid int, gid int) bool + ResolvePath(sftpPath, rootPath string) (string, error) + IsNotExist(err error) bool + IsPermission(err error) bool + ScanDirContents(dirPath string) (int, int64, error) + GetAtomicUploadPath(name string) string + GetRelativePath(name, rootPath string) string + Join(elem ...string) string +} + +// IsDirectory checks if a path exists and is a directory +func IsDirectory(fs Fs, path string) (bool, error) { + fileInfo, err := fs.Stat(path) + if err != nil { + return false, err + } + return fileInfo.IsDir(), err +} + +// GetSFTPError returns an sftp error from a filesystem error +func GetSFTPError(fs Fs, err error) error { + if fs.IsNotExist(err) { + return sftp.ErrSSHFxNoSuchFile + } else if fs.IsPermission(err) { + return sftp.ErrSSHFxPermissionDenied + } else if err != nil { + return sftp.ErrSSHFxFailure + } + return nil +} + +// IsLocalOsFs returns true if fs is the local filesystem implementation +func IsLocalOsFs(fs Fs) bool { + return fs.Name() == osFsName +} + +// ValidateS3FsConfig returns nil if the specified s3 config is valid, otherwise an error +func ValidateS3FsConfig(config *S3FsConfig) error { + if len(config.Bucket) == 0 { + return errors.New("bucket cannot be empty") + } + if len(config.Region) == 0 { + return errors.New("region cannot be empty") + } + if len(config.AccessKey) == 0 { + return errors.New("access_key cannot be empty") + } + if len(config.AccessSecret) == 0 { + return errors.New("access_secret cannot be empty") + } + return nil +} + +// SetPathPermissions calls fs.Chown. +// It does nothing for local filesystem on windows +func SetPathPermissions(fs Fs, path string, uid int, gid int) { + if IsLocalOsFs(fs) { + if runtime.GOOS == "windows" { + return + } + } + if err := fs.Chown(path, uid, gid); err != nil { + fsLog(fs, logger.LevelWarn, "error chowning path %v: %v", path, err) + } +} + +func fsLog(fs Fs, level logger.LogLevel, format string, v ...interface{}) { + logger.Log(level, fs.Name(), fs.ConnectionID(), format, v...) +}