beenull/sftpgo-mirror
1
0
Fork 0
mirror of https://github.com/drakkan/sftpgo.git synced 2024-11-24 16:40:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add basic S3-Compatible Object Storage support

Browse source 
we have now an interface for filesystem backeds, this make easy to add
new filesystem backends
This commit is contained in:
Nicola Murino 2020-01-19 07:41:05 +01:00
parent 0b42dbc3c3
commit a4834f4a83
40 changed files with 2315 additions and 420 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.travis.yml
View file

@ -11,7 +11,7 @@ env:
- GO111MODULE=on
before_script:
- sqlite3 sftpgo.db 'CREATE TABLE "users" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE, "password" varchar(255) NULL, "public_keys" text NULL, "home_dir" varchar(255) NOT NULL, "uid" integer NOT NULL, "gid" integer NOT NULL, "max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "permissions" text NOT NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL, "expiration_date" bigint NOT NULL, "last_login" bigint NOT NULL, "status" integer NOT NULL, "filters" TEXT NULL);'
- sqlite3 sftpgo.db 'CREATE TABLE "users" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE, "password" varchar(255) NULL, "public_keys" text NULL, "home_dir" varchar(255) NOT NULL, "uid" integer NOT NULL, "gid" integer NOT NULL, "max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "permissions" text NOT NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL, "expiration_date" bigint NOT NULL, "last_login" bigint NOT NULL, "status" integer NOT NULL, "filters" TEXT NULL, "filesystem" text NULL);'
install:
- go get -v -t ./...

58
README.md
View file

@ -21,6 +21,7 @@ Full featured and highly configurable SFTP server
- Atomic uploads are configurable.
- Support for Git repositories over SSH.
- SCP and rsync are supported.
- Support for serving S3 Compatible Object Storage over SFTP.
- Prometheus metrics are exposed.
- REST API for users management, backup, restore and real time reports of the active connections with possibility of forcibly closing a connection.
- Web based interface to easily manage users and connections.
@ -136,7 +137,7 @@ The `sftpgo` configuration file contains the following sections:
- `idle_timeout`, integer. Time in minutes after which an idle client will be disconnected. 0 menas disabled. Default: 15
- `max_auth_tries` integer. Maximum number of authentication attempts permitted per connection. If set to a negative number, the number of attempts are unlimited. If set to zero, the number of attempts are limited to 6.
- `umask`, string. Umask for the new files and directories. This setting has no effect on Windows. Default: "0022"
- `banner`, string. Identification string used by the server. Leave empty to use the default banner. Default "SFTPGo_version"
- `banner`, string. Identification string used by the server. Leave empty to use the default banner. Default "SFTPGo_<version>"
- `upload_mode` integer. 0 means standard, the files are uploaded directly to the requested path. 1 means atomic: files are uploaded to a temporary path and renamed to the requested path when the client ends the upload. Atomic mode avoids problems such as a web server that serves partial files when the files are being uploaded. In atomic mode if there is an upload error the temporary file is deleted and so the requested upload path will not contain a partial file. 2 means atomic with resume support: as atomic but if there is an upload error the temporary file is renamed to the requested path and not deleted, this way a client can reconnect and resume the upload.
- `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions. See the "Custom Actions" paragraph for more details
- `execute_on`, list of strings. Valid values are `download`, `upload`, `delete`, `rename`, `ssh_cmd`. Leave empty to disable actions.
@ -150,7 +151,7 @@ The `sftpgo` configuration file contains the following sections:
- `macs`, list of strings. available MAC (message authentication code) algorithms in preference order. Leave empty to use default values. The supported values can be found here: [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/common.go#L84 "Supported MACs")
- `login_banner_file`, path to the login banner file. The contents of the specified file, if any, are sent to the remote user before authentication is allowed. It can be a path relative to the config dir or an absolute one. Leave empty to send no login banner
- `setstat_mode`, integer. 0 means "normal mode": requests for changing permissions, owner/group and access/modification times are executed. 1 means "ignore mode": requests for changing permissions, owner/group and access/modification times are silently ignored.
- `enabled_ssh_commands`, list of enabled SSH commands. These SSH commands are enabled by default: `md5sum`, `sha1sum`, `cd`, `pwd`. `*` enables all supported commands. Some commands are implemented directly inside SFTPGo, while for other commands we use system commands that need to be installed and in your system's `PATH`. For system commands we have no direct control on file creation/deletion and so quota check is suboptimal: if quota is enabled, the number of files is checked at the command begin and not while new files are created. The allowed size is calculated as the difference between the max quota and the used one and it is checked against the bytes transferred via SSH. The command is aborted if it uploads more bytes than the remaining allowed size calculated at the command start. Anyway we see the bytes that the remote command send to the local command via SSH, these bytes contain both protocol commands and files and so the size of the files is different from the size trasferred via SSH: for example a command can send compressed files or a protocol command (few bytes) could delete a big file. To mitigate this issue quotas are recalculated at the command end with a full home directory scan, this could be heavy for big directories. If you need system commands and quotas you could consider to disable quota restrictions and periodically update quota usage yourself using the REST API. We support the following SSH commands:
- `enabled_ssh_commands`, list of enabled SSH commands. These SSH commands are enabled by default: `md5sum`, `sha1sum`, `cd`, `pwd`. `*` enables all supported commands. Some commands are implemented directly inside SFTPGo, while for other commands we use system commands that need to be installed and in your system's `PATH`. For system commands we have no direct control on file creation/deletion and so we cannot support remote filesystems, such as S3, and quota check is suboptimal: if quota is enabled, the number of files is checked at the command begin and not while new files are created. The allowed size is calculated as the difference between the max quota and the used one and it is checked against the bytes transferred via SSH. The command is aborted if it uploads more bytes than the remaining allowed size calculated at the command start. Anyway we see the bytes that the remote command send to the local command via SSH, these bytes contain both protocol commands and files and so the size of the files is different from the size trasferred via SSH: for example a command can send compressed files or a protocol command (few bytes) could delete a big file. To mitigate this issue quotas are recalculated at the command end with a full home directory scan, this could be heavy for big directories. If you need system commands and quotas you could consider to disable quota restrictions and periodically update quota usage yourself using the REST API. We support the following SSH commands:
- `scp`, SCP is an experimental feature, we have our own SCP implementation since we can't rely on "scp" system command to proper handle quotas and user's home dir restrictions. The SCP protocol is quite simple but there is no official docs about it, so we need more testing and feedbacks before enabling it by default. We may not handle some borderline cases or have sneaky bugs. Please do accurate tests yourself before enabling SCP and let us known if something does not work as expected for your use cases. SCP between two remote hosts is supported using the `-3` scp option.
- `md5sum`, `sha1sum`, `sha256sum`, `sha384sum`, `sha512sum`. Useful to check message digests for uploaded files. These commands are implemented inside SFTPGo so they work even if the matching system commands are not available, for example on Windows.
- `cd`, `pwd`. Some SFTP clients does not support the SFTP SSH_FXP_REALPATH packet type and so they use `cd` and `pwd` SSH commands to get the initial directory. Currently `cd` do nothing and `pwd` always returns the `/` path.
@ -416,6 +417,25 @@ The `http_notification_url`, if defined, will be called invoked as http POST. Th
The HTTP request has a 15 seconds timeout.
## S3 Compabible Object Storage backends
Each user can be mapped with an S3-Compatible bucket, this way the mapped bucket is exposed over SFTP/SCP.
SFTPGo uses multipart uploads and parallel downloads for storing and retrieving files from S3 and automatically try to create the mapped bucket if it does not exists.
Some SFTP commands doesn't work over S3:
- `symlink` and `chtimes` will fail
- `chown`, `chmod` are silently ignored
- upload resume is not supported
- upload mode `atomic` is ignored since S3 uploads are already atomic
Other notes:
- `rename` is a two steps operation: server-side copy and then deletion. So it is not atomic as for local filesystem
- We don't support renaming non empty directories since we should rename all the contents too and this could take long time: think about directories with thousands of files, for each file we should do an AWS API call.
- For server side encryption you have to configure the mapped bucket to automatically encrypt objects.
- A local home directory is still required to store temporary files.
## Portable mode
SFTPGo allows to share a single directory on demand using the `portable` subcommand:
@ -432,17 +452,24 @@ Usage:
sftpgo portable [flags]
Flags:
-C, --advertise-credentials If the SFTP service is advertised via multicast DNS this flag allows to put username/password inside the advertised TXT record
-S, --advertise-service Advertise SFTP service using multicast DNS (default true)
-d, --directory string Path to the directory to serve. This can be an absolute path or a path relative to the current directory (default ".")
-h, --help help for portable
-l, --log-file-path string Leave empty to disable logging
-p, --password string Leave empty to use an auto generated value
-g, --permissions strings User's permissions. "*" means any permission (default [list,download])
-C, --advertise-credentials If the SFTP service is advertised via multicast DNS this flag allows to put username/password inside the advertised TXT record
-S, --advertise-service Advertise SFTP service using multicast DNS (default true)
-d, --directory string Path to the directory to serve. This can be an absolute path or a path relative to the current directory (default ".")
-f, --fs-provider int 0 means local filesystem, 1 S3 compatible
-h, --help help for portable
-l, --log-file-path string Leave empty to disable logging
-p, --password string Leave empty to use an auto generated value
-g, --permissions strings User's permissions. "*" means any permission (default [list,download])
-k, --public-key strings
-s, --sftpd-port int 0 means a random non privileged port
-c, --ssh-commands strings SSH commands to enable. "*" means any supported SSH command including scp (default [md5sum,sha1sum,cd,pwd])
-u, --username string Leave empty to use an auto generated value
--s3-access-key string
--s3-access-secret string
--s3-bucket string
--s3-endpoint string
--s3-region string
--s3-storage-class string
-s, --sftpd-port int 0 means a random non privileged port
-c, --ssh-commands strings SSH commands to enable. "*" means any supported SSH command including scp (default [md5sum,sha1sum,cd,pwd])
-u, --username string Leave empty to use an auto generated value
```
In portable mode SFTPGo can advertise the SFTP service and, optionally, the credentials via multicast DNS, so there is a standard way to discover the service and to automatically connect to it.
@ -488,6 +515,13 @@ For each account the following properties can be configured:
- `download_bandwidth` maximum download bandwidth as KB/s, 0 means unlimited.
- `allowed_ip`, List of IP/Mask allowed to login. Any IP address not contained in this list cannot login. IP/Mask must be in CIDR notation as defined in RFC 4632 and RFC 4291, for example "192.0.2.0/24" or "2001:db8::/32"
- `denied_ip`, List of IP/Mask not allowed to login. If an IP address is both allowed and denied then login will be denied
- `fs_provider`, filesystem to serve via SFTP. Local filesystem and S3 Compatible Object Storage are supported
- `s3_bucket`, required for S3 filesystem
- `s3_region`, required for S3 filesystem
- `s3_access_key`, required for S3 filesystem
- `s3_access_secret`, required for S3 filesystem. It is stored encrypted (AES-256-GCM)
- `s3_endpoint`, specifies s3 endpoint (server) different from AWS
- `s3_storage_class`
These properties are stored inside the data provider.

26
cmd/portable.go
View file

@ -6,6 +6,7 @@ import (
"github.com/drakkan/sftpgo/dataprovider"
"github.com/drakkan/sftpgo/service"
"github.com/drakkan/sftpgo/sftpd"
"github.com/drakkan/sftpgo/vfs"
"github.com/spf13/cobra"
)
@ -20,6 +21,13 @@ var (
portablePublicKeys []string
portablePermissions []string
portableSSHCommands []string
portableFsProvider int
portableS3Bucket string
portableS3Region string
portableS3AccessKey string
portableS3AccessSecret string
portableS3Endpoint string
portableS3StorageClass string
portableCmd = &cobra.Command{
Use: "portable",
Short: "Serve a single directory",
@ -53,6 +61,17 @@ Please take a look at the usage below to customize the serving parameters`,
Permissions: permissions,
HomeDir: portableDir,
Status: 1,
FsConfig: dataprovider.Filesystem{
Provider: portableFsProvider,
S3Config: vfs.S3FsConfig{
Bucket: portableS3Bucket,
Region: portableS3Region,
AccessKey: portableS3AccessKey,
AccessSecret: portableS3AccessSecret,
Endpoint: portableS3Endpoint,
StorageClass: portableS3StorageClass,
},
},
},
}
if err := service.StartPortableMode(portableSFTPDPort, portableSSHCommands, portableAdvertiseService,
@ -79,5 +98,12 @@ func init() {
"Advertise SFTP service using multicast DNS")
portableCmd.Flags().BoolVarP(&portableAdvertiseCredentials, "advertise-credentials", "C", false,
"If the SFTP service is advertised via multicast DNS this flag allows to put username/password inside the advertised TXT record")
portableCmd.Flags().IntVarP(&portableFsProvider, "fs-provider", "f", 0, "0 means local filesystem, 1 S3 compatible")
portableCmd.Flags().StringVar(&portableS3Bucket, "s3-bucket", "", "")
portableCmd.Flags().StringVar(&portableS3Region, "s3-region", "", "")
portableCmd.Flags().StringVar(&portableS3AccessKey, "s3-access-key", "", "")
portableCmd.Flags().StringVar(&portableS3AccessSecret, "s3-access-secret", "", "")
portableCmd.Flags().StringVar(&portableS3Endpoint, "s3-endpoint", "", "")
portableCmd.Flags().StringVar(&portableS3StorageClass, "s3-storage-class", "", "")
rootCmd.AddCommand(portableCmd)
}

11
dataprovider/bolt.go
View file

@ -336,7 +336,7 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str
if offset == 0 {
user, err := p.userExists(username)
if err == nil {
users = append(users, getUserNoCredentials(&user))
users = append(users, HideUserSensitiveData(&user))
}
}
return users, err
@ -357,7 +357,7 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str
var user User
err = json.Unmarshal(v, &user)
if err == nil {
users = append(users, getUserNoCredentials(&user))
users = append(users, HideUserSensitiveData(&user))
}
if len(users) >= limit {
break
@ -372,7 +372,7 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str
var user User
err = json.Unmarshal(v, &user)
if err == nil {
users = append(users, getUserNoCredentials(&user))
users = append(users, HideUserSensitiveData(&user))
}
if len(users) >= limit {
break
@ -388,11 +388,6 @@ func (p BoltProvider) close() error {
return p.dbHandle.Close()
}
func getUserNoCredentials(user *User) User {
user.Password = ""
return *user
}
// itob returns an 8-byte big endian representation of v.
func itob(v int64) []byte {
b := make([]byte, 8)

37
dataprovider/dataprovider.go
View file

@ -34,6 +34,7 @@ import (
"github.com/drakkan/sftpgo/logger"
"github.com/drakkan/sftpgo/metrics"
"github.com/drakkan/sftpgo/utils"
"github.com/drakkan/sftpgo/vfs"
unixcrypt "github.com/nathanaelle/password"
)
@ -479,6 +480,27 @@ func validateFilters(user *User) error {
return nil
}
func validateFilesystemConfig(user *User) error {
if user.FsConfig.Provider == 1 {
err := vfs.ValidateS3FsConfig(&user.FsConfig.S3Config)
if err != nil {
return &ValidationError{err: fmt.Sprintf("Could not validate s3config: %v", err)}
}
vals := strings.Split(user.FsConfig.S3Config.AccessSecret, "$")
if !strings.HasPrefix(user.FsConfig.S3Config.AccessSecret, "$aes$") || len(vals) != 4 {
accessSecret, err := utils.EncryptData(user.FsConfig.S3Config.AccessSecret)
if err != nil {
return &ValidationError{err: fmt.Sprintf("Could encrypt s3 access secret: %v", err)}
}
user.FsConfig.S3Config.AccessSecret = accessSecret
}
return nil
}
user.FsConfig.Provider = 0
user.FsConfig.S3Config = vfs.S3FsConfig{}
return nil
}
func validateUser(user *User) error {
buildUserHomeDir(user)
if len(user.Username) == 0 || len(user.HomeDir) == 0 {
@ -493,6 +515,9 @@ func validateUser(user *User) error {
if err := validatePermissions(user); err != nil {
return err
}
if err := validateFilesystemConfig(user); err != nil {
return err
}
if user.Status < 0 || user.Status > 1 {
return &ValidationError{err: fmt.Sprintf("invalid user status: %v", user.Status)}
}
@ -645,6 +670,15 @@ func comparePbkdf2PasswordAndHash(password, hashedPassword string) (bool, error)
return subtle.ConstantTimeCompare(buf, []byte(expected)) == 1, nil
}
// HideUserSensitiveData hides user sensitive data
func HideUserSensitiveData(user *User) User {
user.Password = ""
if user.FsConfig.Provider == 1 {
user.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(user.FsConfig.S3Config.AccessSecret)
}
return *user
}
func getSSLMode() string {
if config.Driver == PGSQLDataProviderName {
if config.SSLMode == 0 {
@ -772,8 +806,7 @@ func executeAction(operation string, user User) {
return
}
}
// hide the hashed password
user.Password = ""
HideUserSensitiveData(&user)
if len(config.Actions.Command) > 0 && filepath.IsAbs(config.Actions.Command) {
// we are in a goroutine but if we have to send an HTTP notification we don't want to wait for the
// end of the command

9
dataprovider/memory.go
View file

@ -244,8 +244,7 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s
if offset == 0 {
user, err := p.userExistsInternal(username)
if err == nil {
user.Password = ""
users = append(users, user)
users = append(users, HideUserSensitiveData(&user))
}
}
return users, err
@ -258,8 +257,7 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s
continue
}
user := p.dbHandle.users[username]
user.Password = ""
users = append(users, user)
users = append(users, HideUserSensitiveData(&user))
if len(users) >= limit {
break
}
@ -272,8 +270,7 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s
}
username := p.dbHandle.usernames[i]
user := p.dbHandle.users[username]
user.Password = ""
users = append(users, user)
users = append(users, HideUserSensitiveData(&user))
if len(users) >= limit {
break
}

33
dataprovider/sqlcommon.go
View file

@ -162,8 +162,13 @@ func sqlCommonAddUser(user User, dbHandle *sql.DB) error {
if err != nil {
return err
}
fsConfig, err := user.GetFsConfigAsJSON()
if err != nil {
return err
}
_, err = stmt.Exec(user.Username, user.Password, string(publicKeys), user.HomeDir, user.UID, user.GID, user.MaxSessions, user.QuotaSize,
user.QuotaFiles, string(permissions), user.UploadBandwidth, user.DownloadBandwidth, user.Status, user.ExpirationDate, string(filters))
user.QuotaFiles, string(permissions), user.UploadBandwidth, user.DownloadBandwidth, user.Status, user.ExpirationDate, string(filters),
string(fsConfig))
return err
}
@ -191,9 +196,13 @@ func sqlCommonUpdateUser(user User, dbHandle *sql.DB) error {
if err != nil {
return err
}
fsConfig, err := user.GetFsConfigAsJSON()
if err != nil {
return err
}
_, err = stmt.Exec(user.Password, string(publicKeys), user.HomeDir, user.UID, user.GID, user.MaxSessions, user.QuotaSize,
user.QuotaFiles, string(permissions), user.UploadBandwidth, user.DownloadBandwidth, user.Status, user.ExpirationDate,
string(filters), user.ID)
string(filters), string(fsConfig), user.ID)
return err
}
@ -253,10 +262,8 @@ func sqlCommonGetUsers(limit int, offset int, order string, username string, dbH
defer rows.Close()
for rows.Next() {
u, err := getUserFromDbRow(nil, rows)
// hide password
if err == nil {
u.Password = ""
users = append(users, u)
users = append(users, HideUserSensitiveData(&u))
} else {
break
}
@ -272,16 +279,17 @@ func getUserFromDbRow(row *sql.Row, rows *sql.Rows) (User, error) {
var password sql.NullString
var publicKey sql.NullString
var filters sql.NullString
var fsConfig sql.NullString
var err error
if row != nil {
err = row.Scan(&user.ID, &user.Username, &password, &publicKey, &user.HomeDir, &user.UID, &user.GID, &user.MaxSessions,
&user.QuotaSize, &user.QuotaFiles, &permissions, &user.UsedQuotaSize, &user.UsedQuotaFiles, &user.LastQuotaUpdate,
&user.UploadBandwidth, &user.DownloadBandwidth, &user.ExpirationDate, &user.LastLogin, &user.Status, &filters)
&user.UploadBandwidth, &user.DownloadBandwidth, &user.ExpirationDate, &user.LastLogin, &user.Status, &filters, &fsConfig)
} else {
err = rows.Scan(&user.ID, &user.Username, &password, &publicKey, &user.HomeDir, &user.UID, &user.GID, &user.MaxSessions,
&user.QuotaSize, &user.QuotaFiles, &permissions, &user.UsedQuotaSize, &user.UsedQuotaFiles, &user.LastQuotaUpdate,
&user.UploadBandwidth, &user.DownloadBandwidth, &user.ExpirationDate, &user.LastLogin, &user.Status, &filters)
&user.UploadBandwidth, &user.DownloadBandwidth, &user.ExpirationDate, &user.LastLogin, &user.Status, &filters, &fsConfig)
}
if err != nil {
if err == sql.ErrNoRows {
@ -326,5 +334,16 @@ func getUserFromDbRow(row *sql.Row, rows *sql.Rows) (User, error) {
DeniedIP: []string{},
}
}
if fsConfig.Valid {
var fs Filesystem
err = json.Unmarshal([]byte(fsConfig.String), &fs)
if err == nil {
user.FsConfig = fs
}
} else {
user.FsConfig = Filesystem{
Provider: 0,
}
}
return user, err
}

17
dataprovider/sqlqueries.go
View file

@ -4,7 +4,7 @@ import "fmt"
const (
selectUserFields = "id,username,password,public_keys,home_dir,uid,gid,max_sessions,quota_size,quota_files,permissions,used_quota_size," +
"used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,expiration_date,last_login,status,filters"
"used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,expiration_date,last_login,status,filters,filesystem"
)
func getSQLPlaceholders() []string {
@ -60,19 +60,20 @@ func getQuotaQuery() string {
func getAddUserQuery() string {
return fmt.Sprintf(`INSERT INTO %v (username,password,public_keys,home_dir,uid,gid,max_sessions,quota_size,quota_files,permissions,
used_quota_size,used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,status,last_login,expiration_date,filters)
VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,0,0,0,%v,%v,%v,0,%v,%v)`, config.UsersTable, sqlPlaceholders[0], sqlPlaceholders[1],
used_quota_size,used_quota_files,last_quota_update,upload_bandwidth,download_bandwidth,status,last_login,expiration_date,filters,
filesystem)
VALUES (%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,0,0,0,%v,%v,%v,0,%v,%v,%v)`, config.UsersTable, sqlPlaceholders[0], sqlPlaceholders[1],
sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7],
sqlPlaceholders[8], sqlPlaceholders[9], sqlPlaceholders[10], sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13],
sqlPlaceholders[14])
sqlPlaceholders[14], sqlPlaceholders[15])
}
func getUpdateUserQuery() string {
return fmt.Sprintf(`UPDATE %v SET password=%v,public_keys=%v,home_dir=%v,uid=%v,gid=%v,max_sessions=%v,quota_size=%v,
quota_files=%v,permissions=%v,upload_bandwidth=%v,download_bandwidth=%v,status=%v,expiration_date=%v,filters=%v WHERE id = %v`,
config.UsersTable, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3], sqlPlaceholders[4],
sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9], sqlPlaceholders[10],
sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13], sqlPlaceholders[14])
quota_files=%v,permissions=%v,upload_bandwidth=%v,download_bandwidth=%v,status=%v,expiration_date=%v,filters=%v,filesystem=%v
WHERE id = %v`, config.UsersTable, sqlPlaceholders[0], sqlPlaceholders[1], sqlPlaceholders[2], sqlPlaceholders[3],
sqlPlaceholders[4], sqlPlaceholders[5], sqlPlaceholders[6], sqlPlaceholders[7], sqlPlaceholders[8], sqlPlaceholders[9],
sqlPlaceholders[10], sqlPlaceholders[11], sqlPlaceholders[12], sqlPlaceholders[13], sqlPlaceholders[14], sqlPlaceholders[15])
}
func getDeleteUserQuery() string {

52
dataprovider/user.go
View file

@ -7,10 +7,10 @@ import (
"path"
"path/filepath"
"strconv"
"strings"
"github.com/drakkan/sftpgo/logger"
"github.com/drakkan/sftpgo/utils"
"github.com/drakkan/sftpgo/vfs"
)
// Available permissions for SFTP users
@ -53,6 +53,13 @@ type UserFilters struct {
DeniedIP []string `json:"denied_ip"`
}
// Filesystem defines cloud storage filesystem details
type Filesystem struct {
// 0 local filesystem, 1 Amazon S3 compatible
Provider int `json:"provider"`
S3Config vfs.S3FsConfig `json:"s3config,omitempty"`
}
// User defines an SFTP user
type User struct {
// Database unique identifier
@ -98,6 +105,16 @@ type User struct {
LastLogin int64 `json:"last_login"`
// Additional restrictions
Filters UserFilters `json:"filters"`
// Filesystem configuration details
FsConfig Filesystem `json:"filesystem"`
}
// GetFilesystem returns the filesystem for this user
func (u *User) GetFilesystem(connectionID string) (vfs.Fs, error) {
if u.FsConfig.Provider == 1 {
return vfs.NewS3Fs(connectionID, u.GetHomeDir(), u.FsConfig.S3Config)
}
return vfs.NewOsFs(connectionID), nil
}
// GetPermissionsForPath returns the permissions for the given path.
@ -211,6 +228,11 @@ func (u *User) GetFiltersAsJSON() ([]byte, error) {
return json.Marshal(u.Filters)
}
// GetFsConfigAsJSON returns the filesystem config as json byte array
func (u *User) GetFsConfigAsJSON() ([]byte, error) {
return json.Marshal(u.FsConfig)
}
// GetUID returns a validate uid, suitable for use with os.Chown
func (u *User) GetUID() int {
if u.UID <= 0 || u.UID > 65535 {
@ -237,19 +259,6 @@ func (u *User) HasQuotaRestrictions() bool {
return u.QuotaFiles > 0 || u.QuotaSize > 0
}
// GetRelativePath returns the path for a file relative to the user's home dir.
// This is the path as seen by SFTP users
func (u *User) GetRelativePath(path string) string {
rel, err := filepath.Rel(u.GetHomeDir(), filepath.Clean(path))
if err != nil {
return ""
}
if rel == "." || strings.HasPrefix(rel, "..") {
rel = ""
}
return "/" + filepath.ToSlash(rel)
}
// GetQuotaSummary returns used quota and limits if defined
func (u *User) GetQuotaSummary() string {
var result string
@ -319,6 +328,9 @@ func (u *User) GetInfoString() string {
t := utils.GetTimeFromMsecSinceEpoch(u.LastLogin)
result += fmt.Sprintf("Last login: %v ", t.Format("2006-01-02 15:04:05")) // YYYY-MM-DD HH:MM:SS
}
if u.FsConfig.Provider == 1 {
result += fmt.Sprintf("Storage: S3")
}
if len(u.PublicKeys) > 0 {
result += fmt.Sprintf("Public keys: %v ", len(u.PublicKeys))
}
@ -387,6 +399,17 @@ func (u *User) getACopy() User {
copy(filters.AllowedIP, u.Filters.AllowedIP)
filters.DeniedIP = make([]string, len(u.Filters.DeniedIP))
copy(filters.DeniedIP, u.Filters.DeniedIP)
fsConfig := Filesystem{
Provider: u.FsConfig.Provider,
S3Config: vfs.S3FsConfig{
Bucket: u.FsConfig.S3Config.Bucket,
Region: u.FsConfig.S3Config.Region,
AccessKey: u.FsConfig.S3Config.AccessKey,
AccessSecret: u.FsConfig.S3Config.AccessSecret,
Endpoint: u.FsConfig.S3Config.Endpoint,
StorageClass: u.FsConfig.S3Config.StorageClass,
},
}
return User{
ID: u.ID,
@ -409,6 +432,7 @@ func (u *User) getACopy() User {
ExpirationDate: u.ExpirationDate,
LastLogin: u.LastLogin,
Filters: filters,
FsConfig: fsConfig,
}
}

4
go.mod
View file

@ -4,7 +4,9 @@ go 1.13
require (
github.com/alexedwards/argon2id v0.0.0-20190612080829-01a59b2b8802
github.com/aws/aws-sdk-go v1.28.3
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/eikenb/pipeat v0.0.0-20190316224601-fb1f3a9aa29f
github.com/go-chi/chi v4.0.2+incompatible
github.com/go-chi/render v1.0.1
github.com/go-sql-driver/mysql v1.5.0
@ -24,3 +26,5 @@ require (
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f
gopkg.in/natefinch/lumberjack.v2 v2.0.0
)
replace github.com/eikenb/pipeat v0.0.0-20190316224601-fb1f3a9aa29f => github.com/drakkan/pipeat v0.0.0-20200114135659-fac71c64d75d

18
go.sum
View file

@ -1,5 +1,4 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@ -9,6 +8,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF
github.com/alexedwards/argon2id v0.0.0-20190612080829-01a59b2b8802 h1:RwMM1q/QSKYIGbHfOkf843hE8sSUJtf1dMwFPtEDmm0=
github.com/alexedwards/argon2id v0.0.0-20190612080829-01a59b2b8802/go.mod h1:4dsm7ufQm1Gwl8S2ss57u+2J7KlxIL2QUmFGlGtWogY=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/aws/aws-sdk-go v1.28.3 h1:FnkDp+fz4JHWUW3Ust2Wh89RpdGif077Wjis/sMrGKM=
github.com/aws/aws-sdk-go v1.28.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@ -32,6 +33,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/drakkan/pipeat v0.0.0-20200114135659-fac71c64d75d h1:+k0oy9bBY9dXlKHriYg6crXpwIrtM1rCrlUehmc/F3M=
github.com/drakkan/pipeat v0.0.0-20200114135659-fac71c64d75d/go.mod h1:wNYvIpR5rIhoezOYcpxcXz4HbIEOu7A45EqlQCA+h+w=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@ -57,10 +60,8 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/grandcat/zeroconf v0.0.0-20190424104450-85eadb44205c h1:svzQzfVE9t7Y1CGULS5PsMWs4/H4Au/ZTJzU/0CKgqc=
@ -70,12 +71,12 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@ -84,10 +85,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@ -148,9 +147,7 @@ github.com/rs/zerolog v1.17.2/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@ -205,7 +202,6 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@ -214,7 +210,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -244,7 +239,6 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

19
httpd/api_maintenance.go
View file

@ -13,6 +13,7 @@ import (
"github.com/drakkan/sftpgo/dataprovider"
"github.com/drakkan/sftpgo/logger"
"github.com/drakkan/sftpgo/sftpd"
)
func dumpData(w http.ResponseWriter, r *http.Request) {
@ -103,10 +104,16 @@ func loadData(w http.ResponseWriter, r *http.Request) {
u, err := dataprovider.UserExists(dataProvider, user.Username)
if err == nil {
user.ID = u.ID
user.LastLogin = u.LastLogin
user.UsedQuotaSize = u.UsedQuotaSize
user.UsedQuotaFiles = u.UsedQuotaFiles
err = dataprovider.UpdateUser(dataProvider, user)
user.Password = "[redacted]"
logger.Debug(logSender, "", "restoring existing user: %+v, dump file: %#v, error: %v", user, inputFile, err)
} else {
user.LastLogin = 0
user.UsedQuotaSize = 0
user.UsedQuotaFiles = 0
err = dataprovider.AddUser(dataProvider, user)
user.Password = "[redacted]"
logger.Debug(logSender, "", "adding new user: %+v, dump file: %#v, error: %v", user, inputFile, err)
@ -115,11 +122,17 @@ func loadData(w http.ResponseWriter, r *http.Request) {
sendAPIResponse(w, r, err, "", getRespStatus(err))
return
}
if scanQuota == 1 || (scanQuota == 2 && user.HasQuotaRestrictions()) {
logger.Debug(logSender, "", "starting quota scan for restored user: %#v", user.Username)
doQuotaScan(user)
if needQuotaScan(scanQuota, &user) {
if sftpd.AddQuotaScan(user.Username) {
logger.Debug(logSender, "", "starting quota scan for restored user: %#v", user.Username)
go doQuotaScan(user)
}
}
}
logger.Debug(logSender, "", "backup restored, users: %v", len(dump.Users))
sendAPIResponse(w, r, err, "Data restored", http.StatusOK)
}
func needQuotaScan(scanQuota int, user *dataprovider.User) bool {
return scanQuota == 1 || (scanQuota == 2 && user.HasQuotaRestrictions())
}

32
httpd/api_quota.go
View file

@ -6,7 +6,6 @@ import (
"github.com/drakkan/sftpgo/dataprovider"
"github.com/drakkan/sftpgo/logger"
"github.com/drakkan/sftpgo/sftpd"
"github.com/drakkan/sftpgo/utils"
"github.com/go-chi/render"
)
@ -26,26 +25,27 @@ func startQuotaScan(w http.ResponseWriter, r *http.Request) {
sendAPIResponse(w, r, err, "", http.StatusNotFound)
return
}
if doQuotaScan(user) {
if sftpd.AddQuotaScan(user.Username) {
go doQuotaScan(user)
sendAPIResponse(w, r, err, "Scan started", http.StatusCreated)
} else {
sendAPIResponse(w, r, err, "Another scan is already in progress", http.StatusConflict)
}
}
func doQuotaScan(user dataprovider.User) bool {
result := sftpd.AddQuotaScan(user.Username)
if result {
go func() {
numFiles, size, _, err := utils.ScanDirContents(user.HomeDir)
if err != nil {
logger.Warn(logSender, "", "error scanning user home dir %#v: %v", user.HomeDir, err)
} else {
err := dataprovider.UpdateUserQuota(dataProvider, user, numFiles, size, true)
logger.Debug(logSender, "", "user home dir scanned, user: %#v, dir: %#v, error: %v", user.Username, user.HomeDir, err)
}
sftpd.RemoveQuotaScan(user.Username)
}()
func doQuotaScan(user dataprovider.User) error {
defer sftpd.RemoveQuotaScan(user.Username)
fs, err := user.GetFilesystem("")
if err != nil {
logger.Warn(logSender, "", "unable scan quota for user %#v error creating filesystem: %v", user.Username, err)
return err
}
return result
numFiles, size, err := fs.ScanDirContents(user.HomeDir)
if err != nil {
logger.Warn(logSender, "", "error scanning user home dir %#v: %v", user.HomeDir, err)
} else {
err = dataprovider.UpdateUserQuota(dataProvider, user, numFiles, size, true)
logger.Debug(logSender, "", "user home dir scanned, user: %#v, dir: %#v, error: %v", user.Username, user.HomeDir, err)
}
return err
}

18
httpd/api_user.go
View file

@ -6,6 +6,7 @@ import (
"strconv"
"github.com/drakkan/sftpgo/dataprovider"
"github.com/drakkan/sftpgo/utils"
"github.com/go-chi/chi"
"github.com/go-chi/render"
)
@ -63,8 +64,7 @@ func getUserByID(w http.ResponseWriter, r *http.Request) {
}
user, err := dataprovider.GetUserByID(dataProvider, userID)
if err == nil {
user.Password = ""
render.JSON(w, r, user)
render.JSON(w, r, dataprovider.HideUserSensitiveData(&user))
} else if _, ok := err.(*dataprovider.RecordNotFoundError); ok {
sendAPIResponse(w, r, err, "", http.StatusNotFound)
} else {
@ -83,8 +83,7 @@ func addUser(w http.ResponseWriter, r *http.Request) {
if err == nil {
user, err = dataprovider.UserExists(dataProvider, user.Username)
if err == nil {
user.Password = ""
render.JSON(w, r, user)
render.JSON(w, r, dataprovider.HideUserSensitiveData(&user))
} else {
sendAPIResponse(w, r, err, "", http.StatusInternalServerError)
}
@ -102,6 +101,10 @@ func updateUser(w http.ResponseWriter, r *http.Request) {
}
user, err := dataprovider.GetUserByID(dataProvider, userID)
oldPermissions := user.Permissions
oldS3AccessSecret := ""
if user.FsConfig.Provider == 1 {
oldS3AccessSecret = user.FsConfig.S3Config.AccessSecret
}
user.Permissions = make(map[string][]string)
if _, ok := err.(*dataprovider.RecordNotFoundError); ok {
sendAPIResponse(w, r, err, "", http.StatusNotFound)
@ -119,6 +122,13 @@ func updateUser(w http.ResponseWriter, r *http.Request) {
if len(user.Permissions) == 0 {
user.Permissions = oldPermissions
}
// we use the new access secret if different from the old one and not empty
if user.FsConfig.Provider == 1 {
if utils.RemoveDecryptionKey(oldS3AccessSecret) == user.FsConfig.S3Config.AccessSecret ||
len(user.FsConfig.S3Config.AccessSecret) == 0 {
user.FsConfig.S3Config.AccessSecret = oldS3AccessSecret
}
}
if user.ID != userID {
sendAPIResponse(w, r, err, "user ID in request body does not match user ID in path parameter", http.StatusBadRequest)
return

56
httpd/api_utils.go
View file

@ -406,10 +406,66 @@ func checkUser(expected *dataprovider.User, actual *dataprovider.User) error {
if err := compareUserFilters(expected, actual); err != nil {
return err
}
if err := compareUserFsConfig(expected, actual); err != nil {
return err
}
return compareEqualsUserFields(expected, actual)
}
func compareUserFsConfig(expected *dataprovider.User, actual *dataprovider.User) error {
if expected.FsConfig.Provider != actual.FsConfig.Provider {
return errors.New("Fs provider mismatch")
}
if expected.FsConfig.S3Config.Bucket != actual.FsConfig.S3Config.Bucket {
return errors.New("S3 bucket mismatch")
}
if expected.FsConfig.S3Config.Region != actual.FsConfig.S3Config.Region {
return errors.New("S3 region mismatch")
}
if expected.FsConfig.S3Config.AccessKey != actual.FsConfig.S3Config.AccessKey {
return errors.New("S3 access key mismatch")
}
if err := checkS3AccessSecret(expected.FsConfig.S3Config.AccessSecret, actual.FsConfig.S3Config.AccessSecret); err != nil {
return err
}
if expected.FsConfig.S3Config.Endpoint != actual.FsConfig.S3Config.Endpoint {
return errors.New("S3 endpoint mismatch")
}
if expected.FsConfig.S3Config.StorageClass != actual.FsConfig.S3Config.StorageClass {
return errors.New("S3 storage class mismatch")
}
return nil
}
func checkS3AccessSecret(expectedAccessSecret, actualAccessSecret string) error {
if len(expectedAccessSecret) > 0 {
vals := strings.Split(expectedAccessSecret, "$")
if strings.HasPrefix(expectedAccessSecret, "$aes$") && len(vals) == 4 {
expectedAccessSecret = utils.RemoveDecryptionKey(expectedAccessSecret)
if expectedAccessSecret != actualAccessSecret {
return fmt.Errorf("S3 access secret mismatch, expected: %v", expectedAccessSecret)
}
} else {
// here we check that actualAccessSecret is aes encrypted without the nonce
parts := strings.Split(actualAccessSecret, "$")
if !strings.HasPrefix(actualAccessSecret, "$aes$") || len(parts) != 3 {
return errors.New("Invalid S3 access secret")
}
if len(parts) == len(vals) {
if expectedAccessSecret != actualAccessSecret {
return errors.New("S3 encrypted access secret mismatch")
}
}
}
} else {
if expectedAccessSecret != actualAccessSecret {
return errors.New("S3 access secret mismatch")
}
}
return nil
}
func compareUserFilters(expected *dataprovider.User, actual *dataprovider.User) error {
if len(expected.Filters.AllowedIP) != len(actual.Filters.AllowedIP) {
return errors.New("AllowedIP mismatch")

137
httpd/httpd_test.go
View file

@ -242,6 +242,16 @@ func TestAddUserInvalidFilters(t *testing.T) {
}
}
func TestAddUserInvalidFsConfig(t *testing.T) {
u := getTestUser()
u.FsConfig.Provider = 1
u.FsConfig.S3Config.Bucket = ""
_, _, err := httpd.AddUser(u, http.StatusBadRequest)
if err != nil {
t.Errorf("unexpected error adding user with invalid fs config: %v", err)
}
}
func TestUserPublicKey(t *testing.T) {
u := getTestUser()
invalidPubKey := "invalid"
@ -299,6 +309,48 @@ func TestUpdateUser(t *testing.T) {
}
}
func TestUserS3Config(t *testing.T) {
user, _, err := httpd.AddUser(getTestUser(), http.StatusOK)
if err != nil {
t.Errorf("unable to add user: %v", err)
}
user.FsConfig.Provider = 1
user.FsConfig.S3Config.Bucket = "test"
user.FsConfig.S3Config.Region = "us-east-1"
user.FsConfig.S3Config.AccessKey = "Server-Access-Key"
user.FsConfig.S3Config.AccessSecret = "Server-Access-Secret"
user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000"
user, _, err = httpd.UpdateUser(user, http.StatusOK)
if err != nil {
t.Errorf("unable to update user: %v", err)
}
_, err = httpd.RemoveUser(user, http.StatusOK)
if err != nil {
t.Errorf("unable to remove: %v", err)
}
user.Password = defaultPassword
user.ID = 0
secret, _ := utils.EncryptData("Server-Access-Secret")
user.FsConfig.S3Config.AccessSecret = secret
user, _, err = httpd.AddUser(user, http.StatusOK)
if err != nil {
t.Errorf("unable to add user: %v", err)
}
user.FsConfig.Provider = 1
user.FsConfig.S3Config.Bucket = "test1"
user.FsConfig.S3Config.Region = "us-east-1"
user.FsConfig.S3Config.AccessKey = "Server-Access-Key1"
user.FsConfig.S3Config.Endpoint = "http://localhost:9000"
user, _, err = httpd.UpdateUser(user, http.StatusOK)
if err != nil {
t.Errorf("unable to update user: %v", err)
}
_, err = httpd.RemoveUser(user, http.StatusOK)
if err != nil {
t.Errorf("unable to remove: %v", err)
}
}
func TestUpdateUserNoCredentials(t *testing.T) {
user, _, err := httpd.AddUser(getTestUser(), http.StatusOK)
if err != nil {
@ -1398,6 +1450,91 @@ func TestWebUserUpdateMock(t *testing.T) {
checkResponseCode(t, http.StatusOK, rr.Code)
}
func TestWebUserS3Mock(t *testing.T) {
user := getTestUser()
userAsJSON := getUserAsJSON(t, user)
req, _ := http.NewRequest(http.MethodPost, userPath, bytes.NewBuffer(userAsJSON))
rr := executeRequest(req)
checkResponseCode(t, http.StatusOK, rr.Code)
err := render.DecodeJSON(rr.Body, &user)
if err != nil {
t.Errorf("Error get user: %v", err)
}
user.FsConfig.Provider = 1
user.FsConfig.S3Config.Bucket = "test"
user.FsConfig.S3Config.Region = "eu-west-1"
user.FsConfig.S3Config.AccessKey = "access-key"
user.FsConfig.S3Config.AccessSecret = "access-secret"
user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?a=b"
user.FsConfig.S3Config.StorageClass = "Standard"
form := make(url.Values)
form.Set("username", user.Username)
form.Set("home_dir", user.HomeDir)
form.Set("uid", "0")
form.Set("gid", strconv.FormatInt(int64(user.GID), 10))
form.Set("max_sessions", strconv.FormatInt(int64(user.MaxSessions), 10))
form.Set("quota_size", strconv.FormatInt(user.QuotaSize, 10))
form.Set("quota_files", strconv.FormatInt(int64(user.QuotaFiles), 10))
form.Set("upload_bandwidth", "0")
form.Set("download_bandwidth", "0")
form.Set("permissions", "*")
form.Set("sub_dirs_permissions", "")
form.Set("status", strconv.Itoa(user.Status))
form.Set("expiration_date", "2020-01-01 00:00:00")
form.Set("allowed_ip", "")
form.Set("denied_ip", "")
form.Set("fs_provider", "1")
form.Set("s3_bucket", user.FsConfig.S3Config.Bucket)
form.Set("s3_region", user.FsConfig.S3Config.Region)
form.Set("s3_access_key", user.FsConfig.S3Config.AccessKey)
form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret)
form.Set("s3_storage_class", user.FsConfig.S3Config.StorageClass)
form.Set("s3_endpoint", user.FsConfig.S3Config.Endpoint)
req, _ = http.NewRequest(http.MethodPost, webUserPath+"/"+strconv.FormatInt(user.ID, 10), strings.NewReader(form.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
rr = executeRequest(req)
checkResponseCode(t, http.StatusSeeOther, rr.Code)
req, _ = http.NewRequest(http.MethodGet, userPath+"?limit=1&offset=0&order=ASC&username="+user.Username, nil)
rr = executeRequest(req)
checkResponseCode(t, http.StatusOK, rr.Code)
var users []dataprovider.User
err = render.DecodeJSON(rr.Body, &users)
if err != nil {
t.Errorf("Error decoding users: %v", err)
}
if len(users) != 1 {
t.Errorf("1 user is expected")
}
updateUser := users[0]
if updateUser.ExpirationDate != 1577836800000 {
t.Errorf("invalid expiration date: %v", updateUser.ExpirationDate)
}
if updateUser.FsConfig.Provider != user.FsConfig.Provider {
t.Error("fs provider mismatch")
}
if updateUser.FsConfig.S3Config.Bucket != user.FsConfig.S3Config.Bucket {
t.Error("s3 bucket mismatch")
}
if updateUser.FsConfig.S3Config.Region != user.FsConfig.S3Config.Region {
t.Error("s3 region mismatch")
}
if updateUser.FsConfig.S3Config.AccessKey != user.FsConfig.S3Config.AccessKey {
t.Error("s3 access key mismatch")
}
if !strings.HasPrefix(updateUser.FsConfig.S3Config.AccessSecret, "$aes$") {
t.Error("s3 access secret is not encrypted")
}
if updateUser.FsConfig.S3Config.StorageClass != user.FsConfig.S3Config.StorageClass {
t.Error("s3 storage class mismatch")
}
if updateUser.FsConfig.S3Config.Endpoint != user.FsConfig.S3Config.Endpoint {
t.Error("s3 endpoint mismatch")
}
req, _ = http.NewRequest(http.MethodDelete, userPath+"/"+strconv.FormatInt(user.ID, 10), nil)
rr = executeRequest(req)
checkResponseCode(t, http.StatusOK, rr.Code)
}
func TestProviderClosedMock(t *testing.T) {
if providerDriverName == dataprovider.BoltDataProviderName {
t.Skip("skipping test provider errors for bolt provider")

91
httpd/internal_test.go
View file

@ -6,9 +6,12 @@ import (
"html/template"
"net/http"
"net/http/httptest"
"os"
"testing"
"github.com/drakkan/sftpgo/dataprovider"
"github.com/drakkan/sftpgo/sftpd"
"github.com/drakkan/sftpgo/utils"
"github.com/go-chi/chi"
)
@ -120,6 +123,13 @@ func TestCheckUser(t *testing.T) {
if err == nil {
t.Errorf("DeniedIP contents are not equal")
}
expected.Filters.DeniedIP = []string{}
actual.Filters.DeniedIP = []string{}
actual.FsConfig.Provider = 1
err = checkUser(expected, actual)
if err == nil {
t.Errorf("Fs providers are not equal")
}
}
func TestCompareUserFields(t *testing.T) {
@ -200,6 +210,72 @@ func TestCompareUserFields(t *testing.T) {
}
}
func TestCompareUserFsConfig(t *testing.T) {
expected := &dataprovider.User{}
actual := &dataprovider.User{}
expected.FsConfig.Provider = 1
err := compareUserFsConfig(expected, actual)
if err == nil {
t.Errorf("Provider does not match")
}
expected.FsConfig.Provider = 0
expected.FsConfig.S3Config.Bucket = "bucket"
err = compareUserFsConfig(expected, actual)
if err == nil {
t.Errorf("S3 bucket does not match")
}
expected.FsConfig.S3Config.Bucket = ""
expected.FsConfig.S3Config.Region = "region"
err = compareUserFsConfig(expected, actual)
if err == nil {
t.Errorf("S3 region does not match")
}
expected.FsConfig.S3Config.Region = ""
expected.FsConfig.S3Config.AccessKey = "access key"
err = compareUserFsConfig(expected, actual)
if err == nil {
t.Errorf("S3 access key does not match")
}
expected.FsConfig.S3Config.AccessKey = ""
actual.FsConfig.S3Config.AccessSecret = "access secret"
err = compareUserFsConfig(expected, actual)
if err == nil {
t.Errorf("S3 access secret does not match")
}
secret, _ := utils.EncryptData("access secret")
actual.FsConfig.S3Config.AccessSecret = ""
expected.FsConfig.S3Config.AccessSecret = secret
err = compareUserFsConfig(expected, actual)
if err == nil {
t.Errorf("S3 access secret does not match")
}
expected.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(secret)
actual.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(secret) + "a"
err = compareUserFsConfig(expected, actual)
if err == nil {
t.Errorf("S3 access secret does not match")
}
expected.FsConfig.S3Config.AccessSecret = "test"
actual.FsConfig.S3Config.AccessSecret = ""
err = compareUserFsConfig(expected, actual)
if err == nil {
t.Errorf("S3 access secret does not match")
}
expected.FsConfig.S3Config.AccessSecret = ""
actual.FsConfig.S3Config.AccessSecret = ""
expected.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/"
err = compareUserFsConfig(expected, actual)
if err == nil {
t.Errorf("S3 endpoint does not match")
}
expected.FsConfig.S3Config.Endpoint = ""
expected.FsConfig.S3Config.StorageClass = "Standard"
err = compareUserFsConfig(expected, actual)
if err == nil {
t.Errorf("S3 storage class does not match")
}
}
func TestApiCallsWithBadURL(t *testing.T) {
oldBaseURL := httpBaseURL
SetBaseURL(invalidURL)
@ -315,3 +391,18 @@ func TestRenderInvalidTemplate(t *testing.T) {
}
}
}
func TestQuotaScanInvalidFs(t *testing.T) {
user := dataprovider.User{
Username: "test",
HomeDir: os.TempDir(),
FsConfig: dataprovider.Filesystem{
Provider: 1,
},
}
sftpd.AddQuotaScan(user.Username)
err := doQuotaScan(user)
if err == nil {
t.Error("quota scan with bad fs must fail")
}
}

48
httpd/schema/openapi.yaml
View file

@ -710,6 +710,50 @@ components:
nullable: true
description: clients connecting from these IP/Mask are not allowed. Denied rules are evaluated before allowed ones
example: [ "172.16.0.0/16" ]
description: Additional restrictions
S3Config:
type: object
properties:
bucket:
type: string
minLength: 1
region:
type: string
minLength: 1
access_key:
type: string
minLength: 1
access_secret:
type: string
minLength: 1
description: the access secret is stored encrypted (AES-256-GCM)
endpoint:
type: string
description: optional endpoint
storage_class:
type: string
required:
- bucket
- region
- access_key
- access_secret
nullable: true
description: S3 Compatible Object Storage configuration details
FilesystemConfig:
type: object
properties:
provider:
type: integer
enum:
- 0
- 1
description: >
Providers:
* `0` - local filesystem
* `1` - S3 Compatible Object Storage
s3config:
$ref: '#/components/schemas/S3Config'
description: Storage filesystem details
User:
type: object
properties:
@ -799,8 +843,8 @@ components:
description: Last user login as unix timestamp in milliseconds
filters:
$ref: '#/components/schemas/UserFilters'
nullable: true
description: Additional restrictions
filesystem:
$ref: '#/components/schemas/FilesystemConfig'
Transfer:
type: object
properties:

19
httpd/web.go
View file

@ -224,6 +224,24 @@ func getFiltersFromUserPostFields(r *http.Request) dataprovider.UserFilters {
return filters
}
func getFsConfigFromUserPostFields(r *http.Request) dataprovider.Filesystem {
var fs dataprovider.Filesystem
provider, err := strconv.Atoi(r.Form.Get("fs_provider"))
if err != nil {
provider = 0
}
fs.Provider = provider
if fs.Provider == 1 {
fs.S3Config.Bucket = r.Form.Get("s3_bucket")
fs.S3Config.Region = r.Form.Get("s3_region")
fs.S3Config.AccessKey = r.Form.Get("s3_access_key")
fs.S3Config.AccessSecret = r.Form.Get("s3_access_secret")
fs.S3Config.Endpoint = r.Form.Get("s3_endpoint")
fs.S3Config.StorageClass = r.Form.Get("s3_storage_class")
}
return fs
}
func getUserFromPostFields(r *http.Request) (dataprovider.User, error) {
var user dataprovider.User
err := r.ParseForm()
@ -289,6 +307,7 @@ func getUserFromPostFields(r *http.Request) (dataprovider.User, error) {
Status: status,
ExpirationDate: expirationDateMillis,
Filters: getFiltersFromUserPostFields(r),
FsConfig: getFsConfigFromUserPostFields(r),
}
return user, err
}

23
scripts/README.md
View file

@ -44,7 +44,7 @@ Let's see a sample usage for each REST API.
Command:
```
python sftpgo_api_cli.py add-user test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 33 --gid 1000 --max-sessions 2 --quota-size 0 --quota-files 3 --permissions "list" "download" "upload" "delete" "rename" "create_dirs" "overwrite" --subdirs-permissions "/dir1:list,download" "/dir2:*" --upload-bandwidth 100 --download-bandwidth 60 --status 0 --expiration-date 2019-01-01 --allowed-ip "192.168.1.1/32"
python sftpgo_api_cli.py add-user test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 33 --gid 1000 --max-sessions 2 --quota-size 0 --quota-files 3 --permissions "list" "download" "upload" "delete" "rename" "create_dirs" "overwrite" --subdirs-permissions "/dir1:list,download" "/dir2:*" --upload-bandwidth 100 --download-bandwidth 60 --status 0 --expiration-date 2019-01-01 --allowed-ip "192.168.1.1/32" --fs S3 --s3-bucket test --s3-region eu-west-1 --s3-access-key accesskey --s3-access-secret secret --s3-endpoint "http://127.0.0.1:9000" --s3-storage-class Standard
```
Output:
@ -53,6 +53,17 @@ Output:
{
"download_bandwidth": 60,
"expiration_date": 1546297200000,
"filesystem": {
"provider": 1,
"s3config": {
"access_key": "accesskey",
"access_secret": "$aes$6c088ba12b0b261247c8cf331c46d9260b8e58002957d89ad1c0495e3af665cd0227",
"bucket": "test",
"endpoint": "http://127.0.0.1:9000",
"region": "eu-west-1",
"storage_class": "Standard"
}
},
"filters": {
"allowed_ip": [
"192.168.1.1/32"
@ -99,7 +110,7 @@ Output:
Command:
```
python sftpgo_api_cli.py update-user 9576 test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 0 --gid 33 --max-sessions 3 --quota-size 0 --quota-files 4 --permissions "*" --subdirs-permissions "/dir1:list,download,create_symlinks" --upload-bandwidth 90 --download-bandwidth 80 --status 1 --expiration-date "" --allowed-ip "" --denied-ip "192.168.1.0/24"
python sftpgo_api_cli.py update-user 9576 test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 0 --gid 33 --max-sessions 3 --quota-size 0 --quota-files 4 --permissions "*" --subdirs-permissions "/dir1:list,download,create_symlinks" --upload-bandwidth 90 --download-bandwidth 80 --status 1 --expiration-date "" --allowed-ip "" --denied-ip "192.168.1.0/24" --fs local
```
Output:
@ -126,6 +137,10 @@ Output:
{
"download_bandwidth": 80,
"expiration_date": 0,
"filesystem": {
"provider": 0,
"s3config": {}
},
"filters": {
"allowed_ip": [],
"denied_ip": [
@ -174,6 +189,10 @@ Output:
{
"download_bandwidth": 80,
"expiration_date": 0,
"filesystem": {
"provider": 0,
"s3config": {}
},
"filters": {
"allowed_ip": [],
"denied_ip": [

67
scripts/sftpgo_api_cli.py
View file

@ -70,9 +70,10 @@ class SFTPGoApiRequests:
else:
print(r.text)
def buildUserObject(self, user_id=0, username="", password="", public_keys=[], home_dir="", uid=0,
gid=0, max_sessions=0, quota_size=0, quota_files=0, permissions={}, upload_bandwidth=0,
download_bandwidth=0, status=1, expiration_date=0, allowed_ip=[], denied_ip=[]):
def buildUserObject(self, user_id=0, username="", password="", public_keys=[], home_dir="", uid=0, gid=0,
max_sessions=0, quota_size=0, quota_files=0, permissions={}, upload_bandwidth=0, download_bandwidth=0,
status=1, expiration_date=0, allowed_ip=[], denied_ip=[], fs_provider='local', s3_bucket='',
s3_region='', s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class=''):
user = {"id":user_id, "username":username, "uid":uid, "gid":gid,
"max_sessions":max_sessions, "quota_size":quota_size, "quota_files":quota_files,
"upload_bandwidth":upload_bandwidth, "download_bandwidth":download_bandwidth,
@ -90,6 +91,8 @@ class SFTPGoApiRequests:
user.update({"permissions":permissions})
if allowed_ip or denied_ip:
user.update({"filters":self.buildFilters(allowed_ip, denied_ip)})
user.update({"filesystem":self.buildFsConfig(fs_provider, s3_bucket, s3_region, s3_access_key,
s3_access_secret, s3_endpoint, s3_storage_class)})
return user
def buildPermissions(self, root_perms, subdirs_perms):
@ -113,16 +116,25 @@ class SFTPGoApiRequests:
filters = {}
if allowed_ip:
if len(allowed_ip) == 1 and not allowed_ip[0]:
filters.update({"allowed_ip":[]})
filters.update({'allowed_ip':[]})
else:
filters.update({"allowed_ip":allowed_ip})
filters.update({'allowed_ip':allowed_ip})
if denied_ip:
if len(denied_ip) == 1 and not denied_ip[0]:
filters.update({"denied_ip":[]})
filters.update({'denied_ip':[]})
else:
filters.update({"denied_ip":denied_ip})
filters.update({'denied_ip':denied_ip})
return filters
def buildFsConfig(self, fs_provider, s3_bucket, s3_region, s3_access_key, s3_access_secret, s3_endpoint,
s3_storage_class):
fs_config = {'provider':0}
if fs_provider == 'S3':
s3config = {'bucket':s3_bucket, 'region':s3_region, 'access_key':s3_access_key, 'access_secret':
s3_access_secret, 'endpoint':s3_endpoint, 'storage_class':s3_storage_class}
fs_config.update({'provider':1, 's3config':s3config})
return fs_config
def getUsers(self, limit=100, offset=0, order="ASC", username=""):
r = requests.get(self.userPath, params={"limit":limit, "offset":offset, "order":order,
"username":username}, auth=self.auth, verify=self.verify)
@ -132,22 +144,25 @@ class SFTPGoApiRequests:
r = requests.get(urlparse.urljoin(self.userPath, "user/" + str(user_id)), auth=self.auth, verify=self.verify)
self.printResponse(r)
def addUser(self, username="", password="", public_keys="", home_dir="", uid=0, gid=0, max_sessions=0,
quota_size=0, quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1,
expiration_date=0, subdirs_permissions=[], allowed_ip=[], denied_ip=[]):
def addUser(self, username="", password="", public_keys="", home_dir="", uid=0, gid=0, max_sessions=0, quota_size=0,
quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1, expiration_date=0,
subdirs_permissions=[], allowed_ip=[], denied_ip=[], fs_provider='local', s3_bucket='', s3_region='',
s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class=''):
u = self.buildUserObject(0, username, password, public_keys, home_dir, uid, gid, max_sessions,
quota_size, quota_files, self.buildPermissions(perms, subdirs_permissions), upload_bandwidth, download_bandwidth,
status, expiration_date, allowed_ip, denied_ip)
status, expiration_date, allowed_ip, denied_ip, fs_provider, s3_bucket, s3_region,
s3_access_key, s3_access_secret, s3_endpoint, s3_storage_class)
r = requests.post(self.userPath, json=u, auth=self.auth, verify=self.verify)
self.printResponse(r)
def updateUser(self, user_id, username="", password="", public_keys="", home_dir="", uid=0, gid=0,
max_sessions=0, quota_size=0, quota_files=0, perms=[], upload_bandwidth=0,
download_bandwidth=0, status=1, expiration_date=0, subdirs_permissions=[],
allowed_ip=[], denied_ip=[]):
def updateUser(self, user_id, username="", password="", public_keys="", home_dir="", uid=0, gid=0, max_sessions=0,
quota_size=0, quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1,
expiration_date=0, subdirs_permissions=[], allowed_ip=[], denied_ip=[], fs_provider='local',
s3_bucket='', s3_region='', s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class=''):
u = self.buildUserObject(user_id, username, password, public_keys, home_dir, uid, gid, max_sessions,
quota_size, quota_files, self.buildPermissions(perms, subdirs_permissions), upload_bandwidth, download_bandwidth,
status, expiration_date, allowed_ip, denied_ip)
status, expiration_date, allowed_ip, denied_ip, fs_provider, s3_bucket, s3_region, s3_access_key,
s3_access_secret, s3_endpoint, s3_storage_class)
r = requests.put(urlparse.urljoin(self.userPath, "user/" + str(user_id)), json=u, auth=self.auth, verify=self.verify)
self.printResponse(r)
@ -238,7 +253,7 @@ class ConvertUsers:
self.convertFromProFTPD()
self.saveUsers()
def isUserValid(self, username, uid, gid):
def isUserValid(self, username, uid):
if self.usernames and not username in self.usernames:
return False
if self.min_uid >= 0 and uid < self.min_uid:
@ -257,7 +272,7 @@ class ConvertUsers:
home_dir = user.pw_dir
status = 1
expiration_date = 0
if not self.isUserValid(username, uid, gid):
if not self.isUserValid(username, uid):
continue
if self.force_uid >= 0:
uid = self.force_uid
@ -375,7 +390,7 @@ def addCommonUserArguments(parser):
parser.add_argument('username', type=str)
parser.add_argument('-P', '--password', type=str, default=None, help='Default: %(default)s')
parser.add_argument('-K', '--public-keys', type=str, nargs='+', default=[], help='Default: %(default)s')
parser.add_argument('-H', '--home-dir', type=str, default="", help='Default: %(default)s')
parser.add_argument('-H', '--home-dir', type=str, default='', help='Default: %(default)s')
parser.add_argument('--uid', type=int, default=0, help='Default: %(default)s')
parser.add_argument('--gid', type=int, default=0, help='Default: %(default)s')
parser.add_argument('-C', '--max-sessions', type=int, default=0,
@ -401,6 +416,14 @@ def addCommonUserArguments(parser):
help='Allowed IP/Mask in CIDR notation. For example "192.168.2.0/24" or "2001:db8::/32". Default: %(default)s')
parser.add_argument('-N', '--denied-ip', type=str, nargs='+', default=[],
help='Denied IP/Mask in CIDR notation. For example "192.168.2.0/24" or "2001:db8::/32". Default: %(default)s')
parser.add_argument('--fs', type=str, default='local', choices=['local', 'S3'],
help='Filesystem provider. Default: %(default)s')
parser.add_argument('--s3-bucket', type=str, default='', help='Default: %(default)s')
parser.add_argument('--s3-region', type=str, default='', help='Default: %(default)s')
parser.add_argument('--s3-access-key', type=str, default='', help='Default: %(default)s')
parser.add_argument('--s3-access-secret', type=str, default='', help='Default: %(default)s')
parser.add_argument('--s3-endpoint', type=str, default='', help='Default: %(default)s')
parser.add_argument('--s3-storage-class', type=str, default='', help='Default: %(default)s')
if __name__ == '__main__':
@ -503,12 +526,14 @@ if __name__ == '__main__':
api.addUser(args.username, args.password, args.public_keys, args.home_dir, args.uid, args.gid, args.max_sessions,
args.quota_size, args.quota_files, args.permissions, args.upload_bandwidth, args.download_bandwidth,
args.status, getDatetimeAsMillisSinceEpoch(args.expiration_date), args.subdirs_permissions, args.allowed_ip,
args.denied_ip)
args.denied_ip, args.fs, args.s3_bucket, args.s3_region, args.s3_access_key, args.s3_access_secret,
args.s3_endpoint, args.s3_storage_class)
elif args.command == 'update-user':
api.updateUser(args.id, args.username, args.password, args.public_keys, args.home_dir, args.uid, args.gid,
args.max_sessions, args.quota_size, args.quota_files, args.permissions, args.upload_bandwidth,
args.download_bandwidth, args.status, getDatetimeAsMillisSinceEpoch(args.expiration_date),
args.subdirs_permissions, args.allowed_ip, args.denied_ip)
args.subdirs_permissions, args.allowed_ip, args.denied_ip, args.fs, args.s3_bucket, args.s3_region,
args.s3_access_key, args.s3_access_secret, args.s3_endpoint, args.s3_storage_class)
elif args.command == 'delete-user':
api.deleteUser(args.id)
elif args.command == 'get-users':

255
sftpd/handler.go
View file

@ -1,19 +1,14 @@
package sftpd
import (
"fmt"
"io"
"io/ioutil"
"net"
"os"
"path"
"path/filepath"
"strings"
"sync"
"time"
"github.com/drakkan/sftpgo/utils"
"github.com/rs/xid"
"github.com/drakkan/sftpgo/vfs"
"golang.org/x/crypto/ssh"
"github.com/drakkan/sftpgo/dataprovider"
@ -40,6 +35,7 @@ type Connection struct {
netConn net.Conn
channel ssh.Channel
command string
fs vfs.Fs
}
// Log outputs a log entry to the configured logger
@ -55,26 +51,29 @@ func (c Connection) Fileread(request *sftp.Request) (io.ReaderAt, error) {
return nil, sftp.ErrSSHFxPermissionDenied
}
p, err := c.buildPath(request.Filepath)
p, err := c.fs.ResolvePath(request.Filepath, c.User.GetHomeDir())
if err != nil {
return nil, getSFTPErrorFromOSError(err)
return nil, vfs.GetSFTPError(c.fs, err)
}
fi, err := os.Stat(p)
fi, err := c.fs.Stat(p)
if err != nil {
return nil, getSFTPErrorFromOSError(err)
return nil, vfs.GetSFTPError(c.fs, err)
}
file, err := os.Open(p)
file, r, cancelFn, err := c.fs.Open(p)
if err != nil {
c.Log(logger.LevelWarn, logSender, "could not open file %#v for reading: %v", p, err)
return nil, getSFTPErrorFromOSError(err)
return nil, vfs.GetSFTPError(c.fs, err)
}
c.Log(logger.LevelDebug, logSender, "fileread requested for path: %#v", p)
transfer := Transfer{
file: file,
readerAt: r,
writerAt: nil,
cancelFn: cancelFn,
path: p,
start: time.Now(),
bytesSent: 0,
@ -98,18 +97,18 @@ func (c Connection) Fileread(request *sftp.Request) (io.ReaderAt, error) {
// Filewrite handles the write actions for a file on the system.
func (c Connection) Filewrite(request *sftp.Request) (io.WriterAt, error) {
updateConnectionActivity(c.ID)
p, err := c.buildPath(request.Filepath)
p, err := c.fs.ResolvePath(request.Filepath, c.User.GetHomeDir())
if err != nil {
return nil, getSFTPErrorFromOSError(err)
return nil, vfs.GetSFTPError(c.fs, err)
}
filePath := p
if isAtomicUploadEnabled() {
filePath = getUploadTempFilePath(p)
if isAtomicUploadEnabled() && c.fs.IsAtomicUploadSupported() {
filePath = c.fs.GetAtomicUploadPath(p)
}
stat, statErr := os.Stat(p)
if os.IsNotExist(statErr) {
stat, statErr := c.fs.Stat(p)
if c.fs.IsNotExist(statErr) {
if !c.User.HasPerm(dataprovider.PermUpload, path.Dir(request.Filepath)) {
return nil, sftp.ErrSSHFxPermissionDenied
}
@ -118,7 +117,7 @@ func (c Connection) Filewrite(request *sftp.Request) (io.WriterAt, error) {
if statErr != nil {
c.Log(logger.LevelError, logSender, "error performing file stat %#v: %v", p, statErr)
return nil, getSFTPErrorFromOSError(err)
return nil, vfs.GetSFTPError(c.fs, statErr)
}
// This happen if we upload a file that has the same name of an existing directory
@ -139,9 +138,9 @@ func (c Connection) Filewrite(request *sftp.Request) (io.WriterAt, error) {
func (c Connection) Filecmd(request *sftp.Request) error {
updateConnectionActivity(c.ID)
p, err := c.buildPath(request.Filepath)
p, err := c.fs.ResolvePath(request.Filepath, c.User.GetHomeDir())
if err != nil {
return getSFTPErrorFromOSError(err)
return vfs.GetSFTPError(c.fs, err)
}
target, err := c.getSFTPCmdTargetPath(request.Target)
if err != nil {
@ -186,7 +185,7 @@ func (c Connection) Filecmd(request *sftp.Request) error {
}
// we return if we remove a file or a dir so source path or target path always exists here
utils.SetPathPermissions(fileLocation, c.User.GetUID(), c.User.GetGID())
vfs.SetPathPermissions(c.fs, fileLocation, c.User.GetUID(), c.User.GetGID())
return sftp.ErrSSHFxOk
}
@ -195,9 +194,9 @@ func (c Connection) Filecmd(request *sftp.Request) error {
// a directory as well as perform file/folder stat calls.
func (c Connection) Filelist(request *sftp.Request) (sftp.ListerAt, error) {
updateConnectionActivity(c.ID)
p, err := c.buildPath(request.Filepath)
p, err := c.fs.ResolvePath(request.Filepath, c.User.GetHomeDir())
if err != nil {
return nil, getSFTPErrorFromOSError(err)
return nil, vfs.GetSFTPError(c.fs, err)
}
switch request.Method {
@ -208,10 +207,10 @@ func (c Connection) Filelist(request *sftp.Request) (sftp.ListerAt, error) {
c.Log(logger.LevelDebug, logSender, "requested list file for dir: %#v", p)
files, err := ioutil.ReadDir(p)
files, err := c.fs.ReadDir(p)
if err != nil {
c.Log(logger.LevelWarn, logSender, "error listing directory: %#v", err)
return nil, getSFTPErrorFromOSError(err)
return nil, vfs.GetSFTPError(c.fs, err)
}
return listerAt(files), nil
@ -221,10 +220,10 @@ func (c Connection) Filelist(request *sftp.Request) (sftp.ListerAt, error) {
}
c.Log(logger.LevelDebug, logSender, "requested stat for path: %#v", p)
s, err := os.Stat(p)
s, err := c.fs.Stat(p)
if err != nil {
c.Log(logger.LevelWarn, logSender, "error running stat on path: %#v", err)
return nil, getSFTPErrorFromOSError(err)
return nil, vfs.GetSFTPError(c.fs, err)
}
return listerAt([]os.FileInfo{s}), nil
@ -239,9 +238,9 @@ func (c Connection) getSFTPCmdTargetPath(requestTarget string) (string, error) {
// location for the server. If it is not, return an error
if len(requestTarget) > 0 {
var err error
target, err = c.buildPath(requestTarget)
target, err = c.fs.ResolvePath(requestTarget, c.User.GetHomeDir())
if err != nil {
return target, getSFTPErrorFromOSError(err)
return target, vfs.GetSFTPError(c.fs, err)
}
}
return target, nil
@ -252,7 +251,7 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er
return nil
}
pathForPerms := request.Filepath
if fi, err := os.Lstat(filePath); err == nil {
if fi, err := c.fs.Lstat(filePath); err == nil {
if fi.IsDir() {
pathForPerms = path.Dir(request.Filepath)
}
@ -263,9 +262,9 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er
return sftp.ErrSSHFxPermissionDenied
}
fileMode := request.Attributes().FileMode()
if err := os.Chmod(filePath, fileMode); err != nil {
if err := c.fs.Chmod(filePath, fileMode); err != nil {
c.Log(logger.LevelWarn, logSender, "failed to chmod path %#v, mode: %v, err: %v", filePath, fileMode.String(), err)
return getSFTPErrorFromOSError(err)
return vfs.GetSFTPError(c.fs, err)
}
logger.CommandLog(chmodLogSender, filePath, "", c.User.Username, fileMode.String(), c.ID, c.protocol, -1, -1, "", "", "")
return nil
@ -275,9 +274,9 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er
}
uid := int(request.Attributes().UID)
gid := int(request.Attributes().GID)
if err := os.Chown(filePath, uid, gid); err != nil {
if err := c.fs.Chown(filePath, uid, gid); err != nil {
c.Log(logger.LevelWarn, logSender, "failed to chown path %#v, uid: %v, gid: %v, err: %v", filePath, uid, gid, err)
return getSFTPErrorFromOSError(err)
return vfs.GetSFTPError(c.fs, err)
}
logger.CommandLog(chownLogSender, filePath, "", c.User.Username, "", c.ID, c.protocol, uid, gid, "", "", "")
return nil
@ -290,10 +289,10 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er
modificationTime := time.Unix(int64(request.Attributes().Mtime), 0)
accessTimeString := accessTime.Format(dateFormat)
modificationTimeString := modificationTime.Format(dateFormat)
if err := os.Chtimes(filePath, accessTime, modificationTime); err != nil {
if err := c.fs.Chtimes(filePath, accessTime, modificationTime); err != nil {
c.Log(logger.LevelWarn, logSender, "failed to chtimes for path %#v, access time: %v, modification time: %v, err: %v",
filePath, accessTime, modificationTime, err)
return getSFTPErrorFromOSError(err)
return vfs.GetSFTPError(c.fs, err)
}
logger.CommandLog(chtimesLogSender, filePath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, accessTimeString,
modificationTimeString, "")
@ -303,16 +302,16 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er
}
func (c Connection) handleSFTPRename(sourcePath string, targetPath string, request *sftp.Request) error {
if c.User.GetRelativePath(sourcePath) == "/" {
if c.fs.GetRelativePath(sourcePath, c.User.GetHomeDir()) == "/" {
c.Log(logger.LevelWarn, logSender, "renaming root dir is not allowed")
return sftp.ErrSSHFxPermissionDenied
}
if !c.User.HasPerm(dataprovider.PermRename, path.Dir(request.Target)) {
return sftp.ErrSSHFxPermissionDenied
}
if err := os.Rename(sourcePath, targetPath); err != nil {
if err := c.fs.Rename(sourcePath, targetPath); err != nil {
c.Log(logger.LevelWarn, logSender, "failed to rename file, source: %#v target: %#v: %v", sourcePath, targetPath, err)
return getSFTPErrorFromOSError(err)
return vfs.GetSFTPError(c.fs, err)
}
logger.CommandLog(renameLogSender, sourcePath, targetPath, c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
go executeAction(operationRename, c.User.Username, sourcePath, targetPath, "", 0)
@ -320,7 +319,7 @@ func (c Connection) handleSFTPRename(sourcePath string, targetPath string, reque
}
func (c Connection) handleSFTPRmdir(dirPath string, request *sftp.Request) error {
if c.User.GetRelativePath(dirPath) == "/" {
if c.fs.GetRelativePath(dirPath, c.User.GetHomeDir()) == "/" {
c.Log(logger.LevelWarn, logSender, "removing root dir is not allowed")
return sftp.ErrSSHFxPermissionDenied
}
@ -330,18 +329,18 @@ func (c Connection) handleSFTPRmdir(dirPath string, request *sftp.Request) error
var fi os.FileInfo
var err error
if fi, err = os.Lstat(dirPath); err != nil {
if fi, err = c.fs.Lstat(dirPath); err != nil {
c.Log(logger.LevelWarn, logSender, "failed to remove a dir %#v: stat error: %v", dirPath, err)
return getSFTPErrorFromOSError(err)
return vfs.GetSFTPError(c.fs, err)
}
if !fi.IsDir() || fi.Mode()&os.ModeSymlink == os.ModeSymlink {
c.Log(logger.LevelDebug, logSender, "cannot remove %#v is not a directory", dirPath)
return sftp.ErrSSHFxFailure
}
if err = os.Remove(dirPath); err != nil {
if err = c.fs.Remove(dirPath, true); err != nil {
c.Log(logger.LevelWarn, logSender, "failed to remove directory %#v: %v", dirPath, err)
return getSFTPErrorFromOSError(err)
return vfs.GetSFTPError(c.fs, err)
}
logger.CommandLog(rmdirLogSender, dirPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
@ -349,16 +348,16 @@ func (c Connection) handleSFTPRmdir(dirPath string, request *sftp.Request) error
}
func (c Connection) handleSFTPSymlink(sourcePath string, targetPath string, request *sftp.Request) error {
if c.User.GetRelativePath(sourcePath) == "/" {
if c.fs.GetRelativePath(sourcePath, c.User.GetHomeDir()) == "/" {
c.Log(logger.LevelWarn, logSender, "symlinking root dir is not allowed")
return sftp.ErrSSHFxPermissionDenied
}
if !c.User.HasPerm(dataprovider.PermCreateSymlinks, path.Dir(request.Target)) {
return sftp.ErrSSHFxPermissionDenied
}
if err := os.Symlink(sourcePath, targetPath); err != nil {
if err := c.fs.Symlink(sourcePath, targetPath); err != nil {
c.Log(logger.LevelWarn, logSender, "failed to create symlink %#v -> %#v: %v", sourcePath, targetPath, err)
return getSFTPErrorFromOSError(err)
return vfs.GetSFTPError(c.fs, err)
}
logger.CommandLog(symlinkLogSender, sourcePath, targetPath, c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
@ -369,11 +368,11 @@ func (c Connection) handleSFTPMkdir(dirPath string, request *sftp.Request) error
if !c.User.HasPerm(dataprovider.PermCreateDirs, path.Dir(request.Filepath)) {
return sftp.ErrSSHFxPermissionDenied
}
if err := os.Mkdir(dirPath, 0777); err != nil {
if err := c.fs.Mkdir(dirPath); err != nil {
c.Log(logger.LevelWarn, logSender, "error creating missing dir: %#v error: %v", dirPath, err)
return getSFTPErrorFromOSError(err)
return vfs.GetSFTPError(c.fs, err)
}
utils.SetPathPermissions(dirPath, c.User.GetUID(), c.User.GetGID())
vfs.SetPathPermissions(c.fs, dirPath, c.User.GetUID(), c.User.GetGID())
logger.CommandLog(mkdirLogSender, dirPath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
return nil
@ -387,18 +386,18 @@ func (c Connection) handleSFTPRemove(filePath string, request *sftp.Request) err
var size int64
var fi os.FileInfo
var err error
if fi, err = os.Lstat(filePath); err != nil {
if fi, err = c.fs.Lstat(filePath); err != nil {
c.Log(logger.LevelWarn, logSender, "failed to remove a file %#v: stat error: %v", filePath, err)
return getSFTPErrorFromOSError(err)
return vfs.GetSFTPError(c.fs, err)
}
if fi.IsDir() && fi.Mode()&os.ModeSymlink != os.ModeSymlink {
c.Log(logger.LevelDebug, logSender, "cannot remove %#v is not a file/symlink", filePath)
return sftp.ErrSSHFxFailure
}
size = fi.Size()
if err := os.Remove(filePath); err != nil {
if err := c.fs.Remove(filePath, false); err != nil {
c.Log(logger.LevelWarn, logSender, "failed to remove a file/symlink %#v: %v", filePath, err)
return getSFTPErrorFromOSError(err)
return vfs.GetSFTPError(c.fs, err)
}
logger.CommandLog(removeLogSender, filePath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
@ -416,16 +415,19 @@ func (c Connection) handleSFTPUploadToNewFile(requestPath, filePath string) (io.
return nil, sftp.ErrSSHFxFailure
}
file, err := os.Create(filePath)
file, w, cancelFn, err := c.fs.Create(filePath, 0)
if err != nil {
c.Log(logger.LevelWarn, logSender, "error creating file %#v: %v", requestPath, err)
return nil, getSFTPErrorFromOSError(err)
return nil, vfs.GetSFTPError(c.fs, err)
}
utils.SetPathPermissions(filePath, c.User.GetUID(), c.User.GetGID())
vfs.SetPathPermissions(c.fs, filePath, c.User.GetUID(), c.User.GetGID())
transfer := Transfer{
file: file,
writerAt: w,
readerAt: nil,
cancelFn: cancelFn,
path: requestPath,
start: time.Now(),
bytesSent: 0,
@ -456,19 +458,25 @@ func (c Connection) handleSFTPUploadToExistingFile(pflags sftp.FileOpenFlags, re
minWriteOffset := int64(0)
osFlags := getOSOpenFlags(pflags)
if isAtomicUploadEnabled() {
err = os.Rename(requestPath, filePath)
if pflags.Append && osFlags&os.O_TRUNC == 0 && !c.fs.IsUploadResumeSupported() {
c.Log(logger.LevelInfo, logSender, "upload resume requested for path: %#v but not supported in fs implementation",
requestPath)
return nil, sftp.ErrSSHFxOpUnsupported
}
if isAtomicUploadEnabled() && c.fs.IsAtomicUploadSupported() {
err = c.fs.Rename(requestPath, filePath)
if err != nil {
c.Log(logger.LevelWarn, logSender, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %v",
requestPath, filePath, err)
return nil, getSFTPErrorFromOSError(err)
return nil, vfs.GetSFTPError(c.fs, err)
}
}
// we use 0666 so the umask is applied
file, err := os.OpenFile(filePath, osFlags, 0666)
file, w, cancelFn, err := c.fs.Create(filePath, osFlags)
if err != nil {
c.Log(logger.LevelWarn, logSender, "error opening existing file, flags: %v, source: %#v, err: %v", pflags, filePath, err)
return nil, getSFTPErrorFromOSError(err)
return nil, vfs.GetSFTPError(c.fs, err)
}
if pflags.Append && osFlags&os.O_TRUNC == 0 {
@ -478,10 +486,13 @@ func (c Connection) handleSFTPUploadToExistingFile(pflags sftp.FileOpenFlags, re
dataprovider.UpdateUserQuota(dataProvider, c.User, 0, -fileSize, false)
}
utils.SetPathPermissions(filePath, c.User.GetUID(), c.User.GetGID())
vfs.SetPathPermissions(c.fs, filePath, c.User.GetUID(), c.User.GetGID())
transfer := Transfer{
file: file,
writerAt: w,
readerAt: nil,
cancelFn: cancelFn,
path: requestPath,
start: time.Now(),
bytesSent: 0,
@ -522,103 +533,6 @@ func (c Connection) hasSpace(checkFiles bool) bool {
return true
}
// Normalizes a file/directory we get from the SFTP request to ensure the user is not able to escape
// from their data directory. After normalization if the file/directory is still within their home
// path it is returned. If they managed to "escape" an error will be returned.
func (c Connection) buildPath(rawPath string) (string, error) {
r := filepath.Clean(filepath.Join(c.User.HomeDir, rawPath))
p, err := filepath.EvalSymlinks(r)
if err != nil && !os.IsNotExist(err) {
return "", err
} else if os.IsNotExist(err) {
// The requested path doesn't exist, so at this point we need to iterate up the
// path chain until we hit a directory that _does_ exist and can be validated.
_, err = c.findFirstExistingDir(r)
if err != nil {
c.Log(logger.LevelWarn, logSender, "error resolving not existent path: %#v", err)
}
return r, err
}
err = c.isSubDir(p)
if err != nil {
c.Log(logger.LevelWarn, logSender, "Invalid path resolution, dir: %#v outside user home: %#v err: %v", p, c.User.HomeDir, err)
}
return r, err
}
// iterate up the path chain until we hit a directory that does exist and can be validated.
// all nonexistent directories will be returned
func (c Connection) findNonexistentDirs(path string) ([]string, error) {
results := []string{}
cleanPath := filepath.Clean(path)
parent := filepath.Dir(cleanPath)
_, err := os.Stat(parent)
for os.IsNotExist(err) {
results = append(results, parent)
parent = filepath.Dir(parent)
_, err = os.Stat(parent)
}
if err != nil {
return results, err
}
p, err := filepath.EvalSymlinks(parent)
if err != nil {
return results, err
}
err = c.isSubDir(p)
if err != nil {
c.Log(logger.LevelWarn, logSender, "Error finding non existing dir: %v", err)
}
return results, err
}
// iterate up the path chain until we hit a directory that does exist and can be validated.
func (c Connection) findFirstExistingDir(path string) (string, error) {
results, err := c.findNonexistentDirs(path)
if err != nil {
c.Log(logger.LevelWarn, logSender, "unable to find non existent dirs: %v", err)
return "", err
}
var parent string
if len(results) > 0 {
lastMissingDir := results[len(results)-1]
parent = filepath.Dir(lastMissingDir)
} else {
parent = c.User.GetHomeDir()
}
p, err := filepath.EvalSymlinks(parent)
if err != nil {
return "", err
}
fileInfo, err := os.Stat(p)
if err != nil {
return "", err
}
if !fileInfo.IsDir() {
return "", fmt.Errorf("resolved path is not a dir: %#v", p)
}
err = c.isSubDir(p)
return p, err
}
// checks if sub is a subpath of the user home dir.
// EvalSymlink must be used on sub before calling this method
func (c Connection) isSubDir(sub string) error {
// home dir must exist and it is already a validated absolute path
parent, err := filepath.EvalSymlinks(c.User.HomeDir)
if err != nil {
c.Log(logger.LevelWarn, logSender, "invalid home dir %#v: %v", c.User.HomeDir, err)
return err
}
if !strings.HasPrefix(sub, parent) {
c.Log(logger.LevelWarn, logSender, "path %#v is not inside: %#v ", sub, parent)
return fmt.Errorf("path %#v is not inside: %#v", sub, parent)
}
return nil
}
func (c Connection) close() error {
if c.channel != nil {
err := c.channel.Close()
@ -649,20 +563,3 @@ func getOSOpenFlags(requestFlags sftp.FileOpenFlags) (flags int) {
}
return osFlags
}
func getUploadTempFilePath(path string) string {
dir := filepath.Dir(path)
guid := xid.New().String()
return filepath.Join(dir, ".sftpgo-upload."+guid+"."+filepath.Base(path))
}
func getSFTPErrorFromOSError(err error) error {
if os.IsNotExist(err) {
return sftp.ErrSSHFxNoSuchFile
} else if os.IsPermission(err) {
return sftp.ErrSSHFxPermissionDenied
} else if err != nil {
return sftp.ErrSSHFxFailure
}
return nil
}

337
sftpd/internal_test.go
View file

@ -8,6 +8,7 @@ import (
"io/ioutil"
"net"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
@ -16,6 +17,8 @@ import (
"github.com/drakkan/sftpgo/dataprovider"
"github.com/drakkan/sftpgo/utils"
"github.com/drakkan/sftpgo/vfs"
"github.com/eikenb/pipeat"
"github.com/pkg/sftp"
)
@ -60,6 +63,61 @@ func (c *MockChannel) Stderr() io.ReadWriter {
return c.StdErrBuffer
}
// MockOsFs mockable OsFs
type MockOsFs struct {
vfs.OsFs
err error
statErr error
isAtomicUploadSupported bool
}
// Name returns the name for the Fs implementation
func (fs MockOsFs) Name() string {
return "mockOsFs"
}
// IsUploadResumeSupported returns true if upload resume is supported
func (MockOsFs) IsUploadResumeSupported() bool {
return false
}
// IsAtomicUploadSupported returns true if atomic upload is supported
func (fs MockOsFs) IsAtomicUploadSupported() bool {
return fs.isAtomicUploadSupported
}
// Stat returns a FileInfo describing the named file
func (fs MockOsFs) Stat(name string) (os.FileInfo, error) {
if fs.statErr != nil {
return nil, fs.statErr
}
return os.Stat(name)
}
// Remove removes the named file or (empty) directory.
func (fs MockOsFs) Remove(name string, isDir bool) error {
if fs.err != nil {
return fs.err
}
return os.Remove(name)
}
// Rename renames (moves) source to target
func (fs MockOsFs) Rename(source, target string) error {
if fs.err != nil {
return fs.err
}
return os.Rename(source, target)
}
func newMockOsFs(err, statErr error, atomicUpload bool) vfs.Fs {
return &MockOsFs{
err: err,
statErr: statErr,
isAtomicUploadSupported: atomicUpload,
}
}
func TestWrongActions(t *testing.T) {
actionsCopy := actions
badCommand := "/bad/command"
@ -218,13 +276,134 @@ func TestReadWriteErrors(t *testing.T) {
if err == nil {
t.Error("upoload must fail the expected size does not match")
}
r, _, _ := pipeat.Pipe()
transfer = Transfer{
readerAt: r,
writerAt: nil,
start: time.Now(),
bytesSent: 0,
bytesReceived: 0,
user: dataprovider.User{
Username: "testuser",
},
connectionID: "",
transferType: transferDownload,
lastActivity: time.Now(),
isNewFile: false,
protocol: protocolSFTP,
transferError: nil,
isFinished: false,
lock: new(sync.Mutex),
}
transfer.closeIO()
_, err = transfer.ReadAt(buf, 0)
if err == nil {
t.Error("reading from a closed pipe must fail")
}
r, w, _ := pipeat.Pipe()
transfer = Transfer{
readerAt: nil,
writerAt: w,
start: time.Now(),
bytesSent: 0,
bytesReceived: 0,
user: dataprovider.User{
Username: "testuser",
},
connectionID: "",
transferType: transferDownload,
lastActivity: time.Now(),
isNewFile: false,
protocol: protocolSFTP,
transferError: nil,
isFinished: false,
lock: new(sync.Mutex),
}
r.Close()
transfer.closeIO()
_, err = transfer.WriteAt([]byte("test"), 0)
if err == nil {
t.Error("writing to closed pipe must fail")
}
os.Remove(testfile)
}
func TestTransferCancelFn(t *testing.T) {
testfile := "testfile"
file, _ := os.Create(testfile)
isCancelled := false
cancelFn := func() {
isCancelled = true
}
transfer := Transfer{
file: file,
cancelFn: cancelFn,
path: file.Name(),
start: time.Now(),
bytesSent: 0,
bytesReceived: 0,
user: dataprovider.User{
Username: "testuser",
},
connectionID: "",
transferType: transferDownload,
lastActivity: time.Now(),
isNewFile: false,
protocol: protocolSFTP,
transferError: nil,
isFinished: false,
minWriteOffset: 0,
expectedSize: 10,
lock: new(sync.Mutex),
}
transfer.TransferError(errors.New("fake error, this will trigger cancelFn"))
transfer.Close()
if !isCancelled {
t.Error("cancelFn not called")
}
os.Remove(testfile)
}
func TestMockFsErrors(t *testing.T) {
errFake := errors.New("fake error")
fs := newMockOsFs(errFake, errFake, false)
u := dataprovider.User{}
u.Username = "test"
u.Permissions = make(map[string][]string)
u.Permissions["/"] = []string{dataprovider.PermAny}
u.HomeDir = os.TempDir()
c := Connection{
fs: fs,
User: u,
}
testfile := filepath.Join(u.HomeDir, "testfile")
request := sftp.NewRequest("Remove", testfile)
ioutil.WriteFile(testfile, []byte("test"), 0666)
err := c.handleSFTPRemove(testfile, request)
if err != sftp.ErrSSHFxFailure {
t.Errorf("unexpected error: %v", err)
}
_, err = c.Filewrite(request)
if err != sftp.ErrSSHFxFailure {
t.Errorf("unexpected error: %v", err)
}
var flags sftp.FileOpenFlags
flags.Write = true
flags.Trunc = false
flags.Append = true
_, err = c.handleSFTPUploadToExistingFile(flags, testfile, testfile, 0)
if err != sftp.ErrSSHFxOpUnsupported {
t.Errorf("unexpected error: %v", err)
}
os.Remove(testfile)
}
func TestUploadFiles(t *testing.T) {
oldUploadMode := uploadMode
uploadMode = uploadModeAtomic
c := Connection{}
c := Connection{
fs: vfs.NewOsFs("123"),
}
var flags sftp.FileOpenFlags
flags.Write = true
flags.Trunc = true
@ -255,10 +434,13 @@ func TestWithInvalidHome(t *testing.T) {
if err == nil {
t.Errorf("login a user with an invalid home_dir must fail")
}
fs, _ := u.GetFilesystem("123")
c := Connection{
User: u,
fs: fs,
}
err = c.isSubDir("dir_rel_path")
u.HomeDir = os.TempDir()
_, err = c.fs.ResolvePath("../upper_path", u.GetHomeDir())
if err == nil {
t.Errorf("tested path is not a home subdir")
}
@ -266,12 +448,18 @@ func TestWithInvalidHome(t *testing.T) {
func TestSFTPCmdTargetPath(t *testing.T) {
u := dataprovider.User{}
u.HomeDir = "home_rel_path"
if runtime.GOOS == "windows" {
u.HomeDir = "C:\\invalid_home"
} else {
u.HomeDir = "/invalid_home"
}
u.Username = "test"
u.Permissions = make(map[string][]string)
u.Permissions["/"] = []string{dataprovider.PermAny}
fs, _ := u.GetFilesystem("123")
connection := Connection{
User: u,
fs: fs,
}
_, err := connection.getSFTPCmdTargetPath("invalid_path")
if err != sftp.ErrSSHFxNoSuchFile {
@ -281,16 +469,17 @@ func TestSFTPCmdTargetPath(t *testing.T) {
func TestGetSFTPErrorFromOSError(t *testing.T) {
err := os.ErrNotExist
err = getSFTPErrorFromOSError(err)
fs := vfs.NewOsFs("")
err = vfs.GetSFTPError(fs, err)
if err != sftp.ErrSSHFxNoSuchFile {
t.Errorf("unexpected error: %v", err)
}
err = os.ErrPermission
err = getSFTPErrorFromOSError(err)
err = vfs.GetSFTPError(fs, err)
if err != sftp.ErrSSHFxPermissionDenied {
t.Errorf("unexpected error: %v", err)
}
err = getSFTPErrorFromOSError(nil)
err = vfs.GetSFTPError(fs, nil)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
@ -418,10 +607,12 @@ func TestSSHCommandErrors(t *testing.T) {
user := dataprovider.User{}
user.Permissions = make(map[string][]string)
user.Permissions["/"] = []string{dataprovider.PermAny}
fs, _ := user.GetFilesystem("123")
connection := Connection{
channel: &mockSSHChannel,
netConn: client,
User: user,
fs: fs,
}
cmd := sshCommand{
command: "md5sum",
@ -499,6 +690,45 @@ func TestSSHCommandErrors(t *testing.T) {
}
}
func TestSSHCommandsRemoteFs(t *testing.T) {
buf := make([]byte, 65535)
stdErrBuf := make([]byte, 65535)
mockSSHChannel := MockChannel{
Buffer: bytes.NewBuffer(buf),
StdErrBuffer: bytes.NewBuffer(stdErrBuf),
}
server, client := net.Pipe()
defer server.Close()
defer client.Close()
user := dataprovider.User{}
user.FsConfig = dataprovider.Filesystem{
Provider: 1}
fs, _ := user.GetFilesystem("123")
connection := Connection{
channel: &mockSSHChannel,
netConn: client,
User: user,
fs: fs,
}
cmd := sshCommand{
command: "md5sum",
connection: connection,
args: []string{},
}
err := cmd.handleHashCommands()
if err == nil {
t.Error("command must fail for a non local filesystem")
}
command, err := cmd.getSystemCommand()
if err != nil {
t.Errorf("unexpected error: %v", err)
}
err = cmd.executeSystemCommand(command)
if err == nil {
t.Error("command must fail for a non local filesystem")
}
}
func TestSSHCommandQuotaScan(t *testing.T) {
buf := make([]byte, 65535)
stdErrBuf := make([]byte, 65535)
@ -513,14 +743,17 @@ func TestSSHCommandQuotaScan(t *testing.T) {
defer client.Close()
permissions := make(map[string][]string)
permissions["/"] = []string{dataprovider.PermAny}
user := dataprovider.User{
Permissions: permissions,
QuotaFiles: 1,
HomeDir: "invalid_path",
}
fs, _ := user.GetFilesystem("123")
connection := Connection{
channel: &mockSSHChannel,
netConn: client,
User: dataprovider.User{
Permissions: permissions,
QuotaFiles: 1,
HomeDir: "invalid_path",
},
User: user,
fs: fs,
}
cmd := sshCommand{
command: "git-receive-pack",
@ -536,11 +769,14 @@ func TestSSHCommandQuotaScan(t *testing.T) {
func TestRsyncOptions(t *testing.T) {
permissions := make(map[string][]string)
permissions["/"] = []string{dataprovider.PermAny}
user := dataprovider.User{
Permissions: permissions,
HomeDir: os.TempDir(),
}
fs, _ := user.GetFilesystem("123")
conn := Connection{
User: dataprovider.User{
Permissions: permissions,
HomeDir: os.TempDir(),
},
User: user,
fs: fs,
}
sshCmd := sshCommand{
command: "rsync",
@ -556,11 +792,11 @@ func TestRsyncOptions(t *testing.T) {
}
permissions["/"] = []string{dataprovider.PermDownload, dataprovider.PermUpload, dataprovider.PermCreateDirs,
dataprovider.PermListItems, dataprovider.PermOverwrite, dataprovider.PermDelete, dataprovider.PermRename}
user.Permissions = permissions
fs, _ = user.GetFilesystem("123")
conn = Connection{
User: dataprovider.User{
Permissions: permissions,
HomeDir: os.TempDir(),
},
User: user,
fs: fs,
}
sshCmd = sshCommand{
command: "rsync",
@ -592,13 +828,16 @@ func TestSystemCommandErrors(t *testing.T) {
defer client.Close()
permissions := make(map[string][]string)
permissions["/"] = []string{dataprovider.PermAny}
user := dataprovider.User{
Permissions: permissions,
HomeDir: os.TempDir(),
}
fs, _ := user.GetFilesystem("123")
connection := Connection{
channel: &mockSSHChannel,
netConn: client,
User: dataprovider.User{
Permissions: permissions,
HomeDir: os.TempDir(),
},
User: user,
fs: fs,
}
sshCmd := sshCommand{
command: "ls",
@ -934,6 +1173,55 @@ func TestSCPCommandHandleErrors(t *testing.T) {
}
}
func TestSCPErrorsMockFs(t *testing.T) {
errFake := errors.New("fake error")
fs := newMockOsFs(errFake, errFake, false)
u := dataprovider.User{}
u.Username = "test"
u.Permissions = make(map[string][]string)
u.Permissions["/"] = []string{dataprovider.PermAny}
u.HomeDir = os.TempDir()
buf := make([]byte, 65535)
stdErrBuf := make([]byte, 65535)
mockSSHChannel := MockChannel{
Buffer: bytes.NewBuffer(buf),
StdErrBuffer: bytes.NewBuffer(stdErrBuf),
}
server, client := net.Pipe()
defer server.Close()
defer client.Close()
connection := Connection{
channel: &mockSSHChannel,
netConn: client,
fs: fs,
User: u,
}
scpCommand := scpCommand{
sshCommand: sshCommand{
command: "scp",
connection: connection,
args: []string{"-r", "-t", "/tmp"},
},
}
err := scpCommand.handleUpload("test", 0)
if err != errFake {
t.Errorf("unexpected error: %v", err)
}
testfile := filepath.Join(u.HomeDir, "testfile")
ioutil.WriteFile(testfile, []byte("test"), 0666)
stat, _ := os.Stat(u.HomeDir)
err = scpCommand.handleRecursiveDownload(u.HomeDir, stat)
if err != errFake {
t.Errorf("unexpected error: %v", err)
}
scpCommand.sshCommand.connection.fs = newMockOsFs(errFake, nil, true)
err = scpCommand.handleUpload(filepath.Base(testfile), 0)
if err != errFake {
t.Errorf("unexpected error: %v", err)
}
os.Remove(testfile)
}
func TestSCPRecursiveDownloadErrors(t *testing.T) {
buf := make([]byte, 65535)
stdErrBuf := make([]byte, 65535)
@ -951,6 +1239,7 @@ func TestSCPRecursiveDownloadErrors(t *testing.T) {
connection := Connection{
channel: &mockSSHChannel,
netConn: client,
fs: vfs.NewOsFs("123"),
}
scpCommand := scpCommand{
sshCommand: sshCommand{
@ -1033,9 +1322,11 @@ func TestSCPCreateDirs(t *testing.T) {
ReadError: nil,
WriteError: nil,
}
fs, _ := u.GetFilesystem("123")
connection := Connection{
User: u,
channel: &mockSSHChannel,
fs: fs,
}
scpCommand := scpCommand{
sshCommand: sshCommand{

50
sftpd/scp.go
View file

@ -3,7 +3,6 @@ package sftpd
import (
"fmt"
"io"
"io/ioutil"
"math"
"os"
"path"
@ -16,6 +15,7 @@ import (
"github.com/drakkan/sftpgo/dataprovider"
"github.com/drakkan/sftpgo/logger"
"github.com/drakkan/sftpgo/utils"
"github.com/drakkan/sftpgo/vfs"
)
var (
@ -116,7 +116,7 @@ func (c *scpCommand) handleRecursiveUpload() error {
func (c *scpCommand) handleCreateDir(dirPath string) error {
updateConnectionActivity(c.connection.ID)
p, err := c.connection.buildPath(dirPath)
p, err := c.connection.fs.ResolvePath(dirPath, c.connection.User.GetHomeDir())
if err != nil {
c.connection.Log(logger.LevelWarn, logSenderSCP, "error creating dir: %#v, invalid file path, err: %v", dirPath, err)
c.sendErrorMessage(err.Error())
@ -189,17 +189,20 @@ func (c *scpCommand) handleUploadFile(requestPath, filePath string, sizeToRead i
return err
}
file, err := os.Create(filePath)
file, w, cancelFn, err := c.connection.fs.Create(filePath, 0)
if err != nil {
c.connection.Log(logger.LevelError, logSenderSCP, "error creating file %#v: %v", requestPath, err)
c.sendErrorMessage(err.Error())
return err
}
utils.SetPathPermissions(filePath, c.connection.User.GetUID(), c.connection.User.GetGID())
vfs.SetPathPermissions(c.connection.fs, filePath, c.connection.User.GetUID(), c.connection.User.GetGID())
transfer := Transfer{
file: file,
readerAt: nil,
writerAt: w,
cancelFn: cancelFn,
path: requestPath,
start: time.Now(),
bytesSent: 0,
@ -225,18 +228,18 @@ func (c *scpCommand) handleUpload(uploadFilePath string, sizeToRead int64) error
updateConnectionActivity(c.connection.ID)
p, err := c.connection.buildPath(uploadFilePath)
p, err := c.connection.fs.ResolvePath(uploadFilePath, c.connection.User.GetHomeDir())
if err != nil {
c.connection.Log(logger.LevelWarn, logSenderSCP, "error uploading file: %#v, err: %v", uploadFilePath, err)
c.sendErrorMessage(err.Error())
return err
}
filePath := p
if isAtomicUploadEnabled() {
filePath = getUploadTempFilePath(p)
if isAtomicUploadEnabled() && c.connection.fs.IsAtomicUploadSupported() {
filePath = c.connection.fs.GetAtomicUploadPath(p)
}
stat, statErr := os.Stat(p)
if os.IsNotExist(statErr) {
stat, statErr := c.connection.fs.Stat(p)
if c.connection.fs.IsNotExist(statErr) {
if !c.connection.User.HasPerm(dataprovider.PermUpload, path.Dir(uploadFilePath)) {
err := fmt.Errorf("Permission denied")
c.connection.Log(logger.LevelWarn, logSenderSCP, "cannot upload file: %#v, permission denied", uploadFilePath)
@ -248,8 +251,8 @@ func (c *scpCommand) handleUpload(uploadFilePath string, sizeToRead int64) error
if statErr != nil {
c.connection.Log(logger.LevelError, logSenderSCP, "error performing file stat %#v: %v", p, statErr)
c.sendErrorMessage(err.Error())
return err
c.sendErrorMessage(statErr.Error())
return statErr
}
if stat.IsDir() {
@ -266,8 +269,8 @@ func (c *scpCommand) handleUpload(uploadFilePath string, sizeToRead int64) error
return err
}
if isAtomicUploadEnabled() {
err = os.Rename(p, filePath)
if isAtomicUploadEnabled() && c.connection.fs.IsAtomicUploadSupported() {
err = c.connection.fs.Rename(p, filePath)
if err != nil {
c.connection.Log(logger.LevelError, logSenderSCP, "error renaming existing file for atomic upload, source: %#v, dest: %#v, err: %v",
p, filePath, err)
@ -315,14 +318,14 @@ func (c *scpCommand) handleRecursiveDownload(dirPath string, stat os.FileInfo) e
if err != nil {
return err
}
files, err := ioutil.ReadDir(dirPath)
files, err := c.connection.fs.ReadDir(dirPath)
if err != nil {
c.sendErrorMessage(err.Error())
return err
}
var dirs []string
for _, file := range files {
filePath := c.connection.User.GetRelativePath(filepath.Join(dirPath, file.Name()))
filePath := c.connection.fs.GetRelativePath(c.connection.fs.Join(dirPath, file.Name()), c.connection.User.GetHomeDir())
if file.Mode().IsRegular() || file.Mode()&os.ModeSymlink == os.ModeSymlink {
err = c.handleDownload(filePath)
if err != nil {
@ -419,7 +422,7 @@ func (c *scpCommand) handleDownload(filePath string) error {
updateConnectionActivity(c.connection.ID)
p, err := c.connection.buildPath(filePath)
p, err := c.connection.fs.ResolvePath(filePath, c.connection.User.GetHomeDir())
if err != nil {
err := fmt.Errorf("Invalid file path")
c.connection.Log(logger.LevelWarn, logSenderSCP, "error downloading file: %#v, invalid file path", filePath)
@ -428,7 +431,7 @@ func (c *scpCommand) handleDownload(filePath string) error {
}
var stat os.FileInfo
if stat, err = os.Stat(p); os.IsNotExist(err) {
if stat, err = c.connection.fs.Stat(p); err != nil {
c.connection.Log(logger.LevelWarn, logSenderSCP, "error downloading file: %#v, err: %v", p, err)
c.sendErrorMessage(err.Error())
return err
@ -452,7 +455,7 @@ func (c *scpCommand) handleDownload(filePath string) error {
return err
}
file, err := os.Open(p)
file, r, cancelFn, err := c.connection.fs.Open(p)
if err != nil {
c.connection.Log(logger.LevelError, logSenderSCP, "could not open file %#v for reading: %v", p, err)
c.sendErrorMessage(err.Error())
@ -461,6 +464,9 @@ func (c *scpCommand) handleDownload(filePath string) error {
transfer := Transfer{
file: file,
readerAt: r,
writerAt: nil,
cancelFn: cancelFn,
path: p,
start: time.Now(),
bytesSent: 0,
@ -608,12 +614,12 @@ func (c *scpCommand) getNextUploadProtocolMessage() (string, error) {
func (c *scpCommand) createDir(dirPath string) error {
var err error
if err = os.Mkdir(dirPath, 0777); err != nil {
if err = c.connection.fs.Mkdir(dirPath); err != nil {
c.connection.Log(logger.LevelError, logSenderSCP, "error creating dir: %v", dirPath)
c.sendErrorMessage(err.Error())
return err
}
utils.SetPathPermissions(dirPath, c.connection.User.GetUID(), c.connection.User.GetGID())
vfs.SetPathPermissions(c.connection.fs, dirPath, c.connection.User.GetUID(), c.connection.User.GetGID())
return err
}
@ -668,8 +674,8 @@ func (c *scpCommand) getFileUploadDestPath(scpDestPath, fileName string) string
// but if scpDestPath is an existing directory then we put the uploaded file
// inside that directory this is as scp command works, for example:
// scp fileName.txt user@127.0.0.1:/existing_dir
if p, err := c.connection.buildPath(scpDestPath); err == nil {
if stat, err := os.Stat(p); err == nil {
if p, err := c.connection.fs.ResolvePath(scpDestPath, c.connection.User.GetHomeDir()); err == nil {
if stat, err := c.connection.fs.Stat(p); err == nil {
if stat.IsDir() {
return path.Join(scpDestPath, fileName)
}

20
sftpd/server.go
View file

@ -266,6 +266,14 @@ func (c Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.Server
loginType = sconn.Permissions.Extensions["login_type"]
connectionID := hex.EncodeToString(sconn.SessionID())
fs, err := user.GetFilesystem(connectionID)
if err != nil {
logger.Warn(logSender, "", "could create filesystem for user %#v err: %v", user.Username, err)
conn.Close()
return
}
connection := Connection{
ID: connectionID,
User: user,
@ -275,7 +283,11 @@ func (c Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.Server
lastActivity: time.Now(),
netConn: conn,
channel: nil,
fs: fs,
}
connection.fs.CheckRootPath(user.GetHomeDir(), user.Username, user.GetUID(), user.GetGID())
connection.Log(logger.LevelInfo, logSender, "User id: %d, logged in with: %#v, username: %#v, home_dir: %#v remote addr: %#v",
user.ID, loginType, user.Username, user.HomeDir, remoteAddr.String())
dataprovider.UpdateLastLogin(dataProvider, user)
@ -368,14 +380,6 @@ func loginUser(user dataprovider.User, loginType string, remoteAddr string) (*ss
logger.Debug(logSender, "", "cannot login user %#v, remote address is not allowed: %v", user.Username, remoteAddr)
return nil, fmt.Errorf("Login is not allowed from this address: %v", remoteAddr)
}
if _, err := os.Stat(user.HomeDir); os.IsNotExist(err) {
err := os.MkdirAll(user.HomeDir, 0777)
logger.Debug(logSender, "", "home directory %#v for user %#v does not exist, try to create, mkdir error: %v",
user.HomeDir, user.Username, err)
if err == nil {
utils.SetPathPermissions(user.HomeDir, user.GetUID(), user.GetGID())
}
}
json, err := json.Marshal(user)
if err != nil {

2
sftpd/sftpd.go
View file

@ -307,7 +307,7 @@ func GetConnectionsStats() []ConnectionStatus {
StartTime: utils.GetTimeAsMsSinceEpoch(t.start),
Size: size,
LastActivity: utils.GetTimeAsMsSinceEpoch(t.lastActivity),
Path: c.User.GetRelativePath(t.path),
Path: c.fs.GetRelativePath(t.path, c.User.GetHomeDir()),
}
conn.Transfers = append(conn.Transfers, connTransfer)
}

61
sftpd/sftpd_test.go
View file

@ -35,6 +35,7 @@ import (
"github.com/drakkan/sftpgo/logger"
"github.com/drakkan/sftpgo/sftpd"
"github.com/drakkan/sftpgo/utils"
"github.com/drakkan/sftpgo/vfs"
"github.com/pkg/sftp"
"github.com/rs/zerolog"
)
@ -1017,6 +1018,45 @@ func TestLoginUserExpiration(t *testing.T) {
os.RemoveAll(user.GetHomeDir())
}
func TestLoginInvalidFs(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("this test is not available on Windows")
}
config.LoadConfig(configDir, "")
providerConf := config.GetProviderConf()
if providerConf.Driver != dataprovider.SQLiteDataProviderName {
t.Skip("this test require sqlite provider")
}
dbPath := providerConf.Name
if !filepath.IsAbs(dbPath) {
dbPath = filepath.Join(configDir, dbPath)
}
usePubKey := true
u := getTestUser(usePubKey)
user, _, err := httpd.AddUser(u, http.StatusOK)
if err != nil {
t.Errorf("unable to add user: %v", err)
}
// we update the database using sqlite3 CLI since we cannot add an user with an invalid config
time.Sleep(150 * time.Millisecond)
updateUserQuery := fmt.Sprintf("UPDATE users SET filesystem='{\"provider\":1}' WHERE id=%v", user.ID)
cmd := exec.Command("sqlite3", dbPath, updateUserQuery)
out, err := cmd.CombinedOutput()
if err != nil {
t.Errorf("unexpected error: %v, cmd out: %v", err, string(out))
}
time.Sleep(200 * time.Millisecond)
_, err = getSftpClient(user, usePubKey)
if err == nil {
t.Error("login must fail, the user has an invalid filesystem config")
}
_, err = httpd.RemoveUser(user, http.StatusOK)
if err != nil {
t.Errorf("unable to remove user: %v", err)
}
os.RemoveAll(user.GetHomeDir())
}
func TestLoginWithIPFilters(t *testing.T) {
usePubKey := true
u := getTestUser(usePubKey)
@ -2864,52 +2904,53 @@ func TestRootDirCommands(t *testing.T) {
func TestRelativePaths(t *testing.T) {
user := getTestUser(true)
path := filepath.Join(user.HomeDir, "/")
rel := user.GetRelativePath(path)
fs := vfs.NewOsFs("")
rel := fs.GetRelativePath(path, user.GetHomeDir())
if rel != "/" {
t.Errorf("Unexpected relative path: %v", rel)
}
path = filepath.Join(user.HomeDir, "//")
rel = user.GetRelativePath(path)
rel = fs.GetRelativePath(path, user.GetHomeDir())
if rel != "/" {
t.Errorf("Unexpected relative path: %v", rel)
}
path = filepath.Join(user.HomeDir, "../..")
rel = user.GetRelativePath(path)
rel = fs.GetRelativePath(path, user.GetHomeDir())
if rel != "/" {
t.Errorf("Unexpected relative path: %v", rel)
}
path = filepath.Join(user.HomeDir, "../../../../../")
rel = user.GetRelativePath(path)
rel = fs.GetRelativePath(path, user.GetHomeDir())
if rel != "/" {
t.Errorf("Unexpected relative path: %v", rel)
}
path = filepath.Join(user.HomeDir, "/..")
rel = user.GetRelativePath(path)
rel = fs.GetRelativePath(path, user.GetHomeDir())
if rel != "/" {
t.Errorf("Unexpected relative path: %v", rel)
}
path = filepath.Join(user.HomeDir, "/../../../..")
rel = user.GetRelativePath(path)
rel = fs.GetRelativePath(path, user.GetHomeDir())
if rel != "/" {
t.Errorf("Unexpected relative path: %v", rel)
}
path = filepath.Join(user.HomeDir, "")
rel = user.GetRelativePath(path)
rel = fs.GetRelativePath(path, user.GetHomeDir())
if rel != "/" {
t.Errorf("Unexpected relative path: %v", rel)
}
path = filepath.Join(user.HomeDir, ".")
rel = user.GetRelativePath(path)
rel = fs.GetRelativePath(path, user.GetHomeDir())
if rel != "/" {
t.Errorf("Unexpected relative path: %v", rel)
}
path = filepath.Join(user.HomeDir, "somedir")
rel = user.GetRelativePath(path)
rel = fs.GetRelativePath(path, user.GetHomeDir())
if rel != "/somedir" {
t.Errorf("Unexpected relative path: %v", rel)
}
path = filepath.Join(user.HomeDir, "/somedir/subdir")
rel = user.GetRelativePath(path)
rel = fs.GetRelativePath(path, user.GetHomeDir())
if rel != "/somedir/subdir" {
t.Errorf("Unexpected relative path: %v", rel)
}

22
sftpd/ssh_cmd.go
View file

@ -21,12 +21,14 @@ import (
"github.com/drakkan/sftpgo/logger"
"github.com/drakkan/sftpgo/metrics"
"github.com/drakkan/sftpgo/utils"
"github.com/drakkan/sftpgo/vfs"
"golang.org/x/crypto/ssh"
)
var (
errQuotaExceeded = errors.New("denying write due to space limit")
errPermissionDenied = errors.New("Permission denied. You don't have the permissions to execute this command")
errQuotaExceeded = errors.New("denying write due to space limit")
errPermissionDenied = errors.New("Permission denied. You don't have the permissions to execute this command")
errUnsupportedConfig = errors.New("command unsupported for this configuration")
)
type sshCommand struct {
@ -101,6 +103,9 @@ func (c *sshCommand) handle() error {
}
func (c *sshCommand) handleHashCommands() error {
if !vfs.IsLocalOsFs(c.connection.fs) {
return c.sendErrorResponse(errUnsupportedConfig)
}
var h hash.Hash
if c.command == "md5sum" {
h = md5.New()
@ -125,14 +130,14 @@ func (c *sshCommand) handleHashCommands() error {
response = fmt.Sprintf("%x -\n", h.Sum(nil))
} else {
sshPath := c.getDestPath()
path, err := c.connection.buildPath(sshPath)
fsPath, err := c.connection.fs.ResolvePath(sshPath, c.connection.User.GetHomeDir())
if err != nil {
return c.sendErrorResponse(err)
}
if !c.connection.User.HasPerm(dataprovider.PermListItems, sshPath) {
return c.sendErrorResponse(errPermissionDenied)
}
hash, err := computeHashForFile(h, path)
hash, err := computeHashForFile(h, fsPath)
if err != nil {
return c.sendErrorResponse(err)
}
@ -144,6 +149,9 @@ func (c *sshCommand) handleHashCommands() error {
}
func (c *sshCommand) executeSystemCommand(command systemCommand) error {
if !vfs.IsLocalOsFs(c.connection.fs) {
return c.sendErrorResponse(errUnsupportedConfig)
}
if c.connection.User.QuotaFiles > 0 && c.connection.User.UsedQuotaFiles > c.connection.User.QuotaFiles {
return c.sendErrorResponse(errQuotaExceeded)
}
@ -288,7 +296,7 @@ func (c *sshCommand) getSystemCommand() (systemCommand, error) {
if len(c.args) > 0 {
var err error
sshPath := c.getDestPath()
path, err = c.connection.buildPath(sshPath)
path, err = c.connection.fs.ResolvePath(sshPath, c.connection.User.GetHomeDir())
if err != nil {
return command, err
}
@ -331,7 +339,7 @@ func (c *sshCommand) rescanHomeDir() error {
var numFiles int
var size int64
if AddQuotaScan(c.connection.User.Username) {
numFiles, size, _, err = utils.ScanDirContents(c.connection.User.HomeDir)
numFiles, size, err = c.connection.fs.ScanDirContents(c.connection.User.HomeDir)
if err != nil {
c.connection.Log(logger.LevelWarn, logSenderSSH, "error scanning user home dir %#v: %v", c.connection.User.HomeDir, err)
} else {
@ -389,7 +397,7 @@ func (c *sshCommand) sendExitStatus(err error) {
if err == nil && c.command != "scp" {
realPath := c.getDestPath()
if len(realPath) > 0 {
p, err := c.connection.buildPath(realPath)
p, err := c.connection.fs.ResolvePath(realPath, c.connection.User.GetHomeDir())
if err == nil {
realPath = p
}

39
sftpd/transfer.go
View file

@ -11,6 +11,7 @@ import (
"github.com/drakkan/sftpgo/dataprovider"
"github.com/drakkan/sftpgo/logger"
"github.com/drakkan/sftpgo/metrics"
"github.com/eikenb/pipeat"
)
const (
@ -26,6 +27,9 @@ var (
// It implements the io Reader and Writer interface to handle files downloads and uploads
type Transfer struct {
file *os.File
writerAt *pipeat.PipeWriterAt
readerAt *pipeat.PipeReaderAt
cancelFn func()
path string
start time.Time
bytesSent int64
@ -52,6 +56,9 @@ func (t *Transfer) TransferError(err error) {
return
}
t.transferError = err
if t.cancelFn != nil {
t.cancelFn()
}
elapsed := time.Since(t.start).Nanoseconds() / 1000000
logger.Warn(logSender, t.connectionID, "Unexpected error for transfer, path: %#v, error: \"%v\" bytes sent: %v, "+
"bytes received: %v transfer running since %v ms", t.path, t.transferError, t.bytesSent, t.bytesReceived, elapsed)
@ -61,7 +68,13 @@ func (t *Transfer) TransferError(err error) {
// It handles download bandwidth throttling too
func (t *Transfer) ReadAt(p []byte, off int64) (n int, err error) {
t.lastActivity = time.Now()
readed, e := t.file.ReadAt(p, off)
var readed int
var e error
if t.readerAt != nil {
readed, e = t.readerAt.ReadAt(p, off)
} else {
readed, e = t.file.ReadAt(p, off)
}
t.lock.Lock()
t.bytesSent += int64(readed)
t.lock.Unlock()
@ -82,7 +95,13 @@ func (t *Transfer) WriteAt(p []byte, off int64) (n int, err error) {
t.TransferError(err)
return 0, err
}
written, e := t.file.WriteAt(p, off)
var written int
var e error
if t.writerAt != nil {
written, e = t.writerAt.WriteAt(p, off)
} else {
written, e = t.file.WriteAt(p, off)
}
t.lock.Lock()
t.bytesReceived += int64(written)
t.lock.Unlock()
@ -105,14 +124,14 @@ func (t *Transfer) Close() error {
if t.isFinished {
return errTransferClosed
}
err := t.file.Close()
err := t.closeIO()
t.isFinished = true
numFiles := 0
if t.isNewFile {
numFiles = 1
}
t.checkDownloadSize()
if t.transferType == transferUpload && t.file.Name() != t.path {
if t.transferType == transferUpload && t.file != nil && t.file.Name() != t.path {
if t.transferError == nil || uploadMode == uploadModeAtomicWithResume {
err = os.Rename(t.file.Name(), t.path)
logger.Debug(logSender, t.connectionID, "atomic upload completed, rename: %#v -> %#v, error: %v",
@ -150,6 +169,18 @@ func (t *Transfer) Close() error {
return err
}
func (t *Transfer) closeIO() error {
var err error
if t.writerAt != nil {
err = t.writerAt.Close()
} else if t.readerAt != nil {
err = t.readerAt.Close()
} else {
err = t.file.Close()
}
return err
}
func (t *Transfer) checkDownloadSize() {
if t.transferType == transferDownload && t.transferError == nil && t.bytesSent < t.expectedSize {
t.transferError = fmt.Errorf("incomplete download: %v/%v bytes transferred", t.bytesSent, t.expectedSize)

6
sql/mysql/20200116.sql Normal file
View file

@ -0,0 +1,6 @@
BEGIN;
--
-- Add field filesystem to user
--
ALTER TABLE `users` ADD COLUMN `filesystem` longtext NULL;
COMMIT;

6
sql/pgsql/20200116.sql Normal file
View file

@ -0,0 +1,6 @@
BEGIN;
--
-- Add field filesystem to user
--
ALTER TABLE "users" ADD COLUMN "filesystem" text NULL;
COMMIT;

9
sql/sqlite/20200116.sql Normal file
View file

@ -0,0 +1,9 @@
BEGIN;
--
-- Add field filesystem to user
--
CREATE TABLE "new__users" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE, "password" varchar(255) NULL, "public_keys" text NULL, "home_dir" varchar(255) NOT NULL, "uid" integer NOT NULL, "gid" integer NOT NULL, "max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "permissions" text NOT NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL, "expiration_date" bigint NOT NULL, "last_login" bigint NOT NULL, "status" integer NOT NULL, "filters" text NULL, "filesystem" text NULL);
INSERT INTO "new__users" ("id", "username", "password", "public_keys", "home_dir", "uid", "gid", "max_sessions", "quota_size", "quota_files", "permissions", "used_quota_size", "used_quota_files", "last_quota_update", "upload_bandwidth", "download_bandwidth", "expiration_date", "last_login", "status", "filters", "filesystem") SELECT "id", "username", "password", "public_keys", "home_dir", "uid", "gid", "max_sessions", "quota_size", "quota_files", "permissions", "used_quota_size", "used_quota_files", "last_quota_update", "upload_bandwidth", "download_bandwidth", "expiration_date", "last_login", "status", "filters", NULL FROM "users";
DROP TABLE "users";
ALTER TABLE "new__users" RENAME TO "users";
COMMIT;

52
templates/user.html
View file

@ -191,6 +191,58 @@
</div>
</div>
<div class="form-group row">
<label for="idFilesystem" class="col-sm-2 col-form-label">Storage</label>
<div class="col-sm-10">
<select class="form-control" id="idFilesystem" name="fs_provider">
<option value="0" {{if eq .User.FsConfig.Provider 0 }}selected{{end}}>local</option>
<option value="1" {{if eq .User.FsConfig.Provider 1 }}selected{{end}}>S3</option>
</select>
</div>
</div>
<div class="form-group row">
<label for="idS3Bucket" class="col-sm-2 col-form-label">S3 Bucket</label>
<div class="col-sm-3">
<input type="text" class="form-control" id="idS3Bucket" name="s3_bucket" placeholder=""
value="{{.User.FsConfig.S3Config.Bucket}}" maxlength="255">
</div>
<div class="col-sm-2"></div>
<label for="idS3Region" class="col-sm-2 col-form-label">S3 Region</label>
<div class="col-sm-3">
<input type="text" class="form-control" id="idS3Region" name="s3_region" placeholder=""
value="{{.User.FsConfig.S3Config.Region}}" maxlength="255">
</div>
</div>
<div class="form-group row">
<label for="idS3AccessKey" class="col-sm-2 col-form-label">S3 Access Key</label>
<div class="col-sm-3">
<input type="text" class="form-control" id="idS3AccessKey" name="s3_access_key" placeholder=""
value="{{.User.FsConfig.S3Config.AccessKey}}" maxlength="255">
</div>
<div class="col-sm-2"></div>
<label for="idS3AccessSecret" class="col-sm-2 col-form-label">S3 Access Secret</label>
<div class="col-sm-3">
<input type="text" class="form-control" id="idS3AccessSecret" name="s3_access_secret" placeholder=""
value="{{.User.FsConfig.S3Config.AccessSecret}}" maxlength="1000">
</div>
</div>
<div class="form-group row">
<label for="idS3StorageClass" class="col-sm-2 col-form-label">S3 Storage Class</label>
<div class="col-sm-3">
<input type="text" class="form-control" id="idS3StorageClass" name="s3_storage_class" placeholder=""
value="{{.User.FsConfig.S3Config.StorageClass}}" maxlength="1000">
</div>
<div class="col-sm-2"></div>
<label for="idS3Endpoint" class="col-sm-2 col-form-label">S3 Endpoint</label>
<div class="col-sm-3">
<input type="text" class="form-control" id="idS3Endpoint" name="s3_endpoint" placeholder=""
value="{{.User.FsConfig.S3Config.Endpoint}}" maxlength="255">
</div>
</div>
<input type="hidden" name="expiration_date" id="hidden_start_datetime" value="">
<button type="submit" class="btn btn-primary float-right mt-3 mb-5 px-5 px-3">Submit</button>
</form>

125
utils/utils.go
View file

@ -2,15 +2,16 @@
package utils
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"net"
"os"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/drakkan/sftpgo/logger"
)
const logSender = "utils"
@ -46,49 +47,6 @@ func GetTimeFromMsecSinceEpoch(msec int64) time.Time {
return time.Unix(0, msec*1000000)
}
// ScanDirContents returns the number of files contained in a directory, their size and a slice with the file paths
func ScanDirContents(path string) (int, int64, []string, error) {
var numFiles int
var size int64
var fileList []string
var err error
numFiles = 0
size = 0
isDir, err := isDirectory(path)
if err == nil && isDir {
err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info != nil && info.Mode().IsRegular() {
size += info.Size()
numFiles++
fileList = append(fileList, path)
}
return err
})
}
return numFiles, size, fileList, err
}
func isDirectory(path string) (bool, error) {
fileInfo, err := os.Stat(path)
if err != nil {
return false, err
}
return fileInfo.IsDir(), err
}
// SetPathPermissions call os.Chown on unix, it does nothing on windows
func SetPathPermissions(path string, uid int, gid int) {
if runtime.GOOS != "windows" {
if err := os.Chown(path, uid, gid); err != nil {
logger.Warn(logSender, "", "error chowning path %v: %v", path, err)
}
}
}
// GetAppVersion returns VersionInfo struct
func GetAppVersion() VersionInfo {
return versionInfo
@ -144,3 +102,74 @@ func GetIPFromRemoteAddress(remoteAddress string) string {
}
return remoteAddress
}
// NilIfEmpty returns nil if the input string is empty
func NilIfEmpty(s string) *string {
if len(s) == 0 {
return nil
}
return &s
}
// EncryptData encrypts data using the given key
func EncryptData(data string) (string, error) {
var result string
key := make([]byte, 16)
if _, err := io.ReadFull(rand.Reader, key); err != nil {
return result, err
}
keyHex := hex.EncodeToString(key)
block, err := aes.NewCipher([]byte(keyHex))
if err != nil {
return result, err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return result, err
}
nonce := make([]byte, gcm.NonceSize())
if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
return result, err
}
ciphertext := gcm.Seal(nonce, nonce, []byte(data), nil)
result = fmt.Sprintf("$aes$%s$%x", keyHex, ciphertext)
return result, err
}
// RemoveDecryptionKey returns encrypted data without the decryption key
func RemoveDecryptionKey(encryptData string) string {
vals := strings.Split(encryptData, "$")
if len(vals) == 4 {
return fmt.Sprintf("$%v$%v", vals[1], vals[3])
}
return encryptData
}
// DecryptData decrypts data encrypted using EncryptData
func DecryptData(data string) (string, error) {
var result string
vals := strings.Split(data, "$")
if len(vals) != 4 {
return "", errors.New("data to decrypt is not in the correct format")
}
key := vals[2]
encrypted, err := hex.DecodeString(vals[3])
if err != nil {
return result, err
}
block, err := aes.NewCipher([]byte(key))
if err != nil {
return result, err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return result, err
}
nonceSize := gcm.NonceSize()
nonce, ciphertext := encrypted[:nonceSize], encrypted[nonceSize:]
plaintext, err := gcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return result, err
}
return string(plaintext), nil
}

288
vfs/osfs.go Normal file
View file

@ -0,0 +1,288 @@
package vfs
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"time"
"github.com/drakkan/sftpgo/logger"
"github.com/eikenb/pipeat"
"github.com/rs/xid"
)
const (
// osFsName is the name for the local Fs implementation
osFsName = "osfs"
)
// OsFs is a Fs implementation that uses functions provided by the os package.
type OsFs struct {
name string
connectionID string
}
// NewOsFs returns an OsFs object that allows to interact with local Os filesystem
func NewOsFs(connectionID string) Fs {
return &OsFs{
name: osFsName,
connectionID: connectionID}
}
// Name returns the name for the Fs implementation
func (fs OsFs) Name() string {
return fs.name
}
// ConnectionID returns the SSH connection ID associated to this Fs implementation
func (fs OsFs) ConnectionID() string {
return fs.connectionID
}
// Stat returns a FileInfo describing the named file
func (OsFs) Stat(name string) (os.FileInfo, error) {
return os.Stat(name)
}
// Lstat returns a FileInfo describing the named file
func (OsFs) Lstat(name string) (os.FileInfo, error) {
return os.Lstat(name)
}
// Open opens the named file for reading
func (OsFs) Open(name string) (*os.File, *pipeat.PipeReaderAt, func(), error) {
f, err := os.Open(name)
return f, nil, nil, err
}
// Create creates or opens the named file for writing
func (OsFs) Create(name string, flag int) (*os.File, *pipeat.PipeWriterAt, func(), error) {
var err error
var f *os.File
if flag == 0 {
f, err = os.Create(name)
} else {
f, err = os.OpenFile(name, flag, 0666)
}
return f, nil, nil, err
}
// Rename renames (moves) source to target
func (OsFs) Rename(source, target string) error {
return os.Rename(source, target)
}
// Remove removes the named file or (empty) directory.
func (OsFs) Remove(name string, isDir bool) error {
return os.Remove(name)
}
// Mkdir creates a new directory with the specified name and default permissions
func (OsFs) Mkdir(name string) error {
return os.Mkdir(name, 0777)
}
// Symlink creates source as a symbolic link to target.
func (OsFs) Symlink(source, target string) error {
return os.Symlink(source, target)
}
// Chown changes the numeric uid and gid of the named file.
func (OsFs) Chown(name string, uid int, gid int) error {
return os.Chown(name, uid, gid)
}
// Chmod changes the mode of the named file to mode
func (OsFs) Chmod(name string, mode os.FileMode) error {
return os.Chmod(name, mode)
}
// Chtimes changes the access and modification times of the named file
func (OsFs) Chtimes(name string, atime, mtime time.Time) error {
return os.Chtimes(name, atime, mtime)
}
// ReadDir reads the directory named by dirname and returns
// a list of directory entries.
func (OsFs) ReadDir(dirname string) ([]os.FileInfo, error) {
return ioutil.ReadDir(dirname)
}
// IsUploadResumeSupported returns true if upload resume is supported
func (OsFs) IsUploadResumeSupported() bool {
return true
}
// IsAtomicUploadSupported returns true if atomic upload is supported
func (OsFs) IsAtomicUploadSupported() bool {
return true
}
// IsNotExist returns a boolean indicating whether the error is known to
// report that a file or directory does not exist
func (OsFs) IsNotExist(err error) bool {
return os.IsNotExist(err)
}
// IsPermission returns a boolean indicating whether the error is known to
// report that permission is denied.
func (OsFs) IsPermission(err error) bool {
return os.IsPermission(err)
}
// CheckRootPath creates the specified root directory if it does not exists
func (fs OsFs) CheckRootPath(rootPath, username string, uid int, gid int) bool {
var err error
if _, err = fs.Stat(rootPath); fs.IsNotExist(err) {
err = os.MkdirAll(rootPath, 0777)
fsLog(fs, logger.LevelDebug, "root directory %#v for user %#v does not exist, try to create, mkdir error: %v",
rootPath, username, err)
if err == nil {
SetPathPermissions(fs, rootPath, uid, gid)
}
}
return (err == nil)
}
// ScanDirContents returns the number of files contained in a directory and
// their size
func (fs OsFs) ScanDirContents(dirPath string) (int, int64, error) {
numFiles := 0
size := int64(0)
isDir, err := IsDirectory(fs, dirPath)
if err == nil && isDir {
err = filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info != nil && info.Mode().IsRegular() {
size += info.Size()
numFiles++
}
return err
})
}
return numFiles, size, err
}
// GetAtomicUploadPath returns the path to use for an atomic upload
func (OsFs) GetAtomicUploadPath(name string) string {
dir := filepath.Dir(name)
guid := xid.New().String()
return filepath.Join(dir, ".sftpgo-upload."+guid+"."+filepath.Base(name))
}
// GetRelativePath returns the path for a file relative to the user's home dir.
// This is the path as seen by SFTP users
func (OsFs) GetRelativePath(name, rootPath string) string {
rel, err := filepath.Rel(rootPath, filepath.Clean(name))
if err != nil {
return ""
}
if rel == "." || strings.HasPrefix(rel, "..") {
rel = ""
}
return "/" + filepath.ToSlash(rel)
}
// Join joins any number of path elements into a single path
func (OsFs) Join(elem ...string) string {
return filepath.Join(elem...)
}
// ResolvePath returns the matching filesystem path for the specified sftp path
func (fs OsFs) ResolvePath(sftpPath, rootPath string) (string, error) {
if !filepath.IsAbs(rootPath) {
return "", fmt.Errorf("Invalid root path: %v", rootPath)
}
r := filepath.Clean(filepath.Join(rootPath, sftpPath))
p, err := filepath.EvalSymlinks(r)
if err != nil && !os.IsNotExist(err) {
return "", err
} else if os.IsNotExist(err) {
// The requested path doesn't exist, so at this point we need to iterate up the
// path chain until we hit a directory that _does_ exist and can be validated.
_, err = fs.findFirstExistingDir(r, rootPath)
if err != nil {
fsLog(fs, logger.LevelWarn, "error resolving not existent path: %#v", err)
}
return r, err
}
err = fs.isSubDir(p, rootPath)
if err != nil {
fsLog(fs, logger.LevelWarn, "Invalid path resolution, dir: %#v outside user home: %#v err: %v", p, rootPath, err)
}
return r, err
}
func (fs *OsFs) findNonexistentDirs(path, rootPath string) ([]string, error) {
results := []string{}
cleanPath := filepath.Clean(path)
parent := filepath.Dir(cleanPath)
_, err := os.Stat(parent)
for os.IsNotExist(err) {
results = append(results, parent)
parent = filepath.Dir(parent)
_, err = os.Stat(parent)
}
if err != nil {
return results, err
}
p, err := filepath.EvalSymlinks(parent)
if err != nil {
return results, err
}
err = fs.isSubDir(p, rootPath)
if err != nil {
fsLog(fs, logger.LevelWarn, "error finding non existing dir: %v", err)
}
return results, err
}
func (fs *OsFs) findFirstExistingDir(path, rootPath string) (string, error) {
results, err := fs.findNonexistentDirs(path, rootPath)
if err != nil {
fsLog(fs, logger.LevelWarn, "unable to find non existent dirs: %v", err)
return "", err
}
var parent string
if len(results) > 0 {
lastMissingDir := results[len(results)-1]
parent = filepath.Dir(lastMissingDir)
} else {
parent = rootPath
}
p, err := filepath.EvalSymlinks(parent)
if err != nil {
return "", err
}
fileInfo, err := os.Stat(p)
if err != nil {
return "", err
}
if !fileInfo.IsDir() {
return "", fmt.Errorf("resolved path is not a dir: %#v", p)
}
err = fs.isSubDir(p, rootPath)
return p, err
}
func (fs *OsFs) isSubDir(sub, rootPath string) error {
// rootPath must exist and it is already a validated absolute path
parent, err := filepath.EvalSymlinks(rootPath)
if err != nil {
fsLog(fs, logger.LevelWarn, "invalid home dir %#v: %v", rootPath, err)
return err
}
if !strings.HasPrefix(sub, parent) {
err = fmt.Errorf("path %#v is not inside: %#v", sub, parent)
fsLog(fs, logger.LevelWarn, "error: %v ", err)
return err
}
return nil
}

60
vfs/s3fileinfo.go Normal file
View file

@ -0,0 +1,60 @@
package vfs
import (
"os"
"time"
)
// S3FileInfo implements os.FileInfo for a file in S3.
type S3FileInfo struct {
name string
sizeInBytes int64
modTime time.Time
mode os.FileMode
sys interface{}
}
// NewS3FileInfo creates file info.
func NewS3FileInfo(name string, isDirectory bool, sizeInBytes int64, modTime time.Time) S3FileInfo {
mode := os.FileMode(0644)
if isDirectory {
mode = os.FileMode(0755) | os.ModeDir
}
return S3FileInfo{
name: name,
sizeInBytes: sizeInBytes,
modTime: modTime,
mode: mode,
}
}
// Name provides the base name of the file.
func (fi S3FileInfo) Name() string {
return fi.name
}
// Size provides the length in bytes for a file.
func (fi S3FileInfo) Size() int64 {
return fi.sizeInBytes
}
// Mode provides the file mode bits
func (fi S3FileInfo) Mode() os.FileMode {
return fi.mode
}
// ModTime provides the last modification time.
func (fi S3FileInfo) ModTime() time.Time {
return fi.modTime
}
// IsDir provides the abbreviation for Mode().IsDir()
func (fi S3FileInfo) IsDir() bool {
return fi.mode&os.ModeDir != 0
}
// Sys provides the underlying data source (can return nil)
func (fi S3FileInfo) Sys() interface{} {
return fi.getFileInfoSys()
}

28
vfs/s3fileinfo_unix.go Normal file
View file

@ -0,0 +1,28 @@
// +build !windows
package vfs
import "syscall"
import "os"
var (
defaultUID, defaultGID int
)
func init() {
defaultUID = os.Getuid()
defaultGID = os.Getuid()
if defaultUID < 0 {
defaultUID = 65534
}
if defaultGID < 0 {
defaultGID = 65534
}
}
func (fi S3FileInfo) getFileInfoSys() interface{} {
return &syscall.Stat_t{
Uid: uint32(defaultUID),
Gid: uint32(defaultGID)}
}

7
vfs/s3fileinfo_windows.go Normal file
View file

@ -0,0 +1,7 @@
package vfs
import "syscall"
func (fi S3FileInfo) getFileInfoSys() interface{} {
return syscall.Win32FileAttributeData{}
}

491
vfs/s3fs.go Normal file
View file

@ -0,0 +1,491 @@
package vfs
import (
"context"
"errors"
"fmt"
"os"
"path"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/drakkan/sftpgo/logger"
"github.com/drakkan/sftpgo/utils"
"github.com/eikenb/pipeat"
)
// S3FsConfig defines the configuration for S3fs
type S3FsConfig struct {
Bucket string `json:"bucket,omitempty"`
Region string `json:"region,omitempty"`
AccessKey string `json:"access_key,omitempty"`
AccessSecret string `json:"access_secret,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
StorageClass string `json:"storage_class,omitempty"`
}
// S3Fs is a Fs implementation for Amazon S3 compatible object storage.
type S3Fs struct {
connectionID string
localTempDir string
config S3FsConfig
svc *s3.S3
ctxTimeout time.Duration
ctxLongTimeout time.Duration
}
// NewS3Fs returns an S3Fs object that allows to interact with an s3 compatible
// object storage
func NewS3Fs(connectionID, localTempDir string, config S3FsConfig) (Fs, error) {
fs := S3Fs{
connectionID: connectionID,
localTempDir: localTempDir,
config: config,
ctxTimeout: 30 * time.Second,
ctxLongTimeout: 300 * time.Second,
}
if err := ValidateS3FsConfig(&fs.config); err != nil {
return fs, err
}
accessSecret, err := utils.DecryptData(fs.config.AccessSecret)
if err != nil {
return fs, err
}
fs.config.AccessSecret = accessSecret
awsConfig := &aws.Config{
Region: aws.String(fs.config.Region),
Credentials: credentials.NewStaticCredentials(fs.config.AccessKey, fs.config.AccessSecret, ""),
}
//config.WithLogLevel(aws.LogDebugWithHTTPBody)
if len(fs.config.Endpoint) > 0 {
awsConfig.Endpoint = aws.String(fs.config.Endpoint)
awsConfig.S3ForcePathStyle = aws.Bool(true)
}
sess, err := session.NewSession(awsConfig)
if err != nil {
return fs, err
}
fs.svc = s3.New(sess)
return fs, nil
}
// Name returns the name for the Fs implementation
func (fs S3Fs) Name() string {
return fmt.Sprintf("S3Fs bucket: %#v", fs.config.Bucket)
}
// ConnectionID returns the SSH connection ID associated to this Fs implementation
func (fs S3Fs) ConnectionID() string {
return fs.connectionID
}
// Stat returns a FileInfo describing the named file
func (fs S3Fs) Stat(name string) (os.FileInfo, error) {
var result S3FileInfo
if name == "/" || name == "." {
err := fs.checkIfBucketExists()
if err != nil {
return result, err
}
return NewS3FileInfo(name, true, 0, time.Time{}), nil
}
prefix := path.Dir(name)
if prefix == "/" || prefix == "." {
prefix = ""
} else {
prefix = strings.TrimPrefix(prefix, "/")
if !strings.HasSuffix(prefix, "/") {
prefix += "/"
}
}
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
defer cancelFn()
err := fs.svc.ListObjectsV2PagesWithContext(ctx, &s3.ListObjectsV2Input{
Bucket: aws.String(fs.config.Bucket),
Prefix: aws.String(prefix),
Delimiter: aws.String("/"),
}, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
for _, p := range page.CommonPrefixes {
if fs.isEqual(p.Prefix, name) {
result = NewS3FileInfo(name, true, 0, time.Time{})
return false
}
}
for _, fileObject := range page.Contents {
if fs.isEqual(fileObject.Key, name) {
objectSize := *fileObject.Size
objectModTime := *fileObject.LastModified
isDir := strings.HasSuffix(*fileObject.Key, "/")
result = NewS3FileInfo(name, isDir, objectSize, objectModTime)
return false
}
}
return true
})
if err == nil && len(result.Name()) == 0 {
err = errors.New("404 no such file or directory")
}
return result, err
}
// Lstat returns a FileInfo describing the named file
func (fs S3Fs) Lstat(name string) (os.FileInfo, error) {
return fs.Stat(name)
}
// Open opens the named file for reading
func (fs S3Fs) Open(name string) (*os.File, *pipeat.PipeReaderAt, func(), error) {
r, w, err := pipeat.AsyncWriterPipeInDir(fs.localTempDir)
if err != nil {
return nil, nil, nil, err
}
ctx, cancelFn := context.WithCancel(context.Background())
downloader := s3manager.NewDownloaderWithClient(fs.svc)
go func() {
defer cancelFn()
key := name
n, err := downloader.DownloadWithContext(ctx, w, &s3.GetObjectInput{
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(key),
})
fsLog(fs, logger.LevelDebug, "download completed, path: %#v size: %v, err: %v", name, n, err)
w.CloseWithError(err)
}()
return nil, r, cancelFn, nil
}
// Create creates or opens the named file for writing
func (fs S3Fs) Create(name string, flag int) (*os.File, *pipeat.PipeWriterAt, func(), error) {
r, w, err := pipeat.PipeInDir(fs.localTempDir)
if err != nil {
return nil, nil, nil, err
}
ctx, cancelFn := context.WithCancel(context.Background())
uploader := s3manager.NewUploaderWithClient(fs.svc)
go func() {
defer cancelFn()
key := name
response, err := uploader.UploadWithContext(ctx, &s3manager.UploadInput{
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(key),
Body: r,
StorageClass: utils.NilIfEmpty(fs.config.StorageClass),
})
fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, response: %v, err: %v", name, response, err)
r.CloseWithError(err)
}()
return nil, w, cancelFn, nil
}
// Rename renames (moves) source to target.
// We don't support renaming non empty directories since we should
// rename all the contents too and this could take long time: think
// about directories with thousands of files, for each file we should
// execute a CopyObject call.
func (fs S3Fs) Rename(source, target string) error {
if source == target {
return nil
}
fi, err := fs.Stat(source)
if err != nil {
return err
}
copySource := fs.Join(fs.config.Bucket, source)
if fi.IsDir() {
contents, err := fs.ReadDir(source)
if err != nil {
return err
}
if len(contents) > 0 {
return fmt.Errorf("Cannot rename non empty directory: %#v", source)
}
if !strings.HasSuffix(copySource, "/") {
copySource += "/"
}
if !strings.HasSuffix(target, "/") {
target += "/"
}
}
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
defer cancelFn()
_, err = fs.svc.CopyObjectWithContext(ctx, &s3.CopyObjectInput{
Bucket: aws.String(fs.config.Bucket),
CopySource: aws.String(copySource),
Key: aws.String(target),
})
if err != nil {
return err
}
return fs.Remove(source, fi.IsDir())
}
// Remove removes the named file or (empty) directory.
func (fs S3Fs) Remove(name string, isDir bool) error {
if isDir {
contents, err := fs.ReadDir(name)
if err != nil {
return err
}
if len(contents) > 0 {
return fmt.Errorf("Cannot remove non empty directory: %#v", name)
}
if !strings.HasSuffix(name, "/") {
name += "/"
}
}
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
defer cancelFn()
_, err := fs.svc.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(name),
})
return err
}
// Mkdir creates a new directory with the specified name and default permissions
func (fs S3Fs) Mkdir(name string) error {
_, err := fs.Stat(name)
if !fs.IsNotExist(err) {
return err
}
if !strings.HasSuffix(name, "/") {
name += "/"
}
_, w, _, err := fs.Create(name, 0)
if err != nil {
return err
}
return w.Close()
}
// Symlink creates source as a symbolic link to target.
func (S3Fs) Symlink(source, target string) error {
return errors.New("403 symlinks are not supported")
}
// Chown changes the numeric uid and gid of the named file.
// Silently ignored.
func (S3Fs) Chown(name string, uid int, gid int) error {
return nil
}
// Chmod changes the mode of the named file to mode.
// Silently ignored.
func (S3Fs) Chmod(name string, mode os.FileMode) error {
return nil
}
// Chtimes changes the access and modification times of the named file.
// Silently ignored.
func (S3Fs) Chtimes(name string, atime, mtime time.Time) error {
return errors.New("403 chtimes is not supported")
}
// ReadDir reads the directory named by dirname and returns
// a list of directory entries.
func (fs S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) {
var result []os.FileInfo
// dirname deve essere già cleaned
prefix := ""
if dirname != "/" && dirname != "." {
prefix = strings.TrimPrefix(dirname, "/")
if !strings.HasSuffix(prefix, "/") {
prefix += "/"
}
}
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
defer cancelFn()
err := fs.svc.ListObjectsV2PagesWithContext(ctx, &s3.ListObjectsV2Input{
Bucket: aws.String(fs.config.Bucket),
Prefix: aws.String(prefix),
Delimiter: aws.String("/"),
}, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
for _, p := range page.CommonPrefixes {
name, isDir := fs.resolve(p.Prefix, prefix)
result = append(result, NewS3FileInfo(name, isDir, 0, time.Time{}))
}
for _, fileObject := range page.Contents {
objectSize := *fileObject.Size
objectModTime := *fileObject.LastModified
name, isDir := fs.resolve(fileObject.Key, prefix)
if len(name) == 0 {
continue
}
result = append(result, NewS3FileInfo(name, isDir, objectSize, objectModTime))
}
return true
})
return result, err
}
// IsUploadResumeSupported returns true if upload resume is supported.
// SFTP Resume is not supported on S3
func (S3Fs) IsUploadResumeSupported() bool {
return false
}
// IsAtomicUploadSupported returns true if atomic upload is supported.
// S3 uploads are already atomic, we don't need to upload to a temporary
// file
func (S3Fs) IsAtomicUploadSupported() bool {
return false
}
// IsNotExist returns a boolean indicating whether the error is known to
// report that a file or directory does not exist
func (S3Fs) IsNotExist(err error) bool {
if err == nil {
return false
}
if aerr, ok := err.(awserr.Error); ok {
if aerr.Code() == s3.ErrCodeNoSuchKey {
return true
}
if aerr.Code() == s3.ErrCodeNoSuchBucket {
return true
}
}
if multierr, ok := err.(s3manager.MultiUploadFailure); ok {
if multierr.Code() == s3.ErrCodeNoSuchKey {
return true
}
if multierr.Code() == s3.ErrCodeNoSuchBucket {
return true
}
}
return strings.Contains(err.Error(), "404")
}
// IsPermission returns a boolean indicating whether the error is known to
// report that permission is denied.
func (S3Fs) IsPermission(err error) bool {
if err == nil {
return false
}
return strings.Contains(err.Error(), "403")
}
// CheckRootPath creates the specified root directory if it does not exists
func (fs S3Fs) CheckRootPath(rootPath, username string, uid int, gid int) bool {
// we need a local directory for temporary files
osFs := NewOsFs(fs.ConnectionID())
osFs.CheckRootPath(fs.localTempDir, username, uid, gid)
err := fs.checkIfBucketExists()
if err == nil {
return true
}
if !fs.IsNotExist(err) {
return false
}
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
defer cancelFn()
input := &s3.CreateBucketInput{
Bucket: aws.String(fs.config.Bucket),
}
_, err = fs.svc.CreateBucketWithContext(ctx, input)
fsLog(fs, logger.LevelDebug, "bucket %#v for user %#v does not exists, try to create, error: %v",
fs.config.Bucket, username, err)
return err == nil
}
// ScanDirContents returns the number of files contained in the bucket,
// and their size
func (fs S3Fs) ScanDirContents(dirPath string) (int, int64, error) {
numFiles := 0
size := int64(0)
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout))
defer cancelFn()
err := fs.svc.ListObjectsV2PagesWithContext(ctx, &s3.ListObjectsV2Input{
Bucket: aws.String(fs.config.Bucket),
Prefix: aws.String(""),
}, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
for _, fileObject := range page.Contents {
numFiles++
size += *fileObject.Size
}
return true
})
return numFiles, size, err
}
// GetAtomicUploadPath returns the path to use for an atomic upload.
// S3 uploads are already atomic, we never call this method for S3
func (S3Fs) GetAtomicUploadPath(name string) string {
return ""
}
// GetRelativePath returns the path for a file relative to the user's home dir.
// This is the path as seen by SFTP users
func (S3Fs) GetRelativePath(name, rootPath string) string {
rel := name
if name == "." {
rel = ""
}
if !strings.HasPrefix(rel, "/") {
return "/" + rel
}
return rel
}
// Join joins any number of path elements into a single path
func (S3Fs) Join(elem ...string) string {
return path.Join(elem...)
}
// ResolvePath returns the matching filesystem path for the specified sftp path
func (fs S3Fs) ResolvePath(sftpPath, rootPath string) (string, error) {
return sftpPath, nil
}
func (fs *S3Fs) resolve(name *string, prefix string) (string, bool) {
result := strings.TrimPrefix(*name, prefix)
isDir := strings.HasSuffix(result, "/")
if isDir {
result = strings.TrimSuffix(result, "/")
}
if strings.Contains(result, "/") {
i := strings.Index(result, "/")
isDir = true
result = result[:i]
}
return result, isDir
}
func (fs *S3Fs) isEqual(s3Key *string, sftpName string) bool {
if *s3Key == sftpName {
return true
}
if "/"+*s3Key == sftpName {
return true
}
if "/"+*s3Key == sftpName+"/" {
return true
}
return false
}
func (fs *S3Fs) checkIfBucketExists() error {
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
defer cancelFn()
_, err := fs.svc.HeadBucketWithContext(ctx, &s3.HeadBucketInput{
Bucket: aws.String(fs.config.Bucket),
})
return err
}
func (fs *S3Fs) getObjectDetails(key string) (*s3.HeadObjectOutput, error) {
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
defer cancelFn()
input := &s3.HeadObjectInput{
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(key),
}
return fs.svc.HeadObjectWithContext(ctx, input)
}

100
vfs/vfs.go Normal file
View file

@ -0,0 +1,100 @@
package vfs
import (
"errors"
"os"
"runtime"
"time"
"github.com/drakkan/sftpgo/logger"
"github.com/eikenb/pipeat"
"github.com/pkg/sftp"
)
// Fs defines the interface for filesystems backends
type Fs interface {
Name() string
ConnectionID() string
Stat(name string) (os.FileInfo, error)
Lstat(name string) (os.FileInfo, error)
Open(name string) (*os.File, *pipeat.PipeReaderAt, func(), error)
Create(name string, flag int) (*os.File, *pipeat.PipeWriterAt, func(), error)
Rename(source, target string) error
Remove(name string, isDir bool) error
Mkdir(name string) error
Symlink(source, target string) error
Chown(name string, uid int, gid int) error
Chmod(name string, mode os.FileMode) error
Chtimes(name string, atime, mtime time.Time) error
ReadDir(dirname string) ([]os.FileInfo, error)
IsUploadResumeSupported() bool
IsAtomicUploadSupported() bool
CheckRootPath(rootPath, username string, uid int, gid int) bool
ResolvePath(sftpPath, rootPath string) (string, error)
IsNotExist(err error) bool
IsPermission(err error) bool
ScanDirContents(dirPath string) (int, int64, error)
GetAtomicUploadPath(name string) string
GetRelativePath(name, rootPath string) string
Join(elem ...string) string
}
// IsDirectory checks if a path exists and is a directory
func IsDirectory(fs Fs, path string) (bool, error) {
fileInfo, err := fs.Stat(path)
if err != nil {
return false, err
}
return fileInfo.IsDir(), err
}
// GetSFTPError returns an sftp error from a filesystem error
func GetSFTPError(fs Fs, err error) error {
if fs.IsNotExist(err) {
return sftp.ErrSSHFxNoSuchFile
} else if fs.IsPermission(err) {
return sftp.ErrSSHFxPermissionDenied
} else if err != nil {
return sftp.ErrSSHFxFailure
}
return nil
}
// IsLocalOsFs returns true if fs is the local filesystem implementation
func IsLocalOsFs(fs Fs) bool {
return fs.Name() == osFsName
}
// ValidateS3FsConfig returns nil if the specified s3 config is valid, otherwise an error
func ValidateS3FsConfig(config *S3FsConfig) error {
if len(config.Bucket) == 0 {
return errors.New("bucket cannot be empty")
}
if len(config.Region) == 0 {
return errors.New("region cannot be empty")
}
if len(config.AccessKey) == 0 {
return errors.New("access_key cannot be empty")
}
if len(config.AccessSecret) == 0 {
return errors.New("access_secret cannot be empty")
}
return nil
}
// SetPathPermissions calls fs.Chown.
// It does nothing for local filesystem on windows
func SetPathPermissions(fs Fs, path string, uid int, gid int) {
if IsLocalOsFs(fs) {
if runtime.GOOS == "windows" {
return
}
}
if err := fs.Chown(path, uid, gid); err != nil {
fsLog(fs, logger.LevelWarn, "error chowning path %v: %v", path, err)
}
}
func fsLog(fs Fs, level logger.LogLevel, format string, v ...interface{}) {
logger.Log(level, fs.Name(), fs.ConnectionID(), format, v...)
}