mirror of
https://github.com/drakkan/sftpgo.git
synced 2024-11-21 15:10:23 +00:00
S3: add support for serving virtual folders
inside the same bucket each user can be assigned to a virtual folder. This is similar to a chroot directory for local filesystem
This commit is contained in:
parent
d75f56b914
commit
4463421028
19 changed files with 241 additions and 86 deletions
11
README.md
11
README.md
|
@ -419,8 +419,13 @@ The HTTP request has a 15 seconds timeout.
|
|||
|
||||
## S3 Compabible Object Storage backends
|
||||
|
||||
Each user can be mapped with an S3-Compatible bucket, this way the mapped bucket is exposed over SFTP/SCP.
|
||||
SFTPGo uses multipart uploads and parallel downloads for storing and retrieving files from S3 and automatically try to create the mapped bucket if it does not exists.
|
||||
Each user can be mapped with an S3-Compatible bucket or a bucket virtual folder, this way the mapped bucket/virtual folder is exposed over SFTP/SCP.
|
||||
|
||||
Specifying a different `key_prefix` you can assign different virtual folders of the same bucket to different users. This is similar to a chroot directory for local filesystem. The virtual folder identified by `key_prefix` does not need to be pre-created.
|
||||
|
||||
SFTPGo uses multipart uploads and parallel downloads for storing and retrieving files from S3.
|
||||
|
||||
SFTPGo tries to automatically create the mapped bucket if it does not exists but it's a better idea to pre-create the bucket and to assign to it the wanted options such as automatic encryption and authorizations.
|
||||
|
||||
Some SFTP commands doesn't work over S3:
|
||||
|
||||
|
@ -465,6 +470,7 @@ Flags:
|
|||
--s3-access-secret string
|
||||
--s3-bucket string
|
||||
--s3-endpoint string
|
||||
--s3-key-prefix string Allows to restrict access to the virtual folder identified by this prefix and its contents
|
||||
--s3-region string
|
||||
--s3-storage-class string
|
||||
-s, --sftpd-port int 0 means a random non privileged port
|
||||
|
@ -522,6 +528,7 @@ For each account the following properties can be configured:
|
|||
- `s3_access_secret`, required for S3 filesystem. It is stored encrypted (AES-256-GCM)
|
||||
- `s3_endpoint`, specifies s3 endpoint (server) different from AWS
|
||||
- `s3_storage_class`
|
||||
- `s3_key_prefix`, allows to restrict access to the virtual folder identified by this prefix and its contents
|
||||
|
||||
These properties are stored inside the data provider.
|
||||
|
||||
|
|
|
@ -28,6 +28,7 @@ var (
|
|||
portableS3AccessSecret string
|
||||
portableS3Endpoint string
|
||||
portableS3StorageClass string
|
||||
portableS3KeyPrefix string
|
||||
portableCmd = &cobra.Command{
|
||||
Use: "portable",
|
||||
Short: "Serve a single directory",
|
||||
|
@ -70,6 +71,7 @@ Please take a look at the usage below to customize the serving parameters`,
|
|||
AccessSecret: portableS3AccessSecret,
|
||||
Endpoint: portableS3Endpoint,
|
||||
StorageClass: portableS3StorageClass,
|
||||
KeyPrefix: portableS3KeyPrefix,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
@ -105,5 +107,7 @@ func init() {
|
|||
portableCmd.Flags().StringVar(&portableS3AccessSecret, "s3-access-secret", "", "")
|
||||
portableCmd.Flags().StringVar(&portableS3Endpoint, "s3-endpoint", "", "")
|
||||
portableCmd.Flags().StringVar(&portableS3StorageClass, "s3-storage-class", "", "")
|
||||
portableCmd.Flags().StringVar(&portableS3KeyPrefix, "s3-key-prefix", "", "Allows to restrict access to the virtual folder "+
|
||||
"identified by this prefix and its contents")
|
||||
rootCmd.AddCommand(portableCmd)
|
||||
}
|
||||
|
|
|
@ -413,19 +413,19 @@ func buildUserHomeDir(user *User) {
|
|||
|
||||
func validatePermissions(user *User) error {
|
||||
if len(user.Permissions) == 0 {
|
||||
return &ValidationError{err: "Please grant some permissions to this user"}
|
||||
return &ValidationError{err: "please grant some permissions to this user"}
|
||||
}
|
||||
permissions := make(map[string][]string)
|
||||
if _, ok := user.Permissions["/"]; !ok {
|
||||
return &ValidationError{err: fmt.Sprintf("Permissions for the root dir \"/\" must be set")}
|
||||
return &ValidationError{err: fmt.Sprintf("permissions for the root dir \"/\" must be set")}
|
||||
}
|
||||
for dir, perms := range user.Permissions {
|
||||
if len(perms) == 0 {
|
||||
return &ValidationError{err: fmt.Sprintf("No permissions granted for the directory: %#v", dir)}
|
||||
return &ValidationError{err: fmt.Sprintf("no permissions granted for the directory: %#v", dir)}
|
||||
}
|
||||
for _, p := range perms {
|
||||
if !utils.IsStringInSlice(p, ValidPerms) {
|
||||
return &ValidationError{err: fmt.Sprintf("Invalid permission: %#v", p)}
|
||||
return &ValidationError{err: fmt.Sprintf("invalid permission: %#v", p)}
|
||||
}
|
||||
}
|
||||
cleanedDir := filepath.ToSlash(path.Clean(dir))
|
||||
|
@ -433,7 +433,7 @@ func validatePermissions(user *User) error {
|
|||
cleanedDir = strings.TrimSuffix(cleanedDir, "/")
|
||||
}
|
||||
if !path.IsAbs(cleanedDir) {
|
||||
return &ValidationError{err: fmt.Sprintf("Cannot set permissions for non absolute path: %#v", dir)}
|
||||
return &ValidationError{err: fmt.Sprintf("cannot set permissions for non absolute path: %#v", dir)}
|
||||
}
|
||||
if utils.IsStringInSlice(PermAny, perms) {
|
||||
permissions[cleanedDir] = []string{PermAny}
|
||||
|
@ -452,7 +452,7 @@ func validatePublicKeys(user *User) error {
|
|||
for i, k := range user.PublicKeys {
|
||||
_, _, _, _, err := ssh.ParseAuthorizedKey([]byte(k))
|
||||
if err != nil {
|
||||
return &ValidationError{err: fmt.Sprintf("Could not parse key nr. %d: %s", i, err)}
|
||||
return &ValidationError{err: fmt.Sprintf("could not parse key nr. %d: %s", i, err)}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
@ -468,13 +468,13 @@ func validateFilters(user *User) error {
|
|||
for _, IPMask := range user.Filters.DeniedIP {
|
||||
_, _, err := net.ParseCIDR(IPMask)
|
||||
if err != nil {
|
||||
return &ValidationError{err: fmt.Sprintf("Could not parse denied IP/Mask %#v : %v", IPMask, err)}
|
||||
return &ValidationError{err: fmt.Sprintf("could not parse denied IP/Mask %#v : %v", IPMask, err)}
|
||||
}
|
||||
}
|
||||
for _, IPMask := range user.Filters.AllowedIP {
|
||||
_, _, err := net.ParseCIDR(IPMask)
|
||||
if err != nil {
|
||||
return &ValidationError{err: fmt.Sprintf("Could not parse allowed IP/Mask %#v : %v", IPMask, err)}
|
||||
return &ValidationError{err: fmt.Sprintf("could not parse allowed IP/Mask %#v : %v", IPMask, err)}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
@ -484,13 +484,13 @@ func validateFilesystemConfig(user *User) error {
|
|||
if user.FsConfig.Provider == 1 {
|
||||
err := vfs.ValidateS3FsConfig(&user.FsConfig.S3Config)
|
||||
if err != nil {
|
||||
return &ValidationError{err: fmt.Sprintf("Could not validate s3config: %v", err)}
|
||||
return &ValidationError{err: fmt.Sprintf("could not validate s3config: %v", err)}
|
||||
}
|
||||
vals := strings.Split(user.FsConfig.S3Config.AccessSecret, "$")
|
||||
if !strings.HasPrefix(user.FsConfig.S3Config.AccessSecret, "$aes$") || len(vals) != 4 {
|
||||
accessSecret, err := utils.EncryptData(user.FsConfig.S3Config.AccessSecret)
|
||||
if err != nil {
|
||||
return &ValidationError{err: fmt.Sprintf("Could encrypt s3 access secret: %v", err)}
|
||||
return &ValidationError{err: fmt.Sprintf("could not encrypt s3 access secret: %v", err)}
|
||||
}
|
||||
user.FsConfig.S3Config.AccessSecret = accessSecret
|
||||
}
|
||||
|
@ -504,10 +504,10 @@ func validateFilesystemConfig(user *User) error {
|
|||
func validateUser(user *User) error {
|
||||
buildUserHomeDir(user)
|
||||
if len(user.Username) == 0 || len(user.HomeDir) == 0 {
|
||||
return &ValidationError{err: "Mandatory parameters missing"}
|
||||
return &ValidationError{err: "mandatory parameters missing"}
|
||||
}
|
||||
if len(user.Password) == 0 && len(user.PublicKeys) == 0 {
|
||||
return &ValidationError{err: "Please set a password or at least a public_key"}
|
||||
return &ValidationError{err: "please set a password or at least a public_key"}
|
||||
}
|
||||
if !filepath.IsAbs(user.HomeDir) {
|
||||
return &ValidationError{err: fmt.Sprintf("home_dir must be an absolute path, actual value: %v", user.HomeDir)}
|
||||
|
|
|
@ -408,6 +408,7 @@ func (u *User) getACopy() User {
|
|||
AccessSecret: u.FsConfig.S3Config.AccessSecret,
|
||||
Endpoint: u.FsConfig.S3Config.Endpoint,
|
||||
StorageClass: u.FsConfig.S3Config.StorageClass,
|
||||
KeyPrefix: u.FsConfig.S3Config.KeyPrefix,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
@ -435,6 +435,10 @@ func compareUserFsConfig(expected *dataprovider.User, actual *dataprovider.User)
|
|||
if expected.FsConfig.S3Config.StorageClass != actual.FsConfig.S3Config.StorageClass {
|
||||
return errors.New("S3 storage class mismatch")
|
||||
}
|
||||
if expected.FsConfig.S3Config.KeyPrefix != actual.FsConfig.S3Config.KeyPrefix &&
|
||||
expected.FsConfig.S3Config.KeyPrefix+"/" != actual.FsConfig.S3Config.KeyPrefix {
|
||||
return errors.New("S3 key prefix mismatch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
@ -250,6 +250,17 @@ func TestAddUserInvalidFsConfig(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Errorf("unexpected error adding user with invalid fs config: %v", err)
|
||||
}
|
||||
u.FsConfig.S3Config.Bucket = "test"
|
||||
u.FsConfig.S3Config.Region = "eu-west-1"
|
||||
u.FsConfig.S3Config.AccessKey = "access-key"
|
||||
u.FsConfig.S3Config.AccessSecret = "access-secret"
|
||||
u.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?a=b"
|
||||
u.FsConfig.S3Config.StorageClass = "Standard"
|
||||
u.FsConfig.S3Config.KeyPrefix = "/somedir/subdir/"
|
||||
_, _, err = httpd.AddUser(u, http.StatusBadRequest)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error adding user with invalid fs config: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserPublicKey(t *testing.T) {
|
||||
|
@ -341,6 +352,7 @@ func TestUserS3Config(t *testing.T) {
|
|||
user.FsConfig.S3Config.Region = "us-east-1"
|
||||
user.FsConfig.S3Config.AccessKey = "Server-Access-Key1"
|
||||
user.FsConfig.S3Config.Endpoint = "http://localhost:9000"
|
||||
user.FsConfig.S3Config.KeyPrefix = "somedir/subdir"
|
||||
user, _, err = httpd.UpdateUser(user, http.StatusOK)
|
||||
if err != nil {
|
||||
t.Errorf("unable to update user: %v", err)
|
||||
|
@ -1467,6 +1479,7 @@ func TestWebUserS3Mock(t *testing.T) {
|
|||
user.FsConfig.S3Config.AccessSecret = "access-secret"
|
||||
user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?a=b"
|
||||
user.FsConfig.S3Config.StorageClass = "Standard"
|
||||
user.FsConfig.S3Config.KeyPrefix = "somedir/subdir/"
|
||||
form := make(url.Values)
|
||||
form.Set("username", user.Username)
|
||||
form.Set("home_dir", user.HomeDir)
|
||||
|
@ -1490,6 +1503,7 @@ func TestWebUserS3Mock(t *testing.T) {
|
|||
form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret)
|
||||
form.Set("s3_storage_class", user.FsConfig.S3Config.StorageClass)
|
||||
form.Set("s3_endpoint", user.FsConfig.S3Config.Endpoint)
|
||||
form.Set("s3_key_prefix", user.FsConfig.S3Config.KeyPrefix)
|
||||
req, _ = http.NewRequest(http.MethodPost, webUserPath+"/"+strconv.FormatInt(user.ID, 10), strings.NewReader(form.Encode()))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
rr = executeRequest(req)
|
||||
|
@ -1530,6 +1544,9 @@ func TestWebUserS3Mock(t *testing.T) {
|
|||
if updateUser.FsConfig.S3Config.Endpoint != user.FsConfig.S3Config.Endpoint {
|
||||
t.Error("s3 endpoint mismatch")
|
||||
}
|
||||
if updateUser.FsConfig.S3Config.KeyPrefix != user.FsConfig.S3Config.KeyPrefix {
|
||||
t.Error("s3 key prefix mismatch")
|
||||
}
|
||||
req, _ = http.NewRequest(http.MethodDelete, userPath+"/"+strconv.FormatInt(user.ID, 10), nil)
|
||||
rr = executeRequest(req)
|
||||
checkResponseCode(t, http.StatusOK, rr.Code)
|
||||
|
|
|
@ -274,6 +274,12 @@ func TestCompareUserFsConfig(t *testing.T) {
|
|||
if err == nil {
|
||||
t.Errorf("S3 storage class does not match")
|
||||
}
|
||||
expected.FsConfig.S3Config.StorageClass = ""
|
||||
expected.FsConfig.S3Config.KeyPrefix = "somedir/subdir"
|
||||
err = compareUserFsConfig(expected, actual)
|
||||
if err == nil {
|
||||
t.Errorf("S3 key prefix does not match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApiCallsWithBadURL(t *testing.T) {
|
||||
|
|
|
@ -732,6 +732,10 @@ components:
|
|||
description: optional endpoint
|
||||
storage_class:
|
||||
type: string
|
||||
key_prefix:
|
||||
type: string
|
||||
description: key_prefix is similar to a chroot directory for a local filesystem. If specified the SFTP user will only see contents that starts with this prefix and so you can restrict access to a specific virtual folder. The prefix, if not empty, must not start with "/" and must end with "/". If empty the whole bucket contents will be available
|
||||
example: folder/subfolder/
|
||||
required:
|
||||
- bucket
|
||||
- region
|
||||
|
|
|
@ -238,6 +238,7 @@ func getFsConfigFromUserPostFields(r *http.Request) dataprovider.Filesystem {
|
|||
fs.S3Config.AccessSecret = r.Form.Get("s3_access_secret")
|
||||
fs.S3Config.Endpoint = r.Form.Get("s3_endpoint")
|
||||
fs.S3Config.StorageClass = r.Form.Get("s3_storage_class")
|
||||
fs.S3Config.KeyPrefix = r.Form.Get("s3_key_prefix")
|
||||
}
|
||||
return fs
|
||||
}
|
||||
|
|
|
@ -44,7 +44,7 @@ Let's see a sample usage for each REST API.
|
|||
Command:
|
||||
|
||||
```
|
||||
python sftpgo_api_cli.py add-user test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 33 --gid 1000 --max-sessions 2 --quota-size 0 --quota-files 3 --permissions "list" "download" "upload" "delete" "rename" "create_dirs" "overwrite" --subdirs-permissions "/dir1:list,download" "/dir2:*" --upload-bandwidth 100 --download-bandwidth 60 --status 0 --expiration-date 2019-01-01 --allowed-ip "192.168.1.1/32" --fs S3 --s3-bucket test --s3-region eu-west-1 --s3-access-key accesskey --s3-access-secret secret --s3-endpoint "http://127.0.0.1:9000" --s3-storage-class Standard
|
||||
python sftpgo_api_cli.py add-user test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 33 --gid 1000 --max-sessions 2 --quota-size 0 --quota-files 3 --permissions "list" "download" "upload" "delete" "rename" "create_dirs" "overwrite" --subdirs-permissions "/dir1:list,download" "/dir2:*" --upload-bandwidth 100 --download-bandwidth 60 --status 0 --expiration-date 2019-01-01 --allowed-ip "192.168.1.1/32" --fs S3 --s3-bucket test --s3-region eu-west-1 --s3-access-key accesskey --s3-access-secret secret --s3-endpoint "http://127.0.0.1:9000" --s3-storage-class Standard --s3-key-prefix "vfolder/"
|
||||
```
|
||||
|
||||
Output:
|
||||
|
@ -60,6 +60,7 @@ Output:
|
|||
"access_secret": "$aes$6c088ba12b0b261247c8cf331c46d9260b8e58002957d89ad1c0495e3af665cd0227",
|
||||
"bucket": "test",
|
||||
"endpoint": "http://127.0.0.1:9000",
|
||||
"key_prefix": "vfolder/",
|
||||
"region": "eu-west-1",
|
||||
"storage_class": "Standard"
|
||||
}
|
||||
|
|
|
@ -73,7 +73,8 @@ class SFTPGoApiRequests:
|
|||
def buildUserObject(self, user_id=0, username="", password="", public_keys=[], home_dir="", uid=0, gid=0,
|
||||
max_sessions=0, quota_size=0, quota_files=0, permissions={}, upload_bandwidth=0, download_bandwidth=0,
|
||||
status=1, expiration_date=0, allowed_ip=[], denied_ip=[], fs_provider='local', s3_bucket='',
|
||||
s3_region='', s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class=''):
|
||||
s3_region='', s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class='',
|
||||
s3_key_prefix=''):
|
||||
user = {"id":user_id, "username":username, "uid":uid, "gid":gid,
|
||||
"max_sessions":max_sessions, "quota_size":quota_size, "quota_files":quota_files,
|
||||
"upload_bandwidth":upload_bandwidth, "download_bandwidth":download_bandwidth,
|
||||
|
@ -92,7 +93,8 @@ class SFTPGoApiRequests:
|
|||
if allowed_ip or denied_ip:
|
||||
user.update({"filters":self.buildFilters(allowed_ip, denied_ip)})
|
||||
user.update({"filesystem":self.buildFsConfig(fs_provider, s3_bucket, s3_region, s3_access_key,
|
||||
s3_access_secret, s3_endpoint, s3_storage_class)})
|
||||
s3_access_secret, s3_endpoint, s3_storage_class,
|
||||
s3_key_prefix)})
|
||||
return user
|
||||
|
||||
def buildPermissions(self, root_perms, subdirs_perms):
|
||||
|
@ -127,11 +129,12 @@ class SFTPGoApiRequests:
|
|||
return filters
|
||||
|
||||
def buildFsConfig(self, fs_provider, s3_bucket, s3_region, s3_access_key, s3_access_secret, s3_endpoint,
|
||||
s3_storage_class):
|
||||
s3_storage_class, s3_key_prefix):
|
||||
fs_config = {'provider':0}
|
||||
if fs_provider == 'S3':
|
||||
s3config = {'bucket':s3_bucket, 'region':s3_region, 'access_key':s3_access_key, 'access_secret':
|
||||
s3_access_secret, 'endpoint':s3_endpoint, 'storage_class':s3_storage_class}
|
||||
s3_access_secret, 'endpoint':s3_endpoint, 'storage_class':s3_storage_class, 'key_prefix':
|
||||
s3_key_prefix}
|
||||
fs_config.update({'provider':1, 's3config':s3config})
|
||||
return fs_config
|
||||
|
||||
|
@ -147,22 +150,23 @@ class SFTPGoApiRequests:
|
|||
def addUser(self, username="", password="", public_keys="", home_dir="", uid=0, gid=0, max_sessions=0, quota_size=0,
|
||||
quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1, expiration_date=0,
|
||||
subdirs_permissions=[], allowed_ip=[], denied_ip=[], fs_provider='local', s3_bucket='', s3_region='',
|
||||
s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class=''):
|
||||
s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class='', s3_key_prefix=''):
|
||||
u = self.buildUserObject(0, username, password, public_keys, home_dir, uid, gid, max_sessions,
|
||||
quota_size, quota_files, self.buildPermissions(perms, subdirs_permissions), upload_bandwidth, download_bandwidth,
|
||||
status, expiration_date, allowed_ip, denied_ip, fs_provider, s3_bucket, s3_region,
|
||||
s3_access_key, s3_access_secret, s3_endpoint, s3_storage_class)
|
||||
s3_access_key, s3_access_secret, s3_endpoint, s3_storage_class, s3_key_prefix)
|
||||
r = requests.post(self.userPath, json=u, auth=self.auth, verify=self.verify)
|
||||
self.printResponse(r)
|
||||
|
||||
def updateUser(self, user_id, username="", password="", public_keys="", home_dir="", uid=0, gid=0, max_sessions=0,
|
||||
quota_size=0, quota_files=0, perms=[], upload_bandwidth=0, download_bandwidth=0, status=1,
|
||||
expiration_date=0, subdirs_permissions=[], allowed_ip=[], denied_ip=[], fs_provider='local',
|
||||
s3_bucket='', s3_region='', s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class=''):
|
||||
s3_bucket='', s3_region='', s3_access_key='', s3_access_secret='', s3_endpoint='', s3_storage_class='',
|
||||
s3_key_prefix=''):
|
||||
u = self.buildUserObject(user_id, username, password, public_keys, home_dir, uid, gid, max_sessions,
|
||||
quota_size, quota_files, self.buildPermissions(perms, subdirs_permissions), upload_bandwidth, download_bandwidth,
|
||||
status, expiration_date, allowed_ip, denied_ip, fs_provider, s3_bucket, s3_region, s3_access_key,
|
||||
s3_access_secret, s3_endpoint, s3_storage_class)
|
||||
s3_access_secret, s3_endpoint, s3_storage_class, s3_key_prefix)
|
||||
r = requests.put(urlparse.urljoin(self.userPath, "user/" + str(user_id)), json=u, auth=self.auth, verify=self.verify)
|
||||
self.printResponse(r)
|
||||
|
||||
|
@ -419,6 +423,9 @@ def addCommonUserArguments(parser):
|
|||
parser.add_argument('--fs', type=str, default='local', choices=['local', 'S3'],
|
||||
help='Filesystem provider. Default: %(default)s')
|
||||
parser.add_argument('--s3-bucket', type=str, default='', help='Default: %(default)s')
|
||||
parser.add_argument('--s3-key-prefix', type=str, default='', help='Virtual root directory. If non empty only this ' +
|
||||
'directory and its contents will be available. Cannot start with "/". For example "folder/subfolder/".' +
|
||||
' Default: %(default)s')
|
||||
parser.add_argument('--s3-region', type=str, default='', help='Default: %(default)s')
|
||||
parser.add_argument('--s3-access-key', type=str, default='', help='Default: %(default)s')
|
||||
parser.add_argument('--s3-access-secret', type=str, default='', help='Default: %(default)s')
|
||||
|
@ -527,13 +534,14 @@ if __name__ == '__main__':
|
|||
args.quota_size, args.quota_files, args.permissions, args.upload_bandwidth, args.download_bandwidth,
|
||||
args.status, getDatetimeAsMillisSinceEpoch(args.expiration_date), args.subdirs_permissions, args.allowed_ip,
|
||||
args.denied_ip, args.fs, args.s3_bucket, args.s3_region, args.s3_access_key, args.s3_access_secret,
|
||||
args.s3_endpoint, args.s3_storage_class)
|
||||
args.s3_endpoint, args.s3_storage_class, args.s3_key_prefix)
|
||||
elif args.command == 'update-user':
|
||||
api.updateUser(args.id, args.username, args.password, args.public_keys, args.home_dir, args.uid, args.gid,
|
||||
args.max_sessions, args.quota_size, args.quota_files, args.permissions, args.upload_bandwidth,
|
||||
args.download_bandwidth, args.status, getDatetimeAsMillisSinceEpoch(args.expiration_date),
|
||||
args.subdirs_permissions, args.allowed_ip, args.denied_ip, args.fs, args.s3_bucket, args.s3_region,
|
||||
args.s3_access_key, args.s3_access_secret, args.s3_endpoint, args.s3_storage_class)
|
||||
args.s3_access_key, args.s3_access_secret, args.s3_endpoint, args.s3_storage_class,
|
||||
args.s3_key_prefix)
|
||||
elif args.command == 'delete-user':
|
||||
api.deleteUser(args.id)
|
||||
elif args.command == 'get-users':
|
||||
|
|
|
@ -302,7 +302,7 @@ func (c Connection) handleSFTPSetstat(filePath string, request *sftp.Request) er
|
|||
}
|
||||
|
||||
func (c Connection) handleSFTPRename(sourcePath string, targetPath string, request *sftp.Request) error {
|
||||
if c.fs.GetRelativePath(sourcePath, c.User.GetHomeDir()) == "/" {
|
||||
if c.fs.GetRelativePath(sourcePath) == "/" {
|
||||
c.Log(logger.LevelWarn, logSender, "renaming root dir is not allowed")
|
||||
return sftp.ErrSSHFxPermissionDenied
|
||||
}
|
||||
|
@ -319,7 +319,7 @@ func (c Connection) handleSFTPRename(sourcePath string, targetPath string, reque
|
|||
}
|
||||
|
||||
func (c Connection) handleSFTPRmdir(dirPath string, request *sftp.Request) error {
|
||||
if c.fs.GetRelativePath(dirPath, c.User.GetHomeDir()) == "/" {
|
||||
if c.fs.GetRelativePath(dirPath) == "/" {
|
||||
c.Log(logger.LevelWarn, logSender, "removing root dir is not allowed")
|
||||
return sftp.ErrSSHFxPermissionDenied
|
||||
}
|
||||
|
@ -348,7 +348,7 @@ func (c Connection) handleSFTPRmdir(dirPath string, request *sftp.Request) error
|
|||
}
|
||||
|
||||
func (c Connection) handleSFTPSymlink(sourcePath string, targetPath string, request *sftp.Request) error {
|
||||
if c.fs.GetRelativePath(sourcePath, c.User.GetHomeDir()) == "/" {
|
||||
if c.fs.GetRelativePath(sourcePath) == "/" {
|
||||
c.Log(logger.LevelWarn, logSender, "symlinking root dir is not allowed")
|
||||
return sftp.ErrSSHFxPermissionDenied
|
||||
}
|
||||
|
|
|
@ -325,7 +325,7 @@ func (c *scpCommand) handleRecursiveDownload(dirPath string, stat os.FileInfo) e
|
|||
}
|
||||
var dirs []string
|
||||
for _, file := range files {
|
||||
filePath := c.connection.fs.GetRelativePath(c.connection.fs.Join(dirPath, file.Name()), c.connection.User.GetHomeDir())
|
||||
filePath := c.connection.fs.GetRelativePath(c.connection.fs.Join(dirPath, file.Name()))
|
||||
if file.Mode().IsRegular() || file.Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||
err = c.handleDownload(filePath)
|
||||
if err != nil {
|
||||
|
|
|
@ -307,7 +307,7 @@ func GetConnectionsStats() []ConnectionStatus {
|
|||
StartTime: utils.GetTimeAsMsSinceEpoch(t.start),
|
||||
Size: size,
|
||||
LastActivity: utils.GetTimeAsMsSinceEpoch(t.lastActivity),
|
||||
Path: c.fs.GetRelativePath(t.path, c.User.GetHomeDir()),
|
||||
Path: c.fs.GetRelativePath(t.path),
|
||||
}
|
||||
conn.Transfers = append(conn.Transfers, connTransfer)
|
||||
}
|
||||
|
|
|
@ -2904,57 +2904,118 @@ func TestRootDirCommands(t *testing.T) {
|
|||
|
||||
func TestRelativePaths(t *testing.T) {
|
||||
user := getTestUser(true)
|
||||
path := filepath.Join(user.HomeDir, "/")
|
||||
fs := vfs.NewOsFs("", user.GetHomeDir())
|
||||
rel := fs.GetRelativePath(path, user.GetHomeDir())
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
var path, rel string
|
||||
filesystems := []vfs.Fs{vfs.NewOsFs("", user.GetHomeDir())}
|
||||
s3config := vfs.S3FsConfig{
|
||||
KeyPrefix: strings.TrimPrefix(user.GetHomeDir(), "/") + "/",
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "//")
|
||||
rel = fs.GetRelativePath(path, user.GetHomeDir())
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
s3fs, _ := vfs.NewS3Fs("", user.GetHomeDir(), s3config)
|
||||
filesystems = append(filesystems, s3fs)
|
||||
for _, fs := range filesystems {
|
||||
path = filepath.Join(user.HomeDir, "/")
|
||||
rel = fs.GetRelativePath(path)
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "//")
|
||||
rel = fs.GetRelativePath(path)
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "../..")
|
||||
rel = fs.GetRelativePath(path)
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v path: %v", rel, path)
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "../../../../../")
|
||||
rel = fs.GetRelativePath(path)
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "/..")
|
||||
rel = fs.GetRelativePath(path)
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v path: %v", rel, path)
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "/../../../..")
|
||||
rel = fs.GetRelativePath(path)
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "")
|
||||
rel = fs.GetRelativePath(path)
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, ".")
|
||||
rel = fs.GetRelativePath(path)
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "somedir")
|
||||
rel = fs.GetRelativePath(path)
|
||||
if rel != "/somedir" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "/somedir/subdir")
|
||||
rel = fs.GetRelativePath(path)
|
||||
if rel != "/somedir/subdir" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
}
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "../..")
|
||||
rel = fs.GetRelativePath(path, user.GetHomeDir())
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
}
|
||||
|
||||
func TestResolvePaths(t *testing.T) {
|
||||
user := getTestUser(true)
|
||||
var path, resolved string
|
||||
var err error
|
||||
filesystems := []vfs.Fs{vfs.NewOsFs("", user.GetHomeDir())}
|
||||
s3config := vfs.S3FsConfig{
|
||||
KeyPrefix: strings.TrimPrefix(user.GetHomeDir(), "/") + "/",
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "../../../../../")
|
||||
rel = fs.GetRelativePath(path, user.GetHomeDir())
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "/..")
|
||||
rel = fs.GetRelativePath(path, user.GetHomeDir())
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "/../../../..")
|
||||
rel = fs.GetRelativePath(path, user.GetHomeDir())
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "")
|
||||
rel = fs.GetRelativePath(path, user.GetHomeDir())
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, ".")
|
||||
rel = fs.GetRelativePath(path, user.GetHomeDir())
|
||||
if rel != "/" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "somedir")
|
||||
rel = fs.GetRelativePath(path, user.GetHomeDir())
|
||||
if rel != "/somedir" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
}
|
||||
path = filepath.Join(user.HomeDir, "/somedir/subdir")
|
||||
rel = fs.GetRelativePath(path, user.GetHomeDir())
|
||||
if rel != "/somedir/subdir" {
|
||||
t.Errorf("Unexpected relative path: %v", rel)
|
||||
os.MkdirAll(user.GetHomeDir(), 0777)
|
||||
s3fs, _ := vfs.NewS3Fs("", user.GetHomeDir(), s3config)
|
||||
filesystems = append(filesystems, s3fs)
|
||||
for _, fs := range filesystems {
|
||||
path = "/"
|
||||
resolved, _ = fs.ResolvePath(filepath.ToSlash(path))
|
||||
if resolved != fs.Join(user.GetHomeDir(), "/") {
|
||||
t.Errorf("Unexpected resolved path: %v for: %v, fs: %v", resolved, path, fs.Name())
|
||||
}
|
||||
path = "."
|
||||
resolved, _ = fs.ResolvePath(filepath.ToSlash(path))
|
||||
if resolved != fs.Join(user.GetHomeDir(), "/") {
|
||||
t.Errorf("Unexpected resolved path: %v for: %v, fs: %v", resolved, path, fs.Name())
|
||||
}
|
||||
path = "test/sub"
|
||||
resolved, _ = fs.ResolvePath(filepath.ToSlash(path))
|
||||
if resolved != fs.Join(user.GetHomeDir(), "/test/sub") {
|
||||
t.Errorf("Unexpected resolved path: %v for: %v, fs: %v", resolved, path, fs.Name())
|
||||
}
|
||||
path = "../test/sub"
|
||||
resolved, err = fs.ResolvePath(filepath.ToSlash(path))
|
||||
if fs.Name() == "osfs" {
|
||||
if err == nil {
|
||||
t.Errorf("Unexpected resolved path: %v for: %v, fs: %v", resolved, path, fs.Name())
|
||||
}
|
||||
} else {
|
||||
if resolved != fs.Join(user.GetHomeDir(), "/test/sub") && err == nil {
|
||||
t.Errorf("Unexpected resolved path: %v for: %v, fs: %v", resolved, path, fs.Name())
|
||||
}
|
||||
}
|
||||
path = "../../../test/../sub"
|
||||
resolved, err = fs.ResolvePath(filepath.ToSlash(path))
|
||||
if fs.Name() == "osfs" {
|
||||
if err == nil {
|
||||
t.Errorf("Unexpected resolved path: %v for: %v, fs: %v", resolved, path, fs.Name())
|
||||
}
|
||||
} else {
|
||||
if resolved != fs.Join(user.GetHomeDir(), "/sub") && err == nil {
|
||||
t.Errorf("Unexpected resolved path: %v for: %v, fs: %v", resolved, path, fs.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
os.RemoveAll(user.GetHomeDir())
|
||||
}
|
||||
|
||||
func TestUserPerms(t *testing.T) {
|
||||
|
|
|
@ -243,6 +243,17 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group row">
|
||||
<label for="idS3KeyPrefix" class="col-sm-2 col-form-label">S3 Key Prefix</label>
|
||||
<div class="col-sm-10">
|
||||
<input type="text" class="form-control" id="idS3KeyPrefix" name="s3_key_prefix" placeholder=""
|
||||
value="{{.User.FsConfig.S3Config.KeyPrefix}}" maxlength="255" aria-describedby="S3KeyPrefixHelpBlock">
|
||||
<small id="S3KeyPrefixHelpBlock" class="form-text text-muted">
|
||||
Similar to a chroot for local filesystem. Cannot start with "/". Example: "somedir/subdir/".
|
||||
</small>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<input type="hidden" name="expiration_date" id="hidden_start_datetime" value="">
|
||||
<button type="submit" class="btn btn-primary float-right mt-3 mb-5 px-5 px-3">Submit</button>
|
||||
</form>
|
||||
|
|
|
@ -180,8 +180,8 @@ func (OsFs) GetAtomicUploadPath(name string) string {
|
|||
|
||||
// GetRelativePath returns the path for a file relative to the user's home dir.
|
||||
// This is the path as seen by SFTP users
|
||||
func (OsFs) GetRelativePath(name, rootPath string) string {
|
||||
rel, err := filepath.Rel(rootPath, filepath.Clean(name))
|
||||
func (fs OsFs) GetRelativePath(name string) string {
|
||||
rel, err := filepath.Rel(fs.rootDir, filepath.Clean(name))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
|
31
vfs/s3fs.go
31
vfs/s3fs.go
|
@ -22,7 +22,14 @@ import (
|
|||
|
||||
// S3FsConfig defines the configuration for S3fs
|
||||
type S3FsConfig struct {
|
||||
Bucket string `json:"bucket,omitempty"`
|
||||
Bucket string `json:"bucket,omitempty"`
|
||||
// KeyPrefix is similar to a chroot directory for local filesystem.
|
||||
// If specified the SFTP user will only see contents that starts with
|
||||
// this prefix and so you can restrict access to a specific virtual
|
||||
// folder. The prefix, if not empty, must not start with "/" and must
|
||||
// end with "/".
|
||||
//If empty the whole bucket contents will be available
|
||||
KeyPrefix string `json:"key_prefix,omitempty"`
|
||||
Region string `json:"region,omitempty"`
|
||||
AccessKey string `json:"access_key,omitempty"`
|
||||
AccessSecret string `json:"access_secret,omitempty"`
|
||||
|
@ -95,6 +102,9 @@ func (fs S3Fs) Stat(name string) (os.FileInfo, error) {
|
|||
}
|
||||
return NewS3FileInfo(name, true, 0, time.Time{}), nil
|
||||
}
|
||||
if "/"+fs.config.KeyPrefix == name+"/" {
|
||||
return NewS3FileInfo(name, true, 0, time.Time{}), nil
|
||||
}
|
||||
prefix := path.Dir(name)
|
||||
if prefix == "/" || prefix == "." {
|
||||
prefix = ""
|
||||
|
@ -403,7 +413,7 @@ func (fs S3Fs) ScanRootDirContents() (int, int64, error) {
|
|||
defer cancelFn()
|
||||
err := fs.svc.ListObjectsV2PagesWithContext(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(fs.config.Bucket),
|
||||
Prefix: aws.String(""),
|
||||
Prefix: aws.String(fs.config.KeyPrefix),
|
||||
}, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
|
||||
for _, fileObject := range page.Contents {
|
||||
numFiles++
|
||||
|
@ -423,14 +433,20 @@ func (S3Fs) GetAtomicUploadPath(name string) string {
|
|||
|
||||
// GetRelativePath returns the path for a file relative to the user's home dir.
|
||||
// This is the path as seen by SFTP users
|
||||
func (S3Fs) GetRelativePath(name, rootPath string) string {
|
||||
rel := name
|
||||
if name == "." {
|
||||
func (fs S3Fs) GetRelativePath(name string) string {
|
||||
rel := path.Clean(name)
|
||||
if rel == "." {
|
||||
rel = ""
|
||||
}
|
||||
if !strings.HasPrefix(rel, "/") {
|
||||
return "/" + rel
|
||||
}
|
||||
if len(fs.config.KeyPrefix) > 0 {
|
||||
if !strings.HasPrefix(rel, "/"+fs.config.KeyPrefix) {
|
||||
rel = "/"
|
||||
}
|
||||
rel = path.Clean("/" + strings.TrimPrefix(rel, "/"+fs.config.KeyPrefix))
|
||||
}
|
||||
return rel
|
||||
}
|
||||
|
||||
|
@ -441,7 +457,10 @@ func (S3Fs) Join(elem ...string) string {
|
|||
|
||||
// ResolvePath returns the matching filesystem path for the specified sftp path
|
||||
func (fs S3Fs) ResolvePath(sftpPath string) (string, error) {
|
||||
return sftpPath, nil
|
||||
if !path.IsAbs(sftpPath) {
|
||||
sftpPath = path.Clean("/" + sftpPath)
|
||||
}
|
||||
return fs.Join("/", fs.config.KeyPrefix, sftpPath), nil
|
||||
}
|
||||
|
||||
func (fs *S3Fs) resolve(name *string, prefix string) (string, bool) {
|
||||
|
|
13
vfs/vfs.go
13
vfs/vfs.go
|
@ -4,7 +4,9 @@ package vfs
|
|||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/drakkan/sftpgo/logger"
|
||||
|
@ -36,7 +38,7 @@ type Fs interface {
|
|||
IsPermission(err error) bool
|
||||
ScanRootDirContents() (int, int64, error)
|
||||
GetAtomicUploadPath(name string) string
|
||||
GetRelativePath(name, rootPath string) string
|
||||
GetRelativePath(name string) string
|
||||
Join(elem ...string) string
|
||||
}
|
||||
|
||||
|
@ -80,6 +82,15 @@ func ValidateS3FsConfig(config *S3FsConfig) error {
|
|||
if len(config.AccessSecret) == 0 {
|
||||
return errors.New("access_secret cannot be empty")
|
||||
}
|
||||
if len(config.KeyPrefix) > 0 {
|
||||
if strings.HasPrefix(config.KeyPrefix, "/") {
|
||||
return errors.New("key_prefix cannot start with /")
|
||||
}
|
||||
config.KeyPrefix = path.Clean(config.KeyPrefix)
|
||||
if !strings.HasSuffix(config.KeyPrefix, "/") {
|
||||
config.KeyPrefix += "/"
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue