beenull/sftpgo-mirror
1
0
Fork 0
mirror of https://github.com/drakkan/sftpgo.git synced 2024-11-21 23:20:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

S3: add support for assume role

Browse source 
Fixes #736

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2022-02-28 20:19:13 +01:00
parent 1ea7429921
commit 4519bffa39
No known key found for this signature in database
GPG key ID: 2F1FB59433D5A8CB
11 changed files with 44 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
docs/s3.md
View file

@ -2,7 +2,7 @@
To connect SFTPGo to AWS, you need to specify credentials, a `bucket` and a `region`. Here is the list of available [AWS regions](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). For example, if your bucket is at `Frankfurt`, you have to set the region to `eu-central-1`. You can specify an AWS [storage class](https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html) too. Leave it blank to use the default AWS storage class. An endpoint is required if you are connecting to a Compatible AWS Storage such as [MinIO](https://min.io/).
AWS SDK has different options for credentials. [More Detail](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html). We support:
AWS SDK has different options for credentials. We support:
1. Providing [Access Keys](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys).
2. Use IAM roles for Amazon EC2
@ -10,6 +10,8 @@ AWS SDK has different options for credentials. [More Detail](https://docs.aws.am
So, you need to provide access keys to activate option 1, or leave them blank to use the other ways to specify credentials.
You can also use a temporary session token or assume a role by setting its ARN.
Specifying a different `key_prefix`, you can assign different "folders" of the same bucket to different users. This is similar to a chroot directory for local filesystem. Each SFTP/SCP user can only access the assigned folder and its contents. The folder identified by `key_prefix` does not need to be pre-created.
SFTPGo uses multipart uploads and parallel downloads for storing and retrieving files from S3.

4
go.mod
View file

@ -41,7 +41,7 @@ require (
github.com/rs/cors v1.8.2
github.com/rs/xid v1.3.0
github.com/rs/zerolog v1.26.2-0.20220227173336-263b0bde3672
github.com/sftpgo/sdk v0.1.1-0.20220225141305-cca7ba31466c
github.com/sftpgo/sdk v0.1.1-0.20220228183957-d7251ba29961
github.com/shirou/gopsutil/v3 v3.22.1
github.com/spf13/afero v1.8.1
github.com/spf13/cobra v1.3.0
@ -130,7 +130,7 @@ require (
golang.org/x/tools v0.1.9 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf // indirect
google.golang.org/genproto v0.0.0-20220228155957-1da8797a5878 // indirect
google.golang.org/grpc v1.44.0 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/ini.v1 v1.66.4 // indirect

7
go.sum
View file

@ -700,8 +700,8 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo=
github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY=
github.com/sftpgo/sdk v0.1.1-0.20220225141305-cca7ba31466c h1:aSWi1VB6DXmPmscawueKEhoyMTZjsMTiRaWFfhHmB4Y=
github.com/sftpgo/sdk v0.1.1-0.20220225141305-cca7ba31466c/go.mod h1:zqCRMcwS28IViwekJHNkFu4GqSfyVmOQTlh8h3icAXE=
github.com/sftpgo/sdk v0.1.1-0.20220228183957-d7251ba29961 h1:XpSoX58U9KR5qbexs3VUBZvgcRogjgbALWzQO4TIZKo=
github.com/sftpgo/sdk v0.1.1-0.20220228183957-d7251ba29961/go.mod h1:zqCRMcwS28IViwekJHNkFu4GqSfyVmOQTlh8h3icAXE=
github.com/shirou/gopsutil/v3 v3.22.1 h1:33y31Q8J32+KstqPfscvFwBlNJ6xLaBy4xqBXzlYV5w=
github.com/shirou/gopsutil/v3 v3.22.1/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
@ -1189,8 +1189,9 @@ google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ6
google.golang.org/genproto v0.0.0-20220211171837-173942840c17/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
google.golang.org/genproto v0.0.0-20220216160803-4663080d8bc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf h1:SVYXkUz2yZS9FWb2Gm8ivSlbNQzL2Z/NpPKE3RG2jWk=
google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
google.golang.org/genproto v0.0.0-20220228155957-1da8797a5878 h1:gERY0VtsF9UyyyCsPSjRk9/RWlcKSa/Gw/aenR/5z48=
google.golang.org/genproto v0.0.0-20220228155957-1da8797a5878/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=

8
httpd/httpd_test.go
View file

@ -1788,6 +1788,7 @@ func TestUserRedactedPassword(t *testing.T) {
u.FsConfig.S3Config.Region = "eu-west-1"
u.FsConfig.S3Config.AccessKey = "access-key"
u.FsConfig.S3Config.SessionToken = "session token"
u.FsConfig.S3Config.RoleARN = "myRoleARN"
u.FsConfig.S3Config.AccessSecret = kms.NewSecret(sdkkms.SecretStatusRedacted, "access-secret", "", "")
u.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?k=m"
u.FsConfig.S3Config.StorageClass = "Standard"
@ -2566,6 +2567,7 @@ func TestUserS3Config(t *testing.T) {
user.FsConfig.S3Config.AccessKey = "Server-Access-Key"
user.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("Server-Access-Secret")
user.FsConfig.S3Config.SessionToken = "Session token"
user.FsConfig.S3Config.RoleARN = "myRoleARN"
user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000"
user.FsConfig.S3Config.UploadPartSize = 8
user.FsConfig.S3Config.DownloadPartMaxTime = 60
@ -15100,6 +15102,7 @@ func TestWebUserS3Mock(t *testing.T) {
user.FsConfig.S3Config.AccessKey = "access-key"
user.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("access-secret")
user.FsConfig.S3Config.SessionToken = "new session token"
user.FsConfig.S3Config.RoleARN = "arn:aws:iam::123456789012:user/Development/product_1234/*"
user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?a=b"
user.FsConfig.S3Config.StorageClass = "Standard"
user.FsConfig.S3Config.KeyPrefix = "somedir/subdir/"
@ -15139,6 +15142,7 @@ func TestWebUserS3Mock(t *testing.T) {
form.Set("s3_access_key", user.FsConfig.S3Config.AccessKey)
form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret.GetPayload())
form.Set("s3_session_token", user.FsConfig.S3Config.SessionToken)
form.Set("s3_role_arn", user.FsConfig.S3Config.RoleARN)
form.Set("s3_storage_class", user.FsConfig.S3Config.StorageClass)
form.Set("s3_acl", user.FsConfig.S3Config.ACL)
form.Set("s3_endpoint", user.FsConfig.S3Config.Endpoint)
@ -15229,6 +15233,7 @@ func TestWebUserS3Mock(t *testing.T) {
assert.Equal(t, updateUser.FsConfig.S3Config.Region, user.FsConfig.S3Config.Region)
assert.Equal(t, updateUser.FsConfig.S3Config.AccessKey, user.FsConfig.S3Config.AccessKey)
assert.Equal(t, updateUser.FsConfig.S3Config.SessionToken, user.FsConfig.S3Config.SessionToken)
assert.Equal(t, updateUser.FsConfig.S3Config.RoleARN, user.FsConfig.S3Config.RoleARN)
assert.Equal(t, updateUser.FsConfig.S3Config.StorageClass, user.FsConfig.S3Config.StorageClass)
assert.Equal(t, updateUser.FsConfig.S3Config.ACL, user.FsConfig.S3Config.ACL)
assert.Equal(t, updateUser.FsConfig.S3Config.Endpoint, user.FsConfig.S3Config.Endpoint)
@ -15930,6 +15935,7 @@ func TestS3WebFolderMock(t *testing.T) {
S3AccessKey := "access-key"
S3AccessSecret := kms.NewPlainSecret("folder-access-secret")
S3SessionToken := "fake session token"
S3RoleARN := "arn:aws:iam::123456789012:user/Development/product_1234/*"
S3Endpoint := "http://127.0.0.1:9000/path?b=c"
S3StorageClass := "Standard"
S3ACL := "public-read-write"
@ -15950,6 +15956,7 @@ func TestS3WebFolderMock(t *testing.T) {
form.Set("s3_access_key", S3AccessKey)
form.Set("s3_access_secret", S3AccessSecret.GetPayload())
form.Set("s3_session_token", S3SessionToken)
form.Set("s3_role_arn", S3RoleARN)
form.Set("s3_storage_class", S3StorageClass)
form.Set("s3_acl", S3ACL)
form.Set("s3_endpoint", S3Endpoint)
@ -16044,6 +16051,7 @@ func TestS3WebFolderMock(t *testing.T) {
assert.Equal(t, S3Region, folder.FsConfig.S3Config.Region)
assert.Equal(t, S3AccessKey, folder.FsConfig.S3Config.AccessKey)
assert.Equal(t, S3SessionToken, folder.FsConfig.S3Config.SessionToken)
assert.Equal(t, S3RoleARN, folder.FsConfig.S3Config.RoleARN)
assert.NotEmpty(t, folder.FsConfig.S3Config.AccessSecret.GetPayload())
assert.Equal(t, S3Endpoint, folder.FsConfig.S3Config.Endpoint)
assert.Equal(t, S3StorageClass, folder.FsConfig.S3Config.StorageClass)

1
httpd/webadmin.go
View file

@ -969,6 +969,7 @@ func getS3Config(r *http.Request) (vfs.S3FsConfig, error) {
config.Region = r.Form.Get("s3_region")
config.AccessKey = r.Form.Get("s3_access_key")
config.SessionToken = strings.TrimSpace(r.Form.Get("s3_session_token"))
config.RoleARN = r.Form.Get("s3_role_arn")
config.AccessSecret = getSecretFromFormField(r, "s3_access_secret")
config.Endpoint = r.Form.Get("s3_endpoint")
config.StorageClass = r.Form.Get("s3_storage_class")

3
httpdtest/httpdtest.go
View file

@ -1278,6 +1278,9 @@ func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error { /
if expected.S3Config.SessionToken != actual.S3Config.SessionToken {
return errors.New("fs S3 session token mismatch")
}
if expected.S3Config.RoleARN != actual.S3Config.RoleARN {
return errors.New("fs S3 role ARN mismatch")
}
if err := checkEncryptedSecret(expected.S3Config.AccessSecret, actual.S3Config.AccessSecret); err != nil {
return fmt.Errorf("fs S3 access secret mismatch: %v", err)
}

3
openapi/openapi.yaml
View file

@ -4720,6 +4720,9 @@ components:
$ref: '#/components/schemas/Secret'
session_token:
type: string
role_arn:
type: string
description: 'IAM Role ARN to assume'
endpoint:
type: string
description: optional endpoint

11
templates/webadmin/fsconfig.html
View file

@ -144,6 +144,17 @@
</div>
</div>
<div class="form-group row fsconfig fsconfig-s3fs">
<label for="idS3RoleARN" class="col-sm-2 col-form-label">Role ARN</label>
<div class="col-sm-10">
<input type="text" class="form-control" id="idS3RoleARN" name="s3_role_arn" placeholder=""
value="{{.S3Config.RoleARN}}" aria-describedby="S3RoleARNHelpBlock">
<small id="S3RoleARNHelpBlock" class="form-text text-muted">
IAM Role ARN to assume
</small>
</div>
</div>
<div class="form-group row fsconfig fsconfig-s3fs">
<label for="idS3ACL" class="col-sm-2 col-form-label">ACL</label>
<div class="col-sm-10">

1
vfs/filesystem.go
View file

@ -245,6 +245,7 @@ func (f *Filesystem) GetACopy() Filesystem {
Region: f.S3Config.Region,
AccessKey: f.S3Config.AccessKey,
SessionToken: f.S3Config.SessionToken,
RoleARN: f.S3Config.RoleARN,
Endpoint: f.S3Config.Endpoint,
StorageClass: f.S3Config.StorageClass,
ACL: f.S3Config.ACL,

5
vfs/s3fs.go
View file

@ -17,6 +17,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
@ -101,6 +102,10 @@ func NewS3Fs(connectionID, localTempDir, mountPath string, config S3FsConfig) (F
if err != nil {
return fs, err
}
if fs.config.RoleARN != "" {
creds := stscreds.NewCredentials(sess, fs.config.RoleARN)
sess.Config.Credentials = creds
}
fs.svc = s3.New(sess)
return fs, nil
}

3
vfs/vfs.go
View file

@ -176,6 +176,9 @@ func (c *S3FsConfig) isEqual(other *S3FsConfig) bool {
if c.SessionToken != other.SessionToken {
return false
}
if c.RoleARN != other.RoleARN {
return false
}
if c.Endpoint != other.Endpoint {
return false
}