From 4519bffa3954ad82528f197691849f5bb05a37d0 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Mon, 28 Feb 2022 20:19:13 +0100 Subject: [PATCH] S3: add support for assume role Fixes #736 Signed-off-by: Nicola Murino --- docs/s3.md | 4 +++- go.mod | 4 ++-- go.sum | 7 ++++--- httpd/httpd_test.go | 8 ++++++++ httpd/webadmin.go | 1 + httpdtest/httpdtest.go | 3 +++ openapi/openapi.yaml | 3 +++ templates/webadmin/fsconfig.html | 11 +++++++++++ vfs/filesystem.go | 1 + vfs/s3fs.go | 5 +++++ vfs/vfs.go | 3 +++ 11 files changed, 44 insertions(+), 6 deletions(-) diff --git a/docs/s3.md b/docs/s3.md index b9447cc6..383d1356 100644 --- a/docs/s3.md +++ b/docs/s3.md @@ -2,7 +2,7 @@ To connect SFTPGo to AWS, you need to specify credentials, a `bucket` and a `region`. Here is the list of available [AWS regions](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions). For example, if your bucket is at `Frankfurt`, you have to set the region to `eu-central-1`. You can specify an AWS [storage class](https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html) too. Leave it blank to use the default AWS storage class. An endpoint is required if you are connecting to a Compatible AWS Storage such as [MinIO](https://min.io/). -AWS SDK has different options for credentials. [More Detail](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html). We support: +AWS SDK has different options for credentials. We support: 1. Providing [Access Keys](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys). 2. Use IAM roles for Amazon EC2 @@ -10,6 +10,8 @@ AWS SDK has different options for credentials. [More Detail](https://docs.aws.am So, you need to provide access keys to activate option 1, or leave them blank to use the other ways to specify credentials. +You can also use a temporary session token or assume a role by setting its ARN. + Specifying a different `key_prefix`, you can assign different "folders" of the same bucket to different users. This is similar to a chroot directory for local filesystem. Each SFTP/SCP user can only access the assigned folder and its contents. The folder identified by `key_prefix` does not need to be pre-created. SFTPGo uses multipart uploads and parallel downloads for storing and retrieving files from S3. diff --git a/go.mod b/go.mod index b4536e96..758fe8c9 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/rs/cors v1.8.2 github.com/rs/xid v1.3.0 github.com/rs/zerolog v1.26.2-0.20220227173336-263b0bde3672 - github.com/sftpgo/sdk v0.1.1-0.20220225141305-cca7ba31466c + github.com/sftpgo/sdk v0.1.1-0.20220228183957-d7251ba29961 github.com/shirou/gopsutil/v3 v3.22.1 github.com/spf13/afero v1.8.1 github.com/spf13/cobra v1.3.0 @@ -130,7 +130,7 @@ require ( golang.org/x/tools v0.1.9 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf // indirect + google.golang.org/genproto v0.0.0-20220228155957-1da8797a5878 // indirect google.golang.org/grpc v1.44.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/ini.v1 v1.66.4 // indirect diff --git a/go.sum b/go.sum index 741b3743..19682728 100644 --- a/go.sum +++ b/go.sum @@ -700,8 +700,8 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= -github.com/sftpgo/sdk v0.1.1-0.20220225141305-cca7ba31466c h1:aSWi1VB6DXmPmscawueKEhoyMTZjsMTiRaWFfhHmB4Y= -github.com/sftpgo/sdk v0.1.1-0.20220225141305-cca7ba31466c/go.mod h1:zqCRMcwS28IViwekJHNkFu4GqSfyVmOQTlh8h3icAXE= +github.com/sftpgo/sdk v0.1.1-0.20220228183957-d7251ba29961 h1:XpSoX58U9KR5qbexs3VUBZvgcRogjgbALWzQO4TIZKo= +github.com/sftpgo/sdk v0.1.1-0.20220228183957-d7251ba29961/go.mod h1:zqCRMcwS28IViwekJHNkFu4GqSfyVmOQTlh8h3icAXE= github.com/shirou/gopsutil/v3 v3.22.1 h1:33y31Q8J32+KstqPfscvFwBlNJ6xLaBy4xqBXzlYV5w= github.com/shirou/gopsutil/v3 v3.22.1/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -1189,8 +1189,9 @@ google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20220211171837-173942840c17/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220216160803-4663080d8bc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf h1:SVYXkUz2yZS9FWb2Gm8ivSlbNQzL2Z/NpPKE3RG2jWk= google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220228155957-1da8797a5878 h1:gERY0VtsF9UyyyCsPSjRk9/RWlcKSa/Gw/aenR/5z48= +google.golang.org/genproto v0.0.0-20220228155957-1da8797a5878/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index 96964d58..3189a12a 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -1788,6 +1788,7 @@ func TestUserRedactedPassword(t *testing.T) { u.FsConfig.S3Config.Region = "eu-west-1" u.FsConfig.S3Config.AccessKey = "access-key" u.FsConfig.S3Config.SessionToken = "session token" + u.FsConfig.S3Config.RoleARN = "myRoleARN" u.FsConfig.S3Config.AccessSecret = kms.NewSecret(sdkkms.SecretStatusRedacted, "access-secret", "", "") u.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?k=m" u.FsConfig.S3Config.StorageClass = "Standard" @@ -2566,6 +2567,7 @@ func TestUserS3Config(t *testing.T) { user.FsConfig.S3Config.AccessKey = "Server-Access-Key" user.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("Server-Access-Secret") user.FsConfig.S3Config.SessionToken = "Session token" + user.FsConfig.S3Config.RoleARN = "myRoleARN" user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000" user.FsConfig.S3Config.UploadPartSize = 8 user.FsConfig.S3Config.DownloadPartMaxTime = 60 @@ -15100,6 +15102,7 @@ func TestWebUserS3Mock(t *testing.T) { user.FsConfig.S3Config.AccessKey = "access-key" user.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("access-secret") user.FsConfig.S3Config.SessionToken = "new session token" + user.FsConfig.S3Config.RoleARN = "arn:aws:iam::123456789012:user/Development/product_1234/*" user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?a=b" user.FsConfig.S3Config.StorageClass = "Standard" user.FsConfig.S3Config.KeyPrefix = "somedir/subdir/" @@ -15139,6 +15142,7 @@ func TestWebUserS3Mock(t *testing.T) { form.Set("s3_access_key", user.FsConfig.S3Config.AccessKey) form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret.GetPayload()) form.Set("s3_session_token", user.FsConfig.S3Config.SessionToken) + form.Set("s3_role_arn", user.FsConfig.S3Config.RoleARN) form.Set("s3_storage_class", user.FsConfig.S3Config.StorageClass) form.Set("s3_acl", user.FsConfig.S3Config.ACL) form.Set("s3_endpoint", user.FsConfig.S3Config.Endpoint) @@ -15229,6 +15233,7 @@ func TestWebUserS3Mock(t *testing.T) { assert.Equal(t, updateUser.FsConfig.S3Config.Region, user.FsConfig.S3Config.Region) assert.Equal(t, updateUser.FsConfig.S3Config.AccessKey, user.FsConfig.S3Config.AccessKey) assert.Equal(t, updateUser.FsConfig.S3Config.SessionToken, user.FsConfig.S3Config.SessionToken) + assert.Equal(t, updateUser.FsConfig.S3Config.RoleARN, user.FsConfig.S3Config.RoleARN) assert.Equal(t, updateUser.FsConfig.S3Config.StorageClass, user.FsConfig.S3Config.StorageClass) assert.Equal(t, updateUser.FsConfig.S3Config.ACL, user.FsConfig.S3Config.ACL) assert.Equal(t, updateUser.FsConfig.S3Config.Endpoint, user.FsConfig.S3Config.Endpoint) @@ -15930,6 +15935,7 @@ func TestS3WebFolderMock(t *testing.T) { S3AccessKey := "access-key" S3AccessSecret := kms.NewPlainSecret("folder-access-secret") S3SessionToken := "fake session token" + S3RoleARN := "arn:aws:iam::123456789012:user/Development/product_1234/*" S3Endpoint := "http://127.0.0.1:9000/path?b=c" S3StorageClass := "Standard" S3ACL := "public-read-write" @@ -15950,6 +15956,7 @@ func TestS3WebFolderMock(t *testing.T) { form.Set("s3_access_key", S3AccessKey) form.Set("s3_access_secret", S3AccessSecret.GetPayload()) form.Set("s3_session_token", S3SessionToken) + form.Set("s3_role_arn", S3RoleARN) form.Set("s3_storage_class", S3StorageClass) form.Set("s3_acl", S3ACL) form.Set("s3_endpoint", S3Endpoint) @@ -16044,6 +16051,7 @@ func TestS3WebFolderMock(t *testing.T) { assert.Equal(t, S3Region, folder.FsConfig.S3Config.Region) assert.Equal(t, S3AccessKey, folder.FsConfig.S3Config.AccessKey) assert.Equal(t, S3SessionToken, folder.FsConfig.S3Config.SessionToken) + assert.Equal(t, S3RoleARN, folder.FsConfig.S3Config.RoleARN) assert.NotEmpty(t, folder.FsConfig.S3Config.AccessSecret.GetPayload()) assert.Equal(t, S3Endpoint, folder.FsConfig.S3Config.Endpoint) assert.Equal(t, S3StorageClass, folder.FsConfig.S3Config.StorageClass) diff --git a/httpd/webadmin.go b/httpd/webadmin.go index c994c15c..612f40d0 100644 --- a/httpd/webadmin.go +++ b/httpd/webadmin.go @@ -969,6 +969,7 @@ func getS3Config(r *http.Request) (vfs.S3FsConfig, error) { config.Region = r.Form.Get("s3_region") config.AccessKey = r.Form.Get("s3_access_key") config.SessionToken = strings.TrimSpace(r.Form.Get("s3_session_token")) + config.RoleARN = r.Form.Get("s3_role_arn") config.AccessSecret = getSecretFromFormField(r, "s3_access_secret") config.Endpoint = r.Form.Get("s3_endpoint") config.StorageClass = r.Form.Get("s3_storage_class") diff --git a/httpdtest/httpdtest.go b/httpdtest/httpdtest.go index 659dd108..abc9b177 100644 --- a/httpdtest/httpdtest.go +++ b/httpdtest/httpdtest.go @@ -1278,6 +1278,9 @@ func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error { / if expected.S3Config.SessionToken != actual.S3Config.SessionToken { return errors.New("fs S3 session token mismatch") } + if expected.S3Config.RoleARN != actual.S3Config.RoleARN { + return errors.New("fs S3 role ARN mismatch") + } if err := checkEncryptedSecret(expected.S3Config.AccessSecret, actual.S3Config.AccessSecret); err != nil { return fmt.Errorf("fs S3 access secret mismatch: %v", err) } diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index e2dc1c9a..eb7fe050 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -4720,6 +4720,9 @@ components: $ref: '#/components/schemas/Secret' session_token: type: string + role_arn: + type: string + description: 'IAM Role ARN to assume' endpoint: type: string description: optional endpoint diff --git a/templates/webadmin/fsconfig.html b/templates/webadmin/fsconfig.html index 5fb04cfb..522f1ab2 100644 --- a/templates/webadmin/fsconfig.html +++ b/templates/webadmin/fsconfig.html @@ -144,6 +144,17 @@ +
+ +
+ + + IAM Role ARN to assume + +
+
+
diff --git a/vfs/filesystem.go b/vfs/filesystem.go index c9f7c919..0950879d 100644 --- a/vfs/filesystem.go +++ b/vfs/filesystem.go @@ -245,6 +245,7 @@ func (f *Filesystem) GetACopy() Filesystem { Region: f.S3Config.Region, AccessKey: f.S3Config.AccessKey, SessionToken: f.S3Config.SessionToken, + RoleARN: f.S3Config.RoleARN, Endpoint: f.S3Config.Endpoint, StorageClass: f.S3Config.StorageClass, ACL: f.S3Config.ACL, diff --git a/vfs/s3fs.go b/vfs/s3fs.go index 267533c1..48ef9c27 100644 --- a/vfs/s3fs.go +++ b/vfs/s3fs.go @@ -17,6 +17,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" @@ -101,6 +102,10 @@ func NewS3Fs(connectionID, localTempDir, mountPath string, config S3FsConfig) (F if err != nil { return fs, err } + if fs.config.RoleARN != "" { + creds := stscreds.NewCredentials(sess, fs.config.RoleARN) + sess.Config.Credentials = creds + } fs.svc = s3.New(sess) return fs, nil } diff --git a/vfs/vfs.go b/vfs/vfs.go index 2d636f32..f093fdcf 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -176,6 +176,9 @@ func (c *S3FsConfig) isEqual(other *S3FsConfig) bool { if c.SessionToken != other.SessionToken { return false } + if c.RoleARN != other.RoleARN { + return false + } if c.Endpoint != other.Endpoint { return false }