From e18ad55067d8b6a822b741820723240a8c274d8b Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Fri, 25 Feb 2022 15:30:04 +0100 Subject: [PATCH] S3: add support for session tokens Fixes #736 Signed-off-by: Nicola Murino --- cmd/portable.go | 3 +++ docs/portable-mode.md | 1 + go.mod | 2 +- go.sum | 4 ++-- httpd/httpd_test.go | 9 +++++++++ httpd/webadmin.go | 1 + httpdtest/httpdtest.go | 3 +++ openapi/openapi.yaml | 2 ++ templates/webadmin/fsconfig.html | 8 ++++++++ vfs/filesystem.go | 1 + vfs/s3fs.go | 3 ++- vfs/vfs.go | 19 +++++++++++++++---- 12 files changed, 48 insertions(+), 8 deletions(-) diff --git a/cmd/portable.go b/cmd/portable.go index 228f664b..83fbae74 100644 --- a/cmd/portable.go +++ b/cmd/portable.go @@ -42,6 +42,7 @@ var ( portableS3Region string portableS3AccessKey string portableS3AccessSecret string + portableS3SessionToken string portableS3Endpoint string portableS3StorageClass string portableS3ACL string @@ -172,6 +173,7 @@ Please take a look at the usage below to customize the serving parameters`, Bucket: portableS3Bucket, Region: portableS3Region, AccessKey: portableS3AccessKey, + SessionToken: portableS3SessionToken, Endpoint: portableS3Endpoint, StorageClass: portableS3StorageClass, ACL: portableS3ACL, @@ -294,6 +296,7 @@ sftpfs => SFTP (legacy: 5)`) portableCmd.Flags().StringVar(&portableS3Region, "s3-region", "", "") portableCmd.Flags().StringVar(&portableS3AccessKey, "s3-access-key", "", "") portableCmd.Flags().StringVar(&portableS3AccessSecret, "s3-access-secret", "", "") + portableCmd.Flags().StringVar(&portableS3SessionToken, "s3-session-token", "", "") portableCmd.Flags().StringVar(&portableS3Endpoint, "s3-endpoint", "", "") portableCmd.Flags().StringVar(&portableS3StorageClass, "s3-storage-class", "", "") portableCmd.Flags().StringVar(&portableS3ACL, "s3-acl", "", "") diff --git a/docs/portable-mode.md b/docs/portable-mode.md index 88a3bcb6..7e155c04 100644 --- a/docs/portable-mode.md +++ b/docs/portable-mode.md @@ -93,6 +93,7 @@ Flags: virtual folder identified by this prefix and its contents --s3-region string + --s3-session-token string --s3-storage-class string --s3-upload-concurrency int How many parts are uploaded in parallel (default 2) diff --git a/go.mod b/go.mod index 9b8fab12..e86c008b 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/rs/cors v1.8.2 github.com/rs/xid v1.3.0 github.com/rs/zerolog v1.26.2-0.20220203140311-fc26014bd4e1 - github.com/sftpgo/sdk v0.1.1-0.20220225104414-9e485ac5bc94 + github.com/sftpgo/sdk v0.1.1-0.20220225141305-cca7ba31466c github.com/shirou/gopsutil/v3 v3.22.1 github.com/spf13/afero v1.8.1 github.com/spf13/cobra v1.3.0 diff --git a/go.sum b/go.sum index 31df53ea..62849027 100644 --- a/go.sum +++ b/go.sum @@ -698,8 +698,8 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= -github.com/sftpgo/sdk v0.1.1-0.20220225104414-9e485ac5bc94 h1:IllQqdyqETJdbik04oorF/oGwSkeY35RTPQjn/eQhO0= -github.com/sftpgo/sdk v0.1.1-0.20220225104414-9e485ac5bc94/go.mod h1:zqCRMcwS28IViwekJHNkFu4GqSfyVmOQTlh8h3icAXE= +github.com/sftpgo/sdk v0.1.1-0.20220225141305-cca7ba31466c h1:aSWi1VB6DXmPmscawueKEhoyMTZjsMTiRaWFfhHmB4Y= +github.com/sftpgo/sdk v0.1.1-0.20220225141305-cca7ba31466c/go.mod h1:zqCRMcwS28IViwekJHNkFu4GqSfyVmOQTlh8h3icAXE= github.com/shirou/gopsutil/v3 v3.22.1 h1:33y31Q8J32+KstqPfscvFwBlNJ6xLaBy4xqBXzlYV5w= github.com/shirou/gopsutil/v3 v3.22.1/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index de880e03..96964d58 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -1787,6 +1787,7 @@ func TestUserRedactedPassword(t *testing.T) { u.FsConfig.S3Config.Bucket = "b" u.FsConfig.S3Config.Region = "eu-west-1" u.FsConfig.S3Config.AccessKey = "access-key" + u.FsConfig.S3Config.SessionToken = "session token" u.FsConfig.S3Config.AccessSecret = kms.NewSecret(sdkkms.SecretStatusRedacted, "access-secret", "", "") u.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?k=m" u.FsConfig.S3Config.StorageClass = "Standard" @@ -2564,6 +2565,7 @@ func TestUserS3Config(t *testing.T) { user.FsConfig.S3Config.Region = "us-east-1" //nolint:goconst user.FsConfig.S3Config.AccessKey = "Server-Access-Key" user.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("Server-Access-Secret") + user.FsConfig.S3Config.SessionToken = "Session token" user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000" user.FsConfig.S3Config.UploadPartSize = 8 user.FsConfig.S3Config.DownloadPartMaxTime = 60 @@ -15097,6 +15099,7 @@ func TestWebUserS3Mock(t *testing.T) { user.FsConfig.S3Config.Region = "eu-west-1" user.FsConfig.S3Config.AccessKey = "access-key" user.FsConfig.S3Config.AccessSecret = kms.NewPlainSecret("access-secret") + user.FsConfig.S3Config.SessionToken = "new session token" user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?a=b" user.FsConfig.S3Config.StorageClass = "Standard" user.FsConfig.S3Config.KeyPrefix = "somedir/subdir/" @@ -15135,6 +15138,7 @@ func TestWebUserS3Mock(t *testing.T) { form.Set("s3_region", user.FsConfig.S3Config.Region) form.Set("s3_access_key", user.FsConfig.S3Config.AccessKey) form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret.GetPayload()) + form.Set("s3_session_token", user.FsConfig.S3Config.SessionToken) form.Set("s3_storage_class", user.FsConfig.S3Config.StorageClass) form.Set("s3_acl", user.FsConfig.S3Config.ACL) form.Set("s3_endpoint", user.FsConfig.S3Config.Endpoint) @@ -15224,6 +15228,7 @@ func TestWebUserS3Mock(t *testing.T) { assert.Equal(t, updateUser.FsConfig.S3Config.Bucket, user.FsConfig.S3Config.Bucket) assert.Equal(t, updateUser.FsConfig.S3Config.Region, user.FsConfig.S3Config.Region) assert.Equal(t, updateUser.FsConfig.S3Config.AccessKey, user.FsConfig.S3Config.AccessKey) + assert.Equal(t, updateUser.FsConfig.S3Config.SessionToken, user.FsConfig.S3Config.SessionToken) assert.Equal(t, updateUser.FsConfig.S3Config.StorageClass, user.FsConfig.S3Config.StorageClass) assert.Equal(t, updateUser.FsConfig.S3Config.ACL, user.FsConfig.S3Config.ACL) assert.Equal(t, updateUser.FsConfig.S3Config.Endpoint, user.FsConfig.S3Config.Endpoint) @@ -15924,6 +15929,7 @@ func TestS3WebFolderMock(t *testing.T) { S3Region := "eu-west-1" S3AccessKey := "access-key" S3AccessSecret := kms.NewPlainSecret("folder-access-secret") + S3SessionToken := "fake session token" S3Endpoint := "http://127.0.0.1:9000/path?b=c" S3StorageClass := "Standard" S3ACL := "public-read-write" @@ -15943,6 +15949,7 @@ func TestS3WebFolderMock(t *testing.T) { form.Set("s3_region", S3Region) form.Set("s3_access_key", S3AccessKey) form.Set("s3_access_secret", S3AccessSecret.GetPayload()) + form.Set("s3_session_token", S3SessionToken) form.Set("s3_storage_class", S3StorageClass) form.Set("s3_acl", S3ACL) form.Set("s3_endpoint", S3Endpoint) @@ -15987,6 +15994,7 @@ func TestS3WebFolderMock(t *testing.T) { assert.Equal(t, S3Bucket, folder.FsConfig.S3Config.Bucket) assert.Equal(t, S3Region, folder.FsConfig.S3Config.Region) assert.Equal(t, S3AccessKey, folder.FsConfig.S3Config.AccessKey) + assert.Equal(t, S3SessionToken, folder.FsConfig.S3Config.SessionToken) assert.NotEmpty(t, folder.FsConfig.S3Config.AccessSecret.GetPayload()) assert.Equal(t, S3Endpoint, folder.FsConfig.S3Config.Endpoint) assert.Equal(t, S3StorageClass, folder.FsConfig.S3Config.StorageClass) @@ -16035,6 +16043,7 @@ func TestS3WebFolderMock(t *testing.T) { assert.Equal(t, S3Bucket, folder.FsConfig.S3Config.Bucket) assert.Equal(t, S3Region, folder.FsConfig.S3Config.Region) assert.Equal(t, S3AccessKey, folder.FsConfig.S3Config.AccessKey) + assert.Equal(t, S3SessionToken, folder.FsConfig.S3Config.SessionToken) assert.NotEmpty(t, folder.FsConfig.S3Config.AccessSecret.GetPayload()) assert.Equal(t, S3Endpoint, folder.FsConfig.S3Config.Endpoint) assert.Equal(t, S3StorageClass, folder.FsConfig.S3Config.StorageClass) diff --git a/httpd/webadmin.go b/httpd/webadmin.go index 42cd8e66..69345f71 100644 --- a/httpd/webadmin.go +++ b/httpd/webadmin.go @@ -964,6 +964,7 @@ func getS3Config(r *http.Request) (vfs.S3FsConfig, error) { config.Bucket = r.Form.Get("s3_bucket") config.Region = r.Form.Get("s3_region") config.AccessKey = r.Form.Get("s3_access_key") + config.SessionToken = strings.TrimSpace(r.Form.Get("s3_session_token")) config.AccessSecret = getSecretFromFormField(r, "s3_access_secret") config.Endpoint = r.Form.Get("s3_endpoint") config.StorageClass = r.Form.Get("s3_storage_class") diff --git a/httpdtest/httpdtest.go b/httpdtest/httpdtest.go index 9f9459e9..659dd108 100644 --- a/httpdtest/httpdtest.go +++ b/httpdtest/httpdtest.go @@ -1275,6 +1275,9 @@ func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error { / if expected.S3Config.AccessKey != actual.S3Config.AccessKey { return errors.New("fs S3 access key mismatch") } + if expected.S3Config.SessionToken != actual.S3Config.SessionToken { + return errors.New("fs S3 session token mismatch") + } if err := checkEncryptedSecret(expected.S3Config.AccessSecret, actual.S3Config.AccessSecret); err != nil { return fmt.Errorf("fs S3 access secret mismatch: %v", err) } diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 528616e2..43b875b1 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -4701,6 +4701,8 @@ components: type: string access_secret: $ref: '#/components/schemas/Secret' + session_token: + type: string endpoint: type: string description: optional endpoint diff --git a/templates/webadmin/fsconfig.html b/templates/webadmin/fsconfig.html index f1697148..5fb04cfb 100644 --- a/templates/webadmin/fsconfig.html +++ b/templates/webadmin/fsconfig.html @@ -166,6 +166,14 @@ +
+ +
+ +
+
+
diff --git a/vfs/filesystem.go b/vfs/filesystem.go index 19254a5a..c9f7c919 100644 --- a/vfs/filesystem.go +++ b/vfs/filesystem.go @@ -244,6 +244,7 @@ func (f *Filesystem) GetACopy() Filesystem { Bucket: f.S3Config.Bucket, Region: f.S3Config.Region, AccessKey: f.S3Config.AccessKey, + SessionToken: f.S3Config.SessionToken, Endpoint: f.S3Config.Endpoint, StorageClass: f.S3Config.StorageClass, ACL: f.S3Config.ACL, diff --git a/vfs/s3fs.go b/vfs/s3fs.go index 97a97902..267533c1 100644 --- a/vfs/s3fs.go +++ b/vfs/s3fs.go @@ -81,7 +81,8 @@ func NewS3Fs(connectionID, localTempDir, mountPath string, config S3FsConfig) (F if err := fs.config.AccessSecret.TryDecrypt(); err != nil { return fs, err } - awsConfig.Credentials = credentials.NewStaticCredentials(fs.config.AccessKey, fs.config.AccessSecret.GetPayload(), "") + awsConfig.Credentials = credentials.NewStaticCredentials(fs.config.AccessKey, + fs.config.AccessSecret.GetPayload(), fs.config.SessionToken) } if fs.config.Endpoint != "" { diff --git a/vfs/vfs.go b/vfs/vfs.go index eee2768a..2d636f32 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -173,6 +173,9 @@ func (c *S3FsConfig) isEqual(other *S3FsConfig) bool { if c.AccessKey != other.AccessKey { return false } + if c.SessionToken != other.SessionToken { + return false + } if c.Endpoint != other.Endpoint { return false } @@ -182,6 +185,17 @@ func (c *S3FsConfig) isEqual(other *S3FsConfig) bool { if c.ACL != other.ACL { return false } + if !c.areMultipartFieldsEqual(other) { + return false + } + + if c.ForcePathStyle != other.ForcePathStyle { + return false + } + return c.isSecretEqual(other) +} + +func (c *S3FsConfig) areMultipartFieldsEqual(other *S3FsConfig) bool { if c.UploadPartSize != other.UploadPartSize { return false } @@ -200,10 +214,7 @@ func (c *S3FsConfig) isEqual(other *S3FsConfig) bool { if c.UploadPartMaxTime != other.UploadPartMaxTime { return false } - if c.ForcePathStyle != other.ForcePathStyle { - return false - } - return c.isSecretEqual(other) + return true } func (c *S3FsConfig) isSecretEqual(other *S3FsConfig) bool {