From ee5c5e033d366a2643943a9e81a7434a0780a539 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sat, 13 Nov 2021 16:05:40 +0100 Subject: [PATCH] S3: add ACL support Fixes #610 --- cmd/portable.go | 3 +++ docs/portable-mode.md | 1 + httpd/httpd_test.go | 7 +++++++ httpd/schema/openapi.yaml | 3 +++ httpd/webadmin.go | 1 + httpdtest/httpdtest.go | 5 ++++- sdk/filesystem.go | 4 ++++ templates/webadmin/fsconfig.html | 14 +++++++++++++- vfs/filesystem.go | 1 + vfs/s3fs.go | 6 ++++-- vfs/vfs.go | 10 ++++++++++ 11 files changed, 51 insertions(+), 4 deletions(-) diff --git a/cmd/portable.go b/cmd/portable.go index e28a9892..fbb8e413 100644 --- a/cmd/portable.go +++ b/cmd/portable.go @@ -44,6 +44,7 @@ var ( portableS3AccessSecret string portableS3Endpoint string portableS3StorageClass string + portableS3ACL string portableS3KeyPrefix string portableS3ULPartSize int portableS3ULConcurrency int @@ -169,6 +170,7 @@ Please take a look at the usage below to customize the serving parameters`, AccessSecret: kms.NewPlainSecret(portableS3AccessSecret), Endpoint: portableS3Endpoint, StorageClass: portableS3StorageClass, + ACL: portableS3ACL, KeyPrefix: portableS3KeyPrefix, UploadPartSize: int64(portableS3ULPartSize), UploadConcurrency: portableS3ULConcurrency, @@ -288,6 +290,7 @@ sftpfs => SFTP (legacy: 5)`) portableCmd.Flags().StringVar(&portableS3AccessSecret, "s3-access-secret", "", "") portableCmd.Flags().StringVar(&portableS3Endpoint, "s3-endpoint", "", "") portableCmd.Flags().StringVar(&portableS3StorageClass, "s3-storage-class", "", "") + portableCmd.Flags().StringVar(&portableS3ACL, "s3-acl", "", "") portableCmd.Flags().StringVar(&portableS3KeyPrefix, "s3-key-prefix", "", `Allows to restrict access to the virtual folder identified by this prefix and its contents`) diff --git a/docs/portable-mode.md b/docs/portable-mode.md index 1bc47f85..9b9fafb1 100644 --- a/docs/portable-mode.md +++ b/docs/portable-mode.md @@ -81,6 +81,7 @@ Flags: -k, --public-key strings --s3-access-key string --s3-access-secret string + --s3-acl string --s3-bucket string --s3-endpoint string --s3-key-prefix string Allows to restrict access to the diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index 0957c23e..00640e60 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -1543,6 +1543,7 @@ func TestUserRedactedPassword(t *testing.T) { u.FsConfig.S3Config.AccessSecret = kms.NewSecret(kms.SecretStatusRedacted, "access-secret", "", "") u.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?k=m" u.FsConfig.S3Config.StorageClass = "Standard" + u.FsConfig.S3Config.ACL = "bucket-owner-full-control" _, resp, err := httpdtest.AddUser(u, http.StatusBadRequest) assert.NoError(t, err, string(resp)) assert.Contains(t, string(resp), "cannot save a user with a redacted secret") @@ -13181,6 +13182,7 @@ func TestWebUserS3Mock(t *testing.T) { user.FsConfig.S3Config.DownloadPartSize = 6 user.FsConfig.S3Config.DownloadConcurrency = 3 user.FsConfig.S3Config.ForcePathStyle = true + user.FsConfig.S3Config.ACL = "public-read" user.Description = "s3 tèst user" form := make(url.Values) form.Set(csrfFormToken, csrfToken) @@ -13205,6 +13207,7 @@ func TestWebUserS3Mock(t *testing.T) { form.Set("s3_access_key", user.FsConfig.S3Config.AccessKey) form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret.GetPayload()) form.Set("s3_storage_class", user.FsConfig.S3Config.StorageClass) + form.Set("s3_acl", user.FsConfig.S3Config.ACL) form.Set("s3_endpoint", user.FsConfig.S3Config.Endpoint) form.Set("s3_key_prefix", user.FsConfig.S3Config.KeyPrefix) form.Set("pattern_path0", "/dir1") @@ -13282,6 +13285,7 @@ func TestWebUserS3Mock(t *testing.T) { assert.Equal(t, updateUser.FsConfig.S3Config.Region, user.FsConfig.S3Config.Region) assert.Equal(t, updateUser.FsConfig.S3Config.AccessKey, user.FsConfig.S3Config.AccessKey) assert.Equal(t, updateUser.FsConfig.S3Config.StorageClass, user.FsConfig.S3Config.StorageClass) + assert.Equal(t, updateUser.FsConfig.S3Config.ACL, user.FsConfig.S3Config.ACL) assert.Equal(t, updateUser.FsConfig.S3Config.Endpoint, user.FsConfig.S3Config.Endpoint) assert.Equal(t, updateUser.FsConfig.S3Config.KeyPrefix, user.FsConfig.S3Config.KeyPrefix) assert.Equal(t, updateUser.FsConfig.S3Config.UploadPartSize, user.FsConfig.S3Config.UploadPartSize) @@ -13931,6 +13935,7 @@ func TestS3WebFolderMock(t *testing.T) { S3AccessSecret := kms.NewPlainSecret("folder-access-secret") S3Endpoint := "http://127.0.0.1:9000/path?b=c" S3StorageClass := "Standard" + S3ACL := "public-read-write" S3KeyPrefix := "somedir/subdir/" S3UploadPartSize := 5 S3UploadConcurrency := 4 @@ -13947,6 +13952,7 @@ func TestS3WebFolderMock(t *testing.T) { form.Set("s3_access_key", S3AccessKey) form.Set("s3_access_secret", S3AccessSecret.GetPayload()) form.Set("s3_storage_class", S3StorageClass) + form.Set("s3_acl", S3ACL) form.Set("s3_endpoint", S3Endpoint) form.Set("s3_key_prefix", S3KeyPrefix) form.Set("s3_upload_part_size", strconv.Itoa(S3UploadPartSize)) @@ -13991,6 +13997,7 @@ func TestS3WebFolderMock(t *testing.T) { assert.NotEmpty(t, folder.FsConfig.S3Config.AccessSecret.GetPayload()) assert.Equal(t, S3Endpoint, folder.FsConfig.S3Config.Endpoint) assert.Equal(t, S3StorageClass, folder.FsConfig.S3Config.StorageClass) + assert.Equal(t, S3ACL, folder.FsConfig.S3Config.ACL) assert.Equal(t, S3KeyPrefix, folder.FsConfig.S3Config.KeyPrefix) assert.Equal(t, S3UploadConcurrency, folder.FsConfig.S3Config.UploadConcurrency) assert.Equal(t, int64(S3UploadPartSize), folder.FsConfig.S3Config.UploadPartSize) diff --git a/httpd/schema/openapi.yaml b/httpd/schema/openapi.yaml index 2551f7e7..9a321cab 100644 --- a/httpd/schema/openapi.yaml +++ b/httpd/schema/openapi.yaml @@ -4181,6 +4181,9 @@ components: description: optional endpoint storage_class: type: string + acl: + type: string + description: 'The canned ACL to apply to uploaded objects. Leave empty to use the default ACL. For more information and available ACLs, see here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl' upload_part_size: type: integer description: 'the buffer size (in MB) to use for multipart uploads. The minimum allowed part size is 5MB, and if this value is set to zero, the default value (5MB) for the AWS SDK will be used. The minimum allowed value is 5.' diff --git a/httpd/webadmin.go b/httpd/webadmin.go index 10fc083b..73b668ab 100644 --- a/httpd/webadmin.go +++ b/httpd/webadmin.go @@ -830,6 +830,7 @@ func getS3Config(r *http.Request) (vfs.S3FsConfig, error) { config.AccessSecret = getSecretFromFormField(r, "s3_access_secret") config.Endpoint = r.Form.Get("s3_endpoint") config.StorageClass = r.Form.Get("s3_storage_class") + config.ACL = r.Form.Get("s3_acl") config.KeyPrefix = r.Form.Get("s3_key_prefix") config.UploadPartSize, err = strconv.ParseInt(r.Form.Get("s3_upload_part_size"), 10, 64) if err != nil { diff --git a/httpdtest/httpdtest.go b/httpdtest/httpdtest.go index 663e78b2..d15a7071 100644 --- a/httpdtest/httpdtest.go +++ b/httpdtest/httpdtest.go @@ -1245,7 +1245,7 @@ func compareFsConfig(expected *vfs.Filesystem, actual *vfs.Filesystem) error { return compareSFTPFsConfig(expected, actual) } -func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error { +func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error { //nolint:gocyclo if expected.S3Config.Bucket != actual.S3Config.Bucket { return errors.New("fs S3 bucket mismatch") } @@ -1264,6 +1264,9 @@ func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error { if expected.S3Config.StorageClass != actual.S3Config.StorageClass { return errors.New("fs S3 storage class mismatch") } + if expected.S3Config.ACL != actual.S3Config.ACL { + return errors.New("fs S3 ACL mismatch") + } if expected.S3Config.UploadPartSize != actual.S3Config.UploadPartSize { return errors.New("fs S3 upload part size mismatch") } diff --git a/sdk/filesystem.go b/sdk/filesystem.go index 570e70ab..a6ff61dd 100644 --- a/sdk/filesystem.go +++ b/sdk/filesystem.go @@ -99,6 +99,10 @@ type S3FsConfig struct { AccessSecret *kms.Secret `json:"access_secret,omitempty"` Endpoint string `json:"endpoint,omitempty"` StorageClass string `json:"storage_class,omitempty"` + // The canned ACL to apply to uploaded objects. Leave empty to use the default ACL. + // For more information and available ACLs, see here: + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl + ACL string `json:"acl,omitempty"` // The buffer size (in MB) to use for multipart uploads. The minimum allowed part size is 5MB, // and if this value is set to zero, the default value (5MB) for the AWS SDK will be used. // The minimum allowed value is 5. diff --git a/templates/webadmin/fsconfig.html b/templates/webadmin/fsconfig.html index b290130e..dd3096c5 100644 --- a/templates/webadmin/fsconfig.html +++ b/templates/webadmin/fsconfig.html @@ -109,8 +109,19 @@
- +
+ + + ACL for uploaded objects. For more info take a look here + +
+ + +
+ +
@@ -119,6 +130,7 @@
+