beenull/sftpgo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

S3: add ACL support

Browse source 
Fixes #610
This commit is contained in:
Nicola Murino 2021-11-13 16:05:40 +01:00
parent 78233ff9a3
commit ee5c5e033d
No known key found for this signature in database
GPG key ID: 2F1FB59433D5A8CB
11 changed files with 51 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
cmd/portable.go
View file

@ -44,6 +44,7 @@ var (
portableS3AccessSecret string
portableS3Endpoint string
portableS3StorageClass string
portableS3ACL string
portableS3KeyPrefix string
portableS3ULPartSize int
portableS3ULConcurrency int
@ -169,6 +170,7 @@ Please take a look at the usage below to customize the serving parameters`,
AccessSecret: kms.NewPlainSecret(portableS3AccessSecret),
Endpoint: portableS3Endpoint,
StorageClass: portableS3StorageClass,
ACL: portableS3ACL,
KeyPrefix: portableS3KeyPrefix,
UploadPartSize: int64(portableS3ULPartSize),
UploadConcurrency: portableS3ULConcurrency,
@ -288,6 +290,7 @@ sftpfs => SFTP (legacy: 5)`)
portableCmd.Flags().StringVar(&portableS3AccessSecret, "s3-access-secret", "", "")
portableCmd.Flags().StringVar(&portableS3Endpoint, "s3-endpoint", "", "")
portableCmd.Flags().StringVar(&portableS3StorageClass, "s3-storage-class", "", "")
portableCmd.Flags().StringVar(&portableS3ACL, "s3-acl", "", "")
portableCmd.Flags().StringVar(&portableS3KeyPrefix, "s3-key-prefix", "", `Allows to restrict access to the
virtual folder identified by this
prefix and its contents`)

1
docs/portable-mode.md
View file

@ -81,6 +81,7 @@ Flags:
-k, --public-key strings
--s3-access-key string
--s3-access-secret string
--s3-acl string
--s3-bucket string
--s3-endpoint string
--s3-key-prefix string Allows to restrict access to the

7
httpd/httpd_test.go
View file

@ -1543,6 +1543,7 @@ func TestUserRedactedPassword(t *testing.T) {
u.FsConfig.S3Config.AccessSecret = kms.NewSecret(kms.SecretStatusRedacted, "access-secret", "", "")
u.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?k=m"
u.FsConfig.S3Config.StorageClass = "Standard"
u.FsConfig.S3Config.ACL = "bucket-owner-full-control"
_, resp, err := httpdtest.AddUser(u, http.StatusBadRequest)
assert.NoError(t, err, string(resp))
assert.Contains(t, string(resp), "cannot save a user with a redacted secret")
@ -13181,6 +13182,7 @@ func TestWebUserS3Mock(t *testing.T) {
user.FsConfig.S3Config.DownloadPartSize = 6
user.FsConfig.S3Config.DownloadConcurrency = 3
user.FsConfig.S3Config.ForcePathStyle = true
user.FsConfig.S3Config.ACL = "public-read"
user.Description = "s3 tèst user"
form := make(url.Values)
form.Set(csrfFormToken, csrfToken)
@ -13205,6 +13207,7 @@ func TestWebUserS3Mock(t *testing.T) {
form.Set("s3_access_key", user.FsConfig.S3Config.AccessKey)
form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret.GetPayload())
form.Set("s3_storage_class", user.FsConfig.S3Config.StorageClass)
form.Set("s3_acl", user.FsConfig.S3Config.ACL)
form.Set("s3_endpoint", user.FsConfig.S3Config.Endpoint)
form.Set("s3_key_prefix", user.FsConfig.S3Config.KeyPrefix)
form.Set("pattern_path0", "/dir1")
@ -13282,6 +13285,7 @@ func TestWebUserS3Mock(t *testing.T) {
assert.Equal(t, updateUser.FsConfig.S3Config.Region, user.FsConfig.S3Config.Region)
assert.Equal(t, updateUser.FsConfig.S3Config.AccessKey, user.FsConfig.S3Config.AccessKey)
assert.Equal(t, updateUser.FsConfig.S3Config.StorageClass, user.FsConfig.S3Config.StorageClass)
assert.Equal(t, updateUser.FsConfig.S3Config.ACL, user.FsConfig.S3Config.ACL)
assert.Equal(t, updateUser.FsConfig.S3Config.Endpoint, user.FsConfig.S3Config.Endpoint)
assert.Equal(t, updateUser.FsConfig.S3Config.KeyPrefix, user.FsConfig.S3Config.KeyPrefix)
assert.Equal(t, updateUser.FsConfig.S3Config.UploadPartSize, user.FsConfig.S3Config.UploadPartSize)
@ -13931,6 +13935,7 @@ func TestS3WebFolderMock(t *testing.T) {
S3AccessSecret := kms.NewPlainSecret("folder-access-secret")
S3Endpoint := "http://127.0.0.1:9000/path?b=c"
S3StorageClass := "Standard"
S3ACL := "public-read-write"
S3KeyPrefix := "somedir/subdir/"
S3UploadPartSize := 5
S3UploadConcurrency := 4
@ -13947,6 +13952,7 @@ func TestS3WebFolderMock(t *testing.T) {
form.Set("s3_access_key", S3AccessKey)
form.Set("s3_access_secret", S3AccessSecret.GetPayload())
form.Set("s3_storage_class", S3StorageClass)
form.Set("s3_acl", S3ACL)
form.Set("s3_endpoint", S3Endpoint)
form.Set("s3_key_prefix", S3KeyPrefix)
form.Set("s3_upload_part_size", strconv.Itoa(S3UploadPartSize))
@ -13991,6 +13997,7 @@ func TestS3WebFolderMock(t *testing.T) {
assert.NotEmpty(t, folder.FsConfig.S3Config.AccessSecret.GetPayload())
assert.Equal(t, S3Endpoint, folder.FsConfig.S3Config.Endpoint)
assert.Equal(t, S3StorageClass, folder.FsConfig.S3Config.StorageClass)
assert.Equal(t, S3ACL, folder.FsConfig.S3Config.ACL)
assert.Equal(t, S3KeyPrefix, folder.FsConfig.S3Config.KeyPrefix)
assert.Equal(t, S3UploadConcurrency, folder.FsConfig.S3Config.UploadConcurrency)
assert.Equal(t, int64(S3UploadPartSize), folder.FsConfig.S3Config.UploadPartSize)

3
httpd/schema/openapi.yaml
View file

@ -4181,6 +4181,9 @@ components:
description: optional endpoint
storage_class:
type: string
acl:
type: string
description: 'The canned ACL to apply to uploaded objects. Leave empty to use the default ACL. For more information and available ACLs, see here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl'
upload_part_size:
type: integer
description: 'the buffer size (in MB) to use for multipart uploads. The minimum allowed part size is 5MB, and if this value is set to zero, the default value (5MB) for the AWS SDK will be used. The minimum allowed value is 5.'

1
httpd/webadmin.go
View file

@ -830,6 +830,7 @@ func getS3Config(r *http.Request) (vfs.S3FsConfig, error) {
config.AccessSecret = getSecretFromFormField(r, "s3_access_secret")
config.Endpoint = r.Form.Get("s3_endpoint")
config.StorageClass = r.Form.Get("s3_storage_class")
config.ACL = r.Form.Get("s3_acl")
config.KeyPrefix = r.Form.Get("s3_key_prefix")
config.UploadPartSize, err = strconv.ParseInt(r.Form.Get("s3_upload_part_size"), 10, 64)
if err != nil {

5
httpdtest/httpdtest.go
View file

@ -1245,7 +1245,7 @@ func compareFsConfig(expected *vfs.Filesystem, actual *vfs.Filesystem) error {
return compareSFTPFsConfig(expected, actual)
}
func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error {
func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error { //nolint:gocyclo
if expected.S3Config.Bucket != actual.S3Config.Bucket {
return errors.New("fs S3 bucket mismatch")
}
@ -1264,6 +1264,9 @@ func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error {
if expected.S3Config.StorageClass != actual.S3Config.StorageClass {
return errors.New("fs S3 storage class mismatch")
}
if expected.S3Config.ACL != actual.S3Config.ACL {
return errors.New("fs S3 ACL mismatch")
}
if expected.S3Config.UploadPartSize != actual.S3Config.UploadPartSize {
return errors.New("fs S3 upload part size mismatch")
}

4
sdk/filesystem.go
View file

@ -99,6 +99,10 @@ type S3FsConfig struct {
AccessSecret *kms.Secret `json:"access_secret,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
StorageClass string `json:"storage_class,omitempty"`
// The canned ACL to apply to uploaded objects. Leave empty to use the default ACL.
// For more information and available ACLs, see here:
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
ACL string `json:"acl,omitempty"`
// The buffer size (in MB) to use for multipart uploads. The minimum allowed part size is 5MB,
// and if this value is set to zero, the default value (5MB) for the AWS SDK will be used.
// The minimum allowed value is 5.

14
templates/webadmin/fsconfig.html
View file

@ -109,8 +109,19 @@
</small>
</div>
<div class="col-sm-2"></div>
<label for="idS3KeyPrefix" class="col-sm-2 col-form-label">Key Prefix</label>
<label for="idS3KeyPrefix" class="col-sm-2 col-form-label">ACL</label>
<div class="col-sm-3">
<input type="text" class="form-control" id="idS3ACL" name="s3_acl" placeholder=""
value="{{.S3Config.ACL}}" maxlength="255" aria-describedby="S3ACLHelpBlock">
<small id="S3ACLHelpBlock" class="form-text text-muted">
ACL for uploaded objects. For more info take a look <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl" target="_blank">here</a>
</small>
</div>
</div>
<div class="form-group row fsconfig fsconfig-s3fs">
<label for="idS3KeyPrefix" class="col-sm-2 col-form-label">Key Prefix</label>
<div class="col-sm-10">
<input type="text" class="form-control" id="idS3KeyPrefix" name="s3_key_prefix" placeholder=""
value="{{.S3Config.KeyPrefix}}" maxlength="255" aria-describedby="S3KeyPrefixHelpBlock">
<small id="S3KeyPrefixHelpBlock" class="form-text text-muted">
@ -119,6 +130,7 @@
</div>
</div>
<div class="form-group fsconfig fsconfig-s3fs">
<div class="form-check">
<input type="checkbox" class="form-check-input" id="idS3ForcePathStyle" name="s3_force_path_style"

1
vfs/filesystem.go
View file

@ -235,6 +235,7 @@ func (f *Filesystem) GetACopy() Filesystem {
AccessSecret: f.S3Config.AccessSecret.Clone(),
Endpoint: f.S3Config.Endpoint,
StorageClass: f.S3Config.StorageClass,
ACL: f.S3Config.ACL,
KeyPrefix: f.S3Config.KeyPrefix,
UploadPartSize: f.S3Config.UploadPartSize,
UploadConcurrency: f.S3Config.UploadConcurrency,

6
vfs/s3fs.go
View file

@ -244,6 +244,7 @@ func (fs *S3Fs) Create(name string, flag int) (File, *PipeWriter, func(), error)
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(key),
Body: r,
ACL: util.NilIfEmpty(fs.config.ACL),
StorageClass: util.NilIfEmpty(fs.config.StorageClass),
ContentType: util.NilIfEmpty(contentType),
}, func(u *s3manager.Uploader) {
@ -252,8 +253,8 @@ func (fs *S3Fs) Create(name string, flag int) (File, *PipeWriter, func(), error)
})
r.CloseWithError(err) //nolint:errcheck
p.Done(err)
fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, response: %v, readed bytes: %v, err: %+v",
name, response, r.GetReadedBytes(), err)
fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, acl: %#v, response: %v, readed bytes: %v, err: %+v",
name, fs.config.ACL, response, r.GetReadedBytes(), err)
metric.S3TransferCompleted(r.GetReadedBytes(), 0, err)
}()
return nil, p, cancelFn, nil
@ -306,6 +307,7 @@ func (fs *S3Fs) Rename(source, target string) error {
CopySource: aws.String(pathEscape(copySource)),
Key: aws.String(target),
StorageClass: util.NilIfEmpty(fs.config.StorageClass),
ACL: util.NilIfEmpty(fs.config.ACL),
ContentType: util.NilIfEmpty(contentType),
})
if err != nil {

10
vfs/vfs.go
View file

@ -169,6 +169,9 @@ func (c *S3FsConfig) isEqual(other *S3FsConfig) bool {
if c.StorageClass != other.StorageClass {
return false
}
if c.ACL != other.ACL {
return false
}
if c.UploadPartSize != other.UploadPartSize {
return false
}
@ -187,6 +190,10 @@ func (c *S3FsConfig) isEqual(other *S3FsConfig) bool {
if c.ForcePathStyle != other.ForcePathStyle {
return false
}
return c.isSecretEqual(other)
}
func (c *S3FsConfig) isSecretEqual(other *S3FsConfig) bool {
if c.AccessSecret == nil {
c.AccessSecret = kms.NewEmptySecret()
}
@ -263,6 +270,8 @@ func (c *S3FsConfig) Validate() error {
c.KeyPrefix += "/"
}
}
c.StorageClass = strings.TrimSpace(c.StorageClass)
c.ACL = strings.TrimSpace(c.ACL)
return c.checkPartSizeAndConcurrency()
}
@ -329,6 +338,7 @@ func (c *GCSFsConfig) Validate(credentialsFilePath string) error {
return errors.New("credentials cannot be empty")
}
}
c.StorageClass = strings.TrimSpace(c.StorageClass)
return nil
}