From 0296e0cafaccabcf1710a44f575899679c87348f Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sun, 18 Dec 2022 11:51:46 +0100 Subject: [PATCH] gcsfs: allow to customize upload part size/time Signed-off-by: Nicola Murino --- go.mod | 2 +- go.sum | 4 +-- internal/common/protocol_test.go | 2 +- internal/httpd/httpd_test.go | 10 ++++++ internal/httpd/webadmin.go | 8 +++++ internal/httpdtest/httpdtest.go | 7 ++++ internal/vfs/filesystem.go | 2 ++ internal/vfs/gcsfs.go | 44 ++++++++++++++++++++++-- internal/vfs/vfs.go | 12 +++++++ openapi/openapi.yaml | 6 ++++ templates/email/password-expiration.html | 2 +- templates/webadmin/fsconfig.html | 21 +++++++++++ 12 files changed, 112 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 4f3a94da..2fc37372 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/rs/cors v1.8.3-0.20220619195839-da52b0701de5 github.com/rs/xid v1.4.0 github.com/rs/zerolog v1.28.0 - github.com/sftpgo/sdk v0.1.3-0.20221211151321-578e45601b27 + github.com/sftpgo/sdk v0.1.3-0.20221217110036-383c1bb50fa0 github.com/shirou/gopsutil/v3 v3.22.11 github.com/spf13/afero v1.9.3 github.com/spf13/cobra v1.6.1 diff --git a/go.sum b/go.sum index 8a9cfdf9..7feffc5f 100644 --- a/go.sum +++ b/go.sum @@ -1452,8 +1452,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= -github.com/sftpgo/sdk v0.1.3-0.20221211151321-578e45601b27 h1:DjNme+rcw3zaiEkWyyrtimqDZd/83GW4qZhUghBkyrI= -github.com/sftpgo/sdk v0.1.3-0.20221211151321-578e45601b27/go.mod h1:3GpW3Qy8IHH6kex0ny+Y6ayeYb9OJxz8Pxh3IZgAs2E= +github.com/sftpgo/sdk v0.1.3-0.20221217110036-383c1bb50fa0 h1:e1OQroqX8SWV06Z270CxG2/v//Wx1026iXKTDRn5J1E= +github.com/sftpgo/sdk v0.1.3-0.20221217110036-383c1bb50fa0/go.mod h1:3GpW3Qy8IHH6kex0ny+Y6ayeYb9OJxz8Pxh3IZgAs2E= github.com/shirou/gopsutil/v3 v3.22.11 h1:kxsPKS+Eeo+VnEQ2XCaGJepeP6KY53QoRTETx3+1ndM= github.com/shirou/gopsutil/v3 v3.22.11/go.mod h1:xl0EeL4vXJ+hQMAGN8B9VFpxukEMA0XdevQOe5MZ1oY= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= diff --git a/internal/common/protocol_test.go b/internal/common/protocol_test.go index d015a729..cf3e2d63 100644 --- a/internal/common/protocol_test.go +++ b/internal/common/protocol_test.go @@ -5870,7 +5870,7 @@ func TestEventRulePasswordExpiration(t *testing.T) { email := lastReceivedEmail.get() assert.Len(t, email.To, 1) assert.Contains(t, email.To, user.Email) - assert.Contains(t, email.Data, "Your SFTPGo password expires in 5 days") + assert.Contains(t, email.Data, "your SFTPGo password expires in 5 days") err = client.RemoveDirectory(dirName) assert.NoError(t, err) } diff --git a/internal/httpd/httpd_test.go b/internal/httpd/httpd_test.go index 94b3516e..8a8ecc02 100644 --- a/internal/httpd/httpd_test.go +++ b/internal/httpd/httpd_test.go @@ -5015,6 +5015,8 @@ func TestUserHiddenFields(t *testing.T) { u2.FsConfig.GCSConfig.Bucket = "test" u2.FsConfig.GCSConfig.Credentials = kms.NewPlainSecret("fake credentials") u2.FsConfig.GCSConfig.ACL = "bucketOwnerRead" + u2.FsConfig.GCSConfig.UploadPartSize = 5 + u2.FsConfig.GCSConfig.UploadPartMaxTime = 20 user2, _, err := httpdtest.AddUser(u2, http.StatusCreated) assert.NoError(t, err) @@ -11719,6 +11721,8 @@ func TestLoginInvalidFs(t *testing.T) { u.Filters.AllowAPIKeyAuth = true u.FsConfig.Provider = sdk.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "test" + u.FsConfig.GCSConfig.UploadPartSize = 1 + u.FsConfig.GCSConfig.UploadPartMaxTime = 10 u.FsConfig.GCSConfig.Credentials = kms.NewPlainSecret("invalid JSON for credentials") user, _, err := httpdtest.AddUser(u, http.StatusCreated) assert.NoError(t, err) @@ -19226,6 +19230,8 @@ func TestWebUserGCSMock(t *testing.T) { user.FsConfig.GCSConfig.KeyPrefix = "somedir/subdir/" user.FsConfig.GCSConfig.StorageClass = "standard" user.FsConfig.GCSConfig.ACL = "publicReadWrite" + user.FsConfig.GCSConfig.UploadPartSize = 16 + user.FsConfig.GCSConfig.UploadPartMaxTime = 32 form := make(url.Values) form.Set(csrfFormToken, csrfToken) form.Set("username", user.Username) @@ -19252,6 +19258,8 @@ func TestWebUserGCSMock(t *testing.T) { form.Set("gcs_storage_class", user.FsConfig.GCSConfig.StorageClass) form.Set("gcs_acl", user.FsConfig.GCSConfig.ACL) form.Set("gcs_key_prefix", user.FsConfig.GCSConfig.KeyPrefix) + form.Set("gcs_upload_part_size", strconv.FormatInt(user.FsConfig.GCSConfig.UploadPartSize, 10)) + form.Set("gcs_upload_part_max_time", strconv.FormatInt(int64(user.FsConfig.GCSConfig.UploadPartMaxTime), 10)) form.Set("pattern_path0", "/dir1") form.Set("patterns0", "*.jpg,*.png") form.Set("pattern_type0", "allowed") @@ -19292,6 +19300,8 @@ func TestWebUserGCSMock(t *testing.T) { assert.Equal(t, user.FsConfig.GCSConfig.StorageClass, updateUser.FsConfig.GCSConfig.StorageClass) assert.Equal(t, user.FsConfig.GCSConfig.ACL, updateUser.FsConfig.GCSConfig.ACL) assert.Equal(t, user.FsConfig.GCSConfig.KeyPrefix, updateUser.FsConfig.GCSConfig.KeyPrefix) + assert.Equal(t, user.FsConfig.GCSConfig.UploadPartSize, updateUser.FsConfig.GCSConfig.UploadPartSize) + assert.Equal(t, user.FsConfig.GCSConfig.UploadPartMaxTime, updateUser.FsConfig.GCSConfig.UploadPartMaxTime) if assert.Len(t, updateUser.Filters.FilePatterns, 1) { assert.Equal(t, "/dir1", updateUser.Filters.FilePatterns[0].Path) assert.Len(t, updateUser.Filters.FilePatterns[0].AllowedPatterns, 2) diff --git a/internal/httpd/webadmin.go b/internal/httpd/webadmin.go index 933096db..b6b27ba4 100644 --- a/internal/httpd/webadmin.go +++ b/internal/httpd/webadmin.go @@ -1513,6 +1513,14 @@ func getGCSConfig(r *http.Request) (vfs.GCSFsConfig, error) { config.StorageClass = strings.TrimSpace(r.Form.Get("gcs_storage_class")) config.ACL = strings.TrimSpace(r.Form.Get("gcs_acl")) config.KeyPrefix = r.Form.Get("gcs_key_prefix") + uploadPartSize, err := strconv.ParseInt(r.Form.Get("gcs_upload_part_size"), 10, 64) + if err == nil { + config.UploadPartSize = uploadPartSize + } + uploadPartMaxTime, err := strconv.Atoi(r.Form.Get("gcs_upload_part_max_time")) + if err == nil { + config.UploadPartMaxTime = uploadPartMaxTime + } autoCredentials := r.Form.Get("gcs_auto_credentials") if autoCredentials != "" { config.AutomaticCredentials = 1 diff --git a/internal/httpdtest/httpdtest.go b/internal/httpdtest/httpdtest.go index a14196d1..ea503517 100644 --- a/internal/httpdtest/httpdtest.go +++ b/internal/httpdtest/httpdtest.go @@ -2037,6 +2037,13 @@ func compareGCSConfig(expected *vfs.Filesystem, actual *vfs.Filesystem) error { if expected.GCSConfig.AutomaticCredentials != actual.GCSConfig.AutomaticCredentials { return errors.New("GCS automatic credentials mismatch") } + if expected.GCSConfig.UploadPartSize != actual.GCSConfig.UploadPartSize { + return errors.New("GCS upload part size mismatch") + } + if expected.GCSConfig.UploadPartMaxTime != actual.GCSConfig.UploadPartMaxTime { + fmt.Printf("aaaaaaaaaa %v, %v", expected.GCSConfig.UploadPartMaxTime, actual.GCSConfig.UploadPartMaxTime) + return errors.New("GCS upload part max time mismatch") + } return nil } diff --git a/internal/vfs/filesystem.go b/internal/vfs/filesystem.go index 20e0c826..df0573de 100644 --- a/internal/vfs/filesystem.go +++ b/internal/vfs/filesystem.go @@ -308,6 +308,8 @@ func (f *Filesystem) GetACopy() Filesystem { StorageClass: f.GCSConfig.StorageClass, ACL: f.GCSConfig.ACL, KeyPrefix: f.GCSConfig.KeyPrefix, + UploadPartSize: f.GCSConfig.UploadPartSize, + UploadPartMaxTime: f.GCSConfig.UploadPartMaxTime, }, Credentials: f.GCSConfig.Credentials.Clone(), }, diff --git a/internal/vfs/gcsfs.go b/internal/vfs/gcsfs.go index 2ac4d760..0f9a4b9c 100644 --- a/internal/vfs/gcsfs.go +++ b/internal/vfs/gcsfs.go @@ -170,8 +170,27 @@ func (fs *GCSFs) Create(name string, flag int) (File, *PipeWriter, func(), error p := NewPipeWriter(w) bkt := fs.svc.Bucket(fs.config.Bucket) obj := bkt.Object(name) + if flag == -1 { + obj = obj.If(storage.Conditions{DoesNotExist: true}) + } else { + attrs, statErr := fs.headObject(name) + if statErr == nil { + obj = obj.If(storage.Conditions{GenerationMatch: attrs.Generation}) + } else if fs.IsNotExist(statErr) { + obj = obj.If(storage.Conditions{DoesNotExist: true}) + } else { + fsLog(fs, logger.LevelWarn, "unable to set precondition for %q, stat err: %v", name, statErr) + } + } + ctx, cancelFn := context.WithCancel(context.Background()) objectWriter := obj.NewWriter(ctx) + if fs.config.UploadPartSize > 0 { + objectWriter.ChunkSize = int(fs.config.UploadPartSize) * 1024 * 1024 + } + if fs.config.UploadPartMaxTime > 0 { + objectWriter.ChunkRetryDeadline = time.Duration(fs.config.UploadPartMaxTime) * time.Second + } var contentType string if flag == -1 { contentType = dirMimeType @@ -231,7 +250,17 @@ func (fs *GCSFs) Rename(source, target string) error { } else { src := fs.svc.Bucket(fs.config.Bucket).Object(realSourceName) dst := fs.svc.Bucket(fs.config.Bucket).Object(target) - ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) + attrs, statErr := fs.headObject(target) + if statErr == nil { + dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation}) + } else if fs.IsNotExist(statErr) { + dst = dst.If(storage.Conditions{DoesNotExist: true}) + } else { + fsLog(fs, logger.LevelWarn, "unable to set precondition for rename, target %q, stat err: %v", + target, statErr) + } + + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout)) defer cancelFn() copier := dst.CopierFrom(src) @@ -276,11 +305,20 @@ func (fs *GCSFs) Remove(name string, isDir bool) error { name += "/" } } + obj := fs.svc.Bucket(fs.config.Bucket).Object(name) + attrs, statErr := fs.headObject(name) + if statErr == nil { + obj = obj.If(storage.Conditions{GenerationMatch: attrs.Generation}) + } else { + fsLog(fs, logger.LevelWarn, "unable to set precondition for deleting %q, stat err: %v", + name, statErr) + } + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) defer cancelFn() - err := fs.svc.Bucket(fs.config.Bucket).Object(name).Delete(ctx) - if fs.IsNotExist(err) && isDir { + err := obj.Delete(ctx) + if isDir && fs.IsNotExist(err) { // we can have directories without a trailing "/" (created using v2.1.0 and before) err = fs.svc.Bucket(fs.config.Bucket).Object(strings.TrimSuffix(name, "/")).Delete(ctx) } diff --git a/internal/vfs/vfs.go b/internal/vfs/vfs.go index 0baab347..127db260 100644 --- a/internal/vfs/vfs.go +++ b/internal/vfs/vfs.go @@ -378,6 +378,12 @@ func (c *GCSFsConfig) isEqual(other GCSFsConfig) bool { if c.ACL != other.ACL { return false } + if c.UploadPartSize != other.UploadPartSize { + return false + } + if c.UploadPartMaxTime != other.UploadPartMaxTime { + return false + } if c.Credentials == nil { c.Credentials = kms.NewEmptySecret() } @@ -416,6 +422,12 @@ func (c *GCSFsConfig) validate() error { } c.StorageClass = strings.TrimSpace(c.StorageClass) c.ACL = strings.TrimSpace(c.ACL) + if c.UploadPartSize < 0 { + c.UploadPartSize = 0 + } + if c.UploadPartMaxTime < 0 { + c.UploadPartMaxTime = 0 + } return nil } diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 0984c186..136c5f7b 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -5107,6 +5107,12 @@ components: type: string description: 'key_prefix is similar to a chroot directory for a local filesystem. If specified the user will only see contents that starts with this prefix and so you can restrict access to a specific virtual folder. The prefix, if not empty, must not start with "/" and must end with "/". If empty the whole bucket contents will be available' example: folder/subfolder/ + upload_part_size: + type: integer + description: 'The buffer size (in MB) to use for multipart uploads. The default value is 16MB. 0 means use the default' + upload_part_max_time: + type: integer + description: 'The maximum time allowed, in seconds, to upload a single chunk. The default value is 32. 0 means use the default' description: 'Google Cloud Storage configuration details. The "credentials" field must be populated only when adding/updating a user. It will be always omitted, since there are sensitive data, when you search/get users' AzureBlobFsConfig: type: object diff --git a/templates/email/password-expiration.html b/templates/email/password-expiration.html index 08a1d4f9..6cd2b82b 100644 --- a/templates/email/password-expiration.html +++ b/templates/email/password-expiration.html @@ -15,5 +15,5 @@ along with this program. If not, see . --> Hi {{.Username}},
-

Your SFTPGo password {{if le .Days 0}}has expired{{else}}expires in {{.Days}} {{if eq .Days 1}}day{{else}}days{{end}}{{end}}.

+

your SFTPGo password {{if le .Days 0}}has expired{{else}}expires in {{.Days}} {{if eq .Days 1}}day{{else}}days{{end}}{{end}}.

Please login to the WebClient and set a new password.

\ No newline at end of file diff --git a/templates/webadmin/fsconfig.html b/templates/webadmin/fsconfig.html index cbb8b7ef..6922d25d 100644 --- a/templates/webadmin/fsconfig.html +++ b/templates/webadmin/fsconfig.html @@ -237,6 +237,27 @@ along with this program. If not, see . +
+ +
+ + + The buffer size for multipart uploads. Zero means the default (16 MB) + +
+
+ +
+ + + Max time limit, in seconds, to upload a single part. 0 means the default (32 secs) + +
+
+