From ca2757d41ec88fffabd120ad24f4359712d39245 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Tue, 12 Mar 2024 08:43:23 +0100 Subject: [PATCH] copy: fix quota for FsFileCopier Signed-off-by: Nicola Murino --- go.mod | 8 +++---- go.sum | 16 ++++++------- internal/common/connection.go | 9 +++++++- internal/vfs/azblobfs.go | 18 +++++++++++++-- internal/vfs/gcsfs.go | 43 ++++++++++++++++++++++++++--------- internal/vfs/s3fs.go | 25 ++++++++++++++++---- internal/vfs/vfs.go | 2 +- 7 files changed, 89 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index 35d45f24..df19a7c1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/drakkan/sftpgo/v2 go 1.22 require ( - cloud.google.com/go/storage v1.39.0 + cloud.google.com/go/storage v1.39.1 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 @@ -175,9 +175,9 @@ require ( golang.org/x/tools v0.19.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20240311132316-a219d84964c2 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect + google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect google.golang.org/grpc v1.62.1 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 402d4e64..ee745397 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,8 @@ cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= cloud.google.com/go/kms v1.15.7 h1:7caV9K3yIxvlQPAcaFffhlT7d1qpxjB1wHBtjWa13SM= cloud.google.com/go/kms v1.15.7/go.mod h1:ub54lbsa6tDkUwnu4W7Yt1aAIFLnspgh0kPGToDukeI= -cloud.google.com/go/storage v1.39.0 h1:brbjUa4hbDHhpQf48tjqMaXEV+f1OGoaTmQau9tmCsA= -cloud.google.com/go/storage v1.39.0/go.mod h1:OAEj/WZwUYjA3YHQ10/YcN9ttGuEpLwvaoyBXIPikEk= +cloud.google.com/go/storage v1.39.1 h1:MvraqHKhogCOTXTlct/9C3K3+Uy2jBmFYb3/Sp6dVtY= +cloud.google.com/go/storage v1.39.1/go.mod h1:xK6xZmxZmo+fyP7+DEF6FhNc24/JAe95OLyOHCXFH1o= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= @@ -535,12 +535,12 @@ google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240311132316-a219d84964c2 h1:rrOOzm+NteCjTNqCnDAdYhvKL1G/9N/Lj1GRxJtQEL0= -google.golang.org/genproto v0.0.0-20240311132316-a219d84964c2/go.mod h1:yA7a1bW1kwl459Ol0m0lV4hLTfrL/7Bkk4Mj2Ir1mWI= -google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 h1:9IZDv+/GcI6u+a4jRFRLxQs0RUCfavGfoOgEW6jpkI0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= +google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 h1:ImUcDPHjTrAqNhlOkSocDLfG9rrNHH7w7uoKWPaWZ8s= +google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7/go.mod h1:/3XmxOjePkvmKrHuBy4zNFw7IzxJXtAgdpXi8Ll990U= +google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 h1:oqta3O3AnlWbmIE3bFnWbu4bRxZjfbWCp0cKSuZh01E= +google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 h1:8EeVk1VKMD+GD/neyEHGmz7pFblqPjHoi+PGQIlLx2s= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= diff --git a/internal/common/connection.go b/internal/common/connection.go index 901b3c13..bfa6ad0e 100644 --- a/internal/common/connection.go +++ b/internal/common/connection.go @@ -628,7 +628,14 @@ func (c *BaseConnection) copyFile(virtualSourcePath, virtualTargetPath string, s if err != nil { return err } - return copier.CopyFile(fsSourcePath, fsTargetPath, srcSize) + startTime := time.Now() + numFiles, sizeDiff, err := copier.CopyFile(fsSourcePath, fsTargetPath, srcSize) + elapsed := time.Since(startTime).Nanoseconds() / 1000000 + updateUserQuotaAfterFileWrite(c, virtualTargetPath, numFiles, sizeDiff) + logger.CommandLog(copyLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1, + "", "", "", srcSize, c.localAddr, c.remoteAddr, elapsed) + ExecuteActionNotification(c, operationCopy, fsSourcePath, virtualSourcePath, fsTargetPath, virtualTargetPath, "", srcSize, err, elapsed, nil) //nolint:errcheck + return err } } diff --git a/internal/vfs/azblobfs.go b/internal/vfs/azblobfs.go index 83e80ae9..85d7597b 100644 --- a/internal/vfs/azblobfs.go +++ b/internal/vfs/azblobfs.go @@ -651,8 +651,22 @@ func (fs *AzureBlobFs) ResolvePath(virtualPath string) (string, error) { } // CopyFile implements the FsFileCopier interface -func (fs *AzureBlobFs) CopyFile(source, target string, _ int64) error { - return fs.copyFileInternal(source, target) +func (fs *AzureBlobFs) CopyFile(source, target string, srcSize int64) (int, int64, error) { + numFiles := 1 + sizeDiff := srcSize + attrs, err := fs.headObject(target) + if err == nil { + sizeDiff -= util.GetIntFromPointer(attrs.ContentLength) + numFiles = 0 + } else { + if !fs.IsNotExist(err) { + return 0, 0, err + } + } + if err := fs.copyFileInternal(source, target); err != nil { + return 0, 0, err + } + return numFiles, sizeDiff, nil } func (fs *AzureBlobFs) headObject(name string) (blob.GetPropertiesResponse, error) { diff --git a/internal/vfs/gcsfs.go b/internal/vfs/gcsfs.go index 994914bf..613e67ff 100644 --- a/internal/vfs/gcsfs.go +++ b/internal/vfs/gcsfs.go @@ -636,8 +636,25 @@ func (fs *GCSFs) ResolvePath(virtualPath string) (string, error) { } // CopyFile implements the FsFileCopier interface -func (fs *GCSFs) CopyFile(source, target string, _ int64) error { - return fs.copyFileInternal(source, target) +func (fs *GCSFs) CopyFile(source, target string, srcSize int64) (int, int64, error) { + numFiles := 1 + sizeDiff := srcSize + var conditions *storage.Conditions + attrs, err := fs.headObject(target) + if err == nil { + sizeDiff -= attrs.Size + numFiles = 0 + conditions = &storage.Conditions{GenerationMatch: attrs.Generation} + } else { + if !fs.IsNotExist(err) { + return 0, 0, err + } + conditions = &storage.Conditions{DoesNotExist: true} + } + if err := fs.copyFileInternal(source, target, conditions); err != nil { + return 0, 0, err + } + return numFiles, sizeDiff, nil } func (fs *GCSFs) resolve(name, prefix, contentType string) (string, bool) { @@ -732,17 +749,21 @@ func (fs *GCSFs) composeObjects(ctx context.Context, dst, partialObject *storage return err } -func (fs *GCSFs) copyFileInternal(source, target string) error { +func (fs *GCSFs) copyFileInternal(source, target string, conditions *storage.Conditions) error { src := fs.svc.Bucket(fs.config.Bucket).Object(source) dst := fs.svc.Bucket(fs.config.Bucket).Object(target) - attrs, statErr := fs.headObject(target) - if statErr == nil { - dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation}) - } else if fs.IsNotExist(statErr) { - dst = dst.If(storage.Conditions{DoesNotExist: true}) + if conditions != nil { + dst = dst.If(*conditions) } else { - fsLog(fs, logger.LevelWarn, "unable to set precondition for rename, target %q, stat err: %v", - target, statErr) + attrs, err := fs.headObject(target) + if err == nil { + dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation}) + } else if fs.IsNotExist(err) { + dst = dst.If(storage.Conditions{DoesNotExist: true}) + } else { + fsLog(fs, logger.LevelWarn, "unable to set precondition for copy, target %q, stat err: %v", + target, err) + } } ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout)) @@ -790,7 +811,7 @@ func (fs *GCSFs) renameInternal(source, target string, fi os.FileInfo, recursion } } } else { - if err := fs.copyFileInternal(source, target); err != nil { + if err := fs.copyFileInternal(source, target, nil); err != nil { return numFiles, filesSize, err } numFiles++ diff --git a/internal/vfs/s3fs.go b/internal/vfs/s3fs.go index a7d7177e..c93a8186 100644 --- a/internal/vfs/s3fs.go +++ b/internal/vfs/s3fs.go @@ -56,8 +56,9 @@ import ( const ( // using this mime type for directories improves compatibility with s3fs-fuse - s3DirMimeType = "application/x-directory" - s3TransferBufferSize = 256 * 1024 + s3DirMimeType = "application/x-directory" + s3TransferBufferSize = 256 * 1024 + s3CopyObjectThreshold = 500 * 1024 * 1024 ) var ( @@ -626,8 +627,22 @@ func (fs *S3Fs) ResolvePath(virtualPath string) (string, error) { } // CopyFile implements the FsFileCopier interface -func (fs *S3Fs) CopyFile(source, target string, srcSize int64) error { - return fs.copyFileInternal(source, target, srcSize) +func (fs *S3Fs) CopyFile(source, target string, srcSize int64) (int, int64, error) { + numFiles := 1 + sizeDiff := srcSize + attrs, err := fs.headObject(target) + if err == nil { + sizeDiff -= util.GetIntFromPointer(attrs.ContentLength) + numFiles = 0 + } else { + if !fs.IsNotExist(err) { + return 0, 0, err + } + } + if err := fs.copyFileInternal(source, target, srcSize); err != nil { + return 0, 0, err + } + return numFiles, sizeDiff, nil } func (fs *S3Fs) resolve(name *string, prefix string) (string, bool) { @@ -666,7 +681,7 @@ func (fs *S3Fs) copyFileInternal(source, target string, fileSize int64) error { contentType := mime.TypeByExtension(path.Ext(source)) copySource := pathEscape(fs.Join(fs.config.Bucket, source)) - if fileSize > 500*1024*1024 { + if fileSize > s3CopyObjectThreshold { fsLog(fs, logger.LevelDebug, "renaming file %q with size %d using multipart copy", source, fileSize) err := fs.doMultipartCopy(copySource, target, contentType, fileSize) diff --git a/internal/vfs/vfs.go b/internal/vfs/vfs.go index fde89ae0..b8edcb34 100644 --- a/internal/vfs/vfs.go +++ b/internal/vfs/vfs.go @@ -159,7 +159,7 @@ type FsRealPather interface { // FsFileCopier is a Fs that implements the CopyFile method. type FsFileCopier interface { Fs - CopyFile(source, target string, srcSize int64) error + CopyFile(source, target string, srcSize int64) (int, int64, error) } // File defines an interface representing a SFTPGo file