copy: fix quota for FsFileCopier

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>

go.mod

@@ -3,7 +3,7 @@ module github.com/drakkan/sftpgo/v2
 go 1.22
 
 require (
-	cloud.google.com/go/storage v1.39.0
+	cloud.google.com/go/storage v1.39.1
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
 	github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5
@@ -175,9 +175,9 @@ require (
 	golang.org/x/tools v0.19.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/genproto v0.0.0-20240311132316-a219d84964c2 // indirect
+	google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect
 	google.golang.org/grpc v1.62.1 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

go.sum

@@ -9,8 +9,8 @@ cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc=
 cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI=
 cloud.google.com/go/kms v1.15.7 h1:7caV9K3yIxvlQPAcaFffhlT7d1qpxjB1wHBtjWa13SM=
 cloud.google.com/go/kms v1.15.7/go.mod h1:ub54lbsa6tDkUwnu4W7Yt1aAIFLnspgh0kPGToDukeI=
-cloud.google.com/go/storage v1.39.0 h1:brbjUa4hbDHhpQf48tjqMaXEV+f1OGoaTmQau9tmCsA=
+cloud.google.com/go/storage v1.39.1 h1:MvraqHKhogCOTXTlct/9C3K3+Uy2jBmFYb3/Sp6dVtY=
-cloud.google.com/go/storage v1.39.0/go.mod h1:OAEj/WZwUYjA3YHQ10/YcN9ttGuEpLwvaoyBXIPikEk=
+cloud.google.com/go/storage v1.39.1/go.mod h1:xK6xZmxZmo+fyP7+DEF6FhNc24/JAe95OLyOHCXFH1o=
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU=
@@ -535,12 +535,12 @@ google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJ
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20240311132316-a219d84964c2 h1:rrOOzm+NteCjTNqCnDAdYhvKL1G/9N/Lj1GRxJtQEL0=
+google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 h1:ImUcDPHjTrAqNhlOkSocDLfG9rrNHH7w7uoKWPaWZ8s=
-google.golang.org/genproto v0.0.0-20240311132316-a219d84964c2/go.mod h1:yA7a1bW1kwl459Ol0m0lV4hLTfrL/7Bkk4Mj2Ir1mWI=
+google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7/go.mod h1:/3XmxOjePkvmKrHuBy4zNFw7IzxJXtAgdpXi8Ll990U=
-google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ=
+google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 h1:oqta3O3AnlWbmIE3bFnWbu4bRxZjfbWCp0cKSuZh01E=
-google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y=
+google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 h1:9IZDv+/GcI6u+a4jRFRLxQs0RUCfavGfoOgEW6jpkI0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 h1:8EeVk1VKMD+GD/neyEHGmz7pFblqPjHoi+PGQIlLx2s=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=

internal/common/connection.go

@@ -628,7 +628,14 @@ func (c *BaseConnection) copyFile(virtualSourcePath, virtualTargetPath string, s
 			if err != nil {
 				return err
 			}
-			return copier.CopyFile(fsSourcePath, fsTargetPath, srcSize)
+			startTime := time.Now()
+			numFiles, sizeDiff, err := copier.CopyFile(fsSourcePath, fsTargetPath, srcSize)
+			elapsed := time.Since(startTime).Nanoseconds() / 1000000
+			updateUserQuotaAfterFileWrite(c, virtualTargetPath, numFiles, sizeDiff)
+			logger.CommandLog(copyLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1,
+				"", "", "", srcSize, c.localAddr, c.remoteAddr, elapsed)
+			ExecuteActionNotification(c, operationCopy, fsSourcePath, virtualSourcePath, fsTargetPath, virtualTargetPath, "", srcSize, err, elapsed, nil) //nolint:errcheck
+			return err
 		}
 	}
 

internal/vfs/azblobfs.go

@@ -651,8 +651,22 @@ func (fs *AzureBlobFs) ResolvePath(virtualPath string) (string, error) {
 }
 
 // CopyFile implements the FsFileCopier interface
-func (fs *AzureBlobFs) CopyFile(source, target string, _ int64) error {
+func (fs *AzureBlobFs) CopyFile(source, target string, srcSize int64) (int, int64, error) {
-	return fs.copyFileInternal(source, target)
+	numFiles := 1
+	sizeDiff := srcSize
+	attrs, err := fs.headObject(target)
+	if err == nil {
+		sizeDiff -= util.GetIntFromPointer(attrs.ContentLength)
+		numFiles = 0
+	} else {
+		if !fs.IsNotExist(err) {
+			return 0, 0, err
+		}
+	}
+	if err := fs.copyFileInternal(source, target); err != nil {
+		return 0, 0, err
+	}
+	return numFiles, sizeDiff, nil
 }
 
 func (fs *AzureBlobFs) headObject(name string) (blob.GetPropertiesResponse, error) {

internal/vfs/gcsfs.go

@@ -636,8 +636,25 @@ func (fs *GCSFs) ResolvePath(virtualPath string) (string, error) {
 }
 
 // CopyFile implements the FsFileCopier interface
-func (fs *GCSFs) CopyFile(source, target string, _ int64) error {
+func (fs *GCSFs) CopyFile(source, target string, srcSize int64) (int, int64, error) {
-	return fs.copyFileInternal(source, target)
+	numFiles := 1
+	sizeDiff := srcSize
+	var conditions *storage.Conditions
+	attrs, err := fs.headObject(target)
+	if err == nil {
+		sizeDiff -= attrs.Size
+		numFiles = 0
+		conditions = &storage.Conditions{GenerationMatch: attrs.Generation}
+	} else {
+		if !fs.IsNotExist(err) {
+			return 0, 0, err
+		}
+		conditions = &storage.Conditions{DoesNotExist: true}
+	}
+	if err := fs.copyFileInternal(source, target, conditions); err != nil {
+		return 0, 0, err
+	}
+	return numFiles, sizeDiff, nil
 }
 
 func (fs *GCSFs) resolve(name, prefix, contentType string) (string, bool) {
@@ -732,17 +749,21 @@ func (fs *GCSFs) composeObjects(ctx context.Context, dst, partialObject *storage
 	return err
 }
 
-func (fs *GCSFs) copyFileInternal(source, target string) error {
+func (fs *GCSFs) copyFileInternal(source, target string, conditions *storage.Conditions) error {
 	src := fs.svc.Bucket(fs.config.Bucket).Object(source)
 	dst := fs.svc.Bucket(fs.config.Bucket).Object(target)
-	attrs, statErr := fs.headObject(target)
+	if conditions != nil {
-	if statErr == nil {
+		dst = dst.If(*conditions)
-		dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation})
-	} else if fs.IsNotExist(statErr) {
-		dst = dst.If(storage.Conditions{DoesNotExist: true})
 	} else {
-		fsLog(fs, logger.LevelWarn, "unable to set precondition for rename, target %q, stat err: %v",
+		attrs, err := fs.headObject(target)
-			target, statErr)
+		if err == nil {
+			dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation})
+		} else if fs.IsNotExist(err) {
+			dst = dst.If(storage.Conditions{DoesNotExist: true})
+		} else {
+			fsLog(fs, logger.LevelWarn, "unable to set precondition for copy, target %q, stat err: %v",
+				target, err)
+		}
 	}
 
 	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout))
@@ -790,7 +811,7 @@ func (fs *GCSFs) renameInternal(source, target string, fi os.FileInfo, recursion
 			}
 		}
 	} else {
-		if err := fs.copyFileInternal(source, target); err != nil {
+		if err := fs.copyFileInternal(source, target, nil); err != nil {
 			return numFiles, filesSize, err
 		}
 		numFiles++

internal/vfs/s3fs.go

@@ -56,8 +56,9 @@ import (
 
 const (
 	// using this mime type for directories improves compatibility with s3fs-fuse
-	s3DirMimeType        = "application/x-directory"
+	s3DirMimeType         = "application/x-directory"
-	s3TransferBufferSize = 256 * 1024
+	s3TransferBufferSize  = 256 * 1024
+	s3CopyObjectThreshold = 500 * 1024 * 1024
 )
 
 var (
@@ -626,8 +627,22 @@ func (fs *S3Fs) ResolvePath(virtualPath string) (string, error) {
 }
 
 // CopyFile implements the FsFileCopier interface
-func (fs *S3Fs) CopyFile(source, target string, srcSize int64) error {
+func (fs *S3Fs) CopyFile(source, target string, srcSize int64) (int, int64, error) {
-	return fs.copyFileInternal(source, target, srcSize)
+	numFiles := 1
+	sizeDiff := srcSize
+	attrs, err := fs.headObject(target)
+	if err == nil {
+		sizeDiff -= util.GetIntFromPointer(attrs.ContentLength)
+		numFiles = 0
+	} else {
+		if !fs.IsNotExist(err) {
+			return 0, 0, err
+		}
+	}
+	if err := fs.copyFileInternal(source, target, srcSize); err != nil {
+		return 0, 0, err
+	}
+	return numFiles, sizeDiff, nil
 }
 
 func (fs *S3Fs) resolve(name *string, prefix string) (string, bool) {
@@ -666,7 +681,7 @@ func (fs *S3Fs) copyFileInternal(source, target string, fileSize int64) error {
 	contentType := mime.TypeByExtension(path.Ext(source))
 	copySource := pathEscape(fs.Join(fs.config.Bucket, source))
 
-	if fileSize > 500*1024*1024 {
+	if fileSize > s3CopyObjectThreshold {
 		fsLog(fs, logger.LevelDebug, "renaming file %q with size %d using multipart copy",
 			source, fileSize)
 		err := fs.doMultipartCopy(copySource, target, contentType, fileSize)

internal/vfs/vfs.go

@@ -159,7 +159,7 @@ type FsRealPather interface {
 // FsFileCopier is a Fs that implements the CopyFile method.
 type FsFileCopier interface {
 	Fs
-	CopyFile(source, target string, srcSize int64) error
+	CopyFile(source, target string, srcSize int64) (int, int64, error)
 }
 
 // File defines an interface representing a SFTPGo file