beenull/sftpgo-mirror
1
0
Fork 0
mirror of https://github.com/drakkan/sftpgo.git synced 2024-11-21 23:20:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

copy: fix quota for FsFileCopier

Browse source 
Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2024-03-12 08:43:23 +01:00
parent f38966c6ac
commit ca2757d41e
No known key found for this signature in database
GPG key ID: 935D2952DEC4EECF
7 changed files with 89 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
go.mod
View file

@ -3,7 +3,7 @@ module github.com/drakkan/sftpgo/v2
go 1.22
require (
cloud.google.com/go/storage v1.39.0
cloud.google.com/go/storage v1.39.1
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5
@ -175,9 +175,9 @@ require (
golang.org/x/tools v0.19.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20240311132316-a219d84964c2 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect
google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect
google.golang.org/grpc v1.62.1 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect

16
go.sum
View file

@ -9,8 +9,8 @@ cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc=
cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI=
cloud.google.com/go/kms v1.15.7 h1:7caV9K3yIxvlQPAcaFffhlT7d1qpxjB1wHBtjWa13SM=
cloud.google.com/go/kms v1.15.7/go.mod h1:ub54lbsa6tDkUwnu4W7Yt1aAIFLnspgh0kPGToDukeI=
cloud.google.com/go/storage v1.39.0 h1:brbjUa4hbDHhpQf48tjqMaXEV+f1OGoaTmQau9tmCsA=
cloud.google.com/go/storage v1.39.0/go.mod h1:OAEj/WZwUYjA3YHQ10/YcN9ttGuEpLwvaoyBXIPikEk=
cloud.google.com/go/storage v1.39.1 h1:MvraqHKhogCOTXTlct/9C3K3+Uy2jBmFYb3/Sp6dVtY=
cloud.google.com/go/storage v1.39.1/go.mod h1:xK6xZmxZmo+fyP7+DEF6FhNc24/JAe95OLyOHCXFH1o=
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU=
@ -535,12 +535,12 @@ google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJ
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20240311132316-a219d84964c2 h1:rrOOzm+NteCjTNqCnDAdYhvKL1G/9N/Lj1GRxJtQEL0=
google.golang.org/genproto v0.0.0-20240311132316-a219d84964c2/go.mod h1:yA7a1bW1kwl459Ol0m0lV4hLTfrL/7Bkk4Mj2Ir1mWI=
google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ=
google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 h1:9IZDv+/GcI6u+a4jRFRLxQs0RUCfavGfoOgEW6jpkI0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs=
google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 h1:ImUcDPHjTrAqNhlOkSocDLfG9rrNHH7w7uoKWPaWZ8s=
google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7/go.mod h1:/3XmxOjePkvmKrHuBy4zNFw7IzxJXtAgdpXi8Ll990U=
google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 h1:oqta3O3AnlWbmIE3bFnWbu4bRxZjfbWCp0cKSuZh01E=
google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 h1:8EeVk1VKMD+GD/neyEHGmz7pFblqPjHoi+PGQIlLx2s=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=

9
internal/common/connection.go
View file

@ -628,7 +628,14 @@ func (c *BaseConnection) copyFile(virtualSourcePath, virtualTargetPath string, s
if err != nil {
return err
}
return copier.CopyFile(fsSourcePath, fsTargetPath, srcSize)
startTime := time.Now()
numFiles, sizeDiff, err := copier.CopyFile(fsSourcePath, fsTargetPath, srcSize)
elapsed := time.Since(startTime).Nanoseconds() / 1000000
updateUserQuotaAfterFileWrite(c, virtualTargetPath, numFiles, sizeDiff)
logger.CommandLog(copyLogSender, fsSourcePath, fsTargetPath, c.User.Username, "", c.ID, c.protocol, -1, -1,
"", "", "", srcSize, c.localAddr, c.remoteAddr, elapsed)
ExecuteActionNotification(c, operationCopy, fsSourcePath, virtualSourcePath, fsTargetPath, virtualTargetPath, "", srcSize, err, elapsed, nil) //nolint:errcheck
return err
}
}

18
internal/vfs/azblobfs.go
View file

@ -651,8 +651,22 @@ func (fs *AzureBlobFs) ResolvePath(virtualPath string) (string, error) {
}
// CopyFile implements the FsFileCopier interface
func (fs *AzureBlobFs) CopyFile(source, target string, _ int64) error {
return fs.copyFileInternal(source, target)
func (fs *AzureBlobFs) CopyFile(source, target string, srcSize int64) (int, int64, error) {
numFiles := 1
sizeDiff := srcSize
attrs, err := fs.headObject(target)
if err == nil {
sizeDiff -= util.GetIntFromPointer(attrs.ContentLength)
numFiles = 0
} else {
if !fs.IsNotExist(err) {
return 0, 0, err
}
}
if err := fs.copyFileInternal(source, target); err != nil {
return 0, 0, err
}
return numFiles, sizeDiff, nil
}
func (fs *AzureBlobFs) headObject(name string) (blob.GetPropertiesResponse, error) {

43
internal/vfs/gcsfs.go
View file

@ -636,8 +636,25 @@ func (fs *GCSFs) ResolvePath(virtualPath string) (string, error) {
}
// CopyFile implements the FsFileCopier interface
func (fs *GCSFs) CopyFile(source, target string, _ int64) error {
return fs.copyFileInternal(source, target)
func (fs *GCSFs) CopyFile(source, target string, srcSize int64) (int, int64, error) {
numFiles := 1
sizeDiff := srcSize
var conditions *storage.Conditions
attrs, err := fs.headObject(target)
if err == nil {
sizeDiff -= attrs.Size
numFiles = 0
conditions = &storage.Conditions{GenerationMatch: attrs.Generation}
} else {
if !fs.IsNotExist(err) {
return 0, 0, err
}
conditions = &storage.Conditions{DoesNotExist: true}
}
if err := fs.copyFileInternal(source, target, conditions); err != nil {
return 0, 0, err
}
return numFiles, sizeDiff, nil
}
func (fs *GCSFs) resolve(name, prefix, contentType string) (string, bool) {
@ -732,17 +749,21 @@ func (fs *GCSFs) composeObjects(ctx context.Context, dst, partialObject *storage
return err
}
func (fs *GCSFs) copyFileInternal(source, target string) error {
func (fs *GCSFs) copyFileInternal(source, target string, conditions *storage.Conditions) error {
src := fs.svc.Bucket(fs.config.Bucket).Object(source)
dst := fs.svc.Bucket(fs.config.Bucket).Object(target)
attrs, statErr := fs.headObject(target)
if statErr == nil {
dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation})
} else if fs.IsNotExist(statErr) {
dst = dst.If(storage.Conditions{DoesNotExist: true})
if conditions != nil {
dst = dst.If(*conditions)
} else {
fsLog(fs, logger.LevelWarn, "unable to set precondition for rename, target %q, stat err: %v",
target, statErr)
attrs, err := fs.headObject(target)
if err == nil {
dst = dst.If(storage.Conditions{GenerationMatch: attrs.Generation})
} else if fs.IsNotExist(err) {
dst = dst.If(storage.Conditions{DoesNotExist: true})
} else {
fsLog(fs, logger.LevelWarn, "unable to set precondition for copy, target %q, stat err: %v",
target, err)
}
}
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout))
@ -790,7 +811,7 @@ func (fs *GCSFs) renameInternal(source, target string, fi os.FileInfo, recursion
}
}
} else {
if err := fs.copyFileInternal(source, target); err != nil {
if err := fs.copyFileInternal(source, target, nil); err != nil {
return numFiles, filesSize, err
}
numFiles++

25
internal/vfs/s3fs.go
View file

@ -56,8 +56,9 @@ import (
const (
// using this mime type for directories improves compatibility with s3fs-fuse
s3DirMimeType = "application/x-directory"
s3TransferBufferSize = 256 * 1024
s3DirMimeType = "application/x-directory"
s3TransferBufferSize = 256 * 1024
s3CopyObjectThreshold = 500 * 1024 * 1024
)
var (
@ -626,8 +627,22 @@ func (fs *S3Fs) ResolvePath(virtualPath string) (string, error) {
}
// CopyFile implements the FsFileCopier interface
func (fs *S3Fs) CopyFile(source, target string, srcSize int64) error {
return fs.copyFileInternal(source, target, srcSize)
func (fs *S3Fs) CopyFile(source, target string, srcSize int64) (int, int64, error) {
numFiles := 1
sizeDiff := srcSize
attrs, err := fs.headObject(target)
if err == nil {
sizeDiff -= util.GetIntFromPointer(attrs.ContentLength)
numFiles = 0
} else {
if !fs.IsNotExist(err) {
return 0, 0, err
}
}
if err := fs.copyFileInternal(source, target, srcSize); err != nil {
return 0, 0, err
}
return numFiles, sizeDiff, nil
}
func (fs *S3Fs) resolve(name *string, prefix string) (string, bool) {
@ -666,7 +681,7 @@ func (fs *S3Fs) copyFileInternal(source, target string, fileSize int64) error {
contentType := mime.TypeByExtension(path.Ext(source))
copySource := pathEscape(fs.Join(fs.config.Bucket, source))
if fileSize > 500*1024*1024 {
if fileSize > s3CopyObjectThreshold {
fsLog(fs, logger.LevelDebug, "renaming file %q with size %d using multipart copy",
source, fileSize)
err := fs.doMultipartCopy(copySource, target, contentType, fileSize)

2
internal/vfs/vfs.go
View file

@ -159,7 +159,7 @@ type FsRealPather interface {
// FsFileCopier is a Fs that implements the CopyFile method.
type FsFileCopier interface {
Fs
CopyFile(source, target string, srcSize int64) error
CopyFile(source, target string, srcSize int64) (int, int64, error)
}
// File defines an interface representing a SFTPGo file