beenull/sftpgo-mirror
1
0
Fork 0
mirror of https://github.com/drakkan/sftpgo.git synced 2024-11-22 07:30:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

S3: fix compatibility with the latest SDK

Browse source 
Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2023-11-18 12:00:53 +01:00
parent bfa17314c6
commit 271d958acf
No known key found for this signature in database
GPG key ID: 935D2952DEC4EECF
3 changed files with 41 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
go.mod
View file

@ -10,14 +10,14 @@ require (
github.com/alexedwards/argon2id v1.0.0 github.com/alexedwards/argon2id v1.0.0
github.com/amoghe/go-crypt v0.0.0-20220222110647-20eada5f5964 github.com/amoghe/go-crypt v0.0.0-20220222110647-20eada5f5964
github.com/aws/aws-sdk-go-v2 v1.23.0 github.com/aws/aws-sdk-go-v2 v1.23.0
github.com/aws/aws-sdk-go-v2/config v1.25.2 github.com/aws/aws-sdk-go-v2/config v1.25.3
github.com/aws/aws-sdk-go-v2/credentials v1.16.1 github.com/aws/aws-sdk-go-v2/credentials v1.16.2
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.9 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.14.0
github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.18.2 github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.18.2
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.2 github.com/aws/aws-sdk-go-v2/service/s3 v1.43.0
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.23.2 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.23.2
github.com/aws/aws-sdk-go-v2/service/sts v1.25.2 github.com/aws/aws-sdk-go-v2/service/sts v1.25.3
github.com/bmatcuk/doublestar/v4 v4.6.1 github.com/bmatcuk/doublestar/v4 v4.6.1
github.com/cockroachdb/cockroach-go/v2 v2.3.5 github.com/cockroachdb/cockroach-go/v2 v2.3.5
github.com/coreos/go-oidc/v3 v3.7.0 github.com/coreos/go-oidc/v3 v3.7.0
@ -95,7 +95,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.17.2 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.17.2 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.2 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.0 // indirect
github.com/aws/smithy-go v1.17.0 // indirect github.com/aws/smithy-go v1.17.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/boombuler/barcode v1.0.1 // indirect github.com/boombuler/barcode v1.0.1 // indirect

24
go.sum
View file

@ -75,14 +75,14 @@ github.com/aws/aws-sdk-go-v2 v1.23.0 h1:PiHAzmiQQr6JULBUdvR8fKlA+UPKLT/8KbiqpFBW
github.com/aws/aws-sdk-go-v2 v1.23.0/go.mod h1:i1XDttT4rnf6vxc9AuskLc6s7XBee8rlLilKlc03uAA= github.com/aws/aws-sdk-go-v2 v1.23.0/go.mod h1:i1XDttT4rnf6vxc9AuskLc6s7XBee8rlLilKlc03uAA=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ=
github.com/aws/aws-sdk-go-v2/config v1.25.2 h1:+Gy7Xe372Tw/PiUw3We94Le9IwU1tmJqCD6cvI4oBJM= github.com/aws/aws-sdk-go-v2/config v1.25.3 h1:E4m9LbwJOoncDNt3e9MPLbz/saxWcGUlZVBydydD6+8=
github.com/aws/aws-sdk-go-v2/config v1.25.2/go.mod h1:6hFlwWQiVOUG0Ej2ql0tG4zPlpDH++HD0WT1MA6l5Q4= github.com/aws/aws-sdk-go-v2/config v1.25.3/go.mod h1:tAByZy03nH5jcq0vZmkcVoo6tRzRHEwSFx3QW4NmDw8=
github.com/aws/aws-sdk-go-v2/credentials v1.16.1 h1:WessyrdgyFN5TB+eLQdrFSlN/3oMnqukIFhDxK6z8h0= github.com/aws/aws-sdk-go-v2/credentials v1.16.2 h1:0sdZ5cwfOAipTzZ7eOL0gw4LAhk/RZnTa16cDqIt8tg=
github.com/aws/aws-sdk-go-v2/credentials v1.16.1/go.mod h1:RQJyPxKcr+m4ArlIG1LUhMOrjposVfzbX6H8oR6oCgE= github.com/aws/aws-sdk-go-v2/credentials v1.16.2/go.mod h1:sDdvGhXrSVT5yzBDR7qXz+rhbpiMpUYfF3vJ01QSdrc=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4 h1:9wKDWEjwSnXZre0/O3+ZwbBl1SmlgWYBbrTV10X/H1s= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4 h1:9wKDWEjwSnXZre0/O3+ZwbBl1SmlgWYBbrTV10X/H1s=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4/go.mod h1:t4i+yGHMCcUNIX1x7YVYa6bH/Do7civ5I6cG/6PMfyA= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4/go.mod h1:t4i+yGHMCcUNIX1x7YVYa6bH/Do7civ5I6cG/6PMfyA=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.9 h1:yG01Big4R5CDxftieMlgZPcHKZbwkRygur4DMGTqSzg= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.14.0 h1:1KdubQbnw76M0Sr8480q6OXBlymBVqpkK+RuCqJz+nQ=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.9/go.mod h1:RV5gmgYb4psddWMPaf4giuGdsK1l0KwlXNFAbzWAIIo= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.14.0/go.mod h1:UcgIwJ9KHquYxs6Q5skC9qXjhYMK+JASDYcXQ4X7JZE=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.3 h1:DUwbD79T8gyQ23qVXFUthjzVMTviSHi3y4z58KvghhM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.3 h1:DUwbD79T8gyQ23qVXFUthjzVMTviSHi3y4z58KvghhM=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.3/go.mod h1:7sGSz1JCKHWWBHq98m6sMtWQikmYPpxjqOydDemiVoM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.3/go.mod h1:7sGSz1JCKHWWBHq98m6sMtWQikmYPpxjqOydDemiVoM=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.3 h1:AplLJCtIaUZDCbr6+gLYdsYNxne4iuaboJhVt9d+WXI= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.3 h1:AplLJCtIaUZDCbr6+gLYdsYNxne4iuaboJhVt9d+WXI=
@ -101,16 +101,16 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3 h1:KV0z2RDc7euMtg
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3/go.mod h1:KZgs2ny8HsxRIRbDwgvJcHHBZPOzQr/+NtGwnP+w2ec= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3/go.mod h1:KZgs2ny8HsxRIRbDwgvJcHHBZPOzQr/+NtGwnP+w2ec=
github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.18.2 h1:UaKsLxZ4sHdYlyX2cRZ+7YznCQS7jwHHgVy1hKkNKfE= github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.18.2 h1:UaKsLxZ4sHdYlyX2cRZ+7YznCQS7jwHHgVy1hKkNKfE=
github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.18.2/go.mod h1:4L1Z3QlQqkGEPSRH9fYPuHLxNEK54VgExjw4J/ShjbM= github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.18.2/go.mod h1:4L1Z3QlQqkGEPSRH9fYPuHLxNEK54VgExjw4J/ShjbM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.2 h1:NnduxUd9+Fq9DcCDdJK8v6l9lR1xDX4usvog+JuQAno= github.com/aws/aws-sdk-go-v2/service/s3 v1.43.0 h1:cwTuq73Tv6jtNJIMgTDKsih5O2YsVrKGpg20H98tbmo=
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.2/go.mod h1:NXRKkiRF+erX2hnybnVU660cYT5/KChRD4iUgJ97cI8= github.com/aws/aws-sdk-go-v2/service/s3 v1.43.0/go.mod h1:NXRKkiRF+erX2hnybnVU660cYT5/KChRD4iUgJ97cI8=
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.23.2 h1:M5NodszNDBfyfFBKoAzJY0flmkkQCg7MGk6+/vBGjCM= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.23.2 h1:M5NodszNDBfyfFBKoAzJY0flmkkQCg7MGk6+/vBGjCM=
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.23.2/go.mod h1:+8dYLQz+I30HIGyhp+6htf3+yyGTqBzzTOG90Ai8lWs= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.23.2/go.mod h1:+8dYLQz+I30HIGyhp+6htf3+yyGTqBzzTOG90Ai8lWs=
github.com/aws/aws-sdk-go-v2/service/sso v1.17.2 h1:V47N5eKgVZoRSvx2+RQ0EpAEit/pqOhqeSQFiS4OFEQ= github.com/aws/aws-sdk-go-v2/service/sso v1.17.2 h1:V47N5eKgVZoRSvx2+RQ0EpAEit/pqOhqeSQFiS4OFEQ=
github.com/aws/aws-sdk-go-v2/service/sso v1.17.2/go.mod h1:/pE21vno3q1h4bbhUOEi+6Zu/aT26UK2WKkDXd+TssQ= github.com/aws/aws-sdk-go-v2/service/sso v1.17.2/go.mod h1:/pE21vno3q1h4bbhUOEi+6Zu/aT26UK2WKkDXd+TssQ=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.2 h1:sMAcO7VHVw28HTAdZpTULDzFirHOsVm/x25CxhUH0jA= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.0 h1:/XiEU7VIFcVWRDQLabyrSjBoKIm8UkYgsvWDuFW8Img=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.2/go.mod h1:dWqm5G767qwKPuayKfzm4rjzFmVjiBFbOJrpSPnAMDs= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.0/go.mod h1:dWqm5G767qwKPuayKfzm4rjzFmVjiBFbOJrpSPnAMDs=
github.com/aws/aws-sdk-go-v2/service/sts v1.25.2 h1:vwyiRTnXLqsak/6WAQ+uTRhVqKI6vxUQ0HJXjKij0zM= github.com/aws/aws-sdk-go-v2/service/sts v1.25.3 h1:M2w4kiMGJCCM6Ljmmx/l6mmpfa3gPJVpBencfnsgvqs=
github.com/aws/aws-sdk-go-v2/service/sts v1.25.2/go.mod h1:4EqRHDCKP78hq3zOnmFXu5k0j4bXbRFfCh/zQ6KnEfQ= github.com/aws/aws-sdk-go-v2/service/sts v1.25.3/go.mod h1:4EqRHDCKP78hq3zOnmFXu5k0j4bXbRFfCh/zQ6KnEfQ=
github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI= github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI=
github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=

34
internal/vfs/s3fs.go
View file

@ -156,11 +156,11 @@ func (fs *S3Fs) Stat(name string) (os.FileInfo, error) {
// Some S3 providers (like SeaweedFS) remove the trailing '/' from object keys. // Some S3 providers (like SeaweedFS) remove the trailing '/' from object keys.
// So we check some common content types to detect if this is a "directory". // So we check some common content types to detect if this is a "directory".
isDir := util.Contains(s3DirMimeTypes, util.GetStringFromPointer(obj.ContentType)) isDir := util.Contains(s3DirMimeTypes, util.GetStringFromPointer(obj.ContentType))
if obj.ContentLength == 0 && !isDir { if util.GetIntFromPointer(obj.ContentLength) == 0 && !isDir {
_, err = fs.headObject(name + "/") _, err = fs.headObject(name + "/")
isDir = err == nil isDir = err == nil
} }
return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir, obj.ContentLength, return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir, util.GetIntFromPointer(obj.ContentLength),
util.GetTimeFromPointer(obj.LastModified), false)) util.GetTimeFromPointer(obj.LastModified), false))
} }
if !fs.IsNotExist(err) { if !fs.IsNotExist(err) {
@ -186,7 +186,7 @@ func (fs *S3Fs) getStatForDir(name string) (os.FileInfo, error) {
if err != nil { if err != nil {
return result, err return result, err
} }
return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, obj.ContentLength, return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, util.GetIntFromPointer(obj.ContentLength),
util.GetTimeFromPointer(obj.LastModified), false)) util.GetTimeFromPointer(obj.LastModified), false))
} }
@ -466,6 +466,7 @@ func (fs *S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) {
} }
for _, fileObject := range page.Contents { for _, fileObject := range page.Contents {
objectModTime := util.GetTimeFromPointer(fileObject.LastModified) objectModTime := util.GetTimeFromPointer(fileObject.LastModified)
objectSize := util.GetIntFromPointer(fileObject.Size)
name, isDir := fs.resolve(fileObject.Key, prefix) name, isDir := fs.resolve(fileObject.Key, prefix)
if name == "" || name == "/" { if name == "" || name == "/" {
continue continue
@ -479,7 +480,8 @@ func (fs *S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) {
if t, ok := modTimes[name]; ok { if t, ok := modTimes[name]; ok {
objectModTime = util.GetTimeFromMsecSinceEpoch(t) objectModTime = util.GetTimeFromMsecSinceEpoch(t)
} }
result = append(result, NewFileInfo(name, (isDir && fileObject.Size == 0), fileObject.Size,
result = append(result, NewFileInfo(name, (isDir && objectSize == 0), objectSize,
objectModTime, false)) objectModTime, false))
} }
} }
@ -627,11 +629,12 @@ func (fs *S3Fs) GetDirSize(dirname string) (int, int64, error) {
} }
for _, fileObject := range page.Contents { for _, fileObject := range page.Contents {
isDir := strings.HasSuffix(util.GetStringFromPointer(fileObject.Key), "/") isDir := strings.HasSuffix(util.GetStringFromPointer(fileObject.Key), "/")
if isDir && fileObject.Size == 0 { objectSize := util.GetIntFromPointer(fileObject.Size)
if isDir && objectSize == 0 {
continue continue
} }
numFiles++ numFiles++
size += fileObject.Size size += objectSize
if numFiles%1000 == 0 { if numFiles%1000 == 0 {
fsLog(fs, logger.LevelDebug, "dirname %q scan in progress, files: %d, size: %d", dirname, numFiles, size) fsLog(fs, logger.LevelDebug, "dirname %q scan in progress, files: %d, size: %d", dirname, numFiles, size)
} }
@ -696,7 +699,8 @@ func (fs *S3Fs) Walk(root string, walkFn filepath.WalkFunc) error {
continue continue
} }
err := walkFn(util.GetStringFromPointer(fileObject.Key), err := walkFn(util.GetStringFromPointer(fileObject.Key),
NewFileInfo(name, isDir, fileObject.Size, util.GetTimeFromPointer(fileObject.LastModified), false), nil) NewFileInfo(name, isDir, util.GetIntFromPointer(fileObject.Size), util.GetTimeFromPointer(fileObject.LastModified),
false), nil)
if err != nil { if err != nil {
return err return err
} }
@ -865,10 +869,11 @@ func (fs *S3Fs) mkdirInternal(name string) error {
func (fs *S3Fs) hasContents(name string) (bool, error) { func (fs *S3Fs) hasContents(name string) (bool, error) {
prefix := fs.getPrefix(name) prefix := fs.getPrefix(name)
maxKeys := int32(2)
paginator := s3.NewListObjectsV2Paginator(fs.svc, &s3.ListObjectsV2Input{ paginator := s3.NewListObjectsV2Paginator(fs.svc, &s3.ListObjectsV2Input{
Bucket: aws.String(fs.config.Bucket), Bucket: aws.String(fs.config.Bucket),
Prefix: aws.String(prefix), Prefix: aws.String(prefix),
MaxKeys: 2, MaxKeys: &maxKeys,
}) })
if paginator.HasMorePages() { if paginator.HasMorePages() {
@ -962,7 +967,7 @@ func (fs *S3Fs) doMultipartCopy(source, target, contentType string, fileSize int
Bucket: aws.String(fs.config.Bucket), Bucket: aws.String(fs.config.Bucket),
CopySource: aws.String(source), CopySource: aws.String(source),
Key: aws.String(target), Key: aws.String(target),
PartNumber: partNum, PartNumber: &partNum,
UploadId: aws.String(uploadID), UploadId: aws.String(uploadID),
CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", partStart, partEnd-1)), CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", partStart, partEnd-1)),
}) })
@ -991,7 +996,7 @@ func (fs *S3Fs) doMultipartCopy(source, target, contentType string, fileSize int
partMutex.Lock() partMutex.Lock()
completedParts = append(completedParts, types.CompletedPart{ completedParts = append(completedParts, types.CompletedPart{
ETag: partResp.CopyPartResult.ETag, ETag: partResp.CopyPartResult.ETag,
PartNumber: partNum, PartNumber: &partNum,
}) })
partMutex.Unlock() partMutex.Unlock()
}(partNumber, start, end) }(partNumber, start, end)
@ -1004,7 +1009,14 @@ func (fs *S3Fs) doMultipartCopy(source, target, contentType string, fileSize int
return copyError return copyError
} }
sort.Slice(completedParts, func(i, j int) bool { sort.Slice(completedParts, func(i, j int) bool {
return completedParts[i].PartNumber < completedParts[j].PartNumber getPartNumber := func(number *int32) int32 {
if number == nil {
return 0
}
return *number
}
return getPartNumber(completedParts[i].PartNumber) < getPartNumber(completedParts[j].PartNumber)
}) })
completeCtx, completeCancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) completeCtx, completeCancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))