From 271d958acf55caaa9904e02c7dd2332ae14df967 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sat, 18 Nov 2023 12:00:53 +0100 Subject: [PATCH] S3: fix compatibility with the latest SDK Signed-off-by: Nicola Murino --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ internal/vfs/s3fs.go | 34 +++++++++++++++++++++++----------- 3 files changed, 41 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index 584a8992..6a5932cc 100644 --- a/go.mod +++ b/go.mod @@ -10,14 +10,14 @@ require ( github.com/alexedwards/argon2id v1.0.0 github.com/amoghe/go-crypt v0.0.0-20220222110647-20eada5f5964 github.com/aws/aws-sdk-go-v2 v1.23.0 - github.com/aws/aws-sdk-go-v2/config v1.25.2 - github.com/aws/aws-sdk-go-v2/credentials v1.16.1 + github.com/aws/aws-sdk-go-v2/config v1.25.3 + github.com/aws/aws-sdk-go-v2/credentials v1.16.2 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.9 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.14.0 github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.18.2 - github.com/aws/aws-sdk-go-v2/service/s3 v1.42.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.43.0 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.23.2 - github.com/aws/aws-sdk-go-v2/service/sts v1.25.2 + github.com/aws/aws-sdk-go-v2/service/sts v1.25.3 github.com/bmatcuk/doublestar/v4 v4.6.1 github.com/cockroachdb/cockroach-go/v2 v2.3.5 github.com/coreos/go-oidc/v3 v3.7.0 @@ -95,7 +95,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.17.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.0 // indirect github.com/aws/smithy-go v1.17.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/boombuler/barcode v1.0.1 // indirect diff --git a/go.sum b/go.sum index e8232143..1a638c82 100644 --- a/go.sum +++ b/go.sum @@ -75,14 +75,14 @@ github.com/aws/aws-sdk-go-v2 v1.23.0 h1:PiHAzmiQQr6JULBUdvR8fKlA+UPKLT/8KbiqpFBW github.com/aws/aws-sdk-go-v2 v1.23.0/go.mod h1:i1XDttT4rnf6vxc9AuskLc6s7XBee8rlLilKlc03uAA= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ= -github.com/aws/aws-sdk-go-v2/config v1.25.2 h1:+Gy7Xe372Tw/PiUw3We94Le9IwU1tmJqCD6cvI4oBJM= -github.com/aws/aws-sdk-go-v2/config v1.25.2/go.mod h1:6hFlwWQiVOUG0Ej2ql0tG4zPlpDH++HD0WT1MA6l5Q4= -github.com/aws/aws-sdk-go-v2/credentials v1.16.1 h1:WessyrdgyFN5TB+eLQdrFSlN/3oMnqukIFhDxK6z8h0= -github.com/aws/aws-sdk-go-v2/credentials v1.16.1/go.mod h1:RQJyPxKcr+m4ArlIG1LUhMOrjposVfzbX6H8oR6oCgE= +github.com/aws/aws-sdk-go-v2/config v1.25.3 h1:E4m9LbwJOoncDNt3e9MPLbz/saxWcGUlZVBydydD6+8= +github.com/aws/aws-sdk-go-v2/config v1.25.3/go.mod h1:tAByZy03nH5jcq0vZmkcVoo6tRzRHEwSFx3QW4NmDw8= +github.com/aws/aws-sdk-go-v2/credentials v1.16.2 h1:0sdZ5cwfOAipTzZ7eOL0gw4LAhk/RZnTa16cDqIt8tg= +github.com/aws/aws-sdk-go-v2/credentials v1.16.2/go.mod h1:sDdvGhXrSVT5yzBDR7qXz+rhbpiMpUYfF3vJ01QSdrc= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4 h1:9wKDWEjwSnXZre0/O3+ZwbBl1SmlgWYBbrTV10X/H1s= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4/go.mod h1:t4i+yGHMCcUNIX1x7YVYa6bH/Do7civ5I6cG/6PMfyA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.9 h1:yG01Big4R5CDxftieMlgZPcHKZbwkRygur4DMGTqSzg= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.9/go.mod h1:RV5gmgYb4psddWMPaf4giuGdsK1l0KwlXNFAbzWAIIo= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.14.0 h1:1KdubQbnw76M0Sr8480q6OXBlymBVqpkK+RuCqJz+nQ= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.14.0/go.mod h1:UcgIwJ9KHquYxs6Q5skC9qXjhYMK+JASDYcXQ4X7JZE= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.3 h1:DUwbD79T8gyQ23qVXFUthjzVMTviSHi3y4z58KvghhM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.3/go.mod h1:7sGSz1JCKHWWBHq98m6sMtWQikmYPpxjqOydDemiVoM= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.3 h1:AplLJCtIaUZDCbr6+gLYdsYNxne4iuaboJhVt9d+WXI= @@ -101,16 +101,16 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3 h1:KV0z2RDc7euMtg github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3/go.mod h1:KZgs2ny8HsxRIRbDwgvJcHHBZPOzQr/+NtGwnP+w2ec= github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.18.2 h1:UaKsLxZ4sHdYlyX2cRZ+7YznCQS7jwHHgVy1hKkNKfE= github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.18.2/go.mod h1:4L1Z3QlQqkGEPSRH9fYPuHLxNEK54VgExjw4J/ShjbM= -github.com/aws/aws-sdk-go-v2/service/s3 v1.42.2 h1:NnduxUd9+Fq9DcCDdJK8v6l9lR1xDX4usvog+JuQAno= -github.com/aws/aws-sdk-go-v2/service/s3 v1.42.2/go.mod h1:NXRKkiRF+erX2hnybnVU660cYT5/KChRD4iUgJ97cI8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.43.0 h1:cwTuq73Tv6jtNJIMgTDKsih5O2YsVrKGpg20H98tbmo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.43.0/go.mod h1:NXRKkiRF+erX2hnybnVU660cYT5/KChRD4iUgJ97cI8= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.23.2 h1:M5NodszNDBfyfFBKoAzJY0flmkkQCg7MGk6+/vBGjCM= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.23.2/go.mod h1:+8dYLQz+I30HIGyhp+6htf3+yyGTqBzzTOG90Ai8lWs= github.com/aws/aws-sdk-go-v2/service/sso v1.17.2 h1:V47N5eKgVZoRSvx2+RQ0EpAEit/pqOhqeSQFiS4OFEQ= github.com/aws/aws-sdk-go-v2/service/sso v1.17.2/go.mod h1:/pE21vno3q1h4bbhUOEi+6Zu/aT26UK2WKkDXd+TssQ= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.2 h1:sMAcO7VHVw28HTAdZpTULDzFirHOsVm/x25CxhUH0jA= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.2/go.mod h1:dWqm5G767qwKPuayKfzm4rjzFmVjiBFbOJrpSPnAMDs= -github.com/aws/aws-sdk-go-v2/service/sts v1.25.2 h1:vwyiRTnXLqsak/6WAQ+uTRhVqKI6vxUQ0HJXjKij0zM= -github.com/aws/aws-sdk-go-v2/service/sts v1.25.2/go.mod h1:4EqRHDCKP78hq3zOnmFXu5k0j4bXbRFfCh/zQ6KnEfQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.0 h1:/XiEU7VIFcVWRDQLabyrSjBoKIm8UkYgsvWDuFW8Img= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.0/go.mod h1:dWqm5G767qwKPuayKfzm4rjzFmVjiBFbOJrpSPnAMDs= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.3 h1:M2w4kiMGJCCM6Ljmmx/l6mmpfa3gPJVpBencfnsgvqs= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.3/go.mod h1:4EqRHDCKP78hq3zOnmFXu5k0j4bXbRFfCh/zQ6KnEfQ= github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI= github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/internal/vfs/s3fs.go b/internal/vfs/s3fs.go index 9081e29c..0f280aa7 100644 --- a/internal/vfs/s3fs.go +++ b/internal/vfs/s3fs.go @@ -156,11 +156,11 @@ func (fs *S3Fs) Stat(name string) (os.FileInfo, error) { // Some S3 providers (like SeaweedFS) remove the trailing '/' from object keys. // So we check some common content types to detect if this is a "directory". isDir := util.Contains(s3DirMimeTypes, util.GetStringFromPointer(obj.ContentType)) - if obj.ContentLength == 0 && !isDir { + if util.GetIntFromPointer(obj.ContentLength) == 0 && !isDir { _, err = fs.headObject(name + "/") isDir = err == nil } - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir, obj.ContentLength, + return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir, util.GetIntFromPointer(obj.ContentLength), util.GetTimeFromPointer(obj.LastModified), false)) } if !fs.IsNotExist(err) { @@ -186,7 +186,7 @@ func (fs *S3Fs) getStatForDir(name string) (os.FileInfo, error) { if err != nil { return result, err } - return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, obj.ContentLength, + return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, util.GetIntFromPointer(obj.ContentLength), util.GetTimeFromPointer(obj.LastModified), false)) } @@ -466,6 +466,7 @@ func (fs *S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) { } for _, fileObject := range page.Contents { objectModTime := util.GetTimeFromPointer(fileObject.LastModified) + objectSize := util.GetIntFromPointer(fileObject.Size) name, isDir := fs.resolve(fileObject.Key, prefix) if name == "" || name == "/" { continue @@ -479,7 +480,8 @@ func (fs *S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) { if t, ok := modTimes[name]; ok { objectModTime = util.GetTimeFromMsecSinceEpoch(t) } - result = append(result, NewFileInfo(name, (isDir && fileObject.Size == 0), fileObject.Size, + + result = append(result, NewFileInfo(name, (isDir && objectSize == 0), objectSize, objectModTime, false)) } } @@ -627,11 +629,12 @@ func (fs *S3Fs) GetDirSize(dirname string) (int, int64, error) { } for _, fileObject := range page.Contents { isDir := strings.HasSuffix(util.GetStringFromPointer(fileObject.Key), "/") - if isDir && fileObject.Size == 0 { + objectSize := util.GetIntFromPointer(fileObject.Size) + if isDir && objectSize == 0 { continue } numFiles++ - size += fileObject.Size + size += objectSize if numFiles%1000 == 0 { fsLog(fs, logger.LevelDebug, "dirname %q scan in progress, files: %d, size: %d", dirname, numFiles, size) } @@ -696,7 +699,8 @@ func (fs *S3Fs) Walk(root string, walkFn filepath.WalkFunc) error { continue } err := walkFn(util.GetStringFromPointer(fileObject.Key), - NewFileInfo(name, isDir, fileObject.Size, util.GetTimeFromPointer(fileObject.LastModified), false), nil) + NewFileInfo(name, isDir, util.GetIntFromPointer(fileObject.Size), util.GetTimeFromPointer(fileObject.LastModified), + false), nil) if err != nil { return err } @@ -865,10 +869,11 @@ func (fs *S3Fs) mkdirInternal(name string) error { func (fs *S3Fs) hasContents(name string) (bool, error) { prefix := fs.getPrefix(name) + maxKeys := int32(2) paginator := s3.NewListObjectsV2Paginator(fs.svc, &s3.ListObjectsV2Input{ Bucket: aws.String(fs.config.Bucket), Prefix: aws.String(prefix), - MaxKeys: 2, + MaxKeys: &maxKeys, }) if paginator.HasMorePages() { @@ -962,7 +967,7 @@ func (fs *S3Fs) doMultipartCopy(source, target, contentType string, fileSize int Bucket: aws.String(fs.config.Bucket), CopySource: aws.String(source), Key: aws.String(target), - PartNumber: partNum, + PartNumber: &partNum, UploadId: aws.String(uploadID), CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", partStart, partEnd-1)), }) @@ -991,7 +996,7 @@ func (fs *S3Fs) doMultipartCopy(source, target, contentType string, fileSize int partMutex.Lock() completedParts = append(completedParts, types.CompletedPart{ ETag: partResp.CopyPartResult.ETag, - PartNumber: partNum, + PartNumber: &partNum, }) partMutex.Unlock() }(partNumber, start, end) @@ -1004,7 +1009,14 @@ func (fs *S3Fs) doMultipartCopy(source, target, contentType string, fileSize int return copyError } sort.Slice(completedParts, func(i, j int) bool { - return completedParts[i].PartNumber < completedParts[j].PartNumber + getPartNumber := func(number *int32) int32 { + if number == nil { + return 0 + } + return *number + } + + return getPartNumber(completedParts[i].PartNumber) < getPartNumber(completedParts[j].PartNumber) }) completeCtx, completeCancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))