S3: fix compatibility with the latest SDK
Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino
parent
bfa17314c6
commit
271d958acf
3 changed files with 41 additions and 29 deletions
12
go.mod
12
go.mod
|
|
@ -10,14 +10,14 @@ require (
|
|||
github.com/alexedwards/argon2id v1.0.0
|
||||
github.com/amoghe/go-crypt v0.0.0-20220222110647-20eada5f5964
|
||||
github.com/aws/aws-sdk-go-v2 v1.23.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.25.2
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.25.3
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.2
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.9
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.14.0
|
||||
github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.18.2
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.2
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.43.0
|
||||
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.23.2
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.25.2
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.25.3
|
||||
github.com/bmatcuk/doublestar/v4 v4.6.1
|
||||
github.com/cockroachdb/cockroach-go/v2 v2.3.5
|
||||
github.com/coreos/go-oidc/v3 v3.7.0
|
||||
|
|
@ -95,7 +95,7 @@ require (
|
|||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.17.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.0 // indirect
|
||||
github.com/aws/smithy-go v1.17.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/boombuler/barcode v1.0.1 // indirect
|
||||
|
|
|
|||
24
go.sum
24
go.sum
|
|
@ -75,14 +75,14 @@ github.com/aws/aws-sdk-go-v2 v1.23.0 h1:PiHAzmiQQr6JULBUdvR8fKlA+UPKLT/8KbiqpFBW
|
|||
github.com/aws/aws-sdk-go-v2 v1.23.0/go.mod h1:i1XDttT4rnf6vxc9AuskLc6s7XBee8rlLilKlc03uAA=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.25.2 h1:+Gy7Xe372Tw/PiUw3We94Le9IwU1tmJqCD6cvI4oBJM=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.25.2/go.mod h1:6hFlwWQiVOUG0Ej2ql0tG4zPlpDH++HD0WT1MA6l5Q4=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.1 h1:WessyrdgyFN5TB+eLQdrFSlN/3oMnqukIFhDxK6z8h0=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.1/go.mod h1:RQJyPxKcr+m4ArlIG1LUhMOrjposVfzbX6H8oR6oCgE=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.25.3 h1:E4m9LbwJOoncDNt3e9MPLbz/saxWcGUlZVBydydD6+8=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.25.3/go.mod h1:tAByZy03nH5jcq0vZmkcVoo6tRzRHEwSFx3QW4NmDw8=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.2 h1:0sdZ5cwfOAipTzZ7eOL0gw4LAhk/RZnTa16cDqIt8tg=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.16.2/go.mod h1:sDdvGhXrSVT5yzBDR7qXz+rhbpiMpUYfF3vJ01QSdrc=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4 h1:9wKDWEjwSnXZre0/O3+ZwbBl1SmlgWYBbrTV10X/H1s=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.4/go.mod h1:t4i+yGHMCcUNIX1x7YVYa6bH/Do7civ5I6cG/6PMfyA=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.9 h1:yG01Big4R5CDxftieMlgZPcHKZbwkRygur4DMGTqSzg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.9/go.mod h1:RV5gmgYb4psddWMPaf4giuGdsK1l0KwlXNFAbzWAIIo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.14.0 h1:1KdubQbnw76M0Sr8480q6OXBlymBVqpkK+RuCqJz+nQ=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.14.0/go.mod h1:UcgIwJ9KHquYxs6Q5skC9qXjhYMK+JASDYcXQ4X7JZE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.3 h1:DUwbD79T8gyQ23qVXFUthjzVMTviSHi3y4z58KvghhM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.3/go.mod h1:7sGSz1JCKHWWBHq98m6sMtWQikmYPpxjqOydDemiVoM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.3 h1:AplLJCtIaUZDCbr6+gLYdsYNxne4iuaboJhVt9d+WXI=
|
||||
|
|
@ -101,16 +101,16 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3 h1:KV0z2RDc7euMtg
|
|||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3/go.mod h1:KZgs2ny8HsxRIRbDwgvJcHHBZPOzQr/+NtGwnP+w2ec=
|
||||
github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.18.2 h1:UaKsLxZ4sHdYlyX2cRZ+7YznCQS7jwHHgVy1hKkNKfE=
|
||||
github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.18.2/go.mod h1:4L1Z3QlQqkGEPSRH9fYPuHLxNEK54VgExjw4J/ShjbM=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.2 h1:NnduxUd9+Fq9DcCDdJK8v6l9lR1xDX4usvog+JuQAno=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.42.2/go.mod h1:NXRKkiRF+erX2hnybnVU660cYT5/KChRD4iUgJ97cI8=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.43.0 h1:cwTuq73Tv6jtNJIMgTDKsih5O2YsVrKGpg20H98tbmo=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.43.0/go.mod h1:NXRKkiRF+erX2hnybnVU660cYT5/KChRD4iUgJ97cI8=
|
||||
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.23.2 h1:M5NodszNDBfyfFBKoAzJY0flmkkQCg7MGk6+/vBGjCM=
|
||||
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.23.2/go.mod h1:+8dYLQz+I30HIGyhp+6htf3+yyGTqBzzTOG90Ai8lWs=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.17.2 h1:V47N5eKgVZoRSvx2+RQ0EpAEit/pqOhqeSQFiS4OFEQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.17.2/go.mod h1:/pE21vno3q1h4bbhUOEi+6Zu/aT26UK2WKkDXd+TssQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.2 h1:sMAcO7VHVw28HTAdZpTULDzFirHOsVm/x25CxhUH0jA=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.2/go.mod h1:dWqm5G767qwKPuayKfzm4rjzFmVjiBFbOJrpSPnAMDs=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.25.2 h1:vwyiRTnXLqsak/6WAQ+uTRhVqKI6vxUQ0HJXjKij0zM=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.25.2/go.mod h1:4EqRHDCKP78hq3zOnmFXu5k0j4bXbRFfCh/zQ6KnEfQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.0 h1:/XiEU7VIFcVWRDQLabyrSjBoKIm8UkYgsvWDuFW8Img=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.0/go.mod h1:dWqm5G767qwKPuayKfzm4rjzFmVjiBFbOJrpSPnAMDs=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.25.3 h1:M2w4kiMGJCCM6Ljmmx/l6mmpfa3gPJVpBencfnsgvqs=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.25.3/go.mod h1:4EqRHDCKP78hq3zOnmFXu5k0j4bXbRFfCh/zQ6KnEfQ=
|
||||
github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI=
|
||||
github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
|
|
|
|||
|
|
@ -156,11 +156,11 @@ func (fs *S3Fs) Stat(name string) (os.FileInfo, error) {
|
|||
// Some S3 providers (like SeaweedFS) remove the trailing '/' from object keys.
|
||||
// So we check some common content types to detect if this is a "directory".
|
||||
isDir := util.Contains(s3DirMimeTypes, util.GetStringFromPointer(obj.ContentType))
|
||||
if obj.ContentLength == 0 && !isDir {
|
||||
if util.GetIntFromPointer(obj.ContentLength) == 0 && !isDir {
|
||||
_, err = fs.headObject(name + "/")
|
||||
isDir = err == nil
|
||||
}
|
||||
return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir, obj.ContentLength,
|
||||
return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir, util.GetIntFromPointer(obj.ContentLength),
|
||||
util.GetTimeFromPointer(obj.LastModified), false))
|
||||
}
|
||||
if !fs.IsNotExist(err) {
|
||||
|
|
@ -186,7 +186,7 @@ func (fs *S3Fs) getStatForDir(name string) (os.FileInfo, error) {
|
|||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, obj.ContentLength,
|
||||
return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, true, util.GetIntFromPointer(obj.ContentLength),
|
||||
util.GetTimeFromPointer(obj.LastModified), false))
|
||||
}
|
||||
|
||||
|
|
@ -466,6 +466,7 @@ func (fs *S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) {
|
|||
}
|
||||
for _, fileObject := range page.Contents {
|
||||
objectModTime := util.GetTimeFromPointer(fileObject.LastModified)
|
||||
objectSize := util.GetIntFromPointer(fileObject.Size)
|
||||
name, isDir := fs.resolve(fileObject.Key, prefix)
|
||||
if name == "" || name == "/" {
|
||||
continue
|
||||
|
|
@ -479,7 +480,8 @@ func (fs *S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) {
|
|||
if t, ok := modTimes[name]; ok {
|
||||
objectModTime = util.GetTimeFromMsecSinceEpoch(t)
|
||||
}
|
||||
result = append(result, NewFileInfo(name, (isDir && fileObject.Size == 0), fileObject.Size,
|
||||
|
||||
result = append(result, NewFileInfo(name, (isDir && objectSize == 0), objectSize,
|
||||
objectModTime, false))
|
||||
}
|
||||
}
|
||||
|
|
@ -627,11 +629,12 @@ func (fs *S3Fs) GetDirSize(dirname string) (int, int64, error) {
|
|||
}
|
||||
for _, fileObject := range page.Contents {
|
||||
isDir := strings.HasSuffix(util.GetStringFromPointer(fileObject.Key), "/")
|
||||
if isDir && fileObject.Size == 0 {
|
||||
objectSize := util.GetIntFromPointer(fileObject.Size)
|
||||
if isDir && objectSize == 0 {
|
||||
continue
|
||||
}
|
||||
numFiles++
|
||||
size += fileObject.Size
|
||||
size += objectSize
|
||||
if numFiles%1000 == 0 {
|
||||
fsLog(fs, logger.LevelDebug, "dirname %q scan in progress, files: %d, size: %d", dirname, numFiles, size)
|
||||
}
|
||||
|
|
@ -696,7 +699,8 @@ func (fs *S3Fs) Walk(root string, walkFn filepath.WalkFunc) error {
|
|||
continue
|
||||
}
|
||||
err := walkFn(util.GetStringFromPointer(fileObject.Key),
|
||||
NewFileInfo(name, isDir, fileObject.Size, util.GetTimeFromPointer(fileObject.LastModified), false), nil)
|
||||
NewFileInfo(name, isDir, util.GetIntFromPointer(fileObject.Size), util.GetTimeFromPointer(fileObject.LastModified),
|
||||
false), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -865,10 +869,11 @@ func (fs *S3Fs) mkdirInternal(name string) error {
|
|||
|
||||
func (fs *S3Fs) hasContents(name string) (bool, error) {
|
||||
prefix := fs.getPrefix(name)
|
||||
maxKeys := int32(2)
|
||||
paginator := s3.NewListObjectsV2Paginator(fs.svc, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(fs.config.Bucket),
|
||||
Prefix: aws.String(prefix),
|
||||
MaxKeys: 2,
|
||||
MaxKeys: &maxKeys,
|
||||
})
|
||||
|
||||
if paginator.HasMorePages() {
|
||||
|
|
@ -962,7 +967,7 @@ func (fs *S3Fs) doMultipartCopy(source, target, contentType string, fileSize int
|
|||
Bucket: aws.String(fs.config.Bucket),
|
||||
CopySource: aws.String(source),
|
||||
Key: aws.String(target),
|
||||
PartNumber: partNum,
|
||||
PartNumber: &partNum,
|
||||
UploadId: aws.String(uploadID),
|
||||
CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", partStart, partEnd-1)),
|
||||
})
|
||||
|
|
@ -991,7 +996,7 @@ func (fs *S3Fs) doMultipartCopy(source, target, contentType string, fileSize int
|
|||
partMutex.Lock()
|
||||
completedParts = append(completedParts, types.CompletedPart{
|
||||
ETag: partResp.CopyPartResult.ETag,
|
||||
PartNumber: partNum,
|
||||
PartNumber: &partNum,
|
||||
})
|
||||
partMutex.Unlock()
|
||||
}(partNumber, start, end)
|
||||
|
|
@ -1004,7 +1009,14 @@ func (fs *S3Fs) doMultipartCopy(source, target, contentType string, fileSize int
|
|||
return copyError
|
||||
}
|
||||
sort.Slice(completedParts, func(i, j int) bool {
|
||||
return completedParts[i].PartNumber < completedParts[j].PartNumber
|
||||
getPartNumber := func(number *int32) int32 {
|
||||
if number == nil {
|
||||
return 0
|
||||
}
|
||||
return *number
|
||||
}
|
||||
|
||||
return getPartNumber(completedParts[i].PartNumber) < getPartNumber(completedParts[j].PartNumber)
|
||||
})
|
||||
|
||||
completeCtx, completeCancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
|
||||
|
|
|
|||
Loading…
Reference in a new issue