diff --git a/go.mod b/go.mod index f8cc5d05..c7bf06ea 100644 --- a/go.mod +++ b/go.mod @@ -8,15 +8,15 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387 - github.com/aws/aws-sdk-go-v2 v1.16.5 - github.com/aws/aws-sdk-go-v2/config v1.15.11 - github.com/aws/aws-sdk-go-v2/credentials v1.12.6 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.16 - github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.6 - github.com/aws/aws-sdk-go-v2/service/s3 v1.26.11 - github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.11 - github.com/aws/aws-sdk-go-v2/service/sts v1.16.7 + github.com/aws/aws-sdk-go-v2 v1.16.6 + github.com/aws/aws-sdk-go-v2/config v1.15.12 + github.com/aws/aws-sdk-go-v2/credentials v1.12.7 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.17 + github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.7 + github.com/aws/aws-sdk-go-v2/service/s3 v1.26.12 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.12 + github.com/aws/aws-sdk-go-v2/service/sts v1.16.8 github.com/cockroachdb/cockroach-go/v2 v2.2.14 github.com/coreos/go-oidc/v3 v3.2.0 github.com/eikenb/pipeat v0.0.0-20210730190139-06b3e6902001 @@ -35,11 +35,11 @@ require ( github.com/hashicorp/go-plugin v1.4.4 github.com/hashicorp/go-retryablehttp v0.7.1 github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126 - github.com/klauspost/compress v1.15.6 + github.com/klauspost/compress v1.15.7 github.com/lestrrat-go/jwx v1.2.25 github.com/lib/pq v1.10.6 github.com/lithammer/shortuuid/v3 v3.0.7 - github.com/mattn/go-sqlite3 v1.14.13 + github.com/mattn/go-sqlite3 v1.14.14 github.com/mhale/smtpd v0.8.0 github.com/minio/sio v0.3.0 github.com/otiai10/copy v1.7.0 @@ -56,7 +56,7 @@ require ( github.com/spf13/afero v1.8.2 github.com/spf13/cobra v1.5.0 github.com/spf13/viper v1.12.0 - github.com/stretchr/testify v1.7.5 + github.com/stretchr/testify v1.8.0 github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 github.com/unrolled/secure v1.10.0 github.com/wagslane/go-password-validator v0.3.0 @@ -67,29 +67,29 @@ require ( gocloud.dev v0.25.0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e - golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 + golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b golang.org/x/time v0.0.0-20220609170525-579cf78fd858 - google.golang.org/api v0.85.0 + google.golang.org/api v0.86.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) require ( - cloud.google.com/go v0.102.1 // indirect + cloud.google.com/go v0.103.0 // indirect cloud.google.com/go/compute v1.7.0 // indirect cloud.google.com/go/iam v0.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.6 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.9 // indirect - github.com/aws/smithy-go v1.11.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.10 // indirect + github.com/aws/smithy-go v1.12.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/boombuler/barcode v1.0.1 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect @@ -103,7 +103,7 @@ require ( github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-test/deep v1.0.8 // indirect - github.com/goccy/go-json v0.9.7 // indirect + github.com/goccy/go-json v0.9.8 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.8 // indirect @@ -155,7 +155,7 @@ require ( golang.org/x/tools v0.1.11 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220627200112-0a929928cb33 // indirect + google.golang.org/genproto v0.0.0-20220630160836-4327a74d660d // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/ini.v1 v1.66.6 // indirect diff --git a/go.sum b/go.sum index 203d0608..dc8ad0ee 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,9 @@ cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2Z cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= -cloud.google.com/go v0.102.1 h1:vpK6iQWv/2uUeFJth4/cBHsQAGjn1iIE6AAlxipRaA0= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.103.0 h1:YXtxp9ymmZjlGzxV7VrYQ8aaQuAgcqxSy6YhDX4I458= +cloud.google.com/go v0.103.0/go.mod h1:vwLx1nqLrzLX/fpwSMOXmFIqBOyHsvHbnAdbGSJ+mKk= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -139,67 +140,67 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= -github.com/aws/aws-sdk-go-v2 v1.16.5 h1:Ah9h1TZD9E2S1LzHpViBO3Jz9FPL5+rmflmb8hXirtI= -github.com/aws/aws-sdk-go-v2 v1.16.5/go.mod h1:Wh7MEsmEApyL5hrWzpDkba4gwAPc5/piwLVLFnCxp48= +github.com/aws/aws-sdk-go-v2 v1.16.6 h1:kzafGZYwkwVgLZ2zEX7P+vTwLli6uIMXF8aGjunN6UI= +github.com/aws/aws-sdk-go-v2 v1.16.6/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.2 h1:LFOGNUQxc/8BlhA4FD+JdYjJKQK6tsz9Xiuh+GUTKAQ= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.2/go.mod h1:u/38zebMi809w7YFnqY/07Tw/FSs6DGhPD95Xiig7XQ= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3 h1:S/ZBwevQkr7gv5YxONYpGQxlMFFYSRfz3RMcjsC9Qhk= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3/go.mod h1:gNsR5CaXKmQSSzrmGxmwmct/r+ZBfbxorAuXYsj/M5Y= github.com/aws/aws-sdk-go-v2/config v1.15.3/go.mod h1:9YL3v07Xc/ohTsxFXzan9ZpFpdTOFl4X65BAKYaz8jg= -github.com/aws/aws-sdk-go-v2/config v1.15.11 h1:qfec8AtiCqVbwMcx51G1yO2PYVfWfhp2lWkDH65V9HA= -github.com/aws/aws-sdk-go-v2/config v1.15.11/go.mod h1:mD5tNFciV7YHNjPpFYqJ6KGpoSfY107oZULvTHIxtbI= +github.com/aws/aws-sdk-go-v2/config v1.15.12 h1:D4mdf0cOSmZRgJe0DDOd1Qm6tkwHJ7r5i1lz0asa+AA= +github.com/aws/aws-sdk-go-v2/config v1.15.12/go.mod h1:oxRNnH11J580bxDEXyfTqfB3Auo2fxzhV052LD4HnyA= github.com/aws/aws-sdk-go-v2/credentials v1.11.2/go.mod h1:j8YsY9TXTm31k4eFhspiQicfXPLZ0gYXA50i4gxPE8g= -github.com/aws/aws-sdk-go-v2/credentials v1.12.6 h1:No1wZFW4bcM/uF6Tzzj6IbaeQJM+xxqXOYmoObm33ws= -github.com/aws/aws-sdk-go-v2/credentials v1.12.6/go.mod h1:mQgnRmBPF2S/M01W4T4Obp3ZaZB6o1s/R8cOUda9vtI= +github.com/aws/aws-sdk-go-v2/credentials v1.12.7 h1:e2DcCR0gP+T2zVj5eQPMQoRdxo+vd2p9BkpJ72BdyzA= +github.com/aws/aws-sdk-go-v2/credentials v1.12.7/go.mod h1:8b1nSHdDaKLho9VEK+K8WivifA/2K5pPm4sfI21NlQ8= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3/go.mod h1:uk1vhHHERfSVCUnqSqz8O48LBYDSC+k6brng09jcMOk= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6 h1:+NZzDh/RpcQTpo9xMFUgkseIam6PC+YJbdhbQp1NOXI= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.6/go.mod h1:ClLMcuQA/wcHPmOIfNzNI4Y1Q0oDbmEkbYhMFOzHDh8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7 h1:8yi2ORCwXpXEPnj0vP3DjYhejwDQD/5klgBoxXcKOxY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7/go.mod h1:81k6q0UUZj6AdQZ1E/VQ27cLrTUpJGraZR6/hVHRxjE= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.3/go.mod h1:0dHuD2HZZSiwfJSy1FO5bX1hQ1TxVV1QXXjpn3XUE44= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.16 h1:W4iOhIRXRMc3L5pWhuvu+RCfPjw8hBVlFRMfoYtpxx4= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.16/go.mod h1:mI+TRQe1NsdDUGcz+hQAAAsllr7XcbhiDJSBIi15rcM= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.17 h1:9Y+OvoIvC8KocGNqbbBNDvMu0zsIgzKg3r+ZllSuH5Y= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.17/go.mod h1:z/7g6Z78jPG0l3HeShseUWzA+aBJDK4Mu5DkKkYdIW0= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9/go.mod h1:AnVH5pvai0pAF4lXRq0bmhbes1u9R8wTE+g+183bZNM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12 h1:Zt7DDk5V7SyQULUUwIKzsROtVzp/kVvcz15uQx/Tkow= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.12/go.mod h1:Afj/U8svX6sJ77Q+FPWMzabJ9QjbwP32YlopgKALUpg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13 h1:WuQ1yGs3TMJgxpGVLspcsU/5q1omSA0SG6Cu0yZ4jkM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13/go.mod h1:wLLesU+LdMZDM3U0PP9vZXJW39zmD/7L4nY2pSrYZ/g= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3/go.mod h1:ssOhaLpRlh88H3UmEcsBoVKq309quMvm3Ds8e9d4eJM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6 h1:eeXdGVtXEe+2Jc49+/vAzna3FAQnUD4AagAw8tzbmfc= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.6/go.mod h1:FwpAKI+FBPIELJIdmQzlLtRe8LQSOreMcM2wBsPMvvc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7 h1:mCeDDYeDXp3loo/xKi7nkx34eeh7q3n1mUBtzptsj8c= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7/go.mod h1:93Uot80ddyVzSl//xEJreNKMhxntr71WtR3v/A1cRYk= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10/go.mod h1:8DcYQcz0+ZJaSxANlHIsbbi6S+zMwjwdDqwW3r9AzaE= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13 h1:L/l0WbIpIadRO7i44jZh1/XeXpNDX0sokFppb4ZnXUI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.13/go.mod h1:hiM/y1XPp3DoEPhoVEYc/CZcS58dP6RKJRDFp99wdX0= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.3 h1:m1vDVDoNK4tZAoWtcetHopEdIeUlrNNpdLZ7cwZke6s= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.3/go.mod h1:annFthsb7FiHQd5X9wKDNst9OJvVFY0l0LjQ8zQniJA= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14 h1:bJv4Y9QOiW0GZPStgLgpGrpdfRDSR3XM4V4M3YCQRZo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14/go.mod h1:R1HF8ZDdcRFfAGF+13En4LSHi2IrrNuPQCaxgWCeGyY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.4 h1:wusoY1MJ9JNrPoX3n4kxY4MTIUivCiXvTYQbYh59yxs= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.4/go.mod h1:cHTMyJVEXRUZ25f8V+pq6CAwoYARarJRFGf3XH4eIxE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.1/go.mod h1:GeUru+8VzrTXV/83XyMJ80KpH8xO89VPoUileyNQ+tc= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.2 h1:T/ywkX1ed+TsZVQccu/8rRJGxKZF/t0Ivgrb4MHTSeo= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.2/go.mod h1:RnloUnyZ4KN9JStGY1LuQ7Wzqh7V0f8FinmRdHYtuaA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.3 h1:4n4KCtv5SUoT5Er5XV41huuzrCqepxlW3SDI9qHQebc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.3/go.mod h1:gkb2qADY+OHaGLKNTYxMaQNacfeyQpZ4csDTQMeFmcw= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.3/go.mod h1:Seb8KNmD6kVTjwRjVEgOT5hPin6sq+v4C2ycJQDwuH8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.7 h1:DYUAx8lWAhIzFiD284oq6RUPKppKk3cyqv/hyUkbWuA= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.7/go.mod h1:6tcs0yjwAW2Z9Yb3Z4X/2tm3u9jNox1dvXxVXTd73Zw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.8 h1:BzBekDihMMeBexBhdK7xS3AIh2Jg/mECyLWO5RRwwHY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.8/go.mod h1:a1BSeQI9IVr1j5Dwn73cdAKi4MdizTaV9YovUaHefGI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3/go.mod h1:wlY6SVjuwvh3TVRpTqdy4I1JpBFLX4UGeKZdWntaocw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6 h1:0ZxYAZ1cn7Swi/US55VKciCE6RhRHIwCKIWaMLdT6pg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.6/go.mod h1:DxAPjquoEHf3rUHh1b9+47RAaXB8/7cB6jkzCt/GOEI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7 h1:M7/BzQNsu0XXiJRe3gUn8UA8tExF6kLMAfvo5PT/KJY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7/go.mod h1:HvVdEh/x4jsPBsjNvDy+MH3CDCPy4gTZEzFe2r4uJY8= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.3/go.mod h1:Bm/v2IaN6rZ+Op7zX+bOUMdL4fsrYZiD0dsjLhNKwZc= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.6 h1:SSrqxZVhrO371eg/C8Fnj6kduzltKHj/mJl2swkTBGc= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.6/go.mod h1:TzDyqDka0783D93yVirkcysbibVRxjX5HFJEWms4kKA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.7 h1:imb0NhTQZaTDSAQvgFyiZbKTwl0F+AkZL1ZNoEHtuQc= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.7/go.mod h1:V952z/yIT247sKya+CB+Ls3sxpB9jeBj5TkLraCGKGU= github.com/aws/aws-sdk-go-v2/service/kms v1.16.3/go.mod h1:QuiHPBqlOFCi4LqdSskYYAWpQlx3PKmohy+rE2F+o5g= -github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.6 h1:QHv9AYaolQo8Tj+PIfIizQ/aD/EHrb7eOlNpeuEKyEU= -github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.6/go.mod h1:M8WgODCojJa7pJRL7vx2bS4NO+NjcRtlvGDr9ls/MAI= +github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.7 h1:/5I4p9aytzW1pQfQaCYAqTUzVE48jgJHVFfQVTU4LB8= +github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.7/go.mod h1:DbtPrtO8rAOefDzgqGCqUV+Ub2UtL3/prfQt9bFFYoU= github.com/aws/aws-sdk-go-v2/service/s3 v1.26.3/go.mod h1:g1qvDuRsJY+XghsV6zg00Z4KJ7DtFFCx8fJD2a491Ak= -github.com/aws/aws-sdk-go-v2/service/s3 v1.26.11 h1:Wt0512f6GfLiMd6a+NuOCC9r3/trmzHMTB697CBDUwg= -github.com/aws/aws-sdk-go-v2/service/s3 v1.26.11/go.mod h1:VMTprbiZWqW44viXgPSQhWdeZ8JTAeJwhO7OXpC/Rsg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.26.12 h1:/JTTdNObz+GygQqnbdBzummuxFIcuB6hbra1mqS+Wic= +github.com/aws/aws-sdk-go-v2/service/s3 v1.26.12/go.mod h1:eas8WnpTDJtCvEjRXAINFuox9TmEGeevxiUKEKv2tQ8= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.4/go.mod h1:PJc8s+lxyU8rrre0/4a0pn2wgwiDvOEzoOjcJUBr67o= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.11 h1:mnL8MXCR3FMw+xeC0+zViYSNuDh7uUhhzGaUsTyCTLs= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.11/go.mod h1:pgtQihVJw8OxQCkC4BmJOuVWT52mBTaj8LcsF5Kr9iA= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.12 h1:Pq2GrbfG74dX/JhY/O+bWBz7DUzdgTvqugofqTWLACQ= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.12/go.mod h1:MfgrkSNjFbMLz19srWgyGJtvDEfXg/ZUJ6AIrxdj65M= github.com/aws/aws-sdk-go-v2/service/sns v1.17.4/go.mod h1:kElt+uCcXxcqFyc+bQqZPFD9DME/eC6oHBXvFzQ9Bcw= github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3/go.mod h1:skmQo0UPvsjsuYYSYMVmrPc1HWCbHUJyrCEp+ZaLzqM= github.com/aws/aws-sdk-go-v2/service/ssm v1.24.1/go.mod h1:NR/xoKjdbRJ+qx0pMR4mI+N/H1I1ynHwXnO6FowXJc0= github.com/aws/aws-sdk-go-v2/service/sso v1.11.3/go.mod h1:7UQ/e69kU7LDPtY40OyoHYgRmgfGM4mgsLYtcObdveU= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.9 h1:Gju1UO3E8ceuoYc/AHcdXLuTZ0WGE1PT2BYDwcYhJg8= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.9/go.mod h1:UqRD9bBt15P0ofRyDZX6CfsIqPpzeHOhZKWzgSuAzpo= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.10 h1:icon5WWg9Yg5nkB0pJF6bfKw6M0xozukeGKSNKtnqzw= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.10/go.mod h1:UHxA35uPrCykRySBV5iSPZhZRlYnWSS2c/aaZVsoU94= github.com/aws/aws-sdk-go-v2/service/sts v1.16.3/go.mod h1:bfBj0iVmsUyUg4weDB4NxktD9rDGeKSVWnjTnwbx9b8= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.7 h1:HLzjwQM9975FQWSF3uENDGHT1gFQm/q3QXu2BYIcI08= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.7/go.mod h1:lVxTdiiSHY3jb1aeg+BBFtDzZGSUCv6qaNOyEGCJ1AY= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.8 h1:GLGfpqX+1bmjNvUJkwB1ZaDpNFXQwJ3z9RkQDA58OBY= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.8/go.mod h1:50YdFq1WIuxA0AGrygvYGucnNYrG24WYzu5fNp7lMgY= github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= -github.com/aws/smithy-go v1.11.3 h1:DQixirEFM9IaKxX1olZ3ke3nvxRS2xMDteKIDWxozW8= -github.com/aws/smithy-go v1.11.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.12.0 h1:gXpeZel/jPoWQ7OEmLIgCUnhkFftqNfwWUwAHSlp1v0= +github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -330,8 +331,9 @@ github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-json v0.7.6/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.9.8 h1:DxXB6MLd6yyel7CLph8EwNIonUtVZd3Ue5iRcL4DQCE= +github.com/goccy/go-json v0.9.8/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= @@ -538,8 +540,8 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY= -github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.7 h1:7cgTQxJCU/vy+oP/E3B9RGbQTgbiVzIJWIKOLoAsPok= +github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.14 h1:QRqdp6bb9M9S5yyKeYteXKuoKE4p0tGlra81fKOpWH8= github.com/klauspost/cpuid/v2 v2.0.14/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= @@ -603,8 +605,8 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.13 h1:1tj15ngiFfcZzii7yd82foL+ks+ouQcj8j/TPq3fk1I= -github.com/mattn/go-sqlite3 v1.14.13/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.14 h1:qZgc/Rwetq+MtyE18WhzjokPD93dNqLGNT3QJuLvBGw= +github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mhale/smtpd v0.8.0 h1:5JvdsehCg33PQrZBvFyDMMUDQmvbzVpZgKob7eYBJc0= @@ -742,8 +744,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q= -github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 h1:b2nJXyPCa9HY7giGM+kYcnQ71m14JnGdQabMPmyt++8= github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs= @@ -866,8 +868,9 @@ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 h1:+jnHzr9VPj32ykQVai5DNahi9+NSp7yYuCsl5eAQtL0= golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0 h1:VnGaRqoLmqZH/3TMLJwYCEWkR4j1nuIU1U9TvbqsDUw= +golang.org/x/oauth2 v0.0.0-20220630143837-2104d58473e0/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -966,6 +969,7 @@ golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b h1:2n253B2r0pYSmEV+UNCQoPfU/FiaizQEK5Gu4Bq4JE8= golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1108,8 +1112,9 @@ google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69 google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= -google.golang.org/api v0.85.0 h1:8rJoHuRxx+vCmZtAO/3k1dRLvYNVyTJtZ5oaFZvhgvc= google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= +google.golang.org/api v0.86.0 h1:ZAnyOHQFIuWso1BodVfSaRyffD74T9ERGFa3k1fNk/U= +google.golang.org/api v0.86.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1214,8 +1219,10 @@ google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220627200112-0a929928cb33 h1:3L4edWcjDHPWGcMl1N0YH1NSoasyvfEcZCe2rUbxHfs= -google.golang.org/genproto v0.0.0-20220627200112-0a929928cb33/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220630160836-4327a74d660d h1:9IbUS1WUO1OPTBCp32JFwX9MqyT5pc3UWXH8plPX8is= +google.golang.org/genproto v0.0.0-20220630160836-4327a74d660d/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/vfs/azblobfs.go b/vfs/azblobfs.go index ebbdd120..d723a5f0 100644 --- a/vfs/azblobfs.go +++ b/vfs/azblobfs.go @@ -17,6 +17,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -904,6 +905,7 @@ func (fs *AzureBlobFs) handleMultipartDownload(ctx context.Context, blockBlob *a finished := false var wg sync.WaitGroup var errOnce sync.Once + var hasError int32 var poolError error poolCtx, poolCancel := context.WithCancel(ctx) @@ -920,7 +922,7 @@ func (fs *AzureBlobFs) handleMultipartDownload(ctx context.Context, blockBlob *a offset = end guard <- struct{}{} - if poolError != nil { + if atomic.LoadInt32(&hasError) == 1 { fsLog(fs, logger.LevelDebug, "pool error, download for part %v not started", part) break } @@ -941,8 +943,9 @@ func (fs *AzureBlobFs) handleMultipartDownload(ctx context.Context, blockBlob *a err := fs.downloadPart(innerCtx, blockBlob, buf, writer, start, count, writeOffset) if err != nil { errOnce.Do(func() { - poolError = err - fsLog(fs, logger.LevelError, "multipart download error: %+v", poolError) + fsLog(fs, logger.LevelError, "multipart download error: %+v", err) + atomic.StoreInt32(&hasError, 1) + poolError = fmt.Errorf("multipart download error: %w", err) poolCancel() }) } @@ -971,6 +974,7 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read var blocks []string var wg sync.WaitGroup var errOnce sync.Once + var hasError int32 var poolError error poolCtx, poolCancel := context.WithCancel(ctx) @@ -998,7 +1002,7 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read blocks = append(blocks, blockID) guard <- struct{}{} - if poolError != nil { + if atomic.LoadInt32(&hasError) == 1 { fsLog(fs, logger.LevelError, "pool error, upload for part %v not started", part) pool.releaseBuffer(buf) break @@ -1021,8 +1025,9 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read _, err := blockBlob.StageBlock(innerCtx, blockID, bufferReader, &azblob.BlockBlobStageBlockOptions{}) if err != nil { errOnce.Do(func() { - poolError = err - fsLog(fs, logger.LevelDebug, "multipart upload error: %+v", poolError) + fsLog(fs, logger.LevelDebug, "multipart upload error: %+v", err) + atomic.StoreInt32(&hasError, 1) + poolError = fmt.Errorf("multipart upload error: %w", err) poolCancel() }) } diff --git a/vfs/s3fs.go b/vfs/s3fs.go index 650c707e..ed605632 100644 --- a/vfs/s3fs.go +++ b/vfs/s3fs.go @@ -14,7 +14,10 @@ import ( "os" "path" "path/filepath" + "sort" "strings" + "sync" + "sync/atomic" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -300,7 +303,7 @@ func (fs *S3Fs) Rename(source, target string) error { copySource = pathEscape(copySource) if fi.Size() > 500*1024*1024 { - fsLog(fs, logger.LevelDebug, "renaming file %#v with size %v, a multipart copy is required, this may take a while", + fsLog(fs, logger.LevelDebug, "renaming file %q with size %d using multipart copy", source, fi.Size()) err = fs.doMultipartCopy(copySource, target, contentType, fi.Size()) } else { @@ -822,43 +825,99 @@ func (fs *S3Fs) doMultipartCopy(source, target, contentType string, fileSize int if uploadID == "" { return errors.New("unable to get multipart copy upload ID") } - maxPartSize := int64(500 * 1024 * 1024) - completedParts := make([]types.CompletedPart, 0) - partNumber := int32(1) - - for copied := int64(0); copied < fileSize; copied += maxPartSize { - innerCtx, innerCancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) - defer innerCancelFn() - - partResp, err := fs.svc.UploadPartCopy(innerCtx, &s3.UploadPartCopyInput{ - Bucket: aws.String(fs.config.Bucket), - CopySource: aws.String(source), - Key: aws.String(target), - PartNumber: partNumber, - UploadId: aws.String(uploadID), - CopySourceRange: aws.String(getMultipartCopyRange(copied, maxPartSize, fileSize)), - }) - if err != nil { - fsLog(fs, logger.LevelError, "unable to copy part number %v: %+v", partNumber, err) - abortCtx, abortCancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) - defer abortCancelFn() - - _, errAbort := fs.svc.AbortMultipartUpload(abortCtx, &s3.AbortMultipartUploadInput{ - Bucket: aws.String(fs.config.Bucket), - Key: aws.String(target), - UploadId: aws.String(uploadID), - }) - if errAbort != nil { - fsLog(fs, logger.LevelError, "unable to abort multipart copy: %+v", errAbort) - } - return fmt.Errorf("error copying part number %v: %w", partNumber, err) - } - completedParts = append(completedParts, types.CompletedPart{ - ETag: partResp.CopyPartResult.ETag, - PartNumber: partNumber, - }) - partNumber++ + // We use 32 MB part size and copy 10 parts in parallel. + // These values are arbitrary. We don't want to start too many goroutines + maxPartSize := int64(32 * 1024 * 1024) + if fileSize > int64(100*1024*1024*1024) { + maxPartSize = int64(500 * 1024 * 1024) } + guard := make(chan struct{}, 10) + finished := false + var completedParts []types.CompletedPart + var partMutex sync.Mutex + var wg sync.WaitGroup + var hasError int32 + var errOnce sync.Once + var copyError error + var partNumber int32 + var offset int64 + + opCtx, opCancel := context.WithCancel(context.Background()) + defer opCancel() + + for partNumber = 1; !finished; partNumber++ { + start := offset + end := offset + maxPartSize + if end >= fileSize { + end = fileSize + finished = true + } + offset = end + + guard <- struct{}{} + if atomic.LoadInt32(&hasError) == 1 { + fsLog(fs, logger.LevelDebug, "previous multipart copy error, copy for part %d not started", partNumber) + break + } + + wg.Add(1) + go func(partNum int32, partStart, partEnd int64) { + defer func() { + <-guard + wg.Done() + }() + + innerCtx, innerCancelFn := context.WithDeadline(opCtx, time.Now().Add(fs.ctxTimeout)) + defer innerCancelFn() + + partResp, err := fs.svc.UploadPartCopy(innerCtx, &s3.UploadPartCopyInput{ + Bucket: aws.String(fs.config.Bucket), + CopySource: aws.String(source), + Key: aws.String(target), + PartNumber: partNum, + UploadId: aws.String(uploadID), + CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", partStart, partEnd-1)), + }) + if err != nil { + errOnce.Do(func() { + fsLog(fs, logger.LevelError, "unable to copy part number %d: %+v", partNum, err) + atomic.StoreInt32(&hasError, 1) + copyError = fmt.Errorf("error copying part number %d: %w", partNum, err) + opCancel() + + abortCtx, abortCancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) + defer abortCancelFn() + + _, errAbort := fs.svc.AbortMultipartUpload(abortCtx, &s3.AbortMultipartUploadInput{ + Bucket: aws.String(fs.config.Bucket), + Key: aws.String(target), + UploadId: aws.String(uploadID), + }) + if errAbort != nil { + fsLog(fs, logger.LevelError, "unable to abort multipart copy: %+v", errAbort) + } + }) + return + } + + partMutex.Lock() + completedParts = append(completedParts, types.CompletedPart{ + ETag: partResp.CopyPartResult.ETag, + PartNumber: partNum, + }) + partMutex.Unlock() + }(partNumber, start, end) + } + + wg.Wait() + close(guard) + + if copyError != nil { + return copyError + } + sort.Slice(completedParts, func(i, j int) bool { + return completedParts[i].PartNumber < completedParts[j].PartNumber + }) completeCtx, completeCancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout)) defer completeCancelFn() @@ -929,15 +988,6 @@ func (fs *S3Fs) getStorageID() string { return fmt.Sprintf("s3://%v", fs.config.Bucket) } -func getMultipartCopyRange(start, maxPartSize, fileSize int64) string { - end := start + maxPartSize - 1 - if end > fileSize { - end = fileSize - 1 - } - - return fmt.Sprintf("bytes=%v-%v", start, end) -} - func getAWSHTTPClient(timeout int, idleConnectionTimeout time.Duration) *awshttp.BuildableClient { c := awshttp.NewBuildableClient(). WithDialerOptions(func(d *net.Dialer) {