From a417df60b3adc7e4c0803c6a7f206f75ec3c58d9 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Fri, 7 Oct 2022 06:54:26 +0200 Subject: [PATCH] azblob: use UUIDs as block IDs Signed-off-by: Nicola Murino --- .github/workflows/release.yml | 2 +- go.mod | 18 +++++++++--------- go.sum | 30 +++++++++++++++--------------- internal/vfs/azblobfs.go | 32 +++++++++++++------------------- 4 files changed, 38 insertions(+), 44 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a41b84fc..d38a3981 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,7 +5,7 @@ on: tags: 'v*' env: - GO_VERSION: 1.19.1 + GO_VERSION: 1.19.2 jobs: prepare-sources-with-deps: diff --git a/go.mod b/go.mod index 2663d580..8f55dc17 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( cloud.google.com/go/storage v1.27.0 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.3 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.0 github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387 @@ -22,7 +22,7 @@ require ( github.com/eikenb/pipeat v0.0.0-20210730190139-06b3e6902001 github.com/fclairamb/ftpserverlib v0.20.0 github.com/fclairamb/go-log v0.4.1 - github.com/go-acme/lego/v4 v4.8.0 + github.com/go-acme/lego/v4 v4.9.0 github.com/go-chi/chi/v5 v5.0.8-0.20220512131524-9e71a0d4b3d6 github.com/go-chi/jwtauth/v5 v5.0.2 github.com/go-chi/render v1.0.2 @@ -65,10 +65,10 @@ require ( go.etcd.io/bbolt v1.3.6 go.uber.org/automaxprocs v1.5.1 gocloud.dev v0.27.0 - golang.org/x/crypto v0.0.0-20220924013350-4ba4fb4dd9e7 - golang.org/x/net v0.0.0-20220923203811-8be639271d50 - golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 - golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec + golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b + golang.org/x/net v0.0.0-20221004154528-8021a29435af + golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1 + golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875 golang.org/x/time v0.0.0-20220922220347-f3bd1da661af google.golang.org/api v0.98.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 @@ -160,7 +160,7 @@ require ( golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220930163606-c98284e70a91 // indirect - google.golang.org/grpc v1.49.0 // indirect + google.golang.org/grpc v1.50.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect @@ -171,6 +171,6 @@ require ( replace ( github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 github.com/pkg/sftp => github.com/drakkan/sftp v0.0.0-20220930161944-e8c89afc13a7 - golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20220925175854-fd2128ac8ea8 - golang.org/x/net => github.com/drakkan/net v0.0.0-20220925175748-018cd5f6a745 + golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20221007045301-0163d4cbe5a2 + golang.org/x/net => github.com/drakkan/net v0.0.0-20221007045029-3c39cb455af9 ) diff --git a/go.sum b/go.sum index 1f8a522c..ce4694c7 100644 --- a/go.sum +++ b/go.sum @@ -98,8 +98,8 @@ github.com/Azure/azure-sdk-for-go v66.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9mo github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.3 h1:8LoU8N2lIUzkmstvwXvVfniMZlFbesfT2AmA1aqvRr8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.3/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0/go.mod h1:+6sju8gk8FRmSajX3Oz4G5Gm7P+mbqE9FVaXXFYTkCM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= @@ -144,8 +144,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 h1:KeNholpO2xKjgaaSyd+DyQRrsQjhbSeS7qe4nEw8aQw= @@ -535,12 +535,12 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/drakkan/crypto v0.0.0-20220925175854-fd2128ac8ea8 h1:3VrLY3sbiilkjtMAWeYbM+KyGJiJi18UP2Rif4ZLU1Q= -github.com/drakkan/crypto v0.0.0-20220925175854-fd2128ac8ea8/go.mod h1:SiM6ypd8Xu1xldObYtbDztuUU7xUzMnUULfphXFZmro= +github.com/drakkan/crypto v0.0.0-20221007045301-0163d4cbe5a2 h1:VW04POuNbYg0uLKufFxgr9RnvvNKVc/Mhqq3EUx5tnE= +github.com/drakkan/crypto v0.0.0-20221007045301-0163d4cbe5a2/go.mod h1:SiM6ypd8Xu1xldObYtbDztuUU7xUzMnUULfphXFZmro= github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 h1:LPH1dEblAOO/LoG7yHPMtBLXhQmjaga91/DDjWk9jWA= github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU= -github.com/drakkan/net v0.0.0-20220925175748-018cd5f6a745 h1:Lgtp3izEEJFPxMLJfLWxc1htWNMZhK7EkBo0/dmEX4E= -github.com/drakkan/net v0.0.0-20220925175748-018cd5f6a745/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +github.com/drakkan/net v0.0.0-20221007045029-3c39cb455af9 h1:LuHrwSqNqg+MNxDUgg+5uSoxADH1z/PMU3zKM4V/bqM= +github.com/drakkan/net v0.0.0-20221007045029-3c39cb455af9/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= github.com/drakkan/sftp v0.0.0-20220930161944-e8c89afc13a7 h1:Hj7AAfZ5yt9QuCxSQDllRygmL33xJ2sZLOmcyyOAdYU= github.com/drakkan/sftp v0.0.0-20220930161944-e8c89afc13a7/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -605,8 +605,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= -github.com/go-acme/lego/v4 v4.8.0 h1:XienkuT6ZKHe0DE/LXeGP4ZY+ft+7ZMlqtiJ7XJs2pI= -github.com/go-acme/lego/v4 v4.8.0/go.mod h1:MXCdgHuQh25bfi/tPpyOV/9k2p1JVu6oxXcylAwkouI= +github.com/go-acme/lego/v4 v4.9.0 h1:8Hjj44IqRS7cigshMyFQ+0pIZvwgkG/+9A0UnNh7G8A= +github.com/go-acme/lego/v4 v4.9.0/go.mod h1:g3JRUyWS3L/VObpp4bCxzJftKyf/Wba8QrSSnoiqjg4= github.com/go-chi/chi/v5 v5.0.4/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/chi/v5 v5.0.8-0.20220512131524-9e71a0d4b3d6 h1:+fT7oFUOersdx+u7uIxOjabDVGxg+qqNV6kRdAXIvaQ= github.com/go-chi/chi/v5 v5.0.8-0.20220512131524-9e71a0d4b3d6/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= @@ -1761,8 +1761,8 @@ golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7Lm golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 h1:lxqLZaMad/dJHMFZH0NiNpiEZI/nhgWhe4wgzpE+MuA= -golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1 h1:3VPzK7eqH25j7GYw5w6g/GzNRc0/fYtrxz27z1gD4W0= +golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1916,8 +1916,8 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875 h1:AzgQNqF+FKwyQ5LbVrVqOcuuFB67N47F9+htZYH0wFM= +golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2249,8 +2249,8 @@ google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.0 h1:fPVVDxY9w++VjTZsYvXWqEf9Rqar/e+9zYfxKK+W+YU= +google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/internal/vfs/azblobfs.go b/internal/vfs/azblobfs.go index b3e866bb..d7fec560 100644 --- a/internal/vfs/azblobfs.go +++ b/internal/vfs/azblobfs.go @@ -38,11 +38,11 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/eikenb/pipeat" + "github.com/google/uuid" "github.com/pkg/sftp" "github.com/drakkan/sftpgo/v2/internal/logger" @@ -103,7 +103,7 @@ func NewAzBlobFs(connectionID, localTempDir, mountPath string, config AzBlobFsCo return fs.initFromSASURL() } - credential, err := azblob.NewSharedKeyCredential(fs.config.AccountName, fs.config.AccountKey.GetPayload()) + credential, err := blob.NewSharedKeyCredential(fs.config.AccountName, fs.config.AccountKey.GetPayload()) if err != nil { return fs, fmt.Errorf("invalid credentials: %v", err) } @@ -123,7 +123,7 @@ func NewAzBlobFs(connectionID, localTempDir, mountPath string, config AzBlobFsCo } func (fs *AzureBlobFs) initFromSASURL() (Fs, error) { - parts, err := azblob.ParseURL(fs.config.SASURL.GetPayload()) + parts, err := blob.ParseURL(fs.config.SASURL.GetPayload()) if err != nil { return fs, fmt.Errorf("invalid SAS URL: %w", err) } @@ -937,7 +937,6 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read // we only need to recycle few byte slices pool := newBufferAllocator(int(partSize)) finished := false - binaryBlockID := make([]byte, 8) var blocks []string var wg sync.WaitGroup var errOnce sync.Once @@ -964,13 +963,20 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read return err } - fs.incrementBlockID(binaryBlockID) - blockID := base64.StdEncoding.EncodeToString(binaryBlockID) + // Block IDs are unique values to avoid issue if 2+ clients are uploading blocks + // at the same time causing CommitBlockList to get a mix of blocks from all the clients. + generatedUUID, err := uuid.NewRandom() + if err != nil { + pool.releaseBuffer(buf) + pool.free() + return fmt.Errorf("unable to generate block ID: %w", err) + } + blockID := base64.StdEncoding.EncodeToString([]byte(generatedUUID.String())) blocks = append(blocks, blockID) guard <- struct{}{} if hasError.Load() { - fsLog(fs, logger.LevelError, "pool error, upload for part %v not started", part) + fsLog(fs, logger.LevelError, "pool error, upload for part %d not started", part) pool.releaseBuffer(buf) break } @@ -1042,18 +1048,6 @@ func (*AzureBlobFs) readFill(r io.Reader, buf []byte) (n int, err error) { return n, err } -// copied from rclone -func (*AzureBlobFs) incrementBlockID(blockID []byte) { - for i, digit := range blockID { - newDigit := digit + 1 - blockID[i] = newDigit - if newDigit >= digit { - // exit if no carry - break - } - } -} - func (fs *AzureBlobFs) preserveModificationTime(source, target string, fi os.FileInfo) { if plugin.Handler.HasMetadater() { if !fi.IsDir() {