beenull/sftpgo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

azblob: use UUIDs as block IDs

Browse source 
Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2022-10-07 07:38:12 +02:00
parent 80fb56bc48
commit e617dc9c0a
No known key found for this signature in database
GPG key ID: 935D2952DEC4EECF
5 changed files with 1233 additions and 158 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/release.yml vendored
View file

@ -5,7 +5,7 @@ on:
tags: 'v*' tags: 'v*'
env: env:
GO_VERSION: 1.18.6 GO_VERSION: 1.18.7
jobs: jobs:
prepare-sources-with-deps: prepare-sources-with-deps:

46
go.mod
View file

@ -4,25 +4,25 @@ go 1.18
require ( require (
cloud.google.com/go/storage v1.27.0 cloud.google.com/go/storage v1.27.0
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.3 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1
github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387 github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387
github.com/aws/aws-sdk-go-v2 v1.16.16 github.com/aws/aws-sdk-go-v2 v1.16.16
github.com/aws/aws-sdk-go-v2/config v1.17.7 github.com/aws/aws-sdk-go-v2/config v1.17.8
github.com/aws/aws-sdk-go-v2/credentials v1.12.20 github.com/aws/aws-sdk-go-v2/credentials v1.12.21
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.34
github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.19 github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.19
github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11 github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.16.1 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.16.2
github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 github.com/aws/aws-sdk-go-v2/service/sts v1.16.19
github.com/cockroachdb/cockroach-go/v2 v2.2.16 github.com/cockroachdb/cockroach-go/v2 v2.2.16
github.com/coreos/go-oidc/v3 v3.4.0 github.com/coreos/go-oidc/v3 v3.4.0
github.com/eikenb/pipeat v0.0.0-20210730190139-06b3e6902001 github.com/eikenb/pipeat v0.0.0-20210730190139-06b3e6902001
github.com/fclairamb/ftpserverlib v0.20.0 github.com/fclairamb/ftpserverlib v0.20.0
github.com/fclairamb/go-log v0.4.1 github.com/fclairamb/go-log v0.4.1
github.com/go-acme/lego/v4 v4.8.0 github.com/go-acme/lego/v4 v4.9.0
github.com/go-chi/chi/v5 v5.0.8-0.20220512131524-9e71a0d4b3d6 github.com/go-chi/chi/v5 v5.0.8-0.20220512131524-9e71a0d4b3d6
github.com/go-chi/jwtauth/v5 v5.0.2 github.com/go-chi/jwtauth/v5 v5.0.2
github.com/go-chi/render v1.0.2 github.com/go-chi/render v1.0.2
@ -35,7 +35,7 @@ require (
github.com/hashicorp/go-plugin v1.4.5 github.com/hashicorp/go-plugin v1.4.5
github.com/hashicorp/go-retryablehttp v0.7.1 github.com/hashicorp/go-retryablehttp v0.7.1
github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126 github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126
github.com/klauspost/compress v1.15.10 github.com/klauspost/compress v1.15.11
github.com/lestrrat-go/jwx v1.2.25 github.com/lestrrat-go/jwx v1.2.25
github.com/lib/pq v1.10.7 github.com/lib/pq v1.10.7
github.com/lithammer/shortuuid/v3 v3.0.7 github.com/lithammer/shortuuid/v3 v3.0.7
@ -52,7 +52,7 @@ require (
github.com/rs/xid v1.4.0 github.com/rs/xid v1.4.0
github.com/rs/zerolog v1.28.0 github.com/rs/zerolog v1.28.0
github.com/sftpgo/sdk v0.1.1 github.com/sftpgo/sdk v0.1.1
github.com/shirou/gopsutil/v3 v3.22.8 github.com/shirou/gopsutil/v3 v3.22.9
github.com/spf13/afero v1.9.2 github.com/spf13/afero v1.9.2
github.com/spf13/cobra v1.5.0 github.com/spf13/cobra v1.5.0
github.com/spf13/viper v1.13.0 github.com/spf13/viper v1.13.0
@ -60,24 +60,24 @@ require (
github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62
github.com/unrolled/secure v1.13.0 github.com/unrolled/secure v1.13.0
github.com/wagslane/go-password-validator v0.3.0 github.com/wagslane/go-password-validator v0.3.0
github.com/xhit/go-simple-mail/v2 v2.11.0 github.com/xhit/go-simple-mail/v2 v2.12.0
github.com/yl2chen/cidranger v1.0.3-0.20210928021809-d1cb2c52f37a github.com/yl2chen/cidranger v1.0.3-0.20210928021809-d1cb2c52f37a
go.etcd.io/bbolt v1.3.6 go.etcd.io/bbolt v1.3.6
go.uber.org/automaxprocs v1.5.1 go.uber.org/automaxprocs v1.5.1
gocloud.dev v0.26.0 gocloud.dev v0.27.0
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b
golang.org/x/net v0.0.0-20220909164309-bea034e7d591 golang.org/x/net v0.0.0-20221004154528-8021a29435af
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1
golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af golang.org/x/time v0.0.0-20220922220347-f3bd1da661af
google.golang.org/api v0.97.0 google.golang.org/api v0.98.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0
) )
require ( require (
cloud.google.com/go v0.104.0 // indirect cloud.google.com/go v0.104.0 // indirect
cloud.google.com/go/compute v1.10.0 // indirect cloud.google.com/go/compute v1.10.0 // indirect
cloud.google.com/go/iam v0.4.0 // indirect cloud.google.com/go/iam v0.5.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect
github.com/ajg/form v1.5.1 // indirect github.com/ajg/form v1.5.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 // indirect
@ -90,7 +90,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6 // indirect
github.com/aws/smithy-go v1.13.3 // indirect github.com/aws/smithy-go v1.13.3 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/boombuler/barcode v1.0.1 // indirect github.com/boombuler/barcode v1.0.1 // indirect
@ -109,7 +109,7 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-cmp v0.5.9 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
github.com/googleapis/gax-go/v2 v2.5.1 // indirect github.com/googleapis/gax-go/v2 v2.5.1 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect
@ -127,7 +127,7 @@ require (
github.com/magiconair/properties v1.8.6 // indirect github.com/magiconair/properties v1.8.6 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-isatty v0.0.16 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
github.com/miekg/dns v1.1.50 // indirect github.com/miekg/dns v1.1.50 // indirect
github.com/minio/sha256-simd v1.0.0 // indirect github.com/minio/sha256-simd v1.0.0 // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect
@ -156,8 +156,8 @@ require (
golang.org/x/tools v0.1.12 // indirect golang.org/x/tools v0.1.12 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220921223823-23cae91e6737 // indirect google.golang.org/genproto v0.0.0-20220930163606-c98284e70a91 // indirect
google.golang.org/grpc v1.49.0 // indirect google.golang.org/grpc v1.50.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect
@ -167,6 +167,6 @@ require (
replace ( replace (
github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9
golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20220831070616-b69bc2ec2993 golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20221007053035-8171b0eebd64
golang.org/x/net => github.com/drakkan/net v0.0.0-20220916171046-9879e3f5b22a golang.org/x/net => github.com/drakkan/net v0.0.0-20221007053106-916f307e3230
) )

1316
go.sum
View file

File diff suppressed because it is too large Load diff

2
pkgs/build.sh
View file

@ -1,6 +1,6 @@
#!/bin/bash #!/bin/bash
NFPM_VERSION=2.18.1 NFPM_VERSION=2.19.2
NFPM_ARCH=${NFPM_ARCH:-amd64} NFPM_ARCH=${NFPM_ARCH:-amd64}
if [ -z ${SFTPGO_VERSION} ] if [ -z ${SFTPGO_VERSION} ]
then then

25
vfs/azblobfs.go
View file

@ -38,6 +38,7 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/eikenb/pipeat" "github.com/eikenb/pipeat"
"github.com/google/uuid"
"github.com/pkg/sftp" "github.com/pkg/sftp"
"github.com/drakkan/sftpgo/v2/logger" "github.com/drakkan/sftpgo/v2/logger"
@ -985,7 +986,6 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read
// we only need to recycle few byte slices // we only need to recycle few byte slices
pool := newBufferAllocator(int(partSize)) pool := newBufferAllocator(int(partSize))
finished := false finished := false
binaryBlockID := make([]byte, 8)
var blocks []string var blocks []string
var wg sync.WaitGroup var wg sync.WaitGroup
var errOnce sync.Once var errOnce sync.Once
@ -1012,8 +1012,15 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read
return err return err
} }
fs.incrementBlockID(binaryBlockID) // Block IDs are unique values to avoid issue if 2+ clients are uploading blocks
blockID := base64.StdEncoding.EncodeToString(binaryBlockID) // at the same time causing CommitBlockList to get a mix of blocks from all the clients.
generatedUUID, err := uuid.NewRandom()
if err != nil {
pool.releaseBuffer(buf)
pool.free()
return fmt.Errorf("unable to generate block ID: %w", err)
}
blockID := base64.StdEncoding.EncodeToString([]byte(generatedUUID.String()))
blocks = append(blocks, blockID) blocks = append(blocks, blockID)
guard <- struct{}{} guard <- struct{}{}
@ -1090,18 +1097,6 @@ func (*AzureBlobFs) readFill(r io.Reader, buf []byte) (n int, err error) {
return n, err return n, err
} }
// copied from rclone
func (*AzureBlobFs) incrementBlockID(blockID []byte) {
for i, digit := range blockID {
newDigit := digit + 1
blockID[i] = newDigit
if newDigit >= digit {
// exit if no carry
break
}
}
}
func (fs *AzureBlobFs) preserveModificationTime(source, target string, fi os.FileInfo) { func (fs *AzureBlobFs) preserveModificationTime(source, target string, fi os.FileInfo) {
if plugin.Handler.HasMetadater() { if plugin.Handler.HasMetadater() {
if !fi.IsDir() { if !fi.IsDir() {