mirror of
https://github.com/drakkan/sftpgo.git
synced 2024-11-24 16:40:26 +00:00
azblob: use UUIDs as block IDs
Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino
parent
80fb56bc48
commit
e617dc9c0a
5 changed files with 1233 additions and 158 deletions
2
.github/workflows/release.yml
vendored
2
.github/workflows/release.yml
vendored
|
|
@ -5,7 +5,7 @@ on:
|
|||
tags: 'v*'
|
||||
|
||||
env:
|
||||
GO_VERSION: 1.18.6
|
||||
GO_VERSION: 1.18.7
|
||||
|
||||
jobs:
|
||||
prepare-sources-with-deps:
|
||||
|
|
|
|||
46
go.mod
46
go.mod
|
|
@ -4,25 +4,25 @@ go 1.18
|
|||
|
||||
require (
|
||||
cloud.google.com/go/storage v1.27.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.3
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1
|
||||
github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
|
||||
github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.16
|
||||
github.com/aws/aws-sdk-go-v2/config v1.17.7
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.12.20
|
||||
github.com/aws/aws-sdk-go-v2/config v1.17.8
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.12.21
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.34
|
||||
github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.19
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11
|
||||
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.16.1
|
||||
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.16.2
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.16.19
|
||||
github.com/cockroachdb/cockroach-go/v2 v2.2.16
|
||||
github.com/coreos/go-oidc/v3 v3.4.0
|
||||
github.com/eikenb/pipeat v0.0.0-20210730190139-06b3e6902001
|
||||
github.com/fclairamb/ftpserverlib v0.20.0
|
||||
github.com/fclairamb/go-log v0.4.1
|
||||
github.com/go-acme/lego/v4 v4.8.0
|
||||
github.com/go-acme/lego/v4 v4.9.0
|
||||
github.com/go-chi/chi/v5 v5.0.8-0.20220512131524-9e71a0d4b3d6
|
||||
github.com/go-chi/jwtauth/v5 v5.0.2
|
||||
github.com/go-chi/render v1.0.2
|
||||
|
|
@ -35,7 +35,7 @@ require (
|
|||
github.com/hashicorp/go-plugin v1.4.5
|
||||
github.com/hashicorp/go-retryablehttp v0.7.1
|
||||
github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126
|
||||
github.com/klauspost/compress v1.15.10
|
||||
github.com/klauspost/compress v1.15.11
|
||||
github.com/lestrrat-go/jwx v1.2.25
|
||||
github.com/lib/pq v1.10.7
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7
|
||||
|
|
@ -52,7 +52,7 @@ require (
|
|||
github.com/rs/xid v1.4.0
|
||||
github.com/rs/zerolog v1.28.0
|
||||
github.com/sftpgo/sdk v0.1.1
|
||||
github.com/shirou/gopsutil/v3 v3.22.8
|
||||
github.com/shirou/gopsutil/v3 v3.22.9
|
||||
github.com/spf13/afero v1.9.2
|
||||
github.com/spf13/cobra v1.5.0
|
||||
github.com/spf13/viper v1.13.0
|
||||
|
|
@ -60,24 +60,24 @@ require (
|
|||
github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62
|
||||
github.com/unrolled/secure v1.13.0
|
||||
github.com/wagslane/go-password-validator v0.3.0
|
||||
github.com/xhit/go-simple-mail/v2 v2.11.0
|
||||
github.com/xhit/go-simple-mail/v2 v2.12.0
|
||||
github.com/yl2chen/cidranger v1.0.3-0.20210928021809-d1cb2c52f37a
|
||||
go.etcd.io/bbolt v1.3.6
|
||||
go.uber.org/automaxprocs v1.5.1
|
||||
gocloud.dev v0.26.0
|
||||
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90
|
||||
golang.org/x/net v0.0.0-20220909164309-bea034e7d591
|
||||
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1
|
||||
golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8
|
||||
gocloud.dev v0.27.0
|
||||
golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b
|
||||
golang.org/x/net v0.0.0-20221004154528-8021a29435af
|
||||
golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1
|
||||
golang.org/x/sys v0.0.0-20221006211917-84dc82d7e875
|
||||
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af
|
||||
google.golang.org/api v0.97.0
|
||||
google.golang.org/api v0.98.0
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.104.0 // indirect
|
||||
cloud.google.com/go/compute v1.10.0 // indirect
|
||||
cloud.google.com/go/iam v0.4.0 // indirect
|
||||
cloud.google.com/go/iam v0.5.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect
|
||||
github.com/ajg/form v1.5.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 // indirect
|
||||
|
|
@ -90,7 +90,7 @@ require (
|
|||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6 // indirect
|
||||
github.com/aws/smithy-go v1.13.3 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/boombuler/barcode v1.0.1 // indirect
|
||||
|
|
@ -109,7 +109,7 @@ require (
|
|||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/google/go-cmp v0.5.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.5.1 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
|
|
@ -127,7 +127,7 @@ require (
|
|||
github.com/magiconair/properties v1.8.6 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.16 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
|
||||
github.com/miekg/dns v1.1.50 // indirect
|
||||
github.com/minio/sha256-simd v1.0.0 // indirect
|
||||
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
|
||||
|
|
@ -156,8 +156,8 @@ require (
|
|||
golang.org/x/tools v0.1.12 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20220921223823-23cae91e6737 // indirect
|
||||
google.golang.org/grpc v1.49.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20220930163606-c98284e70a91 // indirect
|
||||
google.golang.org/grpc v1.50.0 // indirect
|
||||
google.golang.org/protobuf v1.28.1 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
|
||||
|
|
@ -167,6 +167,6 @@ require (
|
|||
|
||||
replace (
|
||||
github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9
|
||||
golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20220831070616-b69bc2ec2993
|
||||
golang.org/x/net => github.com/drakkan/net v0.0.0-20220916171046-9879e3f5b22a
|
||||
golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20221007053035-8171b0eebd64
|
||||
golang.org/x/net => github.com/drakkan/net v0.0.0-20221007053106-916f307e3230
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
NFPM_VERSION=2.18.1
|
||||
NFPM_VERSION=2.19.2
|
||||
NFPM_ARCH=${NFPM_ARCH:-amd64}
|
||||
if [ -z ${SFTPGO_VERSION} ]
|
||||
then
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ import (
|
|||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
"github.com/eikenb/pipeat"
|
||||
"github.com/google/uuid"
|
||||
"github.com/pkg/sftp"
|
||||
|
||||
"github.com/drakkan/sftpgo/v2/logger"
|
||||
|
|
@ -985,7 +986,6 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read
|
|||
// we only need to recycle few byte slices
|
||||
pool := newBufferAllocator(int(partSize))
|
||||
finished := false
|
||||
binaryBlockID := make([]byte, 8)
|
||||
var blocks []string
|
||||
var wg sync.WaitGroup
|
||||
var errOnce sync.Once
|
||||
|
|
@ -1012,8 +1012,15 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read
|
|||
return err
|
||||
}
|
||||
|
||||
fs.incrementBlockID(binaryBlockID)
|
||||
blockID := base64.StdEncoding.EncodeToString(binaryBlockID)
|
||||
// Block IDs are unique values to avoid issue if 2+ clients are uploading blocks
|
||||
// at the same time causing CommitBlockList to get a mix of blocks from all the clients.
|
||||
generatedUUID, err := uuid.NewRandom()
|
||||
if err != nil {
|
||||
pool.releaseBuffer(buf)
|
||||
pool.free()
|
||||
return fmt.Errorf("unable to generate block ID: %w", err)
|
||||
}
|
||||
blockID := base64.StdEncoding.EncodeToString([]byte(generatedUUID.String()))
|
||||
blocks = append(blocks, blockID)
|
||||
|
||||
guard <- struct{}{}
|
||||
|
|
@ -1090,18 +1097,6 @@ func (*AzureBlobFs) readFill(r io.Reader, buf []byte) (n int, err error) {
|
|||
return n, err
|
||||
}
|
||||
|
||||
// copied from rclone
|
||||
func (*AzureBlobFs) incrementBlockID(blockID []byte) {
|
||||
for i, digit := range blockID {
|
||||
newDigit := digit + 1
|
||||
blockID[i] = newDigit
|
||||
if newDigit >= digit {
|
||||
// exit if no carry
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (fs *AzureBlobFs) preserveModificationTime(source, target string, fi os.FileInfo) {
|
||||
if plugin.Handler.HasMetadater() {
|
||||
if !fi.IsDir() {
|
||||
|
|
|
|||
Loading…
Reference in a new issue