From f2acde789d66ed2c5e07771474fbce744bf1b9da Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sun, 25 Oct 2020 21:42:43 +0100 Subject: [PATCH] portable mode: add Azure Blob support --- cmd/portable.go | 41 ++++++++++++++++++++++++++++++++++--- docs/portable-mode.md | 21 ++++++++++++++++--- service/service.go | 2 +- service/service_portable.go | 5 ++++- vfs/azblobfs.go | 1 + 5 files changed, 62 insertions(+), 8 deletions(-) diff --git a/cmd/portable.go b/cmd/portable.go index 6b6c8ebb..a22abdbf 100644 --- a/cmd/portable.go +++ b/cmd/portable.go @@ -55,6 +55,15 @@ var ( portableWebDAVPort int portableWebDAVCert string portableWebDAVKey string + portableAzContainer string + portableAzAccountName string + portableAzAccountKey string + portableAzEndpoint string + portableAzSASURL string + portableAzKeyPrefix string + portableAzULPartSize int + portableAzULConcurrency int + portableAzUseEmulator bool portableCmd = &cobra.Command{ Use: "portable", Short: "Serve a single directory", @@ -150,6 +159,17 @@ Please take a look at the usage below to customize the serving parameters`, StorageClass: portableGCSStorageClass, KeyPrefix: portableGCSKeyPrefix, }, + AzBlobConfig: vfs.AzBlobFsConfig{ + Container: portableAzContainer, + AccountName: portableAzAccountName, + AccountKey: portableAzAccountKey, + Endpoint: portableAzEndpoint, + SASURL: portableAzSASURL, + KeyPrefix: portableAzKeyPrefix, + UseEmulator: portableAzUseEmulator, + UploadPartSize: int64(portableAzULPartSize), + UploadConcurrency: portableAzULConcurrency, + }, }, Filters: dataprovider.UserFilters{ FileExtensions: parseFileExtensionsFilters(), @@ -213,9 +233,10 @@ multicast DNS`) advertised via multicast DNS, this flag allows to put username/password inside the advertised TXT record`) - portableCmd.Flags().IntVarP(&portableFsProvider, "fs-provider", "f", int(dataprovider.LocalFilesystemProvider), `0 means local filesystem, -1 Amazon S3 compatible, -2 Google Cloud Storage`) + portableCmd.Flags().IntVarP(&portableFsProvider, "fs-provider", "f", int(dataprovider.LocalFilesystemProvider), `0 => local filesystem +1 => Amazon S3 compatible +2 => Google Cloud Storage +3 => Azure Blob Storage`) portableCmd.Flags().StringVar(&portableS3Bucket, "s3-bucket", "", "") portableCmd.Flags().StringVar(&portableS3Region, "s3-region", "", "") portableCmd.Flags().StringVar(&portableS3AccessKey, "s3-access-key", "", "") @@ -245,6 +266,20 @@ a JSON credentials file, 1 automatic over HTTPS`) portableCmd.Flags().StringVar(&portableWebDAVKey, "webdav-key", "", `Path to the key file for WebDAV over HTTPS`) + portableCmd.Flags().StringVar(&portableAzContainer, "az-container", "", "") + portableCmd.Flags().StringVar(&portableAzAccountName, "az-account-name", "", "") + portableCmd.Flags().StringVar(&portableAzAccountKey, "az-account-key", "", "") + portableCmd.Flags().StringVar(&portableAzSASURL, "az-sas-url", "", `Shared access signature URL`) + portableCmd.Flags().StringVar(&portableAzEndpoint, "az-endpoint", "", `Leave empty to use the default: +"blob.core.windows.net"`) + portableCmd.Flags().StringVar(&portableAzKeyPrefix, "az-key-prefix", "", `Allows to restrict access to the +virtual folder identified by this +prefix and its contents`) + portableCmd.Flags().IntVar(&portableAzULPartSize, "az-upload-part-size", 4, `The buffer size for multipart uploads +(MB)`) + portableCmd.Flags().IntVar(&portableAzULConcurrency, "az-upload-concurrency", 2, `How many parts are uploaded in +parallel`) + portableCmd.Flags().BoolVar(&portableAzUseEmulator, "az-use-emulator", false, "") rootCmd.AddCommand(portableCmd) } diff --git a/docs/portable-mode.md b/docs/portable-mode.md index ce8cf138..238a5dd2 100644 --- a/docs/portable-mode.md +++ b/docs/portable-mode.md @@ -25,6 +25,20 @@ Flags: insensitive. The format is /dir::ext1,ext2. For example: "/somedir::.jpg,.png" + --az-account-key string + --az-account-name string + --az-container string + --az-endpoint string Leave empty to use the default: + "blob.core.windows.net" + --az-key-prefix string Allows to restrict access to the + virtual folder identified by this + prefix and its contents + --az-sas-url string Shared access signature URL + --az-upload-concurrency int How many parts are uploaded in + parallel (default 2) + --az-upload-part-size int The buffer size for multipart uploads + (MB) (default 4) + --az-use-emulator --denied-extensions stringArray Denied file extensions case insensitive. The format is /dir::ext1,ext2. @@ -33,9 +47,10 @@ Flags: This can be an absolute path or a path relative to the current directory (default ".") - -f, --fs-provider int 0 means local filesystem, - 1 Amazon S3 compatible, - 2 Google Cloud Storage + -f, --fs-provider int 0 => local filesystem + 1 => Amazon S3 compatible + 2 => Google Cloud Storage + 3 => Azure Blob Storage --ftpd-cert string Path to the certificate file for FTPS --ftpd-key string Path to the key file for FTPS --ftpd-port int 0 means a random unprivileged port, diff --git a/service/service.go b/service/service.go index 0b13d8d4..c20214fc 100644 --- a/service/service.go +++ b/service/service.go @@ -60,7 +60,7 @@ func (s *Service) Start() error { logger.InitLogger(s.LogFilePath, s.LogMaxSize, s.LogMaxBackups, s.LogMaxAge, s.LogCompress, logLevel) if s.PortableMode == 1 { logger.EnableConsoleLogger(logLevel) - if len(s.LogFilePath) == 0 { + if s.LogFilePath == "" { logger.DisableLogger() } } diff --git a/service/service_portable.go b/service/service_portable.go index 6085b130..0f06af11 100644 --- a/service/service_portable.go +++ b/service/service_portable.go @@ -27,8 +27,11 @@ func (s *Service) StartPortableMode(sftpdPort, ftpPort, webdavPort int, enabledS if s.PortableMode != 1 { return fmt.Errorf("service is not configured for portable mode") } - var err error rand.Seed(time.Now().UnixNano()) + err := config.LoadConfig(s.ConfigDir, s.ConfigFile) + if err != nil { + fmt.Printf("error loading configuration file: %v using defaults\n", err) + } if len(s.PortableUser.Username) == 0 { s.PortableUser.Username = "user" } diff --git a/vfs/azblobfs.go b/vfs/azblobfs.go index 14de0c93..2190485d 100644 --- a/vfs/azblobfs.go +++ b/vfs/azblobfs.go @@ -767,6 +767,7 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read finished = true } else if err != nil { pool.releaseBuffer(buf) + pool.free() return err }