|
|
@@ -0,0 +1,724 @@
|
|
|
+// +build !noazblob
|
|
|
+
|
|
|
+package vfs
|
|
|
+
|
|
|
+import (
|
|
|
+ "context"
|
|
|
+ "errors"
|
|
|
+ "fmt"
|
|
|
+ "io"
|
|
|
+ "mime"
|
|
|
+ "net/http"
|
|
|
+ "net/url"
|
|
|
+ "os"
|
|
|
+ "path"
|
|
|
+ "path/filepath"
|
|
|
+ "strings"
|
|
|
+ "time"
|
|
|
+
|
|
|
+ "github.com/Azure/azure-storage-blob-go/azblob"
|
|
|
+ "github.com/eikenb/pipeat"
|
|
|
+
|
|
|
+ "github.com/drakkan/sftpgo/logger"
|
|
|
+ "github.com/drakkan/sftpgo/utils"
|
|
|
+ "github.com/drakkan/sftpgo/version"
|
|
|
+)
|
|
|
+
|
|
|
+const azureDefaultEndpoint = "blob.core.windows.net"
|
|
|
+
|
|
|
+// max time of an azure web request response window (whether or not data is flowing)
|
|
|
+// this is the same value used in rclone
|
|
|
+var maxTryTimeout = time.Hour * 24 * 365
|
|
|
+
|
|
|
+// AzureBlobFs is a Fs implementation for Azure Blob storage.
|
|
|
+type AzureBlobFs struct {
|
|
|
+ connectionID string
|
|
|
+ localTempDir string
|
|
|
+ config AzBlobFsConfig
|
|
|
+ svc *azblob.ServiceURL
|
|
|
+ containerURL azblob.ContainerURL
|
|
|
+ ctxTimeout time.Duration
|
|
|
+ ctxLongTimeout time.Duration
|
|
|
+}
|
|
|
+
|
|
|
+func init() {
|
|
|
+ version.AddFeature("+azblob")
|
|
|
+}
|
|
|
+
|
|
|
+// NewAzBlobFs returns an AzBlobFs object that allows to interact with Azure Blob storage
|
|
|
+func NewAzBlobFs(connectionID, localTempDir string, config AzBlobFsConfig) (Fs, error) {
|
|
|
+ fs := AzureBlobFs{
|
|
|
+ connectionID: connectionID,
|
|
|
+ localTempDir: localTempDir,
|
|
|
+ config: config,
|
|
|
+ ctxTimeout: 30 * time.Second,
|
|
|
+ ctxLongTimeout: 300 * time.Second,
|
|
|
+ }
|
|
|
+ if err := ValidateAzBlobFsConfig(&fs.config); err != nil {
|
|
|
+ return fs, err
|
|
|
+ }
|
|
|
+ if fs.config.AccountKey != "" {
|
|
|
+ accountKey, err := utils.DecryptData(fs.config.AccountKey)
|
|
|
+ if err != nil {
|
|
|
+ return fs, err
|
|
|
+ }
|
|
|
+ fs.config.AccountKey = accountKey
|
|
|
+ }
|
|
|
+ setConfigDefaults(&fs)
|
|
|
+
|
|
|
+ if fs.config.SASURL != "" {
|
|
|
+ u, err := url.Parse(fs.config.SASURL)
|
|
|
+ if err != nil {
|
|
|
+ return fs, fmt.Errorf("invalid credentials: %v", err)
|
|
|
+ }
|
|
|
+ pipeline := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{
|
|
|
+ Retry: azblob.RetryOptions{
|
|
|
+ TryTimeout: maxTryTimeout,
|
|
|
+ },
|
|
|
+ Telemetry: azblob.TelemetryOptions{
|
|
|
+ Value: "SFTPGo",
|
|
|
+ },
|
|
|
+ })
|
|
|
+ // Check if we have container level SAS or account level SAS
|
|
|
+ parts := azblob.NewBlobURLParts(*u)
|
|
|
+ if parts.ContainerName != "" {
|
|
|
+ if fs.config.Container != "" && fs.config.Container != parts.ContainerName {
|
|
|
+ return fs, fmt.Errorf("Container name in SAS URL %#v and container provided %#v do not match",
|
|
|
+ parts.ContainerName, fs.config.Container)
|
|
|
+ }
|
|
|
+ fs.svc = nil
|
|
|
+ fs.containerURL = azblob.NewContainerURL(*u, pipeline)
|
|
|
+ } else {
|
|
|
+ if fs.config.Container == "" {
|
|
|
+ return fs, errors.New("container is required with this SAS URL")
|
|
|
+ }
|
|
|
+ serviceURL := azblob.NewServiceURL(*u, pipeline)
|
|
|
+ fs.svc = &serviceURL
|
|
|
+ fs.containerURL = fs.svc.NewContainerURL(fs.config.Container)
|
|
|
+ }
|
|
|
+ return fs, nil
|
|
|
+ }
|
|
|
+
|
|
|
+ credential, err := azblob.NewSharedKeyCredential(fs.config.AccountName, fs.config.AccountKey)
|
|
|
+ if err != nil {
|
|
|
+ return fs, fmt.Errorf("invalid credentials: %v", err)
|
|
|
+ }
|
|
|
+ var u *url.URL
|
|
|
+ if fs.config.UseEmulator {
|
|
|
+ // for the emulator we expect the endpoint prefixed with the protocol, for example:
|
|
|
+ // http://127.0.0.1:10000
|
|
|
+ u, err = url.Parse(fmt.Sprintf("%s/%s", fs.config.Endpoint, fs.config.AccountName))
|
|
|
+ } else {
|
|
|
+ u, err = url.Parse(fmt.Sprintf("https://%s.%s", fs.config.AccountName, fs.config.Endpoint))
|
|
|
+ }
|
|
|
+ if err != nil {
|
|
|
+ return fs, fmt.Errorf("invalid credentials: %v", err)
|
|
|
+ }
|
|
|
+ pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{
|
|
|
+ Retry: azblob.RetryOptions{
|
|
|
+ TryTimeout: maxTryTimeout,
|
|
|
+ },
|
|
|
+ Telemetry: azblob.TelemetryOptions{
|
|
|
+ Value: "SFTPGo",
|
|
|
+ },
|
|
|
+ })
|
|
|
+ serviceURL := azblob.NewServiceURL(*u, pipeline)
|
|
|
+ fs.svc = &serviceURL
|
|
|
+ fs.containerURL = fs.svc.NewContainerURL(fs.config.Container)
|
|
|
+ return fs, nil
|
|
|
+}
|
|
|
+
|
|
|
+func setConfigDefaults(fs *AzureBlobFs) {
|
|
|
+ if fs.config.Endpoint == "" {
|
|
|
+ fs.config.Endpoint = azureDefaultEndpoint
|
|
|
+ }
|
|
|
+ if fs.config.UploadPartSize == 0 {
|
|
|
+ fs.config.UploadPartSize = 4
|
|
|
+ }
|
|
|
+ fs.config.UploadPartSize *= 1024 * 1024
|
|
|
+ if fs.config.UploadConcurrency == 0 {
|
|
|
+ fs.config.UploadConcurrency = 2
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+// Name returns the name for the Fs implementation
|
|
|
+func (fs AzureBlobFs) Name() string {
|
|
|
+ if fs.config.SASURL != "" {
|
|
|
+ return fmt.Sprintf("Azure Blob SAS URL %#v", fs.config.Container)
|
|
|
+ }
|
|
|
+ return fmt.Sprintf("Azure Blob container %#v", fs.config.Container)
|
|
|
+}
|
|
|
+
|
|
|
+// ConnectionID returns the connection ID associated to this Fs implementation
|
|
|
+func (fs AzureBlobFs) ConnectionID() string {
|
|
|
+ return fs.connectionID
|
|
|
+}
|
|
|
+
|
|
|
+// Stat returns a FileInfo describing the named file
|
|
|
+func (fs AzureBlobFs) Stat(name string) (os.FileInfo, error) {
|
|
|
+ if name == "" || name == "." {
|
|
|
+ err := fs.checkIfBucketExists()
|
|
|
+ if err != nil {
|
|
|
+ return nil, err
|
|
|
+ }
|
|
|
+ return NewFileInfo(name, true, 0, time.Now(), false), nil
|
|
|
+ }
|
|
|
+ if fs.config.KeyPrefix == name+"/" {
|
|
|
+ return NewFileInfo(name, true, 0, time.Now(), false), nil
|
|
|
+ }
|
|
|
+ prefix := fs.getPrefixForStat(name)
|
|
|
+ ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
|
|
|
+ defer cancelFn()
|
|
|
+
|
|
|
+ for marker := (azblob.Marker{}); marker.NotDone(); {
|
|
|
+ listBlob, err := fs.containerURL.ListBlobsHierarchySegment(ctx, marker, "/", azblob.ListBlobsSegmentOptions{
|
|
|
+ Details: azblob.BlobListingDetails{
|
|
|
+ Copy: false,
|
|
|
+ Metadata: false,
|
|
|
+ Snapshots: false,
|
|
|
+ UncommittedBlobs: false,
|
|
|
+ Deleted: false,
|
|
|
+ },
|
|
|
+ Prefix: prefix,
|
|
|
+ })
|
|
|
+ if err != nil {
|
|
|
+ return nil, err
|
|
|
+ }
|
|
|
+ marker = listBlob.NextMarker
|
|
|
+ for _, blobPrefix := range listBlob.Segment.BlobPrefixes {
|
|
|
+ if fs.isEqual(blobPrefix.Name, name) {
|
|
|
+ return NewFileInfo(name, true, 0, time.Now(), false), nil
|
|
|
+ }
|
|
|
+ }
|
|
|
+ for _, blobInfo := range listBlob.Segment.BlobItems {
|
|
|
+ if fs.isEqual(blobInfo.Name, name) {
|
|
|
+ isDir := false
|
|
|
+ if blobInfo.Properties.ContentType != nil {
|
|
|
+ isDir = (*blobInfo.Properties.ContentType == dirMimeType)
|
|
|
+ }
|
|
|
+ size := int64(0)
|
|
|
+ if blobInfo.Properties.ContentLength != nil {
|
|
|
+ size = *blobInfo.Properties.ContentLength
|
|
|
+ }
|
|
|
+ return NewFileInfo(name, isDir, size, blobInfo.Properties.LastModified, false), nil
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return nil, errors.New("404 no such file or directory")
|
|
|
+}
|
|
|
+
|
|
|
+// Lstat returns a FileInfo describing the named file
|
|
|
+func (fs AzureBlobFs) Lstat(name string) (os.FileInfo, error) {
|
|
|
+ return fs.Stat(name)
|
|
|
+}
|
|
|
+
|
|
|
+// Open opens the named file for reading
|
|
|
+func (fs AzureBlobFs) Open(name string, offset int64) (*os.File, *pipeat.PipeReaderAt, func(), error) {
|
|
|
+ r, w, err := pipeat.PipeInDir(fs.localTempDir)
|
|
|
+ if err != nil {
|
|
|
+ return nil, nil, nil, err
|
|
|
+ }
|
|
|
+ blobBlockURL := fs.containerURL.NewBlockBlobURL(name)
|
|
|
+ ctx, cancelFn := context.WithCancel(context.Background())
|
|
|
+ blobDownloadResponse, err := blobBlockURL.Download(ctx, offset, azblob.CountToEnd, azblob.BlobAccessConditions{}, false)
|
|
|
+ if err != nil {
|
|
|
+ r.Close()
|
|
|
+ w.Close()
|
|
|
+ cancelFn()
|
|
|
+ return nil, nil, nil, err
|
|
|
+ }
|
|
|
+ body := blobDownloadResponse.Body(azblob.RetryReaderOptions{
|
|
|
+ MaxRetryRequests: 3,
|
|
|
+ })
|
|
|
+
|
|
|
+ go func() {
|
|
|
+ defer cancelFn()
|
|
|
+ defer body.Close()
|
|
|
+
|
|
|
+ n, err := io.Copy(w, body)
|
|
|
+ w.CloseWithError(err) //nolint:errcheck
|
|
|
+ fsLog(fs, logger.LevelDebug, "download completed, path: %#v size: %v, err: %v", name, n, err)
|
|
|
+ }()
|
|
|
+
|
|
|
+ return nil, r, cancelFn, nil
|
|
|
+}
|
|
|
+
|
|
|
+// Create creates or opens the named file for writing
|
|
|
+func (fs AzureBlobFs) Create(name string, flag int) (*os.File, *PipeWriter, func(), error) {
|
|
|
+ r, w, err := pipeat.PipeInDir(fs.localTempDir)
|
|
|
+ if err != nil {
|
|
|
+ return nil, nil, nil, err
|
|
|
+ }
|
|
|
+ p := NewPipeWriter(w)
|
|
|
+ blobBlockURL := fs.containerURL.NewBlockBlobURL(name)
|
|
|
+ ctx, cancelFn := context.WithCancel(context.Background())
|
|
|
+
|
|
|
+ headers := azblob.BlobHTTPHeaders{}
|
|
|
+ var contentType string
|
|
|
+ if flag == -1 {
|
|
|
+ contentType = dirMimeType
|
|
|
+ } else {
|
|
|
+ contentType = mime.TypeByExtension(path.Ext(name))
|
|
|
+ }
|
|
|
+ if contentType != "" {
|
|
|
+ headers.ContentType = contentType
|
|
|
+ }
|
|
|
+
|
|
|
+ go func() {
|
|
|
+ defer cancelFn()
|
|
|
+
|
|
|
+ uploadOptions := azblob.UploadStreamToBlockBlobOptions{
|
|
|
+ BufferSize: int(fs.config.UploadPartSize),
|
|
|
+ BlobHTTPHeaders: headers,
|
|
|
+ MaxBuffers: fs.config.UploadConcurrency,
|
|
|
+ }
|
|
|
+ _, err := azblob.UploadStreamToBlockBlob(ctx, r, blobBlockURL, uploadOptions)
|
|
|
+ r.CloseWithError(err) //nolint:errcheck
|
|
|
+ p.Done(err)
|
|
|
+ fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, readed bytes: %v, err: %v", name, r.GetReadedBytes(), err)
|
|
|
+ }()
|
|
|
+
|
|
|
+ return nil, p, cancelFn, nil
|
|
|
+}
|
|
|
+
|
|
|
+// Rename renames (moves) source to target.
|
|
|
+// We don't support renaming non empty directories since we should
|
|
|
+// rename all the contents too and this could take long time: think
|
|
|
+// about directories with thousands of files, for each file we should
|
|
|
+// execute a StartCopyFromURL call.
|
|
|
+func (fs AzureBlobFs) Rename(source, target string) error {
|
|
|
+ if source == target {
|
|
|
+ return nil
|
|
|
+ }
|
|
|
+ fi, err := fs.Stat(source)
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if fi.IsDir() {
|
|
|
+ contents, err := fs.ReadDir(source)
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if len(contents) > 0 {
|
|
|
+ return fmt.Errorf("Cannot rename non empty directory: %#v", source)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ dstBlobURL := fs.containerURL.NewBlobURL(target)
|
|
|
+ srcURL := fs.containerURL.NewBlobURL(source).URL()
|
|
|
+
|
|
|
+ md := azblob.Metadata{}
|
|
|
+ mac := azblob.ModifiedAccessConditions{}
|
|
|
+ bac := azblob.BlobAccessConditions{}
|
|
|
+ ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
|
|
|
+ defer cancelFn()
|
|
|
+
|
|
|
+ resp, err := dstBlobURL.StartCopyFromURL(ctx, srcURL, md, mac, bac)
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ copyStatus := resp.CopyStatus()
|
|
|
+ nErrors := 0
|
|
|
+ for copyStatus == azblob.CopyStatusPending {
|
|
|
+ // Poll until the copy is complete.
|
|
|
+ time.Sleep(500 * time.Millisecond)
|
|
|
+ propertiesResp, err := dstBlobURL.GetProperties(ctx, azblob.BlobAccessConditions{})
|
|
|
+ if err != nil {
|
|
|
+ // A GetProperties failure may be transient, so allow a couple
|
|
|
+ // of them before giving up.
|
|
|
+ nErrors++
|
|
|
+ if ctx.Err() != nil || nErrors == 3 {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ copyStatus = propertiesResp.CopyStatus()
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if copyStatus != azblob.CopyStatusSuccess {
|
|
|
+ return fmt.Errorf("Copy failed with status: %s", copyStatus)
|
|
|
+ }
|
|
|
+ return fs.Remove(source, fi.IsDir())
|
|
|
+}
|
|
|
+
|
|
|
+// Remove removes the named file or (empty) directory.
|
|
|
+func (fs AzureBlobFs) Remove(name string, isDir bool) error {
|
|
|
+ if isDir {
|
|
|
+ contents, err := fs.ReadDir(name)
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if len(contents) > 0 {
|
|
|
+ return fmt.Errorf("Cannot remove non empty directory: %#v", name)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ blobBlockURL := fs.containerURL.NewBlockBlobURL(name)
|
|
|
+ ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
|
|
|
+ defer cancelFn()
|
|
|
+
|
|
|
+ _, err := blobBlockURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
|
|
|
+ return err
|
|
|
+}
|
|
|
+
|
|
|
+// Mkdir creates a new directory with the specified name and default permissions
|
|
|
+func (fs AzureBlobFs) Mkdir(name string) error {
|
|
|
+ _, err := fs.Stat(name)
|
|
|
+ if !fs.IsNotExist(err) {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ _, w, _, err := fs.Create(name, -1)
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ return w.Close()
|
|
|
+}
|
|
|
+
|
|
|
+// Symlink creates source as a symbolic link to target.
|
|
|
+func (AzureBlobFs) Symlink(source, target string) error {
|
|
|
+ return errors.New("403 symlinks are not supported")
|
|
|
+}
|
|
|
+
|
|
|
+// Readlink returns the destination of the named symbolic link
|
|
|
+func (AzureBlobFs) Readlink(name string) (string, error) {
|
|
|
+ return "", errors.New("403 readlink is not supported")
|
|
|
+}
|
|
|
+
|
|
|
+// Chown changes the numeric uid and gid of the named file.
|
|
|
+// Silently ignored.
|
|
|
+func (AzureBlobFs) Chown(name string, uid int, gid int) error {
|
|
|
+ return nil
|
|
|
+}
|
|
|
+
|
|
|
+// Chmod changes the mode of the named file to mode.
|
|
|
+// Silently ignored.
|
|
|
+func (AzureBlobFs) Chmod(name string, mode os.FileMode) error {
|
|
|
+ return nil
|
|
|
+}
|
|
|
+
|
|
|
+// Chtimes changes the access and modification times of the named file.
|
|
|
+// Silently ignored.
|
|
|
+func (AzureBlobFs) Chtimes(name string, atime, mtime time.Time) error {
|
|
|
+ return errors.New("403 chtimes is not supported")
|
|
|
+}
|
|
|
+
|
|
|
+// Truncate changes the size of the named file.
|
|
|
+// Truncate by path is not supported, while truncating an opened
|
|
|
+// file is handled inside base transfer
|
|
|
+func (AzureBlobFs) Truncate(name string, size int64) error {
|
|
|
+ return errors.New("403 truncate is not supported")
|
|
|
+}
|
|
|
+
|
|
|
+// ReadDir reads the directory named by dirname and returns
|
|
|
+// a list of directory entries.
|
|
|
+func (fs AzureBlobFs) ReadDir(dirname string) ([]os.FileInfo, error) {
|
|
|
+ var result []os.FileInfo
|
|
|
+ // dirname must be already cleaned
|
|
|
+ prefix := ""
|
|
|
+ if dirname != "" && dirname != "." {
|
|
|
+ prefix = strings.TrimPrefix(dirname, "/")
|
|
|
+ if !strings.HasSuffix(prefix, "/") {
|
|
|
+ prefix += "/"
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ prefixes := make(map[string]bool)
|
|
|
+
|
|
|
+ for marker := (azblob.Marker{}); marker.NotDone(); {
|
|
|
+ ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
|
|
|
+ defer cancelFn()
|
|
|
+
|
|
|
+ listBlob, err := fs.containerURL.ListBlobsHierarchySegment(ctx, marker, "/", azblob.ListBlobsSegmentOptions{
|
|
|
+ Details: azblob.BlobListingDetails{
|
|
|
+ Copy: false,
|
|
|
+ Metadata: false,
|
|
|
+ Snapshots: false,
|
|
|
+ UncommittedBlobs: false,
|
|
|
+ Deleted: false,
|
|
|
+ },
|
|
|
+ Prefix: prefix,
|
|
|
+ })
|
|
|
+ if err != nil {
|
|
|
+ return nil, err
|
|
|
+ }
|
|
|
+ marker = listBlob.NextMarker
|
|
|
+ for _, blobPrefix := range listBlob.Segment.BlobPrefixes {
|
|
|
+ // we don't support prefixes == "/" this will be sent if a key starts with "/"
|
|
|
+ if blobPrefix.Name == "/" {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ name := strings.TrimPrefix(blobPrefix.Name, prefix)
|
|
|
+ result = append(result, NewFileInfo(name, true, 0, time.Now(), false))
|
|
|
+ prefixes[strings.TrimSuffix(name, "/")] = true
|
|
|
+ }
|
|
|
+ for _, blobInfo := range listBlob.Segment.BlobItems {
|
|
|
+ name := strings.TrimPrefix(blobInfo.Name, prefix)
|
|
|
+ size := int64(0)
|
|
|
+ if blobInfo.Properties.ContentLength != nil {
|
|
|
+ size = *blobInfo.Properties.ContentLength
|
|
|
+ }
|
|
|
+ isDir := false
|
|
|
+ if blobInfo.Properties.ContentType != nil {
|
|
|
+ isDir = (*blobInfo.Properties.ContentType == dirMimeType)
|
|
|
+ if isDir {
|
|
|
+ // check if the dir is already included, it will be sent as blob prefix if it contains at least one item
|
|
|
+ if _, ok := prefixes[name]; ok {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ result = append(result, NewFileInfo(name, isDir, size, blobInfo.Properties.LastModified, false))
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return result, nil
|
|
|
+}
|
|
|
+
|
|
|
+// IsUploadResumeSupported returns true if upload resume is supported.
|
|
|
+// Upload Resume is not supported on Azure Blob
|
|
|
+func (AzureBlobFs) IsUploadResumeSupported() bool {
|
|
|
+ return false
|
|
|
+}
|
|
|
+
|
|
|
+// IsAtomicUploadSupported returns true if atomic upload is supported.
|
|
|
+// Azure Blob uploads are already atomic, we don't need to upload to a temporary
|
|
|
+// file
|
|
|
+func (AzureBlobFs) IsAtomicUploadSupported() bool {
|
|
|
+ return false
|
|
|
+}
|
|
|
+
|
|
|
+// IsNotExist returns a boolean indicating whether the error is known to
|
|
|
+// report that a file or directory does not exist
|
|
|
+func (AzureBlobFs) IsNotExist(err error) bool {
|
|
|
+ if err == nil {
|
|
|
+ return false
|
|
|
+ }
|
|
|
+
|
|
|
+ if storageErr, ok := err.(azblob.StorageError); ok {
|
|
|
+ if storageErr.Response().StatusCode == http.StatusNotFound { //nolint:bodyclose
|
|
|
+ return true
|
|
|
+ }
|
|
|
+ if storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound ||
|
|
|
+ storageErr.ServiceCode() == azblob.ServiceCodeBlobNotFound {
|
|
|
+ return true
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return strings.Contains(err.Error(), "404")
|
|
|
+}
|
|
|
+
|
|
|
+// IsPermission returns a boolean indicating whether the error is known to
|
|
|
+// report that permission is denied.
|
|
|
+func (AzureBlobFs) IsPermission(err error) bool {
|
|
|
+ if err == nil {
|
|
|
+ return false
|
|
|
+ }
|
|
|
+ if storageErr, ok := err.(azblob.StorageError); ok {
|
|
|
+ code := storageErr.Response().StatusCode //nolint:bodyclose
|
|
|
+ if code == http.StatusForbidden || code == http.StatusUnauthorized {
|
|
|
+ return true
|
|
|
+ }
|
|
|
+ if storageErr.ServiceCode() == azblob.ServiceCodeInsufficientAccountPermissions ||
|
|
|
+ storageErr.ServiceCode() == azblob.ServiceCodeInvalidAuthenticationInfo ||
|
|
|
+ storageErr.ServiceCode() == azblob.ServiceCodeUnauthorizedBlobOverwrite {
|
|
|
+ return true
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return strings.Contains(err.Error(), "403")
|
|
|
+}
|
|
|
+
|
|
|
+// CheckRootPath creates the specified local root directory if it does not exists
|
|
|
+func (fs AzureBlobFs) CheckRootPath(username string, uid int, gid int) bool {
|
|
|
+ // we need a local directory for temporary files
|
|
|
+ osFs := NewOsFs(fs.ConnectionID(), fs.localTempDir, nil)
|
|
|
+ return osFs.CheckRootPath(username, uid, gid)
|
|
|
+}
|
|
|
+
|
|
|
+// ScanRootDirContents returns the number of files contained in the bucket,
|
|
|
+// and their size
|
|
|
+func (fs AzureBlobFs) ScanRootDirContents() (int, int64, error) {
|
|
|
+ numFiles := 0
|
|
|
+ size := int64(0)
|
|
|
+
|
|
|
+ for marker := (azblob.Marker{}); marker.NotDone(); {
|
|
|
+ ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
|
|
|
+ defer cancelFn()
|
|
|
+
|
|
|
+ listBlob, err := fs.containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{
|
|
|
+ Details: azblob.BlobListingDetails{
|
|
|
+ Copy: false,
|
|
|
+ Metadata: false,
|
|
|
+ Snapshots: false,
|
|
|
+ UncommittedBlobs: false,
|
|
|
+ Deleted: false,
|
|
|
+ },
|
|
|
+ Prefix: fs.config.KeyPrefix,
|
|
|
+ })
|
|
|
+ if err != nil {
|
|
|
+ return numFiles, size, err
|
|
|
+ }
|
|
|
+ marker = listBlob.NextMarker
|
|
|
+ for _, blobInfo := range listBlob.Segment.BlobItems {
|
|
|
+ isDir := false
|
|
|
+ if blobInfo.Properties.ContentType != nil {
|
|
|
+ isDir = (*blobInfo.Properties.ContentType == dirMimeType)
|
|
|
+ }
|
|
|
+ blobSize := int64(0)
|
|
|
+ if blobInfo.Properties.ContentLength != nil {
|
|
|
+ blobSize = *blobInfo.Properties.ContentLength
|
|
|
+ }
|
|
|
+ if isDir && blobSize == 0 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ numFiles++
|
|
|
+ size += blobSize
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return numFiles, size, nil
|
|
|
+}
|
|
|
+
|
|
|
+// GetDirSize returns the number of files and the size for a folder
|
|
|
+// including any subfolders
|
|
|
+func (AzureBlobFs) GetDirSize(dirname string) (int, int64, error) {
|
|
|
+ return 0, 0, errUnsupported
|
|
|
+}
|
|
|
+
|
|
|
+// GetAtomicUploadPath returns the path to use for an atomic upload.
|
|
|
+// Azure Blob Storage uploads are already atomic, we never call this method
|
|
|
+func (AzureBlobFs) GetAtomicUploadPath(name string) string {
|
|
|
+ return ""
|
|
|
+}
|
|
|
+
|
|
|
+// GetRelativePath returns the path for a file relative to the user's home dir.
|
|
|
+// This is the path as seen by SFTPGo users
|
|
|
+func (fs AzureBlobFs) GetRelativePath(name string) string {
|
|
|
+ rel := path.Clean(name)
|
|
|
+ if rel == "." {
|
|
|
+ rel = ""
|
|
|
+ }
|
|
|
+ if !path.IsAbs(rel) {
|
|
|
+ rel = "/" + rel
|
|
|
+ }
|
|
|
+ if len(fs.config.KeyPrefix) > 0 {
|
|
|
+ if !strings.HasPrefix(rel, "/"+fs.config.KeyPrefix) {
|
|
|
+ rel = "/"
|
|
|
+ }
|
|
|
+ rel = path.Clean("/" + strings.TrimPrefix(rel, "/"+fs.config.KeyPrefix))
|
|
|
+ }
|
|
|
+ return rel
|
|
|
+}
|
|
|
+
|
|
|
+// Walk walks the file tree rooted at root, calling walkFn for each file or
|
|
|
+// directory in the tree, including root
|
|
|
+func (fs AzureBlobFs) Walk(root string, walkFn filepath.WalkFunc) error {
|
|
|
+ prefix := ""
|
|
|
+ if root != "" && root != "." {
|
|
|
+ prefix = strings.TrimPrefix(root, "/")
|
|
|
+ if !strings.HasSuffix(prefix, "/") {
|
|
|
+ prefix += "/"
|
|
|
+ }
|
|
|
+ }
|
|
|
+ for marker := (azblob.Marker{}); marker.NotDone(); {
|
|
|
+ ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
|
|
|
+ defer cancelFn()
|
|
|
+
|
|
|
+ listBlob, err := fs.containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{
|
|
|
+ Details: azblob.BlobListingDetails{
|
|
|
+ Copy: false,
|
|
|
+ Metadata: false,
|
|
|
+ Snapshots: false,
|
|
|
+ UncommittedBlobs: false,
|
|
|
+ Deleted: false,
|
|
|
+ },
|
|
|
+ Prefix: prefix,
|
|
|
+ })
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ marker = listBlob.NextMarker
|
|
|
+ for _, blobInfo := range listBlob.Segment.BlobItems {
|
|
|
+ isDir := false
|
|
|
+ if blobInfo.Properties.ContentType != nil {
|
|
|
+ isDir = (*blobInfo.Properties.ContentType == dirMimeType)
|
|
|
+ }
|
|
|
+ name := path.Clean(blobInfo.Name)
|
|
|
+ if len(name) == 0 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ blobSize := int64(0)
|
|
|
+ if blobInfo.Properties.ContentLength != nil {
|
|
|
+ blobSize = *blobInfo.Properties.ContentLength
|
|
|
+ }
|
|
|
+ err = walkFn(blobInfo.Name, NewFileInfo(name, isDir, blobSize, blobInfo.Properties.LastModified, false), nil)
|
|
|
+ if err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return walkFn(root, NewFileInfo(root, true, 0, time.Now(), false), nil)
|
|
|
+}
|
|
|
+
|
|
|
+// Join joins any number of path elements into a single path
|
|
|
+func (AzureBlobFs) Join(elem ...string) string {
|
|
|
+ return strings.TrimPrefix(path.Join(elem...), "/")
|
|
|
+}
|
|
|
+
|
|
|
+// HasVirtualFolders returns true if folders are emulated
|
|
|
+func (AzureBlobFs) HasVirtualFolders() bool {
|
|
|
+ return true
|
|
|
+}
|
|
|
+
|
|
|
+// ResolvePath returns the matching filesystem path for the specified sftp path
|
|
|
+func (fs AzureBlobFs) ResolvePath(virtualPath string) (string, error) {
|
|
|
+ if !path.IsAbs(virtualPath) {
|
|
|
+ virtualPath = path.Clean("/" + virtualPath)
|
|
|
+ }
|
|
|
+ return fs.Join(fs.config.KeyPrefix, strings.TrimPrefix(virtualPath, "/")), nil
|
|
|
+}
|
|
|
+
|
|
|
+// GetMimeType implements MimeTyper interface
|
|
|
+func (fs AzureBlobFs) GetMimeType(name string) (string, error) {
|
|
|
+ ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
|
|
|
+ defer cancelFn()
|
|
|
+
|
|
|
+ blobBlockURL := fs.containerURL.NewBlockBlobURL(name)
|
|
|
+ response, err := blobBlockURL.GetProperties(ctx, azblob.BlobAccessConditions{})
|
|
|
+ if err != nil {
|
|
|
+ return "", err
|
|
|
+ }
|
|
|
+ return response.ContentType(), nil
|
|
|
+}
|
|
|
+
|
|
|
+func (fs *AzureBlobFs) isEqual(key string, virtualName string) bool {
|
|
|
+ if key == virtualName {
|
|
|
+ return true
|
|
|
+ }
|
|
|
+ if key == virtualName+"/" {
|
|
|
+ return true
|
|
|
+ }
|
|
|
+ if key+"/" == virtualName {
|
|
|
+ return true
|
|
|
+ }
|
|
|
+ return false
|
|
|
+}
|
|
|
+
|
|
|
+func (fs *AzureBlobFs) checkIfBucketExists() error {
|
|
|
+ ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
|
|
|
+ defer cancelFn()
|
|
|
+
|
|
|
+ _, err := fs.containerURL.GetProperties(ctx, azblob.LeaseAccessConditions{})
|
|
|
+ return err
|
|
|
+}
|
|
|
+
|
|
|
+func (fs *AzureBlobFs) getPrefixForStat(name string) string {
|
|
|
+ prefix := path.Dir(name)
|
|
|
+ if prefix == "/" || prefix == "." || prefix == "" {
|
|
|
+ prefix = ""
|
|
|
+ } else {
|
|
|
+ prefix = strings.TrimPrefix(prefix, "/")
|
|
|
+ if !strings.HasSuffix(prefix, "/") {
|
|
|
+ prefix += "/"
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return prefix
|
|
|
+}
|
Nicola Murino