azblob: handle dirs metadata

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>

go.mod

@@ -9,15 +9,15 @@ require (
 	github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
 	github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387
 	github.com/aws/aws-sdk-go-v2 v1.17.1
-	github.com/aws/aws-sdk-go-v2/config v1.18.2
+	github.com/aws/aws-sdk-go-v2/config v1.18.3
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.2
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.3
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19
-	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.41
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.42
-	github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.23
+	github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.24
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.29.3
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.29.4
-	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.16.7
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.16.8
-	github.com/aws/aws-sdk-go-v2/service/sts v1.17.4
+	github.com/aws/aws-sdk-go-v2/service/sts v1.17.5
-	github.com/cockroachdb/cockroach-go/v2 v2.2.18
+	github.com/cockroachdb/cockroach-go/v2 v2.2.19
 	github.com/coreos/go-oidc/v3 v3.4.0
 	github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b
 	github.com/eikenb/pipeat v0.0.0-20210730190139-06b3e6902001

go.sum

@@ -233,17 +233,17 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3/go.mod h1:gNsR5CaXK
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9 h1:RKci2D7tMwpvGpDNZnGQw9wk6v7o/xSwFcUAuNPoB8k=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9/go.mod h1:vCmV1q1VK8eoQJ5+aYE7PkK1K6v41qJ5pJdK3ggCDvg=
 github.com/aws/aws-sdk-go-v2/config v1.15.15/go.mod h1:A1Lzyy/o21I5/s2FbyX5AevQfSVXpvvIDCoVFD0BC4E=
-github.com/aws/aws-sdk-go-v2/config v1.18.2 h1:tRhTb3xMZsB0gW0sXWpqs9FeIP8iQp5SvnvwiPXzHwo=
+github.com/aws/aws-sdk-go-v2/config v1.18.3 h1:3kfBKcX3votFX84dm00U8RGA1sCCh3eRMOGzg5dCWfU=
-github.com/aws/aws-sdk-go-v2/config v1.18.2/go.mod h1:9XVoZTdD8ICjrgI5ddb8j918q6lEZkFYpb7uohgvU6c=
+github.com/aws/aws-sdk-go-v2/config v1.18.3/go.mod h1:BYdrbeCse3ZnOD5+2/VE/nATOK8fEUpBtmPMdKSyhMU=
 github.com/aws/aws-sdk-go-v2/credentials v1.12.10/go.mod h1:g5eIM5XRs/OzIIK81QMBl+dAuDyoLN0VYaLP+tBqEOk=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.2 h1:F/v1w0XcFDZjL0bCdi9XWJenoPKjGbzljBhDKcryzEQ=
+github.com/aws/aws-sdk-go-v2/credentials v1.13.3 h1:ur+FHdp4NbVIv/49bUjBW+FE7e57HOo03ELodttmagk=
-github.com/aws/aws-sdk-go-v2/credentials v1.13.2/go.mod h1:eAT5aj/WJ2UDIA0IVNFc2byQLeD89SDEi4cjzH/MKoQ=
+github.com/aws/aws-sdk-go-v2/credentials v1.13.3/go.mod h1:/rOMmqYBcFfNbRPU0iN9IgGqD5+V2yp3iWNmIlz0wI4=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.9/go.mod h1:KDCCm4ONIdHtUloDcFvK2+vshZvx4Zmj7UMDfusuz5s=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 h1:E3PXZSI3F2bzyj6XxUXdTIfvp425HHhwKsFvmzBwHgs=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.21/go.mod h1:iIYPrQ2rYfZiB/iADYlhj9HHZ9TTi6PqKQPAqygohbE=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.41 h1:ssgdsNm11dvFtO7F/AeiW4dAO3eGsDeg5fwpag/JP/I=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.42 h1:bxgBYvvBh+W1RnNYP4ROXEB8N+HSSucDszfE7Rb+kfU=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.41/go.mod h1:CS+AbDFAaPU9TQOo7U6mVV23YvqCOElnqmh0XQjgJ1g=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.42/go.mod h1:LHOsygMiW/14CkFxdXxvzKyMh3jbk/QfZVaDtCbLkl8=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.15/go.mod h1:pWrr2OoHlT7M/Pd2y4HV3gJyPb3qj5qMmnPkKSNPYK4=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 h1:nBO/RFxeq/IS5G9Of+ZrgucRciie2qpLy++3UGZ+q2E=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY=
@@ -269,14 +269,14 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.9/go.mod h1:Rc5+wn2
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.19 h1:piDBAaWkaxkkVV3xJJbTehXCZRXYs49kvpi/LG6LR2o=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.19/go.mod h1:BmQWRVkLTmyNzYPFAZgon53qKLWBNSvonugD1MrSWUs=
 github.com/aws/aws-sdk-go-v2/service/kms v1.18.1/go.mod h1:4PZMUkc9rXHWGVB5J9vKaZy3D7Nai79ORworQ3ASMiM=
-github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.23 h1:NZKyAfM6J2OPehuSZI8woNGTxYi5jMkTDWGEFwR6/FA=
+github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.24 h1:DYr+X4xrRzcthq2OLJzsiS/uSJhZ/HHxXG0yUgGZceU=
-github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.23/go.mod h1:mRGY+k3s1yt7yQA3AfzJhnr68OCs1xDfQfIABFUk+ek=
+github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.24/go.mod h1:mRGY+k3s1yt7yQA3AfzJhnr68OCs1xDfQfIABFUk+ek=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.27.2/go.mod h1:u+566cosFI+d+motIz3USXEh6sN8Nq4GrNXSg2RXVMo=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.29.3 h1:F6wgg8aHGNyhaAy2ONnWBThiPdLa386qNA0j33FIuSM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.29.4 h1:QgmmWifaYZZcpaw3y1+ccRlgH6jAvLm4K/MBGUc7cNM=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.29.3/go.mod h1:/NHbqPRiwxSPVOB2Xr+StDEH+GWV/64WwnUjv4KYzV0=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.29.4/go.mod h1:/NHbqPRiwxSPVOB2Xr+StDEH+GWV/64WwnUjv4KYzV0=
 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.14/go.mod h1:xakbH8KMsQQKqzX87uyyzTHshc/0/Df8bsTneTS5pFU=
-github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.16.7 h1:bfC2Q8ABNbYYm9mh3NfPy5kvnWOPtiqS018NBGDwPl8=
+github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.16.8 h1:Zw48FHykP40fKMxPmagkuzklpEuDPLhvUjKP8Ygrds0=
-github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.16.7/go.mod h1:k6CPuxyzO247nYEM1baEwHH1kRtosRCvgahAepaaShw=
+github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.16.8/go.mod h1:k6CPuxyzO247nYEM1baEwHH1kRtosRCvgahAepaaShw=
 github.com/aws/aws-sdk-go-v2/service/sns v1.17.10/go.mod h1:uITsRNVMeCB3MkWpXxXw0eDz8pW4TYLzj+eyQtbhSxM=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.19.1/go.mod h1:A94o564Gj+Yn+7QO1eLFeI7UVv3riy/YBFOfICVqFvU=
 github.com/aws/aws-sdk-go-v2/service/ssm v1.27.6/go.mod h1:fiFzQgj4xNOg4/wqmAiPvzgDMXPD+cUEplX/CYn+0j0=
@@ -286,8 +286,8 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.11.25/go.mod h1:IARHuzTXmj1C0KS35vbo
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 h1:jcw6kKZrtNfBPJkaHrscDOZoe5gvi9wjudnxvozYFJo=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8/go.mod h1:er2JHN+kBY6FcMfcBBKNGCT3CarImmdFzishsqBmSRI=
 github.com/aws/aws-sdk-go-v2/service/sts v1.16.10/go.mod h1:cftkHYN6tCDNfkSasAmclSfl4l7cySoay8vz7p/ce0E=
-github.com/aws/aws-sdk-go-v2/service/sts v1.17.4 h1:YNncBj5dVYd05i4ZQ+YicOotSXo0ufc9P8kTioi13EM=
+github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 h1:60SJ4lhvn///8ygCzYy2l53bFW/Q15bVfyjyAWo6zuw=
-github.com/aws/aws-sdk-go-v2/service/sts v1.17.4/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4=
+github.com/aws/aws-sdk-go-v2/service/sts v1.17.5/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4=
 github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk=
 github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
@@ -359,8 +359,8 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
-github.com/cockroachdb/cockroach-go/v2 v2.2.18 h1:bRNZWqzRSZesVYFjDAl1jgFb1jhIFIEreWIC2kPbcdY=
+github.com/cockroachdb/cockroach-go/v2 v2.2.19 h1:YIHyz17jZumBeXPuoZKq/0nrITsqDoDD8/KQt3/xiyc=
-github.com/cockroachdb/cockroach-go/v2 v2.2.18/go.mod h1:mzlIDDBALQfEjv/7DU12fb2AfQ/MUYTlychcMpWp9QI=
+github.com/cockroachdb/cockroach-go/v2 v2.2.19/go.mod h1:mzlIDDBALQfEjv/7DU12fb2AfQ/MUYTlychcMpWp9QI=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=

internal/vfs/azblobfs.go

@@ -55,6 +55,7 @@ import (
 const (
 	azureDefaultEndpoint = "blob.core.windows.net"
 	azBlobFsName         = "AzureBlobFs"
+	azFolderKey          = "hdi_isfolder"
 )
 
 // AzureBlobFs is a Fs implementation for Azure Blob storage.
@@ -182,6 +183,14 @@ func (fs *AzureBlobFs) Stat(name string) (os.FileInfo, error) {
 	if err == nil {
 		contentType := util.GetStringFromPointer(attrs.ContentType)
 		isDir := contentType == dirMimeType
+		if !isDir {
+			for k, v := range attrs.Metadata {
+				if strings.ToLower(k) == azFolderKey {
+					isDir = (v == "true")
+					break
+				}
+			}
+		}
 		metric.AZListObjectsCompleted(nil)
 		return updateFileInfoModTime(fs.getStorageID(), name, NewFileInfo(name, isDir,
 			util.GetIntFromPointer(attrs.ContentLength),
@@ -217,7 +226,7 @@ func (fs *AzureBlobFs) Open(name string, offset int64) (File, *pipeat.PipeReader
 	go func() {
 		defer cancelFn()
 
-		blockBlob := fs.containerClient.NewBlockBlobClient(name)
+		blockBlob := fs.containerClient.NewBlockBlobClient(url.PathEscape(name))
 		err := fs.handleMultipartDownload(ctx, blockBlob, offset, w)
 		w.CloseWithError(err) //nolint:errcheck
 		fsLog(fs, logger.LevelDebug, "download completed, path: %#v size: %v, err: %+v", name, w.GetWrittenBytes(), err)
@@ -238,8 +247,12 @@ func (fs *AzureBlobFs) Create(name string, flag int) (File, *PipeWriter, func(),
 	p := NewPipeWriter(w)
 	headers := blob.HTTPHeaders{}
 	var contentType string
+	var metadata map[string]string
 	if flag == -1 {
 		contentType = dirMimeType
+		metadata = map[string]string{
+			azFolderKey: "true",
+		}
 	} else {
 		contentType = mime.TypeByExtension(path.Ext(name))
 	}
@@ -250,8 +263,8 @@ func (fs *AzureBlobFs) Create(name string, flag int) (File, *PipeWriter, func(),
 	go func() {
 		defer cancelFn()
 
-		blockBlob := fs.containerClient.NewBlockBlobClient(name)
+		blockBlob := fs.containerClient.NewBlockBlobClient(url.PathEscape(name))
-		err := fs.handleMultipartUpload(ctx, r, blockBlob, &headers)
+		err := fs.handleMultipartUpload(ctx, r, blockBlob, &headers, metadata)
 		r.CloseWithError(err) //nolint:errcheck
 		p.Done(err)
 		fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, readed bytes: %v, err: %+v", name, r.GetReadedBytes(), err)
@@ -282,43 +295,47 @@ func (fs *AzureBlobFs) Rename(source, target string) error {
 		if hasContents {
 			return fmt.Errorf("cannot rename non empty directory: %#v", source)
 		}
-	}
+		if err := fs.mkdirInternal(target); err != nil {
-	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout))
+			return err
-	defer cancelFn()
-
-	srcBlob := fs.containerClient.NewBlockBlobClient(url.PathEscape(source))
-	dstBlob := fs.containerClient.NewBlockBlobClient(target)
-	resp, err := dstBlob.StartCopyFromURL(ctx, srcBlob.URL(), fs.getCopyOptions())
-	if err != nil {
-		metric.AZCopyObjectCompleted(err)
-		return err
-	}
-	copyStatus := blob.CopyStatusType(util.GetStringFromPointer((*string)(resp.CopyStatus)))
-	nErrors := 0
-	for copyStatus == blob.CopyStatusTypePending {
-		// Poll until the copy is complete.
-		time.Sleep(500 * time.Millisecond)
-		resp, err := dstBlob.GetProperties(ctx, &blob.GetPropertiesOptions{})
-		if err != nil {
-			// A GetProperties failure may be transient, so allow a couple
-			// of them before giving up.
-			nErrors++
-			if ctx.Err() != nil || nErrors == 3 {
-				metric.AZCopyObjectCompleted(err)
-				return err
-			}
-		} else {
-			copyStatus = blob.CopyStatusType(util.GetStringFromPointer((*string)(resp.CopyStatus)))
 		}
-	}
+	} else {
-	if copyStatus != blob.CopyStatusTypeSuccess {
+		ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxLongTimeout))
-		err := fmt.Errorf("copy failed with status: %s", copyStatus)
+		defer cancelFn()
-		metric.AZCopyObjectCompleted(err)
-		return err
-	}
 
-	metric.AZCopyObjectCompleted(nil)
+		srcBlob := fs.containerClient.NewBlockBlobClient(url.PathEscape(source))
-	fs.preserveModificationTime(source, target, fi)
+		dstBlob := fs.containerClient.NewBlockBlobClient(url.PathEscape(target))
+		resp, err := dstBlob.StartCopyFromURL(ctx, srcBlob.URL(), fs.getCopyOptions())
+		if err != nil {
+			metric.AZCopyObjectCompleted(err)
+			return err
+		}
+		copyStatus := blob.CopyStatusType(util.GetStringFromPointer((*string)(resp.CopyStatus)))
+		nErrors := 0
+		for copyStatus == blob.CopyStatusTypePending {
+			// Poll until the copy is complete.
+			time.Sleep(500 * time.Millisecond)
+			resp, err := dstBlob.GetProperties(ctx, &blob.GetPropertiesOptions{})
+			if err != nil {
+				// A GetProperties failure may be transient, so allow a couple
+				// of them before giving up.
+				nErrors++
+				if ctx.Err() != nil || nErrors == 3 {
+					metric.AZCopyObjectCompleted(err)
+					return err
+				}
+			} else {
+				copyStatus = blob.CopyStatusType(util.GetStringFromPointer((*string)(resp.CopyStatus)))
+			}
+		}
+		if copyStatus != blob.CopyStatusTypeSuccess {
+			err := fmt.Errorf("copy failed with status: %s", copyStatus)
+			metric.AZCopyObjectCompleted(err)
+			return err
+		}
+
+		metric.AZCopyObjectCompleted(nil)
+		fs.preserveModificationTime(source, target, fi)
+	}
 	return fs.Remove(source, fi.IsDir())
 }
 
@@ -337,11 +354,22 @@ func (fs *AzureBlobFs) Remove(name string, isDir bool) error {
 	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
 	defer cancelFn()
 
-	blobBlock := fs.containerClient.NewBlockBlobClient(name)
+	blobBlock := fs.containerClient.NewBlockBlobClient(url.PathEscape(name))
-	deletSnapshots := blob.DeleteSnapshotsOptionTypeInclude
+	var deletSnapshots blob.DeleteSnapshotsOptionType
+	if !isDir {
+		deletSnapshots = blob.DeleteSnapshotsOptionTypeInclude
+	}
 	_, err := blobBlock.Delete(ctx, &blob.DeleteOptions{
 		DeleteSnapshots: &deletSnapshots,
 	})
+	if err != nil && isDir {
+		if fs.isBadRequestError(err) {
+			deletSnapshots = blob.DeleteSnapshotsOptionTypeInclude
+			_, err = blobBlock.Delete(ctx, &blob.DeleteOptions{
+				DeleteSnapshots: &deletSnapshots,
+			})
+		}
+	}
 	metric.AZDeleteObjectCompleted(err)
 	if plugin.Handler.HasMetadater() && err == nil && !isDir {
 		if errMetadata := plugin.Handler.RemoveMetadata(fs.getStorageID(), ensureAbsPath(name)); errMetadata != nil {
@@ -357,11 +385,7 @@ func (fs *AzureBlobFs) Mkdir(name string) error {
 	if !fs.IsNotExist(err) {
 		return err
 	}
-	_, w, _, err := fs.Create(name, -1)
+	return fs.mkdirInternal(name)
-	if err != nil {
-		return err
-	}
-	return w.Close()
 }
 
 // Symlink creates source as a symbolic link to target.
@@ -424,8 +448,10 @@ func (fs *AzureBlobFs) ReadDir(dirname string) ([]os.FileInfo, error) {
 	prefixes := make(map[string]bool)
 
 	pager := fs.containerClient.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{
-		Include: container.ListBlobsInclude{},
+		Include: container.ListBlobsInclude{
-		Prefix:  &prefix,
+			//Metadata: true,
+		},
+		Prefix: &prefix,
 	})
 
 	for pager.More() {
@@ -462,7 +488,7 @@ func (fs *AzureBlobFs) ReadDir(dirname string) ([]os.FileInfo, error) {
 				size = util.GetIntFromPointer(blobItem.Properties.ContentLength)
 				modTime = util.GetTimeFromPointer(blobItem.Properties.LastModified)
 				contentType := util.GetStringFromPointer(blobItem.Properties.ContentType)
-				isDir = (contentType == dirMimeType)
+				isDir = checkDirectoryMarkers(contentType, blobItem.Metadata)
 				if isDir {
 					// check if the dir is already included, it will be sent as blob prefix if it contains at least one item
 					if _, ok := prefixes[name]; ok {
@@ -530,6 +556,17 @@ func (*AzureBlobFs) IsNotSupported(err error) bool {
 	return err == ErrVfsUnsupported
 }
 
+func (*AzureBlobFs) isBadRequestError(err error) bool {
+	if err == nil {
+		return false
+	}
+	var respErr *azcore.ResponseError
+	if errors.As(err, &respErr) {
+		return respErr.StatusCode == http.StatusBadRequest
+	}
+	return false
+}
+
 // CheckRootPath creates the specified local root directory if it does not exists
 func (fs *AzureBlobFs) CheckRootPath(username string, uid int, gid int) bool {
 	// we need a local directory for temporary files
@@ -544,6 +581,9 @@ func (fs *AzureBlobFs) ScanRootDirContents() (int, int64, error) {
 	size := int64(0)
 
 	pager := fs.containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
+		Include: container.ListBlobsInclude{
+			Metadata: true,
+		},
 		Prefix: &fs.config.KeyPrefix,
 	})
 
@@ -559,7 +599,7 @@ func (fs *AzureBlobFs) ScanRootDirContents() (int, int64, error) {
 		for _, blobItem := range resp.ListBlobsFlatSegmentResponse.Segment.BlobItems {
 			if blobItem.Properties != nil {
 				contentType := util.GetStringFromPointer(blobItem.Properties.ContentType)
-				isDir := (contentType == dirMimeType)
+				isDir := checkDirectoryMarkers(contentType, blobItem.Metadata)
 				blobSize := util.GetIntFromPointer(blobItem.Properties.ContentLength)
 				if isDir && blobSize == 0 {
 					continue
@@ -585,8 +625,10 @@ func (fs *AzureBlobFs) getFileNamesInPrefix(fsPrefix string) (map[string]bool, e
 	}
 
 	pager := fs.containerClient.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{
-		Include: container.ListBlobsInclude{},
+		Include: container.ListBlobsInclude{
-		Prefix:  &prefix,
+			//Metadata: true,
+		},
+		Prefix: &prefix,
 	})
 
 	for pager.More() {
@@ -603,7 +645,7 @@ func (fs *AzureBlobFs) getFileNamesInPrefix(fsPrefix string) (map[string]bool, e
 			name = strings.TrimPrefix(name, prefix)
 			if blobItem.Properties != nil {
 				contentType := util.GetStringFromPointer(blobItem.Properties.ContentType)
-				isDir := (contentType == dirMimeType)
+				isDir := checkDirectoryMarkers(contentType, blobItem.Metadata)
 				if isDir {
 					continue
 				}
@@ -660,7 +702,10 @@ func (fs *AzureBlobFs) GetRelativePath(name string) string {
 func (fs *AzureBlobFs) Walk(root string, walkFn filepath.WalkFunc) error {
 	prefix := fs.getPrefix(root)
 	pager := fs.containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
-		Prefix: &fs.config.KeyPrefix,
+		Include: container.ListBlobsInclude{
+			Metadata: true,
+		},
+		Prefix: &prefix,
 	})
 
 	for pager.More() {
@@ -682,7 +727,7 @@ func (fs *AzureBlobFs) Walk(root string, walkFn filepath.WalkFunc) error {
 			isDir := false
 			if blobItem.Properties != nil {
 				contentType := util.GetStringFromPointer(blobItem.Properties.ContentType)
-				isDir = (contentType == dirMimeType)
+				isDir = checkDirectoryMarkers(contentType, blobItem.Metadata)
 				blobSize = util.GetIntFromPointer(blobItem.Properties.ContentLength)
 				lastModified = util.GetTimeFromPointer(blobItem.Properties.LastModified)
 			}
@@ -722,7 +767,7 @@ func (fs *AzureBlobFs) headObject(name string) (blob.GetPropertiesResponse, erro
 	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
 	defer cancelFn()
 
-	resp, err := fs.containerClient.NewBlockBlobClient(name).GetProperties(ctx, &blob.GetPropertiesOptions{})
+	resp, err := fs.containerClient.NewBlockBlobClient(url.PathEscape(name)).GetProperties(ctx, &blob.GetPropertiesOptions{})
 
 	metric.AZHeadObjectCompleted(err)
 	return resp, err
@@ -795,6 +840,14 @@ func (fs *AzureBlobFs) setConfigDefaults() {
 	}
 }
 
+func (fs *AzureBlobFs) mkdirInternal(name string) error {
+	_, w, _, err := fs.Create(name, -1)
+	if err != nil {
+		return err
+	}
+	return w.Close()
+}
+
 func (fs *AzureBlobFs) hasContents(name string) (bool, error) {
 	result := false
 	prefix := fs.getPrefix(name)
@@ -931,7 +984,7 @@ func (fs *AzureBlobFs) handleMultipartDownload(ctx context.Context, blockBlob *b
 }
 
 func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Reader,
-	blockBlob *blockblob.Client, httpHeaders *blob.HTTPHeaders,
+	blockBlob *blockblob.Client, httpHeaders *blob.HTTPHeaders, metadata map[string]string,
 ) error {
 	partSize := fs.config.UploadPartSize
 	guard := make(chan struct{}, fs.config.UploadConcurrency)
@@ -1021,6 +1074,7 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read
 
 	commitOptions := blockblob.CommitBlockListOptions{
 		HTTPHeaders: httpHeaders,
+		Metadata:    metadata,
 	}
 	if fs.config.AccessTier != "" {
 		commitOptions.Tier = (*blob.AccessTier)(&fs.config.AccessTier)
@@ -1083,6 +1137,18 @@ func (fs *AzureBlobFs) getStorageID() string {
 	return fmt.Sprintf("azblob://%v", fs.config.Container)
 }
 
+func checkDirectoryMarkers(contentType string, metadata map[string]*string) bool {
+	if contentType == dirMimeType {
+		return true
+	}
+	for k, v := range metadata {
+		if strings.ToLower(k) == azFolderKey {
+			return util.GetStringFromPointer(v) == "true"
+		}
+	}
+	return false
+}
+
 func getAzContainerClientOptions() *container.ClientOptions {
 	version := version.Get()
 	return &container.ClientOptions{

internal/vfs/gcsfs.go

@@ -226,38 +226,32 @@ func (fs *GCSFs) Rename(source, target string) error {
 		if hasContents {
 			return fmt.Errorf("cannot rename non empty directory: %#v", source)
 		}
-		if !strings.HasSuffix(target, "/") {
+		if err := fs.mkdirInternal(target); err != nil {
-			target += "/"
+			return err
 		}
-	}
-	src := fs.svc.Bucket(fs.config.Bucket).Object(realSourceName)
-	dst := fs.svc.Bucket(fs.config.Bucket).Object(target)
-	ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
-	defer cancelFn()
-
-	copier := dst.CopierFrom(src)
-	if fs.config.StorageClass != "" {
-		copier.StorageClass = fs.config.StorageClass
-	}
-	if fs.config.ACL != "" {
-		copier.PredefinedACL = fs.config.ACL
-	}
-	var contentType string
-	if fi.IsDir() {
-		contentType = dirMimeType
 	} else {
-		contentType = mime.TypeByExtension(path.Ext(source))
+		src := fs.svc.Bucket(fs.config.Bucket).Object(realSourceName)
-	}
+		dst := fs.svc.Bucket(fs.config.Bucket).Object(target)
-	if contentType != "" {
+		ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(fs.ctxTimeout))
-		copier.ContentType = contentType
+		defer cancelFn()
-	}
+
-	_, err = copier.Run(ctx)
+		copier := dst.CopierFrom(src)
-	metric.GCSCopyObjectCompleted(err)
+		if fs.config.StorageClass != "" {
-	if err != nil {
+			copier.StorageClass = fs.config.StorageClass
-		return err
+		}
-	}
+		if fs.config.ACL != "" {
-	if plugin.Handler.HasMetadater() {
+			copier.PredefinedACL = fs.config.ACL
-		if !fi.IsDir() {
+		}
+		contentType := mime.TypeByExtension(path.Ext(source))
+		if contentType != "" {
+			copier.ContentType = contentType
+		}
+		_, err = copier.Run(ctx)
+		metric.GCSCopyObjectCompleted(err)
+		if err != nil {
+			return err
+		}
+		if plugin.Handler.HasMetadater() {
 			err = plugin.Handler.SetModificationTime(fs.getStorageID(), ensureAbsPath(target),
 				util.GetTimeAsMsSinceEpoch(fi.ModTime()))
 			if err != nil {
@@ -306,14 +300,7 @@ func (fs *GCSFs) Mkdir(name string) error {
 	if !fs.IsNotExist(err) {
 		return err
 	}
-	if !strings.HasSuffix(name, "/") {
+	return fs.mkdirInternal(name)
-		name += "/"
-	}
-	_, w, _, err := fs.Create(name, -1)
-	if err != nil {
-		return err
-	}
-	return w.Close()
 }
 
 // Symlink creates source as a symbolic link to target.
@@ -762,6 +749,17 @@ func (fs *GCSFs) getObjectStat(name string) (string, os.FileInfo, error) {
 	return name + "/", info, err
 }
 
+func (fs *GCSFs) mkdirInternal(name string) error {
+	if !strings.HasSuffix(name, "/") {
+		name += "/"
+	}
+	_, w, _, err := fs.Create(name, -1)
+	if err != nil {
+		return err
+	}
+	return w.Close()
+}
+
 func (fs *GCSFs) hasContents(name string) (bool, error) {
 	result := false
 	prefix := fs.getPrefix(name)