diff --git a/cmd/portable.go b/cmd/portable.go
index a22abdbf..2db055a8 100644
--- a/cmd/portable.go
+++ b/cmd/portable.go
@@ -234,7 +234,7 @@ advertised via multicast DNS, this
flag allows to put username/password
inside the advertised TXT record`)
portableCmd.Flags().IntVarP(&portableFsProvider, "fs-provider", "f", int(dataprovider.LocalFilesystemProvider), `0 => local filesystem
-1 => Amazon S3 compatible
+1 => AWS S3 compatible
2 => Google Cloud Storage
3 => Azure Blob Storage`)
portableCmd.Flags().StringVar(&portableS3Bucket, "s3-bucket", "", "")
diff --git a/dataprovider/user.go b/dataprovider/user.go
index b3001818..a2107856 100644
--- a/dataprovider/user.go
+++ b/dataprovider/user.go
@@ -125,7 +125,7 @@ type FilesystemProvider int
// supported values for FilesystemProvider
const (
LocalFilesystemProvider FilesystemProvider = iota // Local
- S3FilesystemProvider // Amazon S3 compatible
+ S3FilesystemProvider // AWS S3 compatible
GCSFilesystemProvider // Google Cloud Storage
AzureBlobFilesystemProvider // Azure Blob Storage
)
diff --git a/docs/howto/postgresql-s3.md b/docs/howto/postgresql-s3.md
index 9e413e19..50da996f 100644
--- a/docs/howto/postgresql-s3.md
+++ b/docs/howto/postgresql-s3.md
@@ -208,7 +208,7 @@ Click `Add` and fill the user details, the minimum required parameters are:
- `Password` or `Public keys`
- `Permissions`
- `Home Dir` can be empty since we defined a default base dir
-- Select `Amazon S3 (Compatible)` as storage and then set `Bucket`, `Region` and optionally a `Key Prefix` if you want to restrict the user to a specific virtual folder in the bucket. The specified virtual folder does not need to be pre-created. You can leave `Access Key` and `Access Secret` empty since we defined global credentials for the `sftpgo` user and we use this system user to run the SFTPGo service.
+- Select `AWS S3 (Compatible)` as storage and then set `Bucket`, `Region` and optionally a `Key Prefix` if you want to restrict the user to a specific virtual folder in the bucket. The specified virtual folder does not need to be pre-created. You can leave `Access Key` and `Access Secret` empty since we defined global credentials for the `sftpgo` user and we use this system user to run the SFTPGo service.
You are done! Now you can connect to you SFTPGo instance using any compatible `sftp` client on port `2022`.
diff --git a/docs/portable-mode.md b/docs/portable-mode.md
index 238a5dd2..0393f712 100644
--- a/docs/portable-mode.md
+++ b/docs/portable-mode.md
@@ -48,7 +48,7 @@ Flags:
relative to the current directory
(default ".")
-f, --fs-provider int 0 => local filesystem
- 1 => Amazon S3 compatible
+ 1 => AWS S3 compatible
2 => Google Cloud Storage
3 => Azure Blob Storage
--ftpd-cert string Path to the certificate file for FTPS
diff --git a/examples/ldapauthserver/httpd/models.go b/examples/ldapauthserver/httpd/models.go
index 40b33367..42aef279 100644
--- a/examples/ldapauthserver/httpd/models.go
+++ b/examples/ldapauthserver/httpd/models.go
@@ -51,7 +51,7 @@ type GCSFsConfig struct {
// SFTPGoFilesystem defines cloud storage filesystem details
type SFTPGoFilesystem struct {
- // 0 local filesystem, 1 Amazon S3 compatible, 2 Google Cloud Storage
+ // 0 local filesystem, 1 AWS S3 compatible, 2 Google Cloud Storage
Provider int `json:"provider"`
S3Config S3FsConfig `json:"s3config,omitempty"`
GCSConfig GCSFsConfig `json:"gcsconfig,omitempty"`
diff --git a/metrics/metrics.go b/metrics/metrics.go
index b82b0147..0c62b34f 100644
--- a/metrics/metrics.go
+++ b/metrics/metrics.go
@@ -306,6 +306,18 @@ var (
Help: "The total number of S3 delete object errors",
})
+ // totalS3HeadObject is the metric that reports the total successful S3 head object requests
+ totalS3HeadObject = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_s3_head_object",
+ Help: "The total number of successful S3 head object requests",
+ })
+
+ // totalS3HeadObjectErrors is the metric that reports the total S3 head object errors
+ totalS3HeadObjectErrors = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_s3_head_object_errors",
+ Help: "The total number of S3 head object errors",
+ })
+
// totalS3HeadBucket is the metric that reports the total successful S3 head bucket requests
totalS3HeadBucket = promauto.NewCounter(prometheus.CounterOpts{
Name: "sftpgo_s3_head_bucket",
@@ -354,7 +366,7 @@ var (
Help: "The total GCS download size as bytes, partial downloads are included",
})
- // totalS3ListObjects is the metric that reports the total successful GCS list objects requests
+ // totalGCSListObjects is the metric that reports the total successful GCS list objects requests
totalGCSListObjects = promauto.NewCounter(prometheus.CounterOpts{
Name: "sftpgo_gcs_list_objects",
Help: "The total number of successful GCS list objects requests",
@@ -366,7 +378,7 @@ var (
Help: "The total number of successful GCS copy object requests",
})
- // totalGCSDeleteObject is the metric that reports the total successful S3 delete object requests
+ // totalGCSDeleteObject is the metric that reports the total successful GCS delete object requests
totalGCSDeleteObject = promauto.NewCounter(prometheus.CounterOpts{
Name: "sftpgo_gcs_delete_object",
Help: "The total number of successful GCS delete object requests",
@@ -390,6 +402,18 @@ var (
Help: "The total number of GCS delete object errors",
})
+ // totalGCSHeadObject is the metric that reports the total successful GCS head object requests
+ totalGCSHeadObject = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_gcs_head_object",
+ Help: "The total number of successful GCS head object requests",
+ })
+
+ // totalGCSHeadObjectErrors is the metric that reports the total GCS head object errors
+ totalGCSHeadObjectErrors = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_gcs_head_object_errors",
+ Help: "The total number of GCS head object errors",
+ })
+
// totalGCSHeadBucket is the metric that reports the total successful GCS head bucket requests
totalGCSHeadBucket = promauto.NewCounter(prometheus.CounterOpts{
Name: "sftpgo_gcs_head_bucket",
@@ -401,6 +425,102 @@ var (
Name: "sftpgo_gcs_head_bucket_errors",
Help: "The total number of GCS head bucket errors",
})
+
+ // totalAZUploads is the metric that reports the total number of successful Azure uploads
+ totalAZUploads = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_uploads_total",
+ Help: "The total number of successful Azure uploads",
+ })
+
+ // totalAZDownloads is the metric that reports the total number of successful Azure downloads
+ totalAZDownloads = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_downloads_total",
+ Help: "The total number of successful Azure downloads",
+ })
+
+ // totalAZUploadErrors is the metric that reports the total number of Azure upload errors
+ totalAZUploadErrors = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_upload_errors_total",
+ Help: "The total number of Azure upload errors",
+ })
+
+ // totalAZDownloadErrors is the metric that reports the total number of Azure download errors
+ totalAZDownloadErrors = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_download_errors_total",
+ Help: "The total number of Azure download errors",
+ })
+
+ // totalAZUploadSize is the metric that reports the total Azure uploads size as bytes
+ totalAZUploadSize = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_upload_size",
+ Help: "The total Azure upload size as bytes, partial uploads are included",
+ })
+
+ // totalAZDownloadSize is the metric that reports the total Azure downloads size as bytes
+ totalAZDownloadSize = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_download_size",
+ Help: "The total Azure download size as bytes, partial downloads are included",
+ })
+
+ // totalAZListObjects is the metric that reports the total successful Azure list objects requests
+ totalAZListObjects = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_list_objects",
+ Help: "The total number of successful Azure list objects requests",
+ })
+
+ // totalAZCopyObject is the metric that reports the total successful Azure copy object requests
+ totalAZCopyObject = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_copy_object",
+ Help: "The total number of successful Azure copy object requests",
+ })
+
+ // totalAZDeleteObject is the metric that reports the total successful Azure delete object requests
+ totalAZDeleteObject = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_delete_object",
+ Help: "The total number of successful Azure delete object requests",
+ })
+
+ // totalAZListObjectsError is the metric that reports the total Azure list objects errors
+ totalAZListObjectsErrors = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_list_objects_errors",
+ Help: "The total number of Azure list objects errors",
+ })
+
+ // totalAZCopyObjectErrors is the metric that reports the total Azure copy object errors
+ totalAZCopyObjectErrors = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_copy_object_errors",
+ Help: "The total number of Azure copy object errors",
+ })
+
+ // totalAZDeleteObjectErrors is the metric that reports the total Azure delete object errors
+ totalAZDeleteObjectErrors = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_delete_object_errors",
+ Help: "The total number of Azure delete object errors",
+ })
+
+ // totalAZHeadObject is the metric that reports the total successful Azure head object requests
+ totalAZHeadObject = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_head_object",
+ Help: "The total number of successful Azure head object requests",
+ })
+
+ // totalAZHeadObjectErrors is the metric that reports the total Azure head object errors
+ totalAZHeadObjectErrors = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_head_object_errors",
+ Help: "The total number of Azure head object errors",
+ })
+
+ // totalAZHeadContainer is the metric that reports the total successful Azure head container requests
+ totalAZHeadContainer = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_head_container",
+ Help: "The total number of successful Azure head container requests",
+ })
+
+ // totalAZHeadContainerErrors is the metric that reports the total Azure head container errors
+ totalAZHeadContainerErrors = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sftpgo_az_head_container_errors",
+ Help: "The total number of Azure head container errors",
+ })
)
// AddMetricsEndpoint exposes metrics to the specified endpoint
@@ -477,7 +597,16 @@ func S3DeleteObjectCompleted(err error) {
}
}
-// S3HeadBucketCompleted updates metrics after an S3 head bucket request terminates
+// S3HeadObjectCompleted updates metrics after a S3 head object request terminates
+func S3HeadObjectCompleted(err error) {
+ if err == nil {
+ totalS3HeadObject.Inc()
+ } else {
+ totalS3HeadObjectErrors.Inc()
+ }
+}
+
+// S3HeadBucketCompleted updates metrics after a S3 head bucket request terminates
func S3HeadBucketCompleted(err error) {
if err == nil {
totalS3HeadBucket.Inc()
@@ -534,6 +663,15 @@ func GCSDeleteObjectCompleted(err error) {
}
}
+// GCSHeadObjectCompleted updates metrics after a GCS head object request terminates
+func GCSHeadObjectCompleted(err error) {
+ if err == nil {
+ totalGCSHeadObject.Inc()
+ } else {
+ totalGCSHeadObjectErrors.Inc()
+ }
+}
+
// GCSHeadBucketCompleted updates metrics after a GCS head bucket request terminates
func GCSHeadBucketCompleted(err error) {
if err == nil {
@@ -543,6 +681,72 @@ func GCSHeadBucketCompleted(err error) {
}
}
+// AZTransferCompleted updates metrics after a Azure upload or a download
+func AZTransferCompleted(bytes int64, transferKind int, err error) {
+ if transferKind == 0 {
+ // upload
+ if err == nil {
+ totalAZUploads.Inc()
+ } else {
+ totalAZUploadErrors.Inc()
+ }
+ totalAZUploadSize.Add(float64(bytes))
+ } else {
+ // download
+ if err == nil {
+ totalAZDownloads.Inc()
+ } else {
+ totalAZDownloadErrors.Inc()
+ }
+ totalAZDownloadSize.Add(float64(bytes))
+ }
+}
+
+// AZListObjectsCompleted updates metrics after a Azure list objects request terminates
+func AZListObjectsCompleted(err error) {
+ if err == nil {
+ totalAZListObjects.Inc()
+ } else {
+ totalAZListObjectsErrors.Inc()
+ }
+}
+
+// AZCopyObjectCompleted updates metrics after a Azure copy object request terminates
+func AZCopyObjectCompleted(err error) {
+ if err == nil {
+ totalAZCopyObject.Inc()
+ } else {
+ totalAZCopyObjectErrors.Inc()
+ }
+}
+
+// AZDeleteObjectCompleted updates metrics after a Azure delete object request terminates
+func AZDeleteObjectCompleted(err error) {
+ if err == nil {
+ totalAZDeleteObject.Inc()
+ } else {
+ totalAZDeleteObjectErrors.Inc()
+ }
+}
+
+// AZHeadObjectCompleted updates metrics after a Azure head object request terminates
+func AZHeadObjectCompleted(err error) {
+ if err == nil {
+ totalAZHeadObject.Inc()
+ } else {
+ totalAZHeadObjectErrors.Inc()
+ }
+}
+
+// AZHeadContainerCompleted updates metrics after a Azure head container request terminates
+func AZHeadContainerCompleted(err error) {
+ if err == nil {
+ totalAZHeadContainer.Inc()
+ } else {
+ totalAZHeadContainerErrors.Inc()
+ }
+}
+
// SSHCommandCompleted update metrics after an SSH command terminates
func SSHCommandCompleted(err error) {
if err == nil {
diff --git a/templates/user.html b/templates/user.html
index b1a6400e..93c2b23e 100644
--- a/templates/user.html
+++ b/templates/user.html
@@ -276,7 +276,7 @@
<div class="col-sm-10">
<select class="form-control" id="idFilesystem" name="fs_provider" onchange="onFilesystemChanged(this.value)">
<option value="0" {{if eq .User.FsConfig.Provider 0 }}selected{{end}}>local</option>
- <option value="1" {{if eq .User.FsConfig.Provider 1 }}selected{{end}}>Amazon S3 (Compatible)</option>
+ <option value="1" {{if eq .User.FsConfig.Provider 1 }}selected{{end}}>AWS S3 (Compatible)</option>
<option value="2" {{if eq .User.FsConfig.Provider 2 }}selected{{end}}>Google Cloud Storage</option>
<option value="3" {{if eq .User.FsConfig.Provider 3 }}selected{{end}}>Azure Blob Storage</option>
</select>
diff --git a/vfs/azblobfs.go b/vfs/azblobfs.go
index 2190485d..74f033b4 100644
--- a/vfs/azblobfs.go
+++ b/vfs/azblobfs.go
@@ -23,6 +23,7 @@ import (
"github.com/eikenb/pipeat"
"github.com/drakkan/sftpgo/logger"
+ "github.com/drakkan/sftpgo/metrics"
"github.com/drakkan/sftpgo/utils"
"github.com/drakkan/sftpgo/version"
)
@@ -185,11 +186,13 @@ func (fs AzureBlobFs) Stat(name string) (os.FileInfo, error) {
Prefix: prefix,
})
if err != nil {
+ metrics.AZListObjectsCompleted(err)
return nil, err
}
marker = listBlob.NextMarker
for _, blobPrefix := range listBlob.Segment.BlobPrefixes {
if fs.isEqual(blobPrefix.Name, name) {
+ metrics.AZListObjectsCompleted(nil)
return NewFileInfo(name, true, 0, time.Now(), false), nil
}
}
@@ -203,11 +206,13 @@ func (fs AzureBlobFs) Stat(name string) (os.FileInfo, error) {
if blobInfo.Properties.ContentLength != nil {
size = *blobInfo.Properties.ContentLength
}
+ metrics.AZListObjectsCompleted(nil)
return NewFileInfo(name, isDir, size, blobInfo.Properties.LastModified, false), nil
}
}
}
+ metrics.AZListObjectsCompleted(nil)
return nil, errors.New("404 no such file or directory")
}
@@ -242,6 +247,7 @@ func (fs AzureBlobFs) Open(name string, offset int64) (*os.File, *pipeat.PipeRea
n, err := io.Copy(w, body)
w.CloseWithError(err) //nolint:errcheck
fsLog(fs, logger.LevelDebug, "download completed, path: %#v size: %v, err: %v", name, n, err)
+ metrics.AZTransferCompleted(n, 1, err)
}()
return nil, r, cancelFn, nil
@@ -284,6 +290,7 @@ func (fs AzureBlobFs) Create(name string, flag int) (*os.File, *PipeWriter, func
r.CloseWithError(err) //nolint:errcheck
p.Done(err)
fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, readed bytes: %v, err: %v", name, r.GetReadedBytes(), err)
+ metrics.AZTransferCompleted(r.GetReadedBytes(), 0, err)
}()
return nil, p, cancelFn, nil
@@ -322,6 +329,7 @@ func (fs AzureBlobFs) Rename(source, target string) error {
resp, err := dstBlobURL.StartCopyFromURL(ctx, srcURL, md, mac, bac)
if err != nil {
+ metrics.AZCopyObjectCompleted(err)
return err
}
copyStatus := resp.CopyStatus()
@@ -335,6 +343,7 @@ func (fs AzureBlobFs) Rename(source, target string) error {
// of them before giving up.
nErrors++
if ctx.Err() != nil || nErrors == 3 {
+ metrics.AZCopyObjectCompleted(err)
return err
}
} else {
@@ -342,8 +351,11 @@ func (fs AzureBlobFs) Rename(source, target string) error {
}
}
if copyStatus != azblob.CopyStatusSuccess {
- return fmt.Errorf("Copy failed with status: %s", copyStatus)
+ err := fmt.Errorf("Copy failed with status: %s", copyStatus)
+ metrics.AZCopyObjectCompleted(err)
+ return err
}
+ metrics.AZCopyObjectCompleted(nil)
return fs.Remove(source, fi.IsDir())
}
@@ -363,6 +375,7 @@ func (fs AzureBlobFs) Remove(name string, isDir bool) error {
defer cancelFn()
_, err := blobBlockURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
+ metrics.AZDeleteObjectCompleted(err)
return err
}
@@ -444,6 +457,7 @@ func (fs AzureBlobFs) ReadDir(dirname string) ([]os.FileInfo, error) {
Prefix: prefix,
})
if err != nil {
+ metrics.AZListObjectsCompleted(err)
return nil, err
}
marker = listBlob.NextMarker
@@ -481,6 +495,7 @@ func (fs AzureBlobFs) ReadDir(dirname string) ([]os.FileInfo, error) {
}
}
+ metrics.AZListObjectsCompleted(nil)
return result, nil
}
@@ -565,6 +580,7 @@ func (fs AzureBlobFs) ScanRootDirContents() (int, int64, error) {
Prefix: fs.config.KeyPrefix,
})
if err != nil {
+ metrics.AZListObjectsCompleted(err)
return numFiles, size, err
}
marker = listBlob.NextMarker
@@ -585,6 +601,7 @@ func (fs AzureBlobFs) ScanRootDirContents() (int, int64, error) {
}
}
+ metrics.AZListObjectsCompleted(nil)
return numFiles, size, nil
}
@@ -644,6 +661,7 @@ func (fs AzureBlobFs) Walk(root string, walkFn filepath.WalkFunc) error {
Prefix: prefix,
})
if err != nil {
+ metrics.AZListObjectsCompleted(err)
return err
}
marker = listBlob.NextMarker
@@ -667,6 +685,7 @@ func (fs AzureBlobFs) Walk(root string, walkFn filepath.WalkFunc) error {
}
}
+ metrics.AZListObjectsCompleted(nil)
return walkFn(root, NewFileInfo(root, true, 0, time.Now(), false), nil)
}
@@ -695,6 +714,7 @@ func (fs AzureBlobFs) GetMimeType(name string) (string, error) {
blobBlockURL := fs.containerURL.NewBlockBlobURL(name)
response, err := blobBlockURL.GetProperties(ctx, azblob.BlobAccessConditions{})
+ metrics.AZHeadObjectCompleted(err)
if err != nil {
return "", err
}
@@ -719,6 +739,7 @@ func (fs *AzureBlobFs) checkIfBucketExists() error {
defer cancelFn()
_, err := fs.containerURL.GetProperties(ctx, azblob.LeaseAccessConditions{})
+ metrics.AZHeadContainerCompleted(err)
return err
}
@@ -790,7 +811,6 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read
defer cancelFn()
_, err := blockBlobURL.StageBlock(innerCtx, blockID, bufferReader, azblob.LeaseAccessConditions{}, nil)
- pool.releaseBuffer(buf)
if err != nil {
errOnce.Do(func() {
poolError = err
@@ -798,6 +818,7 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read
poolCancel()
})
}
+ pool.releaseBuffer(buf)
<-guard
}(blockID, buf, n)
}
@@ -840,11 +861,13 @@ type bufferAllocator struct {
sync.Mutex
available [][]byte
bufferSize int
+ finalized bool
}
func newBufferAllocator(size int) *bufferAllocator {
return &bufferAllocator{
bufferSize: size,
+ finalized: false,
}
}
@@ -871,7 +894,7 @@ func (b *bufferAllocator) releaseBuffer(buf []byte) {
b.Lock()
defer b.Unlock()
- if len(buf) != b.bufferSize {
+ if b.finalized || len(buf) != b.bufferSize {
return
}
@@ -883,4 +906,5 @@ func (b *bufferAllocator) free() {
defer b.Unlock()
b.available = nil
+ b.finalized = true
}
diff --git a/vfs/gcsfs.go b/vfs/gcsfs.go
index 88438f93..bdf23382 100644
--- a/vfs/gcsfs.go
+++ b/vfs/gcsfs.go
@@ -610,6 +610,7 @@ func (fs GCSFs) GetMimeType(name string) (string, error) {
bkt := fs.svc.Bucket(fs.config.Bucket)
obj := bkt.Object(name)
attrs, err := obj.Attrs(ctx)
+ metrics.GCSHeadObjectCompleted(err)
if err != nil {
return "", err
}
diff --git a/vfs/s3fs.go b/vfs/s3fs.go
index a606e599..0ade641c 100644
--- a/vfs/s3fs.go
+++ b/vfs/s3fs.go
@@ -27,7 +27,7 @@ import (
"github.com/drakkan/sftpgo/version"
)
-// S3Fs is a Fs implementation for Amazon S3 compatible object storage.
+// S3Fs is a Fs implementation for AWS S3 compatible object storages
type S3Fs struct {
connectionID string
localTempDir string
@@ -598,6 +598,7 @@ func (fs S3Fs) GetMimeType(name string) (string, error) {
Bucket: aws.String(fs.config.Bucket),
Key: aws.String(name),
})
+ metrics.S3HeadObjectCompleted(err)
if err != nil {
return "", err
}