diff --git a/cmd/portable.go b/cmd/portable.go index a22abdbf..2db055a8 100644 --- a/cmd/portable.go +++ b/cmd/portable.go @@ -234,7 +234,7 @@ advertised via multicast DNS, this flag allows to put username/password inside the advertised TXT record`) portableCmd.Flags().IntVarP(&portableFsProvider, "fs-provider", "f", int(dataprovider.LocalFilesystemProvider), `0 => local filesystem -1 => Amazon S3 compatible +1 => AWS S3 compatible 2 => Google Cloud Storage 3 => Azure Blob Storage`) portableCmd.Flags().StringVar(&portableS3Bucket, "s3-bucket", "", "") diff --git a/dataprovider/user.go b/dataprovider/user.go index b3001818..a2107856 100644 --- a/dataprovider/user.go +++ b/dataprovider/user.go @@ -125,7 +125,7 @@ type FilesystemProvider int // supported values for FilesystemProvider const ( LocalFilesystemProvider FilesystemProvider = iota // Local - S3FilesystemProvider // Amazon S3 compatible + S3FilesystemProvider // AWS S3 compatible GCSFilesystemProvider // Google Cloud Storage AzureBlobFilesystemProvider // Azure Blob Storage ) diff --git a/docs/howto/postgresql-s3.md b/docs/howto/postgresql-s3.md index 9e413e19..50da996f 100644 --- a/docs/howto/postgresql-s3.md +++ b/docs/howto/postgresql-s3.md @@ -208,7 +208,7 @@ Click `Add` and fill the user details, the minimum required parameters are: - `Password` or `Public keys` - `Permissions` - `Home Dir` can be empty since we defined a default base dir -- Select `Amazon S3 (Compatible)` as storage and then set `Bucket`, `Region` and optionally a `Key Prefix` if you want to restrict the user to a specific virtual folder in the bucket. The specified virtual folder does not need to be pre-created. You can leave `Access Key` and `Access Secret` empty since we defined global credentials for the `sftpgo` user and we use this system user to run the SFTPGo service. +- Select `AWS S3 (Compatible)` as storage and then set `Bucket`, `Region` and optionally a `Key Prefix` if you want to restrict the user to a specific virtual folder in the bucket. The specified virtual folder does not need to be pre-created. You can leave `Access Key` and `Access Secret` empty since we defined global credentials for the `sftpgo` user and we use this system user to run the SFTPGo service. You are done! Now you can connect to you SFTPGo instance using any compatible `sftp` client on port `2022`. diff --git a/docs/portable-mode.md b/docs/portable-mode.md index 238a5dd2..0393f712 100644 --- a/docs/portable-mode.md +++ b/docs/portable-mode.md @@ -48,7 +48,7 @@ Flags: relative to the current directory (default ".") -f, --fs-provider int 0 => local filesystem - 1 => Amazon S3 compatible + 1 => AWS S3 compatible 2 => Google Cloud Storage 3 => Azure Blob Storage --ftpd-cert string Path to the certificate file for FTPS diff --git a/examples/ldapauthserver/httpd/models.go b/examples/ldapauthserver/httpd/models.go index 40b33367..42aef279 100644 --- a/examples/ldapauthserver/httpd/models.go +++ b/examples/ldapauthserver/httpd/models.go @@ -51,7 +51,7 @@ type GCSFsConfig struct { // SFTPGoFilesystem defines cloud storage filesystem details type SFTPGoFilesystem struct { - // 0 local filesystem, 1 Amazon S3 compatible, 2 Google Cloud Storage + // 0 local filesystem, 1 AWS S3 compatible, 2 Google Cloud Storage Provider int `json:"provider"` S3Config S3FsConfig `json:"s3config,omitempty"` GCSConfig GCSFsConfig `json:"gcsconfig,omitempty"` diff --git a/metrics/metrics.go b/metrics/metrics.go index b82b0147..0c62b34f 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -306,6 +306,18 @@ var ( Help: "The total number of S3 delete object errors", }) + // totalS3HeadObject is the metric that reports the total successful S3 head object requests + totalS3HeadObject = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_head_object", + Help: "The total number of successful S3 head object requests", + }) + + // totalS3HeadObjectErrors is the metric that reports the total S3 head object errors + totalS3HeadObjectErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_head_object_errors", + Help: "The total number of S3 head object errors", + }) + // totalS3HeadBucket is the metric that reports the total successful S3 head bucket requests totalS3HeadBucket = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_s3_head_bucket", @@ -354,7 +366,7 @@ var ( Help: "The total GCS download size as bytes, partial downloads are included", }) - // totalS3ListObjects is the metric that reports the total successful GCS list objects requests + // totalGCSListObjects is the metric that reports the total successful GCS list objects requests totalGCSListObjects = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_gcs_list_objects", Help: "The total number of successful GCS list objects requests", @@ -366,7 +378,7 @@ var ( Help: "The total number of successful GCS copy object requests", }) - // totalGCSDeleteObject is the metric that reports the total successful S3 delete object requests + // totalGCSDeleteObject is the metric that reports the total successful GCS delete object requests totalGCSDeleteObject = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_gcs_delete_object", Help: "The total number of successful GCS delete object requests", @@ -390,6 +402,18 @@ var ( Help: "The total number of GCS delete object errors", }) + // totalGCSHeadObject is the metric that reports the total successful GCS head object requests + totalGCSHeadObject = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_gcs_head_object", + Help: "The total number of successful GCS head object requests", + }) + + // totalGCSHeadObjectErrors is the metric that reports the total GCS head object errors + totalGCSHeadObjectErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_gcs_head_object_errors", + Help: "The total number of GCS head object errors", + }) + // totalGCSHeadBucket is the metric that reports the total successful GCS head bucket requests totalGCSHeadBucket = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_gcs_head_bucket", @@ -401,6 +425,102 @@ var ( Name: "sftpgo_gcs_head_bucket_errors", Help: "The total number of GCS head bucket errors", }) + + // totalAZUploads is the metric that reports the total number of successful Azure uploads + totalAZUploads = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_uploads_total", + Help: "The total number of successful Azure uploads", + }) + + // totalAZDownloads is the metric that reports the total number of successful Azure downloads + totalAZDownloads = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_downloads_total", + Help: "The total number of successful Azure downloads", + }) + + // totalAZUploadErrors is the metric that reports the total number of Azure upload errors + totalAZUploadErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_upload_errors_total", + Help: "The total number of Azure upload errors", + }) + + // totalAZDownloadErrors is the metric that reports the total number of Azure download errors + totalAZDownloadErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_download_errors_total", + Help: "The total number of Azure download errors", + }) + + // totalAZUploadSize is the metric that reports the total Azure uploads size as bytes + totalAZUploadSize = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_upload_size", + Help: "The total Azure upload size as bytes, partial uploads are included", + }) + + // totalAZDownloadSize is the metric that reports the total Azure downloads size as bytes + totalAZDownloadSize = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_download_size", + Help: "The total Azure download size as bytes, partial downloads are included", + }) + + // totalAZListObjects is the metric that reports the total successful Azure list objects requests + totalAZListObjects = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_list_objects", + Help: "The total number of successful Azure list objects requests", + }) + + // totalAZCopyObject is the metric that reports the total successful Azure copy object requests + totalAZCopyObject = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_copy_object", + Help: "The total number of successful Azure copy object requests", + }) + + // totalAZDeleteObject is the metric that reports the total successful Azure delete object requests + totalAZDeleteObject = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_delete_object", + Help: "The total number of successful Azure delete object requests", + }) + + // totalAZListObjectsError is the metric that reports the total Azure list objects errors + totalAZListObjectsErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_list_objects_errors", + Help: "The total number of Azure list objects errors", + }) + + // totalAZCopyObjectErrors is the metric that reports the total Azure copy object errors + totalAZCopyObjectErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_copy_object_errors", + Help: "The total number of Azure copy object errors", + }) + + // totalAZDeleteObjectErrors is the metric that reports the total Azure delete object errors + totalAZDeleteObjectErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_delete_object_errors", + Help: "The total number of Azure delete object errors", + }) + + // totalAZHeadObject is the metric that reports the total successful Azure head object requests + totalAZHeadObject = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_head_object", + Help: "The total number of successful Azure head object requests", + }) + + // totalAZHeadObjectErrors is the metric that reports the total Azure head object errors + totalAZHeadObjectErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_head_object_errors", + Help: "The total number of Azure head object errors", + }) + + // totalAZHeadContainer is the metric that reports the total successful Azure head container requests + totalAZHeadContainer = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_head_container", + Help: "The total number of successful Azure head container requests", + }) + + // totalAZHeadContainerErrors is the metric that reports the total Azure head container errors + totalAZHeadContainerErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_az_head_container_errors", + Help: "The total number of Azure head container errors", + }) ) // AddMetricsEndpoint exposes metrics to the specified endpoint @@ -477,7 +597,16 @@ func S3DeleteObjectCompleted(err error) { } } -// S3HeadBucketCompleted updates metrics after an S3 head bucket request terminates +// S3HeadObjectCompleted updates metrics after a S3 head object request terminates +func S3HeadObjectCompleted(err error) { + if err == nil { + totalS3HeadObject.Inc() + } else { + totalS3HeadObjectErrors.Inc() + } +} + +// S3HeadBucketCompleted updates metrics after a S3 head bucket request terminates func S3HeadBucketCompleted(err error) { if err == nil { totalS3HeadBucket.Inc() @@ -534,6 +663,15 @@ func GCSDeleteObjectCompleted(err error) { } } +// GCSHeadObjectCompleted updates metrics after a GCS head object request terminates +func GCSHeadObjectCompleted(err error) { + if err == nil { + totalGCSHeadObject.Inc() + } else { + totalGCSHeadObjectErrors.Inc() + } +} + // GCSHeadBucketCompleted updates metrics after a GCS head bucket request terminates func GCSHeadBucketCompleted(err error) { if err == nil { @@ -543,6 +681,72 @@ func GCSHeadBucketCompleted(err error) { } } +// AZTransferCompleted updates metrics after a Azure upload or a download +func AZTransferCompleted(bytes int64, transferKind int, err error) { + if transferKind == 0 { + // upload + if err == nil { + totalAZUploads.Inc() + } else { + totalAZUploadErrors.Inc() + } + totalAZUploadSize.Add(float64(bytes)) + } else { + // download + if err == nil { + totalAZDownloads.Inc() + } else { + totalAZDownloadErrors.Inc() + } + totalAZDownloadSize.Add(float64(bytes)) + } +} + +// AZListObjectsCompleted updates metrics after a Azure list objects request terminates +func AZListObjectsCompleted(err error) { + if err == nil { + totalAZListObjects.Inc() + } else { + totalAZListObjectsErrors.Inc() + } +} + +// AZCopyObjectCompleted updates metrics after a Azure copy object request terminates +func AZCopyObjectCompleted(err error) { + if err == nil { + totalAZCopyObject.Inc() + } else { + totalAZCopyObjectErrors.Inc() + } +} + +// AZDeleteObjectCompleted updates metrics after a Azure delete object request terminates +func AZDeleteObjectCompleted(err error) { + if err == nil { + totalAZDeleteObject.Inc() + } else { + totalAZDeleteObjectErrors.Inc() + } +} + +// AZHeadObjectCompleted updates metrics after a Azure head object request terminates +func AZHeadObjectCompleted(err error) { + if err == nil { + totalAZHeadObject.Inc() + } else { + totalAZHeadObjectErrors.Inc() + } +} + +// AZHeadContainerCompleted updates metrics after a Azure head container request terminates +func AZHeadContainerCompleted(err error) { + if err == nil { + totalAZHeadContainer.Inc() + } else { + totalAZHeadContainerErrors.Inc() + } +} + // SSHCommandCompleted update metrics after an SSH command terminates func SSHCommandCompleted(err error) { if err == nil { diff --git a/templates/user.html b/templates/user.html index b1a6400e..93c2b23e 100644 --- a/templates/user.html +++ b/templates/user.html @@ -276,7 +276,7 @@
diff --git a/vfs/azblobfs.go b/vfs/azblobfs.go index 2190485d..74f033b4 100644 --- a/vfs/azblobfs.go +++ b/vfs/azblobfs.go @@ -23,6 +23,7 @@ import ( "github.com/eikenb/pipeat" "github.com/drakkan/sftpgo/logger" + "github.com/drakkan/sftpgo/metrics" "github.com/drakkan/sftpgo/utils" "github.com/drakkan/sftpgo/version" ) @@ -185,11 +186,13 @@ func (fs AzureBlobFs) Stat(name string) (os.FileInfo, error) { Prefix: prefix, }) if err != nil { + metrics.AZListObjectsCompleted(err) return nil, err } marker = listBlob.NextMarker for _, blobPrefix := range listBlob.Segment.BlobPrefixes { if fs.isEqual(blobPrefix.Name, name) { + metrics.AZListObjectsCompleted(nil) return NewFileInfo(name, true, 0, time.Now(), false), nil } } @@ -203,11 +206,13 @@ func (fs AzureBlobFs) Stat(name string) (os.FileInfo, error) { if blobInfo.Properties.ContentLength != nil { size = *blobInfo.Properties.ContentLength } + metrics.AZListObjectsCompleted(nil) return NewFileInfo(name, isDir, size, blobInfo.Properties.LastModified, false), nil } } } + metrics.AZListObjectsCompleted(nil) return nil, errors.New("404 no such file or directory") } @@ -242,6 +247,7 @@ func (fs AzureBlobFs) Open(name string, offset int64) (*os.File, *pipeat.PipeRea n, err := io.Copy(w, body) w.CloseWithError(err) //nolint:errcheck fsLog(fs, logger.LevelDebug, "download completed, path: %#v size: %v, err: %v", name, n, err) + metrics.AZTransferCompleted(n, 1, err) }() return nil, r, cancelFn, nil @@ -284,6 +290,7 @@ func (fs AzureBlobFs) Create(name string, flag int) (*os.File, *PipeWriter, func r.CloseWithError(err) //nolint:errcheck p.Done(err) fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, readed bytes: %v, err: %v", name, r.GetReadedBytes(), err) + metrics.AZTransferCompleted(r.GetReadedBytes(), 0, err) }() return nil, p, cancelFn, nil @@ -322,6 +329,7 @@ func (fs AzureBlobFs) Rename(source, target string) error { resp, err := dstBlobURL.StartCopyFromURL(ctx, srcURL, md, mac, bac) if err != nil { + metrics.AZCopyObjectCompleted(err) return err } copyStatus := resp.CopyStatus() @@ -335,6 +343,7 @@ func (fs AzureBlobFs) Rename(source, target string) error { // of them before giving up. nErrors++ if ctx.Err() != nil || nErrors == 3 { + metrics.AZCopyObjectCompleted(err) return err } } else { @@ -342,8 +351,11 @@ func (fs AzureBlobFs) Rename(source, target string) error { } } if copyStatus != azblob.CopyStatusSuccess { - return fmt.Errorf("Copy failed with status: %s", copyStatus) + err := fmt.Errorf("Copy failed with status: %s", copyStatus) + metrics.AZCopyObjectCompleted(err) + return err } + metrics.AZCopyObjectCompleted(nil) return fs.Remove(source, fi.IsDir()) } @@ -363,6 +375,7 @@ func (fs AzureBlobFs) Remove(name string, isDir bool) error { defer cancelFn() _, err := blobBlockURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + metrics.AZDeleteObjectCompleted(err) return err } @@ -444,6 +457,7 @@ func (fs AzureBlobFs) ReadDir(dirname string) ([]os.FileInfo, error) { Prefix: prefix, }) if err != nil { + metrics.AZListObjectsCompleted(err) return nil, err } marker = listBlob.NextMarker @@ -481,6 +495,7 @@ func (fs AzureBlobFs) ReadDir(dirname string) ([]os.FileInfo, error) { } } + metrics.AZListObjectsCompleted(nil) return result, nil } @@ -565,6 +580,7 @@ func (fs AzureBlobFs) ScanRootDirContents() (int, int64, error) { Prefix: fs.config.KeyPrefix, }) if err != nil { + metrics.AZListObjectsCompleted(err) return numFiles, size, err } marker = listBlob.NextMarker @@ -585,6 +601,7 @@ func (fs AzureBlobFs) ScanRootDirContents() (int, int64, error) { } } + metrics.AZListObjectsCompleted(nil) return numFiles, size, nil } @@ -644,6 +661,7 @@ func (fs AzureBlobFs) Walk(root string, walkFn filepath.WalkFunc) error { Prefix: prefix, }) if err != nil { + metrics.AZListObjectsCompleted(err) return err } marker = listBlob.NextMarker @@ -667,6 +685,7 @@ func (fs AzureBlobFs) Walk(root string, walkFn filepath.WalkFunc) error { } } + metrics.AZListObjectsCompleted(nil) return walkFn(root, NewFileInfo(root, true, 0, time.Now(), false), nil) } @@ -695,6 +714,7 @@ func (fs AzureBlobFs) GetMimeType(name string) (string, error) { blobBlockURL := fs.containerURL.NewBlockBlobURL(name) response, err := blobBlockURL.GetProperties(ctx, azblob.BlobAccessConditions{}) + metrics.AZHeadObjectCompleted(err) if err != nil { return "", err } @@ -719,6 +739,7 @@ func (fs *AzureBlobFs) checkIfBucketExists() error { defer cancelFn() _, err := fs.containerURL.GetProperties(ctx, azblob.LeaseAccessConditions{}) + metrics.AZHeadContainerCompleted(err) return err } @@ -790,7 +811,6 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read defer cancelFn() _, err := blockBlobURL.StageBlock(innerCtx, blockID, bufferReader, azblob.LeaseAccessConditions{}, nil) - pool.releaseBuffer(buf) if err != nil { errOnce.Do(func() { poolError = err @@ -798,6 +818,7 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read poolCancel() }) } + pool.releaseBuffer(buf) <-guard }(blockID, buf, n) } @@ -840,11 +861,13 @@ type bufferAllocator struct { sync.Mutex available [][]byte bufferSize int + finalized bool } func newBufferAllocator(size int) *bufferAllocator { return &bufferAllocator{ bufferSize: size, + finalized: false, } } @@ -871,7 +894,7 @@ func (b *bufferAllocator) releaseBuffer(buf []byte) { b.Lock() defer b.Unlock() - if len(buf) != b.bufferSize { + if b.finalized || len(buf) != b.bufferSize { return } @@ -883,4 +906,5 @@ func (b *bufferAllocator) free() { defer b.Unlock() b.available = nil + b.finalized = true } diff --git a/vfs/gcsfs.go b/vfs/gcsfs.go index 88438f93..bdf23382 100644 --- a/vfs/gcsfs.go +++ b/vfs/gcsfs.go @@ -610,6 +610,7 @@ func (fs GCSFs) GetMimeType(name string) (string, error) { bkt := fs.svc.Bucket(fs.config.Bucket) obj := bkt.Object(name) attrs, err := obj.Attrs(ctx) + metrics.GCSHeadObjectCompleted(err) if err != nil { return "", err } diff --git a/vfs/s3fs.go b/vfs/s3fs.go index a606e599..0ade641c 100644 --- a/vfs/s3fs.go +++ b/vfs/s3fs.go @@ -27,7 +27,7 @@ import ( "github.com/drakkan/sftpgo/version" ) -// S3Fs is a Fs implementation for Amazon S3 compatible object storage. +// S3Fs is a Fs implementation for AWS S3 compatible object storages type S3Fs struct { connectionID string localTempDir string @@ -598,6 +598,7 @@ func (fs S3Fs) GetMimeType(name string) (string, error) { Bucket: aws.String(fs.config.Bucket), Key: aws.String(name), }) + metrics.S3HeadObjectCompleted(err) if err != nil { return "", err }