From 5f4efc9148e659fb95a92aa9935bf504126e72fb Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Thu, 23 Jan 2020 23:17:00 +0100 Subject: [PATCH] S3: add metrics --- README.md | 6 +- go.mod | 2 +- go.sum | 4 +- metrics/metrics.go | 192 +++++++++++++++++++++++++++++++++++++++++---- sftpd/transfer.go | 2 +- vfs/s3fs.go | 16 +++- 6 files changed, 197 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 4d828065..5d97eb7e 100644 --- a/README.md +++ b/README.md @@ -372,7 +372,7 @@ The program must write the questions on its standard output, in a single line, u - `instruction`, string. A short description to show to the user that is trying to authenticate. Can be empty or omitted - `questions`, list of questions to be asked to the user - `echos` list of boolean flags corresponding to the questions (so the lengths of both lists must be the same) and indicating whether user's reply for a particular question should be echoed on the screen while they are typing: true if it should be echoed, or false if it should be hidden. -- `auth_result`, integer. Set this field to 1 to indicate successfull authentication, 0 is ignored, any other value means authentication error. If this fields is found and it is different from 0 then SFTPGo does not read any other questions from the external program and finalize the authentication. +- `auth_result`, integer. Set this field to 1 to indicate successful authentication, 0 is ignored, any other value means authentication error. If this fields is found and it is different from 0 then SFTPGo does not read any other questions from the external program and finalize the authentication. SFTPGo writes the user answers to the program standard input, one per line, in the same order of the questions. Please be sure that your program receive the answers for all the issued questions before asking for the next ones. @@ -411,7 +411,7 @@ Actions will not be executed if an error is detected and so a partial file is up The `command`, if defined, is invoked with the following arguments: -- `action`, string, possibile values are: `download`, `upload`, `delete`, `rename`, `ssh_cmd` +- `action`, string, possible values are: `download`, `upload`, `delete`, `rename`, `ssh_cmd` - `username` - `path` is the full filesystem path, can be empty for some ssh commands - `target_path`, non empty for `rename` action @@ -446,7 +446,7 @@ Actions will not be fired for internal updates such as the last login or the use The `command`, if defined, is invoked with the following arguments: -- `action`, string, possibile values are: `add`, `update`, `delete` +- `action`, string, possible values are: `add`, `update`, `delete` - `username` - `ID` - `status` diff --git a/go.mod b/go.mod index 9b2d5bc8..ad1d004f 100644 --- a/go.mod +++ b/go.mod @@ -27,4 +27,4 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) -replace github.com/eikenb/pipeat v0.0.0-20190316224601-fb1f3a9aa29f => github.com/drakkan/pipeat v0.0.0-20200122173221-ea03f92ba172 +replace github.com/eikenb/pipeat v0.0.0-20190316224601-fb1f3a9aa29f => github.com/drakkan/pipeat v0.0.0-20200123131427-11c048cfc0ec diff --git a/go.sum b/go.sum index 65a06d2c..fc1a180a 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/drakkan/pipeat v0.0.0-20200122173221-ea03f92ba172 h1:0mSDVf/0IPuuy3c5Qg+ceoxgbV7KUoLN3Ircswglf7A= -github.com/drakkan/pipeat v0.0.0-20200122173221-ea03f92ba172/go.mod h1:wNYvIpR5rIhoezOYcpxcXz4HbIEOu7A45EqlQCA+h+w= +github.com/drakkan/pipeat v0.0.0-20200123131427-11c048cfc0ec h1:DXfzg1NXoesnFzdCyyi2uU3o1o0XiWTN2ZcpWDE7MCk= +github.com/drakkan/pipeat v0.0.0-20200123131427-11c048cfc0ec/go.mod h1:wNYvIpR5rIhoezOYcpxcXz4HbIEOu7A45EqlQCA+h+w= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= diff --git a/metrics/metrics.go b/metrics/metrics.go index 3516cdae..b5ded44d 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -19,40 +19,40 @@ var ( Help: "Total number of logged in users", }) - // totalUploads is the metric that reports the total number of uploads + // totalUploads is the metric that reports the total number of successful SFTP/SCP uploads totalUploads = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_uploads_total", - Help: "The total number of uploads", + Help: "The total number of successful SFTP/SCP uploads", }) - // totalDownloads is the metric that reports the total number of downloads + // totalDownloads is the metric that reports the total number of successful SFTP/SCP downloads totalDownloads = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_downloads_total", - Help: "The total number of downloads", + Help: "The total number of successful SFTP/SCP downloads", }) - // totalUploadErrors is the metric that reports the total number of upload errors + // totalUploadErrors is the metric that reports the total number of SFTP/SCP upload errors totalUploadErrors = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_upload_errors_total", - Help: "The total number of upload errors", + Help: "The total number of SFTP/SCP upload errors", }) - // totalDownloadErrors is the metric that reports the total number of download errors + // totalDownloadErrors is the metric that reports the total number of SFTP/SCP download errors totalDownloadErrors = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_download_errors_total", - Help: "The total number of download errors", + Help: "The total number of SFTP/SCP download errors", }) - // totalUploadSize is the metric that reports the total uploads size as bytes + // totalUploadSize is the metric that reports the total SFTP/SCP uploads size as bytes totalUploadSize = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_upload_size", - Help: "The total upload size as bytes", + Help: "The total SFTP/SCP upload size as bytes, partial uploads are included", }) - // totalDownloadSize is the metric that reports the total downloads size as bytes + // totalDownloadSize is the metric that reports the total SFTP/SCP downloads size as bytes totalDownloadSize = promauto.NewCounter(prometheus.CounterOpts{ Name: "sftpgo_download_size", - Help: "The total download size as bytes", + Help: "The total SFTP/SCP download size as bytes, partial downloads are included", }) // totalSSHCommands is the metric that reports the total number of executed SSH commands @@ -167,26 +167,188 @@ var ( Name: "sftpgo_http_server_errors_total", Help: "The total number of HTTP requests served with 5xx status code", }) + + // totalS3Uploads is the metric that reports the total number of successful S3 uploads + totalS3Uploads = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_uploads_total", + Help: "The total number of successful S3 uploads", + }) + + // totalS3Downloads is the metric that reports the total number of successful S3 downloads + totalS3Downloads = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_downloads_total", + Help: "The total number of successful S3 downloads", + }) + + // totalS3UploadErrors is the metric that reports the total number of S3 upload errors + totalS3UploadErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_upload_errors_total", + Help: "The total number of S3 upload errors", + }) + + // totalS3DownloadErrors is the metric that reports the total number of S3 download errors + totalS3DownloadErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_download_errors_total", + Help: "The total number of S3 download errors", + }) + + // totalS3UploadSize is the metric that reports the total S3 uploads size as bytes + totalS3UploadSize = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_upload_size", + Help: "The total S3 upload size as bytes, partial uploads are included", + }) + + // totalS3DownloadSize is the metric that reports the total S3 downloads size as bytes + totalS3DownloadSize = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_download_size", + Help: "The total S3 download size as bytes, partial downloads are included", + }) + + // totalS3ListObjects is the metric that reports the total successful S3 list objects requests + totalS3ListObjects = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_list_objects", + Help: "The total number of successful S3 list objects requests", + }) + + // totalS3CopyObject is the metric that reports the total successful S3 copy object requests + totalS3CopyObject = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_copy_object", + Help: "The total number of successful S3 copy object requests", + }) + + // totalS3DeleteObject is the metric that reports the total successful S3 delete object requests + totalS3DeleteObject = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_delete_object", + Help: "The total number of successful S3 delete object requests", + }) + + // totalS3ListObjectsError is the metric that reports the total S3 list objects errors + totalS3ListObjectsErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_list_objects_errors", + Help: "The total number of S3 list objects errors", + }) + + // totalS3CopyObjectErrors is the metric that reports the total S3 copy object errors + totalS3CopyObjectErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_copy_object_errors", + Help: "The total number of S3 copy object errors", + }) + + // totalS3DeleteObjectErrors is the metric that reports the total S3 delete object errors + totalS3DeleteObjectErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_delete_object_errors", + Help: "The total number of S3 delete object errors", + }) + + // totalS3HeadBucket is the metric that reports the total successful S3 head bucket requests + totalS3HeadBucket = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_head_bucket", + Help: "The total number of successful S3 head bucket requests", + }) + + // totalS3CreateBucket is the metric that reports the total successful S3 create bucket requests + totalS3CreateBucket = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_create_bucket", + Help: "The total number of successful S3 create bucket requests", + }) + + // totalS3HeadBucketErrors is the metric that reports the total S3 head bucket errors + totalS3HeadBucketErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_head_bucket_errors", + Help: "The total number of S3 head bucket errors", + }) + + // totalS3CreateBucketErrors is the metric that reports the total S3 create bucket errors + totalS3CreateBucketErrors = promauto.NewCounter(prometheus.CounterOpts{ + Name: "sftpgo_s3_create_bucket_errors", + Help: "The total number of S3 create bucket errors", + }) ) -// TransferCompleted update metrics after an upload or a download +// TransferCompleted updates metrics after an upload or a download func TransferCompleted(bytesSent, bytesReceived int64, transferKind int, err error) { if transferKind == 0 { // upload if err == nil { totalUploads.Inc() - totalUploadSize.Add(float64(bytesReceived)) } else { totalUploadErrors.Inc() } + totalUploadSize.Add(float64(bytesReceived)) } else { // download if err == nil { totalDownloads.Inc() - totalDownloadSize.Add(float64(bytesSent)) } else { totalDownloadErrors.Inc() } + totalDownloadSize.Add(float64(bytesSent)) + } +} + +// S3TransferCompleted updates metrics after an S3 upload or a download +func S3TransferCompleted(bytes int64, transferKind int, err error) { + if transferKind == 0 { + // upload + if err == nil { + totalS3Uploads.Inc() + } else { + totalS3UploadErrors.Inc() + } + totalS3UploadSize.Add(float64(bytes)) + } else { + // download + if err == nil { + totalS3Downloads.Inc() + } else { + totalS3DownloadErrors.Inc() + } + totalS3DownloadSize.Add(float64(bytes)) + } +} + +// S3ListObjectsCompleted updates metrics after an S3 list objects request terminates +func S3ListObjectsCompleted(err error) { + if err == nil { + totalS3ListObjects.Inc() + } else { + totalS3ListObjectsErrors.Inc() + } +} + +// S3CopyObjectCompleted updates metrics after an S3 copy object request terminates +func S3CopyObjectCompleted(err error) { + if err == nil { + totalS3CopyObject.Inc() + } else { + totalS3CopyObjectErrors.Inc() + } +} + +// S3DeleteObjectCompleted updates metrics after an S3 delete object request terminates +func S3DeleteObjectCompleted(err error) { + if err == nil { + totalS3DeleteObject.Inc() + } else { + totalS3DeleteObjectErrors.Inc() + } +} + +// S3HeadBucketCompleted updates metrics after an S3 head bucket request terminates +func S3HeadBucketCompleted(err error) { + if err == nil { + totalS3HeadBucket.Inc() + } else { + totalS3HeadBucketErrors.Inc() + } +} + +// S3CreateBucketCompleted updates metrics after an S3 create bucket request terminates +func S3CreateBucketCompleted(err error) { + if err == nil { + totalS3CreateBucket.Inc() + } else { + totalS3CreateBucketErrors.Inc() } } diff --git a/sftpd/transfer.go b/sftpd/transfer.go index 5e691323..8cf46cfa 100644 --- a/sftpd/transfer.go +++ b/sftpd/transfer.go @@ -132,6 +132,7 @@ func (t *Transfer) Close() error { numFiles = 1 } t.checkDownloadSize() + metrics.TransferCompleted(t.bytesSent, t.bytesReceived, t.transferType, t.transferError) if t.transferType == transferUpload && t.file != nil && t.file.Name() != t.path { if t.transferError == nil || uploadMode == uploadModeAtomicWithResume { err = os.Rename(t.file.Name(), t.path) @@ -162,7 +163,6 @@ func (t *Transfer) Close() error { err = t.transferError } } - metrics.TransferCompleted(t.bytesSent, t.bytesReceived, t.transferType, t.transferError) removeTransfer(t) t.updateQuota(numFiles) return err diff --git a/vfs/s3fs.go b/vfs/s3fs.go index 3f40e9fe..3a350bcc 100644 --- a/vfs/s3fs.go +++ b/vfs/s3fs.go @@ -16,6 +16,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/drakkan/sftpgo/logger" + "github.com/drakkan/sftpgo/metrics" "github.com/drakkan/sftpgo/utils" "github.com/eikenb/pipeat" ) @@ -138,6 +139,7 @@ func (fs S3Fs) Stat(name string) (os.FileInfo, error) { } return true }) + metrics.S3ListObjectsCompleted(err) if err == nil && len(result.Name()) == 0 { err = errors.New("404 no such file or directory") } @@ -164,8 +166,9 @@ func (fs S3Fs) Open(name string) (*os.File, *pipeat.PipeReaderAt, func(), error) Bucket: aws.String(fs.config.Bucket), Key: aws.String(key), }) - fsLog(fs, logger.LevelDebug, "download completed, path: %#v size: %v, err: %v", name, n, err) w.CloseWithError(err) + fsLog(fs, logger.LevelDebug, "download completed, path: %#v size: %v, err: %v", name, n, err) + metrics.S3TransferCompleted(n, 1, err) }() return nil, r, cancelFn, nil } @@ -187,8 +190,10 @@ func (fs S3Fs) Create(name string, flag int) (*os.File, *pipeat.PipeWriterAt, fu Body: r, StorageClass: utils.NilIfEmpty(fs.config.StorageClass), }) - fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, response: %v, err: %v", name, response, err) r.CloseWithError(err) + fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, response: %v, readed bytes: %v, err: %v", + name, response, r.GetReadedBytes(), err) + metrics.S3TransferCompleted(r.GetReadedBytes(), 0, err) }() return nil, w, cancelFn, nil } @@ -229,6 +234,7 @@ func (fs S3Fs) Rename(source, target string) error { CopySource: aws.String(copySource), Key: aws.String(target), }) + metrics.S3CopyObjectCompleted(err) if err != nil { return err } @@ -255,6 +261,7 @@ func (fs S3Fs) Remove(name string, isDir bool) error { Bucket: aws.String(fs.config.Bucket), Key: aws.String(name), }) + metrics.S3DeleteObjectCompleted(err) return err } @@ -331,6 +338,7 @@ func (fs S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) { } return true }) + metrics.S3ListObjectsCompleted(err) return result, err } @@ -401,6 +409,7 @@ func (fs S3Fs) CheckRootPath(username string, uid int, gid int) bool { _, err = fs.svc.CreateBucketWithContext(ctx, input) fsLog(fs, logger.LevelDebug, "bucket %#v for user %#v does not exists, try to create, error: %v", fs.config.Bucket, username, err) + metrics.S3CreateBucketCompleted(err) return err == nil } @@ -421,7 +430,7 @@ func (fs S3Fs) ScanRootDirContents() (int, int64, error) { } return true }) - + metrics.S3ListObjectsCompleted(err) return numFiles, size, err } @@ -496,6 +505,7 @@ func (fs *S3Fs) checkIfBucketExists() error { _, err := fs.svc.HeadBucketWithContext(ctx, &s3.HeadBucketInput{ Bucket: aws.String(fs.config.Bucket), }) + metrics.S3HeadBucketCompleted(err) return err }