mirror of
https://github.com/drakkan/sftpgo.git
synced 2024-11-21 15:10:23 +00:00
S3: add metrics
This commit is contained in:
Nicola Murino
parent
d481294519
commit
5f4efc9148
6 changed files with 197 additions and 25 deletions
|
|
@ -372,7 +372,7 @@ The program must write the questions on its standard output, in a single line, u
|
|||
- `instruction`, string. A short description to show to the user that is trying to authenticate. Can be empty or omitted
|
||||
- `questions`, list of questions to be asked to the user
|
||||
- `echos` list of boolean flags corresponding to the questions (so the lengths of both lists must be the same) and indicating whether user's reply for a particular question should be echoed on the screen while they are typing: true if it should be echoed, or false if it should be hidden.
|
||||
- `auth_result`, integer. Set this field to 1 to indicate successfull authentication, 0 is ignored, any other value means authentication error. If this fields is found and it is different from 0 then SFTPGo does not read any other questions from the external program and finalize the authentication.
|
||||
- `auth_result`, integer. Set this field to 1 to indicate successful authentication, 0 is ignored, any other value means authentication error. If this fields is found and it is different from 0 then SFTPGo does not read any other questions from the external program and finalize the authentication.
|
||||
|
||||
SFTPGo writes the user answers to the program standard input, one per line, in the same order of the questions.
|
||||
Please be sure that your program receive the answers for all the issued questions before asking for the next ones.
|
||||
|
|
@ -411,7 +411,7 @@ Actions will not be executed if an error is detected and so a partial file is up
|
|||
|
||||
The `command`, if defined, is invoked with the following arguments:
|
||||
|
||||
- `action`, string, possibile values are: `download`, `upload`, `delete`, `rename`, `ssh_cmd`
|
||||
- `action`, string, possible values are: `download`, `upload`, `delete`, `rename`, `ssh_cmd`
|
||||
- `username`
|
||||
- `path` is the full filesystem path, can be empty for some ssh commands
|
||||
- `target_path`, non empty for `rename` action
|
||||
|
|
@ -446,7 +446,7 @@ Actions will not be fired for internal updates such as the last login or the use
|
|||
|
||||
The `command`, if defined, is invoked with the following arguments:
|
||||
|
||||
- `action`, string, possibile values are: `add`, `update`, `delete`
|
||||
- `action`, string, possible values are: `add`, `update`, `delete`
|
||||
- `username`
|
||||
- `ID`
|
||||
- `status`
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -27,4 +27,4 @@ require (
|
|||
gopkg.in/natefinch/lumberjack.v2 v2.0.0
|
||||
)
|
||||
|
||||
replace github.com/eikenb/pipeat v0.0.0-20190316224601-fb1f3a9aa29f => github.com/drakkan/pipeat v0.0.0-20200122173221-ea03f92ba172
|
||||
replace github.com/eikenb/pipeat v0.0.0-20190316224601-fb1f3a9aa29f => github.com/drakkan/pipeat v0.0.0-20200123131427-11c048cfc0ec
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -32,8 +32,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
|
|||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/drakkan/pipeat v0.0.0-20200122173221-ea03f92ba172 h1:0mSDVf/0IPuuy3c5Qg+ceoxgbV7KUoLN3Ircswglf7A=
|
||||
github.com/drakkan/pipeat v0.0.0-20200122173221-ea03f92ba172/go.mod h1:wNYvIpR5rIhoezOYcpxcXz4HbIEOu7A45EqlQCA+h+w=
|
||||
github.com/drakkan/pipeat v0.0.0-20200123131427-11c048cfc0ec h1:DXfzg1NXoesnFzdCyyi2uU3o1o0XiWTN2ZcpWDE7MCk=
|
||||
github.com/drakkan/pipeat v0.0.0-20200123131427-11c048cfc0ec/go.mod h1:wNYvIpR5rIhoezOYcpxcXz4HbIEOu7A45EqlQCA+h+w=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
|
|
|
|||
|
|
@ -19,40 +19,40 @@ var (
|
|||
Help: "Total number of logged in users",
|
||||
})
|
||||
|
||||
// totalUploads is the metric that reports the total number of uploads
|
||||
// totalUploads is the metric that reports the total number of successful SFTP/SCP uploads
|
||||
totalUploads = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_uploads_total",
|
||||
Help: "The total number of uploads",
|
||||
Help: "The total number of successful SFTP/SCP uploads",
|
||||
})
|
||||
|
||||
// totalDownloads is the metric that reports the total number of downloads
|
||||
// totalDownloads is the metric that reports the total number of successful SFTP/SCP downloads
|
||||
totalDownloads = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_downloads_total",
|
||||
Help: "The total number of downloads",
|
||||
Help: "The total number of successful SFTP/SCP downloads",
|
||||
})
|
||||
|
||||
// totalUploadErrors is the metric that reports the total number of upload errors
|
||||
// totalUploadErrors is the metric that reports the total number of SFTP/SCP upload errors
|
||||
totalUploadErrors = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_upload_errors_total",
|
||||
Help: "The total number of upload errors",
|
||||
Help: "The total number of SFTP/SCP upload errors",
|
||||
})
|
||||
|
||||
// totalDownloadErrors is the metric that reports the total number of download errors
|
||||
// totalDownloadErrors is the metric that reports the total number of SFTP/SCP download errors
|
||||
totalDownloadErrors = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_download_errors_total",
|
||||
Help: "The total number of download errors",
|
||||
Help: "The total number of SFTP/SCP download errors",
|
||||
})
|
||||
|
||||
// totalUploadSize is the metric that reports the total uploads size as bytes
|
||||
// totalUploadSize is the metric that reports the total SFTP/SCP uploads size as bytes
|
||||
totalUploadSize = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_upload_size",
|
||||
Help: "The total upload size as bytes",
|
||||
Help: "The total SFTP/SCP upload size as bytes, partial uploads are included",
|
||||
})
|
||||
|
||||
// totalDownloadSize is the metric that reports the total downloads size as bytes
|
||||
// totalDownloadSize is the metric that reports the total SFTP/SCP downloads size as bytes
|
||||
totalDownloadSize = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_download_size",
|
||||
Help: "The total download size as bytes",
|
||||
Help: "The total SFTP/SCP download size as bytes, partial downloads are included",
|
||||
})
|
||||
|
||||
// totalSSHCommands is the metric that reports the total number of executed SSH commands
|
||||
|
|
@ -167,26 +167,188 @@ var (
|
|||
Name: "sftpgo_http_server_errors_total",
|
||||
Help: "The total number of HTTP requests served with 5xx status code",
|
||||
})
|
||||
|
||||
// totalS3Uploads is the metric that reports the total number of successful S3 uploads
|
||||
totalS3Uploads = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_uploads_total",
|
||||
Help: "The total number of successful S3 uploads",
|
||||
})
|
||||
|
||||
// totalS3Downloads is the metric that reports the total number of successful S3 downloads
|
||||
totalS3Downloads = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_downloads_total",
|
||||
Help: "The total number of successful S3 downloads",
|
||||
})
|
||||
|
||||
// totalS3UploadErrors is the metric that reports the total number of S3 upload errors
|
||||
totalS3UploadErrors = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_upload_errors_total",
|
||||
Help: "The total number of S3 upload errors",
|
||||
})
|
||||
|
||||
// totalS3DownloadErrors is the metric that reports the total number of S3 download errors
|
||||
totalS3DownloadErrors = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_download_errors_total",
|
||||
Help: "The total number of S3 download errors",
|
||||
})
|
||||
|
||||
// totalS3UploadSize is the metric that reports the total S3 uploads size as bytes
|
||||
totalS3UploadSize = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_upload_size",
|
||||
Help: "The total S3 upload size as bytes, partial uploads are included",
|
||||
})
|
||||
|
||||
// totalS3DownloadSize is the metric that reports the total S3 downloads size as bytes
|
||||
totalS3DownloadSize = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_download_size",
|
||||
Help: "The total S3 download size as bytes, partial downloads are included",
|
||||
})
|
||||
|
||||
// totalS3ListObjects is the metric that reports the total successful S3 list objects requests
|
||||
totalS3ListObjects = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_list_objects",
|
||||
Help: "The total number of successful S3 list objects requests",
|
||||
})
|
||||
|
||||
// totalS3CopyObject is the metric that reports the total successful S3 copy object requests
|
||||
totalS3CopyObject = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_copy_object",
|
||||
Help: "The total number of successful S3 copy object requests",
|
||||
})
|
||||
|
||||
// totalS3DeleteObject is the metric that reports the total successful S3 delete object requests
|
||||
totalS3DeleteObject = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_delete_object",
|
||||
Help: "The total number of successful S3 delete object requests",
|
||||
})
|
||||
|
||||
// totalS3ListObjectsError is the metric that reports the total S3 list objects errors
|
||||
totalS3ListObjectsErrors = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_list_objects_errors",
|
||||
Help: "The total number of S3 list objects errors",
|
||||
})
|
||||
|
||||
// totalS3CopyObjectErrors is the metric that reports the total S3 copy object errors
|
||||
totalS3CopyObjectErrors = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_copy_object_errors",
|
||||
Help: "The total number of S3 copy object errors",
|
||||
})
|
||||
|
||||
// totalS3DeleteObjectErrors is the metric that reports the total S3 delete object errors
|
||||
totalS3DeleteObjectErrors = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_delete_object_errors",
|
||||
Help: "The total number of S3 delete object errors",
|
||||
})
|
||||
|
||||
// totalS3HeadBucket is the metric that reports the total successful S3 head bucket requests
|
||||
totalS3HeadBucket = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_head_bucket",
|
||||
Help: "The total number of successful S3 head bucket requests",
|
||||
})
|
||||
|
||||
// totalS3CreateBucket is the metric that reports the total successful S3 create bucket requests
|
||||
totalS3CreateBucket = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_create_bucket",
|
||||
Help: "The total number of successful S3 create bucket requests",
|
||||
})
|
||||
|
||||
// totalS3HeadBucketErrors is the metric that reports the total S3 head bucket errors
|
||||
totalS3HeadBucketErrors = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_head_bucket_errors",
|
||||
Help: "The total number of S3 head bucket errors",
|
||||
})
|
||||
|
||||
// totalS3CreateBucketErrors is the metric that reports the total S3 create bucket errors
|
||||
totalS3CreateBucketErrors = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sftpgo_s3_create_bucket_errors",
|
||||
Help: "The total number of S3 create bucket errors",
|
||||
})
|
||||
)
|
||||
|
||||
// TransferCompleted update metrics after an upload or a download
|
||||
// TransferCompleted updates metrics after an upload or a download
|
||||
func TransferCompleted(bytesSent, bytesReceived int64, transferKind int, err error) {
|
||||
if transferKind == 0 {
|
||||
// upload
|
||||
if err == nil {
|
||||
totalUploads.Inc()
|
||||
totalUploadSize.Add(float64(bytesReceived))
|
||||
} else {
|
||||
totalUploadErrors.Inc()
|
||||
}
|
||||
totalUploadSize.Add(float64(bytesReceived))
|
||||
} else {
|
||||
// download
|
||||
if err == nil {
|
||||
totalDownloads.Inc()
|
||||
totalDownloadSize.Add(float64(bytesSent))
|
||||
} else {
|
||||
totalDownloadErrors.Inc()
|
||||
}
|
||||
totalDownloadSize.Add(float64(bytesSent))
|
||||
}
|
||||
}
|
||||
|
||||
// S3TransferCompleted updates metrics after an S3 upload or a download
|
||||
func S3TransferCompleted(bytes int64, transferKind int, err error) {
|
||||
if transferKind == 0 {
|
||||
// upload
|
||||
if err == nil {
|
||||
totalS3Uploads.Inc()
|
||||
} else {
|
||||
totalS3UploadErrors.Inc()
|
||||
}
|
||||
totalS3UploadSize.Add(float64(bytes))
|
||||
} else {
|
||||
// download
|
||||
if err == nil {
|
||||
totalS3Downloads.Inc()
|
||||
} else {
|
||||
totalS3DownloadErrors.Inc()
|
||||
}
|
||||
totalS3DownloadSize.Add(float64(bytes))
|
||||
}
|
||||
}
|
||||
|
||||
// S3ListObjectsCompleted updates metrics after an S3 list objects request terminates
|
||||
func S3ListObjectsCompleted(err error) {
|
||||
if err == nil {
|
||||
totalS3ListObjects.Inc()
|
||||
} else {
|
||||
totalS3ListObjectsErrors.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
// S3CopyObjectCompleted updates metrics after an S3 copy object request terminates
|
||||
func S3CopyObjectCompleted(err error) {
|
||||
if err == nil {
|
||||
totalS3CopyObject.Inc()
|
||||
} else {
|
||||
totalS3CopyObjectErrors.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
// S3DeleteObjectCompleted updates metrics after an S3 delete object request terminates
|
||||
func S3DeleteObjectCompleted(err error) {
|
||||
if err == nil {
|
||||
totalS3DeleteObject.Inc()
|
||||
} else {
|
||||
totalS3DeleteObjectErrors.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
// S3HeadBucketCompleted updates metrics after an S3 head bucket request terminates
|
||||
func S3HeadBucketCompleted(err error) {
|
||||
if err == nil {
|
||||
totalS3HeadBucket.Inc()
|
||||
} else {
|
||||
totalS3HeadBucketErrors.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
// S3CreateBucketCompleted updates metrics after an S3 create bucket request terminates
|
||||
func S3CreateBucketCompleted(err error) {
|
||||
if err == nil {
|
||||
totalS3CreateBucket.Inc()
|
||||
} else {
|
||||
totalS3CreateBucketErrors.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -132,6 +132,7 @@ func (t *Transfer) Close() error {
|
|||
numFiles = 1
|
||||
}
|
||||
t.checkDownloadSize()
|
||||
metrics.TransferCompleted(t.bytesSent, t.bytesReceived, t.transferType, t.transferError)
|
||||
if t.transferType == transferUpload && t.file != nil && t.file.Name() != t.path {
|
||||
if t.transferError == nil || uploadMode == uploadModeAtomicWithResume {
|
||||
err = os.Rename(t.file.Name(), t.path)
|
||||
|
|
@ -162,7 +163,6 @@ func (t *Transfer) Close() error {
|
|||
err = t.transferError
|
||||
}
|
||||
}
|
||||
metrics.TransferCompleted(t.bytesSent, t.bytesReceived, t.transferType, t.transferError)
|
||||
removeTransfer(t)
|
||||
t.updateQuota(numFiles)
|
||||
return err
|
||||
|
|
|
|||
16
vfs/s3fs.go
16
vfs/s3fs.go
|
|
@ -16,6 +16,7 @@ import (
|
|||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/drakkan/sftpgo/logger"
|
||||
"github.com/drakkan/sftpgo/metrics"
|
||||
"github.com/drakkan/sftpgo/utils"
|
||||
"github.com/eikenb/pipeat"
|
||||
)
|
||||
|
|
@ -138,6 +139,7 @@ func (fs S3Fs) Stat(name string) (os.FileInfo, error) {
|
|||
}
|
||||
return true
|
||||
})
|
||||
metrics.S3ListObjectsCompleted(err)
|
||||
if err == nil && len(result.Name()) == 0 {
|
||||
err = errors.New("404 no such file or directory")
|
||||
}
|
||||
|
|
@ -164,8 +166,9 @@ func (fs S3Fs) Open(name string) (*os.File, *pipeat.PipeReaderAt, func(), error)
|
|||
Bucket: aws.String(fs.config.Bucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
fsLog(fs, logger.LevelDebug, "download completed, path: %#v size: %v, err: %v", name, n, err)
|
||||
w.CloseWithError(err)
|
||||
fsLog(fs, logger.LevelDebug, "download completed, path: %#v size: %v, err: %v", name, n, err)
|
||||
metrics.S3TransferCompleted(n, 1, err)
|
||||
}()
|
||||
return nil, r, cancelFn, nil
|
||||
}
|
||||
|
|
@ -187,8 +190,10 @@ func (fs S3Fs) Create(name string, flag int) (*os.File, *pipeat.PipeWriterAt, fu
|
|||
Body: r,
|
||||
StorageClass: utils.NilIfEmpty(fs.config.StorageClass),
|
||||
})
|
||||
fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, response: %v, err: %v", name, response, err)
|
||||
r.CloseWithError(err)
|
||||
fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, response: %v, readed bytes: %v, err: %v",
|
||||
name, response, r.GetReadedBytes(), err)
|
||||
metrics.S3TransferCompleted(r.GetReadedBytes(), 0, err)
|
||||
}()
|
||||
return nil, w, cancelFn, nil
|
||||
}
|
||||
|
|
@ -229,6 +234,7 @@ func (fs S3Fs) Rename(source, target string) error {
|
|||
CopySource: aws.String(copySource),
|
||||
Key: aws.String(target),
|
||||
})
|
||||
metrics.S3CopyObjectCompleted(err)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -255,6 +261,7 @@ func (fs S3Fs) Remove(name string, isDir bool) error {
|
|||
Bucket: aws.String(fs.config.Bucket),
|
||||
Key: aws.String(name),
|
||||
})
|
||||
metrics.S3DeleteObjectCompleted(err)
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -331,6 +338,7 @@ func (fs S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) {
|
|||
}
|
||||
return true
|
||||
})
|
||||
metrics.S3ListObjectsCompleted(err)
|
||||
return result, err
|
||||
}
|
||||
|
||||
|
|
@ -401,6 +409,7 @@ func (fs S3Fs) CheckRootPath(username string, uid int, gid int) bool {
|
|||
_, err = fs.svc.CreateBucketWithContext(ctx, input)
|
||||
fsLog(fs, logger.LevelDebug, "bucket %#v for user %#v does not exists, try to create, error: %v",
|
||||
fs.config.Bucket, username, err)
|
||||
metrics.S3CreateBucketCompleted(err)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
|
|
@ -421,7 +430,7 @@ func (fs S3Fs) ScanRootDirContents() (int, int64, error) {
|
|||
}
|
||||
return true
|
||||
})
|
||||
|
||||
metrics.S3ListObjectsCompleted(err)
|
||||
return numFiles, size, err
|
||||
}
|
||||
|
||||
|
|
@ -496,6 +505,7 @@ func (fs *S3Fs) checkIfBucketExists() error {
|
|||
_, err := fs.svc.HeadBucketWithContext(ctx, &s3.HeadBucketInput{
|
||||
Bucket: aws.String(fs.config.Bucket),
|
||||
})
|
||||
metrics.S3HeadBucketCompleted(err)
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue