From 48451a992429d1514a4007ee7041a3c9c1c62838 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sat, 27 Jul 2019 09:38:09 +0200 Subject: [PATCH] add support for custom actions Configurable custom commands and/or HTTP notifications on SFTP upload, download, delete or rename --- README.md | 61 +++++++++++++++++++++++------------- api/quota.go | 2 +- config/config.go | 5 +++ config/config_test.go | 2 +- sftpd/handler.go | 8 +++-- sftpd/internal_test.go | 40 ++++++++++++++++++++++++ sftpd/server.go | 12 ++++--- sftpd/sftpd.go | 71 ++++++++++++++++++++++++++++++++++++++++-- sftpd/sftpd_test.go | 18 ++++++++--- sftpd/transfer.go | 13 ++++---- sftpgo.conf | 49 ++++++++++++++++------------- utils/utils.go | 7 +++-- 12 files changed, 219 insertions(+), 69 deletions(-) create mode 100644 sftpd/internal_test.go diff --git a/README.md b/README.md index af5de62e..de315ab8 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Full featured and highly configurable SFTP server software - Per user maximum concurrent sessions - Per user permissions: list directories content, upload, download, delete, rename, create directories, create symlinks can be enabled or disabled - Per user files/folders ownership: you can map all the users to the system account that runs SFTPGo (all platforms are supported) or you can run SFTPGo as root user and map each user or group of users to a different system account (*NIX only) +- Configurable custom commands and/or HTTP notifications on SFTP upload, download, delete or rename - REST API for users and quota management and real time reports for the active connections with possibility of forcibly closing a connection - Log files are accurate and they are saved in the easily parsable JSON format - Automatically terminating idle connections @@ -63,6 +64,18 @@ The `sftpgo.conf` configuration file contains the following sections: - `idle_timeout`, integer. Time in minutes after which an idle client will be disconnected. Default: 15 - `max_auth_tries` integer. Maximum number of authentication attempts permitted per connection. If set to a negative number, the number of attempts are unlimited. If set to zero, the number of attempts are limited to 6. - `umask`, string. Umask for the new files and directories. This setting has no effect on Windows. Default: "0022" + - `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions + - `execute_on`, list of strings. Valid values are `download`, `upload`, `delete`, `rename`. Leave empty to disable actions. + - `command`, string. Absolute path to the command to execute. Leave empty to disable. The command is invoked with the following arguments: + - `action`, any valid `execute_on` string + - `username`, user who did the action + - `path` to the affected file. For `rename` action this is the old file name + - `target_path`, non empty for `rename` action, this is the new file name + - `http_notification_url`, a valid URL. An HTTP GET request will be executed to this URL. Leave empty to disable. The query string will contain the following parameters that have the same meaning of the command's arguments: + - `action` + - `username` + - `path` + - `target_path`, added for `rename` action only - **"data_provider"**, the configuration for the data provider - `driver`, string. Supported drivers are `sqlite`, `mysql`, `postgresql` - `name`, string. Database name @@ -83,28 +96,34 @@ Here is a full example showing the default config: ```{ "sftpd":{ - "bind_port":2022, - "bind_address": "", - "idle_timeout": 15, - "umask": "0022" + "bind_port":2022, + "bind_address":"", + "idle_timeout":15, + "max_auth_tries":0, + "umask":"0022", + "actions":{ + "execute_on":["upload"], + "command":"/usr/bin/uploadscript", + "http_notification_url":"" + } }, - "data_provider": { - "driver": "sqlite", - "name": "sftpgo.db", - "host": "", - "port": 5432, - "username": "", - "password": "", - "sslmode": 0, - "connection_string": "", - "users_table": "users", - "manage_users": 1, - "track_quota": 1 - }, - "httpd":{ - "bind_port":8080, - "bind_address": "127.0.0.1" - } + "data_provider":{ + "driver":"sqlite", + "name":"sftpgo.db", + "host":"", + "port":5432, + "username":"", + "password":"", + "sslmode":0, + "connection_string":"", + "users_table":"users", + "manage_users":1, + "track_quota":1 + }, + "httpd":{ + "bind_port":8080, + "bind_address":"127.0.0.1" + } } ``` diff --git a/api/quota.go b/api/quota.go index 449b0098..fe8aaaab 100644 --- a/api/quota.go +++ b/api/quota.go @@ -29,7 +29,7 @@ func startQuotaScan(w http.ResponseWriter, r *http.Request) { if sftpd.AddQuotaScan(user.Username) { sendAPIResponse(w, r, err, "Scan started", http.StatusCreated) go func() { - numFiles, size, err := utils.ScanDirContents(user.HomeDir) + numFiles, size, _, err := utils.ScanDirContents(user.HomeDir) if err != nil { logger.Warn(logSender, "error scanning user home dir %v: %v", user.HomeDir, err) } else { diff --git a/config/config.go b/config/config.go index f5188c11..345a2b6d 100644 --- a/config/config.go +++ b/config/config.go @@ -33,6 +33,11 @@ func init() { IdleTimeout: 15, MaxAuthTries: 0, Umask: "0022", + Actions: sftpd.Actions{ + ExecuteOn: []string{}, + Command: "", + HTTPNotificationURL: "", + }, }, ProviderConf: dataprovider.Config{ Driver: "sqlite", diff --git a/config/config_test.go b/config/config_test.go index 32246381..81ed8134 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -29,7 +29,7 @@ func TestLoadConfigTest(t *testing.T) { t.Errorf("error loading provider conf") } emptySFTPDConf := sftpd.Configuration{} - if config.GetSFTPDConfig() == emptySFTPDConf { + if config.GetSFTPDConfig().BindPort == emptySFTPDConf.BindPort { t.Errorf("error loading SFTPD conf") } confName = "sftpgo.conf.missing" diff --git a/sftpd/handler.go b/sftpd/handler.go index 7f7ff8ff..3b17280e 100644 --- a/sftpd/handler.go +++ b/sftpd/handler.go @@ -322,6 +322,7 @@ func (c Connection) handleSFTPRename(sourcePath string, targetPath string) error return sftp.ErrSshFxFailure } logger.CommandLog(sftpdRenameLogSender, sourcePath, targetPath, c.User.Username, c.ID) + executeAction(operationRename, c.User.Username, sourcePath, targetPath) return nil } @@ -330,7 +331,7 @@ func (c Connection) handleSFTPRmdir(path string) error { return sftp.ErrSshFxPermissionDenied } - numFiles, size, err := utils.ScanDirContents(path) + numFiles, size, fileList, err := utils.ScanDirContents(path) if err != nil { logger.Error(logSender, "failed to remove directory %v, scanning error: %v", path, err) return sftp.ErrSshFxFailure @@ -342,7 +343,9 @@ func (c Connection) handleSFTPRmdir(path string) error { logger.CommandLog(sftpdRmdirLogSender, path, "", c.User.Username, c.ID) dataprovider.UpdateUserQuota(dataProvider, c.User.Username, -numFiles, -size, false) - + for _, p := range fileList { + executeAction(operationDelete, c.User.Username, p, "") + } return sftp.ErrSshFxOk } @@ -394,6 +397,7 @@ func (c Connection) handleSFTPRemove(path string) error { if fi.Mode()&os.ModeSymlink != os.ModeSymlink { dataprovider.UpdateUserQuota(dataProvider, c.User.Username, -1, -size, false) } + executeAction(operationDelete, c.User.Username, path, "") return sftp.ErrSshFxOk } diff --git a/sftpd/internal_test.go b/sftpd/internal_test.go new file mode 100644 index 00000000..ff8fe954 --- /dev/null +++ b/sftpd/internal_test.go @@ -0,0 +1,40 @@ +package sftpd + +import ( + "testing" +) + +func TestWrongActions(t *testing.T) { + actionsCopy := actions + actions = Actions{ + ExecuteOn: []string{operationDownload}, + Command: "/bad/command", + HTTPNotificationURL: "", + } + err := executeAction(operationDownload, "username", "path", "") + if err == nil { + t.Errorf("action with bad command must fail") + } + actions.Command = "" + actions.HTTPNotificationURL = "http://foo\x7f.com/" + err = executeAction(operationDownload, "username", "path", "") + if err == nil { + t.Errorf("action with bad url must fail") + } + actions = actionsCopy +} + +func TestRemoveNonexistentTransfer(t *testing.T) { + transfer := Transfer{} + err := removeTransfer(&transfer) + if err == nil { + t.Errorf("remove nonexistent transfer must fail") + } +} + +func TestRemoveNonexistentQuotaScan(t *testing.T) { + err := RemoveQuotaScan("username") + if err == nil { + t.Errorf("remove nonexistent transfer must fail") + } +} diff --git a/sftpd/server.go b/sftpd/server.go index 85a347b9..fa8b845c 100644 --- a/sftpd/server.go +++ b/sftpd/server.go @@ -27,11 +27,12 @@ import ( // Configuration server configuration type Configuration struct { - BindPort int `json:"bind_port"` - BindAddress string `json:"bind_address"` - IdleTimeout int `json:"idle_timeout"` - MaxAuthTries int `json:"max_auth_tries"` - Umask string `json:"umask"` + BindPort int `json:"bind_port"` + BindAddress string `json:"bind_address"` + IdleTimeout int `json:"idle_timeout"` + MaxAuthTries int `json:"max_auth_tries"` + Umask string `json:"umask"` + Actions Actions `json:"actions"` } // Initialize the SFTP server and add a persistent listener to handle inbound SFTP connections. @@ -42,6 +43,7 @@ func (c Configuration) Initialize(configDir string) error { } else { logger.Warn(logSender, "error reading umask, please fix your config file: %v", err) } + actions = c.Actions serverConfig := &ssh.ServerConfig{ NoClientAuth: false, MaxAuthTries: c.MaxAuthTries, diff --git a/sftpd/sftpd.go b/sftpd/sftpd.go index ae56d3f1..12e6920d 100644 --- a/sftpd/sftpd.go +++ b/sftpd/sftpd.go @@ -1,6 +1,11 @@ package sftpd import ( + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" "sync" "time" @@ -20,6 +25,8 @@ const ( sftpdRemoveLogSender = "SFTPRemove" operationDownload = "download" operationUpload = "upload" + operationDelete = "delete" + operationRename = "rename" ) var ( @@ -30,6 +37,7 @@ var ( idleTimeout time.Duration activeQuotaScans []ActiveQuotaScan dataProvider dataprovider.Provider + actions Actions ) type connectionTransfer struct { @@ -45,6 +53,14 @@ type ActiveQuotaScan struct { StartTime int64 `json:"start_time"` } +// Actions configuration for external script to execute on create, download, delete. +// A rename trigger delete script for the old file and create script for the new one +type Actions struct { + ExecuteOn []string `json:"execute_on"` + Command string `json:"command"` + HTTPNotificationURL string `json:"http_notification_url"` +} + // ConnectionStatus status for an active connection type ConnectionStatus struct { Username string `json:"username"` @@ -58,6 +74,7 @@ type ConnectionStatus struct { func init() { openConnections = make(map[string]Connection) + idleConnectionTicker = time.NewTicker(5 * time.Minute) } // SetDataProvider sets the data provider @@ -104,9 +121,10 @@ func AddQuotaScan(username string) bool { } // RemoveQuotaScan remove and user from the ones with active quota scans -func RemoveQuotaScan(username string) { +func RemoveQuotaScan(username string) error { mutex.Lock() defer mutex.Unlock() + var err error indexToRemove := -1 for i, s := range activeQuotaScans { if s.Username == username { @@ -117,7 +135,11 @@ func RemoveQuotaScan(username string) { if indexToRemove >= 0 { activeQuotaScans[indexToRemove] = activeQuotaScans[len(activeQuotaScans)-1] activeQuotaScans = activeQuotaScans[:len(activeQuotaScans)-1] + } else { + logger.Warn(logSender, "quota scan to remove not found for user: %v", username) + err = fmt.Errorf("quota scan to remove not found for user: %v", username) } + return err } // CloseActiveConnection close an active SFTP connection, returns true on success @@ -180,7 +202,6 @@ func GetConnectionsStats() []ConnectionStatus { } func startIdleTimer(maxIdleTime time.Duration) { - idleConnectionTicker = time.NewTicker(5 * time.Minute) idleTimeout = maxIdleTime go func() { for t := range idleConnectionTicker.C { @@ -237,9 +258,10 @@ func addTransfer(transfer *Transfer) { activeTransfers = append(activeTransfers, transfer) } -func removeTransfer(transfer *Transfer) { +func removeTransfer(transfer *Transfer) error { mutex.Lock() defer mutex.Unlock() + var err error indexToRemove := -1 for i, v := range activeTransfers { if v == transfer { @@ -252,7 +274,9 @@ func removeTransfer(transfer *Transfer) { activeTransfers = activeTransfers[:len(activeTransfers)-1] } else { logger.Warn(logSender, "transfer to remove not found!") + err = fmt.Errorf("transfer to remove not found") } + return err } func updateConnectionActivity(id string) { @@ -263,3 +287,44 @@ func updateConnectionActivity(id string) { openConnections[id] = c } } + +func executeAction(operation string, username string, path string, target string) error { + if !utils.IsStringInSlice(operation, actions.ExecuteOn) { + return nil + } + var err error + if len(actions.Command) > 0 && filepath.IsAbs(actions.Command) { + if _, err = os.Stat(actions.Command); err == nil { + command := exec.Command(actions.Command, operation, username, path, target) + err = command.Start() + logger.Debug(logSender, "executed command \"%v\" with arguments: %v, %v, %v, error: %v", + actions.Command, operation, path, target, err) + } else { + logger.Warn(logSender, "Invalid action command \"%v\" : %v", actions.Command, err) + } + } + if len(actions.HTTPNotificationURL) > 0 { + var req *http.Request + req, err = http.NewRequest(http.MethodGet, actions.HTTPNotificationURL, nil) + if err == nil { + q := req.URL.Query() + q.Add("action", operation) + q.Add("username", username) + q.Add("path", path) + if len(target) > 0 { + q.Add("target_path", target) + } + req.URL.RawQuery = q.Encode() + resp, err := http.DefaultClient.Do(req) + respCode := 0 + if err == nil { + respCode = resp.StatusCode + resp.Body.Close() + } + logger.Debug(logSender, "notified action to URL: %v status code: %v err: %v", req.URL.RequestURI(), respCode, err) + } else { + logger.Warn(logSender, "Invalid http_notification_url \"%v\" : %v", actions.HTTPNotificationURL, err) + } + } + return err +} diff --git a/sftpd/sftpd_test.go b/sftpd/sftpd_test.go index 79736bde..c4ef8da8 100644 --- a/sftpd/sftpd_test.go +++ b/sftpd/sftpd_test.go @@ -80,11 +80,6 @@ var ( ) func TestMain(m *testing.M) { - if runtime.GOOS == "windows" { - homeBasePath = "C:\\" - } else { - homeBasePath = "/tmp" - } configDir := ".." logfilePath := filepath.Join(configDir, "sftpgo_sftpd_test.log") confName := "sftpgo.conf" @@ -102,6 +97,14 @@ func TestMain(m *testing.M) { sftpdConf := config.GetSFTPDConfig() httpdConf := config.GetHTTPDConfig() router := api.GetHTTPRouter() + if runtime.GOOS == "windows" { + homeBasePath = "C:\\" + } else { + homeBasePath = "/tmp" + sftpdConf.Actions.ExecuteOn = []string{"download", "upload", "delete"} + sftpdConf.Actions.Command = "/bin/true" + sftpdConf.Actions.HTTPNotificationURL = "http://127.0.0.1:8080/" + } sftpd.SetDataProvider(dataProvider) api.SetDataProvider(dataProvider) @@ -721,6 +724,10 @@ func TestBandwidthAndConnections(t *testing.T) { localDownloadPath := filepath.Join(homeBasePath, "test_download.dat") c := sftpDownloadNonBlocking(testFileName, localDownloadPath, testFileSize, client) waitForActiveTransfer() + // wait some additional arbitrary time to wait for transfer activity to happen + // it is need to reach all the code in CheckIdleConnections + time.Sleep(100 * time.Millisecond) + sftpd.CheckIdleConnections() err = <-c if err != nil { t.Errorf("file download error: %v", err) @@ -732,6 +739,7 @@ func TestBandwidthAndConnections(t *testing.T) { // test disconnection c = sftpUploadNonBlocking(testFilePath, testFileName, testFileSize, client) waitForActiveTransfer() + time.Sleep(100 * time.Millisecond) sftpd.CheckIdleConnections() stats := sftpd.GetConnectionsStats() for _, stat := range stats { diff --git a/sftpd/transfer.go b/sftpd/transfer.go index 22e9bf4e..00bd2cb4 100644 --- a/sftpd/transfer.go +++ b/sftpd/transfer.go @@ -29,18 +29,18 @@ type Transfer struct { // ReadAt update sent bytes func (t *Transfer) ReadAt(p []byte, off int64) (n int, err error) { + t.lastActivity = time.Now() readed, e := t.file.ReadAt(p, off) t.bytesSent += int64(readed) - t.lastActivity = time.Now() t.handleThrottle() return readed, e } // WriteAt update received bytes func (t *Transfer) WriteAt(p []byte, off int64) (n int, err error) { + t.lastActivity = time.Now() written, e := t.file.WriteAt(p, off) t.bytesReceived += int64(written) - t.lastActivity = time.Now() t.handleThrottle() return written, e } @@ -50,15 +50,15 @@ func (t *Transfer) Close() error { elapsed := time.Since(t.start).Nanoseconds() / 1000000 if t.transferType == transferDownload { logger.TransferLog(sftpdDownloadLogSender, t.path, elapsed, t.bytesSent, t.user.Username, t.connectionID) + executeAction(operationDownload, t.user.Username, t.path, "") } else { logger.TransferLog(sftpUploadLogSender, t.path, elapsed, t.bytesReceived, t.user.Username, t.connectionID) + executeAction(operationUpload, t.user.Username, t.path, "") } removeTransfer(t) - if t.transferType == transferUpload && t.bytesReceived > 0 { + if t.transferType == transferUpload && t.bytesReceived > 0 && t.isNewFile { numFiles := 0 - if t.isNewFile { - numFiles++ - } + numFiles++ dataprovider.UpdateUserQuota(dataProvider, t.user.Username, numFiles, t.bytesReceived, false) } return t.file.Close() @@ -77,6 +77,7 @@ func (t *Transfer) handleThrottle() { if wantedBandwidth > 0 { // real and wanted elapsed as milliseconds, bytes as kilobytes realElapsed := time.Since(t.start).Nanoseconds() / 1000000 + // trasferredBytes / 1000 = KB/s, we multiply for 1000 to get milliseconds wantedElapsed := 1000 * (trasferredBytes / 1000) / wantedBandwidth if wantedElapsed > realElapsed { toSleep := time.Duration(wantedElapsed - realElapsed) diff --git a/sftpgo.conf b/sftpgo.conf index 387f579d..5de10a32 100644 --- a/sftpgo.conf +++ b/sftpgo.conf @@ -1,26 +1,31 @@ { "sftpd":{ - "bind_port":2022, - "bind_address": "", - "idle_timeout": 15, - "max_auth_tries": 0, - "umask": "0022" + "bind_port":2022, + "bind_address":"", + "idle_timeout":15, + "max_auth_tries":0, + "umask":"0022", + "actions":{ + "execute_on":[], + "command":"", + "http_notification_url":"" + } }, - "data_provider": { - "driver": "sqlite", - "name": "sftpgo.db", - "host": "", - "port": 5432, - "username": "", - "password": "", - "sslmode": 0, - "connection_string": "", - "users_table": "users", - "manage_users": 1, - "track_quota": 1 - }, - "httpd":{ - "bind_port":8080, - "bind_address": "127.0.0.1" - } + "data_provider":{ + "driver":"sqlite", + "name":"sftpgo.db", + "host":"", + "port":5432, + "username":"", + "password":"", + "sslmode":0, + "connection_string":"", + "users_table":"users", + "manage_users":1, + "track_quota":1 + }, + "httpd":{ + "bind_port":8080, + "bind_address":"127.0.0.1" + } } diff --git a/utils/utils.go b/utils/utils.go index 56b79fa1..b46fa0cc 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -27,9 +27,10 @@ func GetTimeAsMsSinceEpoch(t time.Time) int64 { } // ScanDirContents returns the number of files contained in a directory and their size -func ScanDirContents(path string) (int, int64, error) { +func ScanDirContents(path string) (int, int64, []string, error) { var numFiles int var size int64 + var fileList []string var err error numFiles = 0 size = 0 @@ -42,13 +43,13 @@ func ScanDirContents(path string) (int, int64, error) { if info != nil && info.Mode().IsRegular() { size += info.Size() numFiles++ + fileList = append(fileList, path) } - return err }) } - return numFiles, size, err + return numFiles, size, fileList, err } func isDirectory(path string) (bool, error) {