add support for custom actions

Configurable custom commands and/or HTTP notifications on SFTP upload, download, delete or rename

README.md

@@ -14,6 +14,7 @@ Full featured and highly configurable SFTP server software
 - Per user maximum concurrent sessions
 - Per user permissions: list directories content, upload, download, delete, rename, create directories, create symlinks can be enabled or disabled
 - Per user files/folders ownership: you can map all the users to the system account that runs SFTPGo (all platforms are supported) or you can run SFTPGo as root user and map each user or group of users to a different system account (*NIX only) 
+- Configurable custom commands and/or HTTP notifications on SFTP upload, download, delete or rename
 - REST API for users and quota management and real time reports for the active connections with possibility of forcibly closing a connection  
 - Log files are accurate and they are saved in the easily parsable JSON format
 - Automatically terminating idle connections
@@ -63,6 +64,18 @@ The `sftpgo.conf` configuration file contains the following sections:
     - `idle_timeout`, integer. Time in minutes after which an idle client will be disconnected. Default: 15
     - `max_auth_tries` integer. Maximum number of authentication attempts permitted per connection. If set to a negative number, the number of attempts are unlimited. If set to zero, the number of attempts are limited to 6.
     - `umask`, string. Umask for the new files and directories. This setting has no effect on Windows. Default: "0022"
+    - `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions
+        - `execute_on`, list of strings. Valid values are `download`, `upload`, `delete`, `rename`. Leave empty to disable actions.
+        - `command`, string. Absolute path to the command to execute. Leave empty to disable. The command is invoked with the following arguments: 
+            - `action`, any valid `execute_on` string
+            - `username`, user who did the action
+            - `path` to the affected file. For `rename` action this is the old file name
+            - `target_path`, non empty for `rename` action, this is the new file name
+        - `http_notification_url`, a valid URL. An HTTP GET request will be executed to this URL. Leave empty to disable. The query string will contain the following parameters that have the same meaning of the command's arguments:
+            - `action`
+            - `username`
+            - `path`
+            - `target_path`, added for `rename` action only
 - **"data_provider"**, the configuration for the data provider
     - `driver`, string. Supported drivers are `sqlite`, `mysql`, `postgresql`
     - `name`, string. Database name
@@ -83,28 +96,34 @@ Here is a full example showing the default config:
 
 ```{
    "sftpd":{
-       "bind_port":2022,
-       "bind_address": "",
-       "idle_timeout": 15,
-       "umask": "0022"
+        "bind_port":2022,
+        "bind_address":"",
+        "idle_timeout":15,
+        "max_auth_tries":0,
+        "umask":"0022",
+        "actions":{
+            "execute_on":["upload"],
+            "command":"/usr/bin/uploadscript",
+            "http_notification_url":""
+        }
    },
-   "data_provider": {
-       "driver": "sqlite",
-       "name": "sftpgo.db",
-       "host": "",
-       "port": 5432,
-       "username": "",
-       "password": "",
-       "sslmode": 0,
-       "connection_string": "",
-       "users_table": "users",
-       "manage_users": 1,
-       "track_quota": 1
-   },
-   "httpd":{
-       "bind_port":8080,
-       "bind_address": "127.0.0.1"
-   }
+   "data_provider":{
+        "driver":"sqlite",
+        "name":"sftpgo.db",
+        "host":"",
+        "port":5432,
+        "username":"",
+        "password":"",
+        "sslmode":0,
+        "connection_string":"",
+        "users_table":"users",
+        "manage_users":1,
+        "track_quota":1
+    },
+    "httpd":{
+        "bind_port":8080,
+        "bind_address":"127.0.0.1"
+    }
 }
 ```
 

api/quota.go

@@ -29,7 +29,7 @@ func startQuotaScan(w http.ResponseWriter, r *http.Request) {
 	if sftpd.AddQuotaScan(user.Username) {
 		sendAPIResponse(w, r, err, "Scan started", http.StatusCreated)
 		go func() {
-			numFiles, size, err := utils.ScanDirContents(user.HomeDir)
+			numFiles, size, _, err := utils.ScanDirContents(user.HomeDir)
 			if err != nil {
 				logger.Warn(logSender, "error scanning user home dir %v: %v", user.HomeDir, err)
 			} else {

config/config.go

@@ -33,6 +33,11 @@ func init() {
 			IdleTimeout:  15,
 			MaxAuthTries: 0,
 			Umask:        "0022",
+			Actions: sftpd.Actions{
+				ExecuteOn:           []string{},
+				Command:             "",
+				HTTPNotificationURL: "",
+			},
 		},
 		ProviderConf: dataprovider.Config{
 			Driver:           "sqlite",

config/config_test.go

@@ -29,7 +29,7 @@ func TestLoadConfigTest(t *testing.T) {
 		t.Errorf("error loading provider conf")
 	}
 	emptySFTPDConf := sftpd.Configuration{}
-	if config.GetSFTPDConfig() == emptySFTPDConf {
+	if config.GetSFTPDConfig().BindPort == emptySFTPDConf.BindPort {
 		t.Errorf("error loading SFTPD conf")
 	}
 	confName = "sftpgo.conf.missing"

sftpd/handler.go

@@ -322,6 +322,7 @@ func (c Connection) handleSFTPRename(sourcePath string, targetPath string) error
 		return sftp.ErrSshFxFailure
 	}
 	logger.CommandLog(sftpdRenameLogSender, sourcePath, targetPath, c.User.Username, c.ID)
+	executeAction(operationRename, c.User.Username, sourcePath, targetPath)
 	return nil
 }
 
@@ -330,7 +331,7 @@ func (c Connection) handleSFTPRmdir(path string) error {
 		return sftp.ErrSshFxPermissionDenied
 	}
 
-	numFiles, size, err := utils.ScanDirContents(path)
+	numFiles, size, fileList, err := utils.ScanDirContents(path)
 	if err != nil {
 		logger.Error(logSender, "failed to remove directory %v, scanning error: %v", path, err)
 		return sftp.ErrSshFxFailure
@@ -342,7 +343,9 @@ func (c Connection) handleSFTPRmdir(path string) error {
 
 	logger.CommandLog(sftpdRmdirLogSender, path, "", c.User.Username, c.ID)
 	dataprovider.UpdateUserQuota(dataProvider, c.User.Username, -numFiles, -size, false)
-
+	for _, p := range fileList {
+		executeAction(operationDelete, c.User.Username, p, "")
+	}
 	return sftp.ErrSshFxOk
 }
 
@@ -394,6 +397,7 @@ func (c Connection) handleSFTPRemove(path string) error {
 	if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
 		dataprovider.UpdateUserQuota(dataProvider, c.User.Username, -1, -size, false)
 	}
+	executeAction(operationDelete, c.User.Username, path, "")
 
 	return sftp.ErrSshFxOk
 }

sftpd/internal_test.go

@@ -0,0 +1,40 @@
+package sftpd
+
+import (
+	"testing"
+)
+
+func TestWrongActions(t *testing.T) {
+	actionsCopy := actions
+	actions = Actions{
+		ExecuteOn:           []string{operationDownload},
+		Command:             "/bad/command",
+		HTTPNotificationURL: "",
+	}
+	err := executeAction(operationDownload, "username", "path", "")
+	if err == nil {
+		t.Errorf("action with bad command must fail")
+	}
+	actions.Command = ""
+	actions.HTTPNotificationURL = "http://foo\x7f.com/"
+	err = executeAction(operationDownload, "username", "path", "")
+	if err == nil {
+		t.Errorf("action with bad url must fail")
+	}
+	actions = actionsCopy
+}
+
+func TestRemoveNonexistentTransfer(t *testing.T) {
+	transfer := Transfer{}
+	err := removeTransfer(&transfer)
+	if err == nil {
+		t.Errorf("remove nonexistent transfer must fail")
+	}
+}
+
+func TestRemoveNonexistentQuotaScan(t *testing.T) {
+	err := RemoveQuotaScan("username")
+	if err == nil {
+		t.Errorf("remove nonexistent transfer must fail")
+	}
+}

sftpd/server.go

@@ -27,11 +27,12 @@ import (
 
 // Configuration server configuration
 type Configuration struct {
-	BindPort     int    `json:"bind_port"`
-	BindAddress  string `json:"bind_address"`
-	IdleTimeout  int    `json:"idle_timeout"`
-	MaxAuthTries int    `json:"max_auth_tries"`
-	Umask        string `json:"umask"`
+	BindPort     int     `json:"bind_port"`
+	BindAddress  string  `json:"bind_address"`
+	IdleTimeout  int     `json:"idle_timeout"`
+	MaxAuthTries int     `json:"max_auth_tries"`
+	Umask        string  `json:"umask"`
+	Actions      Actions `json:"actions"`
 }
 
 // Initialize the SFTP server and add a persistent listener to handle inbound SFTP connections.
@@ -42,6 +43,7 @@ func (c Configuration) Initialize(configDir string) error {
 	} else {
 		logger.Warn(logSender, "error reading umask, please fix your config file: %v", err)
 	}
+	actions = c.Actions
 	serverConfig := &ssh.ServerConfig{
 		NoClientAuth: false,
 		MaxAuthTries: c.MaxAuthTries,

sftpd/sftpd.go

@@ -1,6 +1,11 @@
 package sftpd
 
 import (
+	"fmt"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
 	"sync"
 	"time"
 
@@ -20,6 +25,8 @@ const (
 	sftpdRemoveLogSender   = "SFTPRemove"
 	operationDownload      = "download"
 	operationUpload        = "upload"
+	operationDelete        = "delete"
+	operationRename        = "rename"
 )
 
 var (
@@ -30,6 +37,7 @@ var (
 	idleTimeout          time.Duration
 	activeQuotaScans     []ActiveQuotaScan
 	dataProvider         dataprovider.Provider
+	actions              Actions
 )
 
 type connectionTransfer struct {
@@ -45,6 +53,14 @@ type ActiveQuotaScan struct {
 	StartTime int64  `json:"start_time"`
 }
 
+// Actions configuration for external script to execute on create, download, delete.
+// A rename trigger delete script for the old file and create script for the new one
+type Actions struct {
+	ExecuteOn           []string `json:"execute_on"`
+	Command             string   `json:"command"`
+	HTTPNotificationURL string   `json:"http_notification_url"`
+}
+
 // ConnectionStatus status for an active connection
 type ConnectionStatus struct {
 	Username       string               `json:"username"`
@@ -58,6 +74,7 @@ type ConnectionStatus struct {
 
 func init() {
 	openConnections = make(map[string]Connection)
+	idleConnectionTicker = time.NewTicker(5 * time.Minute)
 }
 
 // SetDataProvider sets the data provider
@@ -104,9 +121,10 @@ func AddQuotaScan(username string) bool {
 }
 
 // RemoveQuotaScan remove and user from the ones with active quota scans
-func RemoveQuotaScan(username string) {
+func RemoveQuotaScan(username string) error {
 	mutex.Lock()
 	defer mutex.Unlock()
+	var err error
 	indexToRemove := -1
 	for i, s := range activeQuotaScans {
 		if s.Username == username {
@@ -117,7 +135,11 @@ func RemoveQuotaScan(username string) {
 	if indexToRemove >= 0 {
 		activeQuotaScans[indexToRemove] = activeQuotaScans[len(activeQuotaScans)-1]
 		activeQuotaScans = activeQuotaScans[:len(activeQuotaScans)-1]
+	} else {
+		logger.Warn(logSender, "quota scan to remove not found for user: %v", username)
+		err = fmt.Errorf("quota scan to remove not found for user: %v", username)
 	}
+	return err
 }
 
 // CloseActiveConnection close an active SFTP connection, returns true on success
@@ -180,7 +202,6 @@ func GetConnectionsStats() []ConnectionStatus {
 }
 
 func startIdleTimer(maxIdleTime time.Duration) {
-	idleConnectionTicker = time.NewTicker(5 * time.Minute)
 	idleTimeout = maxIdleTime
 	go func() {
 		for t := range idleConnectionTicker.C {
@@ -237,9 +258,10 @@ func addTransfer(transfer *Transfer) {
 	activeTransfers = append(activeTransfers, transfer)
 }
 
-func removeTransfer(transfer *Transfer) {
+func removeTransfer(transfer *Transfer) error {
 	mutex.Lock()
 	defer mutex.Unlock()
+	var err error
 	indexToRemove := -1
 	for i, v := range activeTransfers {
 		if v == transfer {
@@ -252,7 +274,9 @@ func removeTransfer(transfer *Transfer) {
 		activeTransfers = activeTransfers[:len(activeTransfers)-1]
 	} else {
 		logger.Warn(logSender, "transfer to remove not found!")
+		err = fmt.Errorf("transfer to remove not found")
 	}
+	return err
 }
 
 func updateConnectionActivity(id string) {
@@ -263,3 +287,44 @@ func updateConnectionActivity(id string) {
 		openConnections[id] = c
 	}
 }
+
+func executeAction(operation string, username string, path string, target string) error {
+	if !utils.IsStringInSlice(operation, actions.ExecuteOn) {
+		return nil
+	}
+	var err error
+	if len(actions.Command) > 0 && filepath.IsAbs(actions.Command) {
+		if _, err = os.Stat(actions.Command); err == nil {
+			command := exec.Command(actions.Command, operation, username, path, target)
+			err = command.Start()
+			logger.Debug(logSender, "executed command \"%v\" with arguments: %v, %v, %v, error: %v",
+				actions.Command, operation, path, target, err)
+		} else {
+			logger.Warn(logSender, "Invalid action command \"%v\" : %v", actions.Command, err)
+		}
+	}
+	if len(actions.HTTPNotificationURL) > 0 {
+		var req *http.Request
+		req, err = http.NewRequest(http.MethodGet, actions.HTTPNotificationURL, nil)
+		if err == nil {
+			q := req.URL.Query()
+			q.Add("action", operation)
+			q.Add("username", username)
+			q.Add("path", path)
+			if len(target) > 0 {
+				q.Add("target_path", target)
+			}
+			req.URL.RawQuery = q.Encode()
+			resp, err := http.DefaultClient.Do(req)
+			respCode := 0
+			if err == nil {
+				respCode = resp.StatusCode
+				resp.Body.Close()
+			}
+			logger.Debug(logSender, "notified action to URL: %v status code: %v err: %v", req.URL.RequestURI(), respCode, err)
+		} else {
+			logger.Warn(logSender, "Invalid http_notification_url \"%v\" : %v", actions.HTTPNotificationURL, err)
+		}
+	}
+	return err
+}

sftpd/sftpd_test.go

@@ -80,11 +80,6 @@ var (
 )
 
 func TestMain(m *testing.M) {
-	if runtime.GOOS == "windows" {
-		homeBasePath = "C:\\"
-	} else {
-		homeBasePath = "/tmp"
-	}
 	configDir := ".."
 	logfilePath := filepath.Join(configDir, "sftpgo_sftpd_test.log")
 	confName := "sftpgo.conf"
@@ -102,6 +97,14 @@ func TestMain(m *testing.M) {
 	sftpdConf := config.GetSFTPDConfig()
 	httpdConf := config.GetHTTPDConfig()
 	router := api.GetHTTPRouter()
+	if runtime.GOOS == "windows" {
+		homeBasePath = "C:\\"
+	} else {
+		homeBasePath = "/tmp"
+		sftpdConf.Actions.ExecuteOn = []string{"download", "upload", "delete"}
+		sftpdConf.Actions.Command = "/bin/true"
+		sftpdConf.Actions.HTTPNotificationURL = "http://127.0.0.1:8080/"
+	}
 
 	sftpd.SetDataProvider(dataProvider)
 	api.SetDataProvider(dataProvider)
@@ -721,6 +724,10 @@ func TestBandwidthAndConnections(t *testing.T) {
 		localDownloadPath := filepath.Join(homeBasePath, "test_download.dat")
 		c := sftpDownloadNonBlocking(testFileName, localDownloadPath, testFileSize, client)
 		waitForActiveTransfer()
+		// wait some additional arbitrary time to wait for transfer activity to happen
+		// it is need to reach all the code in CheckIdleConnections
+		time.Sleep(100 * time.Millisecond)
+		sftpd.CheckIdleConnections()
 		err = <-c
 		if err != nil {
 			t.Errorf("file download error: %v", err)
@@ -732,6 +739,7 @@ func TestBandwidthAndConnections(t *testing.T) {
 		// test disconnection
 		c = sftpUploadNonBlocking(testFilePath, testFileName, testFileSize, client)
 		waitForActiveTransfer()
+		time.Sleep(100 * time.Millisecond)
 		sftpd.CheckIdleConnections()
 		stats := sftpd.GetConnectionsStats()
 		for _, stat := range stats {

sftpd/transfer.go

@@ -29,18 +29,18 @@ type Transfer struct {
 
 // ReadAt update sent bytes
 func (t *Transfer) ReadAt(p []byte, off int64) (n int, err error) {
+	t.lastActivity = time.Now()
 	readed, e := t.file.ReadAt(p, off)
 	t.bytesSent += int64(readed)
-	t.lastActivity = time.Now()
 	t.handleThrottle()
 	return readed, e
 }
 
 // WriteAt update received bytes
 func (t *Transfer) WriteAt(p []byte, off int64) (n int, err error) {
+	t.lastActivity = time.Now()
 	written, e := t.file.WriteAt(p, off)
 	t.bytesReceived += int64(written)
-	t.lastActivity = time.Now()
 	t.handleThrottle()
 	return written, e
 }
@@ -50,15 +50,15 @@ func (t *Transfer) Close() error {
 	elapsed := time.Since(t.start).Nanoseconds() / 1000000
 	if t.transferType == transferDownload {
 		logger.TransferLog(sftpdDownloadLogSender, t.path, elapsed, t.bytesSent, t.user.Username, t.connectionID)
+		executeAction(operationDownload, t.user.Username, t.path, "")
 	} else {
 		logger.TransferLog(sftpUploadLogSender, t.path, elapsed, t.bytesReceived, t.user.Username, t.connectionID)
+		executeAction(operationUpload, t.user.Username, t.path, "")
 	}
 	removeTransfer(t)
-	if t.transferType == transferUpload && t.bytesReceived > 0 {
+	if t.transferType == transferUpload && t.bytesReceived > 0 && t.isNewFile {
 		numFiles := 0
-		if t.isNewFile {
-			numFiles++
-		}
+		numFiles++
 		dataprovider.UpdateUserQuota(dataProvider, t.user.Username, numFiles, t.bytesReceived, false)
 	}
 	return t.file.Close()
@@ -77,6 +77,7 @@ func (t *Transfer) handleThrottle() {
 	if wantedBandwidth > 0 {
 		// real and wanted elapsed as milliseconds, bytes as kilobytes
 		realElapsed := time.Since(t.start).Nanoseconds() / 1000000
+		// trasferredBytes / 1000 = KB/s, we multiply for 1000 to get milliseconds
 		wantedElapsed := 1000 * (trasferredBytes / 1000) / wantedBandwidth
 		if wantedElapsed > realElapsed {
 			toSleep := time.Duration(wantedElapsed - realElapsed)

sftpgo.conf

@@ -1,26 +1,31 @@
 {
    "sftpd":{
-       "bind_port":2022,
-       "bind_address": "",
-       "idle_timeout": 15,
-       "max_auth_tries": 0,
-       "umask": "0022"
+        "bind_port":2022,
+        "bind_address":"",
+        "idle_timeout":15,
+        "max_auth_tries":0,
+        "umask":"0022",
+        "actions":{
+            "execute_on":[],
+            "command":"",
+            "http_notification_url":""
+        }
    },
-   "data_provider": {
-       "driver": "sqlite",
-       "name": "sftpgo.db",
-       "host": "",
-       "port": 5432,
-       "username": "",
-       "password": "",
-       "sslmode": 0,
-       "connection_string": "",
-       "users_table": "users",
-       "manage_users": 1,
-       "track_quota": 1
-   },
-   "httpd":{
-       "bind_port":8080,
-       "bind_address": "127.0.0.1"
-   }
+   "data_provider":{
+        "driver":"sqlite",
+        "name":"sftpgo.db",
+        "host":"",
+        "port":5432,
+        "username":"",
+        "password":"",
+        "sslmode":0,
+        "connection_string":"",
+        "users_table":"users",
+        "manage_users":1,
+        "track_quota":1
+    },
+    "httpd":{
+        "bind_port":8080,
+        "bind_address":"127.0.0.1"
+    }
 }

utils/utils.go

@@ -27,9 +27,10 @@ func GetTimeAsMsSinceEpoch(t time.Time) int64 {
 }
 
 // ScanDirContents returns the number of files contained in a directory and their size
-func ScanDirContents(path string) (int, int64, error) {
+func ScanDirContents(path string) (int, int64, []string, error) {
 	var numFiles int
 	var size int64
+	var fileList []string
 	var err error
 	numFiles = 0
 	size = 0
@@ -42,13 +43,13 @@ func ScanDirContents(path string) (int, int64, error) {
 			if info != nil && info.Mode().IsRegular() {
 				size += info.Size()
 				numFiles++
+				fileList = append(fileList, path)
 			}
-
 			return err
 		})
 	}
 
-	return numFiles, size, err
+	return numFiles, size, fileList, err
 }
 
 func isDirectory(path string) (bool, error) {