dataprovider: add custom command and/or HTTP notifications on users add, update and delete

This way custom logic can be implemented for example to create a UNIX user
as asked in #58

README.md

@@ -14,7 +14,7 @@ Full featured and highly configurable SFTP server
 - Per user maximum concurrent sessions.
 - Per user permissions: list directories content, upload, overwrite, download, delete, rename, create directories, create symlinks can be enabled or disabled.
 - Per user files/folders ownership: you can map all the users to the system account that runs SFTPGo (all platforms are supported) or you can run SFTPGo as root user and map each user or group of users to a different system account (*NIX only).
-- Configurable custom commands and/or HTTP notifications on upload, download, delete or rename.
+- Configurable custom commands and/or HTTP notifications on files upload, download, delete, rename and on users add, update and delete.
 - Automatically terminating idle connections.
 - Atomic uploads are configurable.
 - Optional SCP support.
@@ -132,7 +132,7 @@ The `sftpgo` configuration file contains the following sections:
     - `banner`, string. Identification string used by the server. Leave empty to use the default banner. Default "SFTPGo_version"
     - `upload_mode` integer. 0 means standard, the files are uploaded directly to the requested path. 1 means atomic: files are uploaded to a temporary path and renamed to the requested path when the client ends the upload. Atomic mode avoids problems such as a web server that serves partial files when the files are being uploaded. In atomic mode if there is an upload error the temporary file is deleted and so the requested upload path will not contain a partial file. 2 means atomic with resume support: as atomic but if there is an upload error the temporary file is renamed to the requested path and not deleted, this way a client can reconnect and resume the upload.
     - `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions
-        - `execute_on`, list of strings. Valid values are `download`, `upload`, `delete`, `rename`. On folder deletion a `delete` notification will be sent for each deleted file. Actions will be not executed if an error is detected and so a partial file is uploaded or downloaded. Leave empty to disable actions. The `upload` condition includes both uploads to new files and overwrite existing files
+        - `execute_on`, list of strings. Valid values are `download`, `upload`, `delete`, `rename`. Actions will be not executed if an error is detected and so a partial file is uploaded or downloaded. The `upload` condition includes both uploads to new files and overwrite of existing files. Leave empty to disable actions.
         - `command`, string. Absolute path to the command to execute. Leave empty to disable. The command is invoked with the following arguments:
             - `action`, any valid `execute_on` string
             - `username`, user who did the action
@@ -167,6 +167,18 @@ The `sftpgo` configuration file contains the following sections:
         - 2, quota is updated each time a user upload or delete a file but only for users with quota restrictions. With this configuration the "quota scan" REST API can still be used to periodically update space usage for users without quota restrictions
     - `pool_size`, integer. Sets the maximum number of open connections for `mysql` and `postgresql` driver. Default 0 (unlimited)
     - `users_base_dir`, string. Users' default base directory. If no home dir is defined while adding a new user, and this value is a valid absolute path, then the user home dir will be automatically defined as the path obtained joining the base dir and the username
+    - `actions`, struct. It contains the command to execute and/or the HTTP URL to notify and the trigger conditions
+        - `execute_on`, list of strings. Valid values are `add`, `update`, `delete`.
+        - `command`, string. Absolute path to the command to execute. Leave empty to disable. The command is invoked with the following arguments that identify the user added, updated or deleted:
+            - `action`, any valid `execute_on` string
+            - `username`
+            - `ID`
+            - `status`
+            - `expiration_date`, as unix timestamp in milliseconds
+            - `home_dir`
+            - `uid`
+            - `gid`
+        - `http_notification_url`, a valid URL. The action is added to the query string. For example `<http_notification_url>?action=update`. An HTTP POST request will be executed to this URL. The user is sent serialized as json inside the POST body. Leave empty to disable.
 - **"httpd"**, the configuration for the HTTP server used to serve REST API
     - `bind_port`, integer. The port used for serving HTTP requests. Set to 0 to disable HTTP server. Default: 8080
     - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: "127.0.0.1"
@@ -210,7 +222,12 @@ Here is a full example showing the default config in JSON format:
     "manage_users": 1,
     "track_quota": 2,
     "pool_size": 0,
-    "users_base_dir": ""
+    "users_base_dir": "",
+    "actions": {
+      "execute_on": [],
+      "command": "",
+      "http_notification_url": ""
+    }
   },
   "httpd": {
     "bind_port": 8080,

config/config.go

@@ -75,6 +75,11 @@ func init() {
 			TrackQuota:       1,
 			PoolSize:         0,
 			UsersBaseDir:     "",
+			Actions: dataprovider.Actions{
+				ExecuteOn:           []string{},
+				Command:             "",
+				HTTPNotificationURL: "",
+			},
 		},
 		HTTPDConfig: httpd.Conf{
 			BindPort:        8080,

config/config_test.go

@@ -29,7 +29,7 @@ func TestLoadConfigTest(t *testing.T) {
 		t.Errorf("error loading httpd conf")
 	}
 	emptyProviderConf := dataprovider.Config{}
-	if config.GetProviderConf() == emptyProviderConf {
+	if config.GetProviderConf().Driver == emptyProviderConf.Driver {
 		t.Errorf("error loading provider conf")
 	}
 	emptySFTPDConf := sftpd.Configuration{}

dataprovider/dataprovider.go

@@ -4,14 +4,20 @@
 package dataprovider
 
 import (
+	"bytes"
 	"crypto/sha1"
 	"crypto/sha256"
 	"crypto/sha512"
 	"crypto/subtle"
 	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"hash"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -48,6 +54,9 @@ const (
 	sha512cryptPwdPrefix     = "$6$"
 	manageUsersDisabledError = "please set manage_users to 1 in your configuration to enable this method"
 	trackQuotaDisabledError  = "please enable track_quota in your configuration to use this method"
+	operationAdd             = "add"
+	operationUpdate          = "update"
+	operationDelete          = "delete"
 )
 
 var (
@@ -68,6 +77,20 @@ var (
 	availabilityTickerDone chan bool
 )
 
+// Actions to execute on user create, update, delete.
+// An external command can be executed and/or an HTTP notification can be fired
+type Actions struct {
+	// Valid values are add, update, delete. Empty slice to disable
+	ExecuteOn []string `json:"execute_on" mapstructure:"execute_on"`
+	// Absolute path to the command to execute, empty to disable
+	Command string `json:"command" mapstructure:"command"`
+	// The URL to notify using an HTTP POST.
+	// The action is added to the query string. For example <url>?action=update.
+	// The user is sent serialized as json inside the POST body.
+	// Empty to disable
+	HTTPNotificationURL string `json:"http_notification_url" mapstructure:"http_notification_url"`
+}
+
 // Config provider configuration
 type Config struct {
 	// Driver name, must be one of the SupportedProviders
@@ -110,6 +133,9 @@ type Config struct {
 	// a valid absolute path, then the user home dir will be automatically
 	// defined as the path obtained joining the base dir and the username
 	UsersBaseDir string `json:"users_base_dir" mapstructure:"users_base_dir"`
+	// Actions to execute on user add, update, delete.
+	// Update action will not be fired for internal updates such as the last login fiels or the user quota.
+	Actions Actions `json:"actions" mapstructure:"actions"`
 }
 
 // ValidationError raised if input data is not valid
@@ -246,7 +272,11 @@ func AddUser(p Provider, user User) error {
 	if config.ManageUsers == 0 {
 		return &MethodDisabledError{err: manageUsersDisabledError}
 	}
-	return p.addUser(user)
+	err := p.addUser(user)
+	if err == nil {
+		go executeAction(operationAdd, user)
+	}
+	return err
 }
 
 // UpdateUser updates an existing SFTP user.
@@ -255,7 +285,11 @@ func UpdateUser(p Provider, user User) error {
 	if config.ManageUsers == 0 {
 		return &MethodDisabledError{err: manageUsersDisabledError}
 	}
-	return p.updateUser(user)
+	err := p.updateUser(user)
+	if err == nil {
+		go executeAction(operationUpdate, user)
+	}
+	return err
 }
 
 // DeleteUser deletes an existing SFTP user.
@@ -264,7 +298,11 @@ func DeleteUser(p Provider, user User) error {
 	if config.ManageUsers == 0 {
 		return &MethodDisabledError{err: manageUsersDisabledError}
 	}
-	return p.deleteUser(user)
+	err := p.deleteUser(user)
+	if err == nil {
+		go executeAction(operationDelete, user)
+	}
+	return err
 }
 
 // GetUsers returns an array of users respecting limit and offset and filtered by username exact match if not empty
@@ -504,3 +542,65 @@ func checkDataprovider() {
 func providerLog(level logger.LogLevel, format string, v ...interface{}) {
 	logger.Log(level, logSender, "", format, v...)
 }
+
+// executed in a goroutine
+func executeAction(operation string, user User) {
+	if !utils.IsStringInSlice(operation, config.Actions.ExecuteOn) {
+		return
+	}
+	if operation != operationDelete {
+		var err error
+		user, err = provider.userExists(user.Username)
+		if err != nil {
+			providerLog(logger.LevelWarn, "unable to get the user to notify operation %#v: %v", operation, err)
+			return
+		}
+	}
+	// hide the hashed password
+	user.Password = ""
+	if len(config.Actions.Command) > 0 && filepath.IsAbs(config.Actions.Command) {
+		if _, err := os.Stat(config.Actions.Command); err == nil {
+			commandArgs := []string{operation}
+			commandArgs = append(commandArgs, user.getNotificationFieldsAsSlice()...)
+			command := exec.Command(config.Actions.Command, commandArgs...)
+			err = command.Start()
+			providerLog(logger.LevelDebug, "start command %#v with arguments: %+v, error: %v",
+				config.Actions.Command, commandArgs, err)
+			if err == nil {
+				// we are in a goroutine but we don't want to block here, this way we can send the
+				// HTTP notification, if configured, without waiting the end of the command
+				go command.Wait()
+			}
+		} else {
+			providerLog(logger.LevelWarn, "Invalid action command %#v for operation %#v: %v", config.Actions.Command, operation, err)
+		}
+	}
+	if len(config.Actions.HTTPNotificationURL) > 0 {
+		var url *url.URL
+		url, err := url.Parse(config.Actions.HTTPNotificationURL)
+		if err != nil {
+			providerLog(logger.LevelWarn, "Invalid http_notification_url %#v for operation %#v: %v", config.Actions.HTTPNotificationURL,
+				operation, err)
+			return
+		}
+		q := url.Query()
+		q.Add("action", operation)
+		url.RawQuery = q.Encode()
+		userAsJSON, err := json.Marshal(user)
+		if err != nil {
+			return
+		}
+		startTime := time.Now()
+		httpClient := &http.Client{
+			Timeout: 15 * time.Second,
+		}
+		resp, err := httpClient.Post(url.String(), "application/json", bytes.NewBuffer(userAsJSON))
+		respCode := 0
+		if err == nil {
+			respCode = resp.StatusCode
+			resp.Body.Close()
+		}
+		providerLog(logger.LevelDebug, "notified operation %#v to URL: %v status code: %v, elapsed: %v err: %v",
+			operation, url.String(), respCode, time.Since(startTime), err)
+	}
+}

dataprovider/user.go

@@ -235,3 +235,14 @@ func (u *User) getACopy() User {
 		LastLogin:         u.LastLogin,
 	}
 }
+
+func (u *User) getNotificationFieldsAsSlice() []string {
+	return []string{u.Username,
+		strconv.FormatInt(u.ID, 10),
+		strconv.FormatInt(int64(u.Status), 10),
+		strconv.FormatInt(int64(u.ExpirationDate), 10),
+		u.HomeDir,
+		strconv.FormatInt(int64(u.UID), 10),
+		strconv.FormatInt(int64(u.GID), 10),
+	}
+}

service/service.go

@@ -189,9 +189,9 @@ func (s *Service) StartPortableMode(sftpdPort int, enableSCP, advertiseService,
 			)
 			if err != nil {
 				mDNSService = nil
-				logger.WarnToConsole("Unable to advertise service via multicast DNS: %v", err)
+				logger.WarnToConsole("Unable to advertise SFTP service via multicast DNS: %v", err)
 			} else {
-				logger.InfoToConsole("Service advertised via multicast DNS")
+				logger.InfoToConsole("SFTP service advertised via multicast DNS")
 			}
 
 		}

sftpd/handler.go

@@ -270,7 +270,7 @@ func (c Connection) handleSFTPRename(sourcePath string, targetPath string) error
 		return sftp.ErrSSHFxFailure
 	}
 	logger.CommandLog(renameLogSender, sourcePath, targetPath, c.User.Username, c.ID, c.protocol)
-	executeAction(operationRename, c.User.Username, sourcePath, targetPath)
+	go executeAction(operationRename, c.User.Username, sourcePath, targetPath)
 	return nil
 }
 
@@ -352,7 +352,7 @@ func (c Connection) handleSFTPRemove(path string) error {
 	if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
 		dataprovider.UpdateUserQuota(dataProvider, c.User, -1, -size, false)
 	}
-	executeAction(operationDelete, c.User.Username, path, "")
+	go executeAction(operationDelete, c.User.Username, path, "")
 
 	return sftp.ErrSSHFxOk
 }

sftpd/sftpd.go

@@ -371,6 +371,7 @@ func isAtomicUploadEnabled() bool {
 	return uploadMode == uploadModeAtomic || uploadMode == uploadModeAtomicWithResume
 }
 
+// executed in a goroutine
 func executeAction(operation string, username string, path string, target string) error {
 	if !utils.IsStringInSlice(operation, actions.ExecuteOn) {
 		return nil
@@ -383,10 +384,12 @@ func executeAction(operation string, username string, path string, target string
 			logger.Debug(logSender, "", "start command %#v with arguments: %v, %v, %v, %v, error: %v",
 				actions.Command, operation, username, path, target, err)
 			if err == nil {
+				// we are in a goroutine but we don't want to block here, this way we can send the
+				// HTTP notification, if configured, without waiting the end of the command
 				go command.Wait()
 			}
 		} else {
-			logger.Warn(logSender, "", "Invalid action command %#v : %v", actions.Command, err)
+			logger.Warn(logSender, "", "Invalid action command %#v for operation %#v: %v", actions.Command, operation, err)
 		}
 	}
 	if len(actions.HTTPNotificationURL) > 0 {
@@ -401,22 +404,21 @@ func executeAction(operation string, username string, path string, target string
 				q.Add("target_path", target)
 			}
 			url.RawQuery = q.Encode()
-			go func() {
+			startTime := time.Now()
-				startTime := time.Now()
+			httpClient := &http.Client{
-				httpClient := &http.Client{
+				Timeout: 15 * time.Second,
-					Timeout: 15 * time.Second,
+			}
-				}
+			resp, err := httpClient.Get(url.String())
-				resp, err := httpClient.Get(url.String())
+			respCode := 0
-				respCode := 0
+			if err == nil {
-				if err == nil {
+				respCode = resp.StatusCode
-					respCode = resp.StatusCode
+				resp.Body.Close()
-					resp.Body.Close()
+			}
-				}
+			logger.Debug(logSender, "", "notified operation %#v to URL: %v status code: %v, elapsed: %v err: %v",
-				logger.Debug(logSender, "", "notified action to URL: %v status code: %v, elapsed: %v err: %v",
+				operation, url.String(), respCode, time.Since(startTime), err)
-					url.String(), respCode, time.Since(startTime), err)
-			}()
 		} else {
-			logger.Warn(logSender, "", "Invalid http_notification_url %#v : %v", actions.HTTPNotificationURL, err)
+			logger.Warn(logSender, "", "Invalid http_notification_url %#v for operation %#v: %v", actions.HTTPNotificationURL,
+				operation, err)
 		}
 	}
 	return err

sftpd/transfer.go

@@ -101,10 +101,10 @@ func (t *Transfer) Close() error {
 		elapsed := time.Since(t.start).Nanoseconds() / 1000000
 		if t.transferType == transferDownload {
 			logger.TransferLog(downloadLogSender, t.path, elapsed, t.bytesSent, t.user.Username, t.connectionID, t.protocol)
-			executeAction(operationDownload, t.user.Username, t.path, "")
+			go executeAction(operationDownload, t.user.Username, t.path, "")
 		} else {
 			logger.TransferLog(uploadLogSender, t.path, elapsed, t.bytesReceived, t.user.Username, t.connectionID, t.protocol)
-			executeAction(operationUpload, t.user.Username, t.path, "")
+			go executeAction(operationUpload, t.user.Username, t.path, "")
 		}
 	}
 	metrics.TransferCompleted(t.bytesSent, t.bytesReceived, t.transferType, t.transferError)

sftpgo.json

@@ -32,7 +32,12 @@
     "manage_users": 1,
     "track_quota": 2,
     "pool_size": 0,
-    "users_base_dir": ""
+    "users_base_dir": "",
+    "actions": {
+      "execute_on": [],
+      "command": "",
+      "http_notification_url": ""
+    }
   },
   "httpd": {
     "bind_port": 8080,