pre-login program: allow to create a new user too

clarify the difference between dynamic user creation/update and external
authentication

README.md

@@ -120,9 +120,9 @@ This authentication method is typically used for multi-factor authentication.
 
 More information can be found [here](./docs/keyboard-interactive.md).
 
-## Dynamic user modification
+## Dynamic user creation or modification
 
-The user configuration, retrieved from the data provider, can be modified by an external program. More information about this can be found [here](./docs/dynamic-user-mod.md).
+A user can be created or modified by an external program just before the login. More information about this can be found [here](./docs/dynamic-user-mod.md).
 
 ## Custom Actions
 

dataprovider/bolt.go

@@ -328,6 +328,20 @@ func (p BoltProvider) dumpUsers() ([]User, error) {
 	return users, err
 }
 
+func (p BoltProvider) getUserWithUsername(username string) ([]User, error) {
+	users := []User{}
+	var user User
+	user, err := p.userExists(username)
+	if err == nil {
+		users = append(users, HideUserSensitiveData(&user))
+		return users, nil
+	}
+	if _, ok := err.(*RecordNotFoundError); ok {
+		err = nil
+	}
+	return users, err
+}
+
 func (p BoltProvider) getUsers(limit int, offset int, order string, username string) ([]User, error) {
 	users := []User{}
 	var err error
@@ -336,15 +350,7 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str
 	}
 	if len(username) > 0 {
 		if offset == 0 {
-			var user User
-			user, err = p.userExists(username)
-			if err == nil {
-				users = append(users, HideUserSensitiveData(&user))
-				return users, nil
-			}
-			if _, ok := err.(*RecordNotFoundError); ok {
-				err = nil
-			}
+			return p.getUserWithUsername(username)
 		}
 		return users, err
 	}

dataprovider/dataprovider.go

@@ -74,7 +74,7 @@ var (
 	// SupportedProviders data provider configured in the sftpgo.conf file must match of these strings
 	SupportedProviders = []string{SQLiteDataProviderName, PGSQLDataProviderName, MySQLDataProviderName,
 		BoltDataProviderName, MemoryDataProviderName}
-	// ValidPerms list that contains all the valid permissions for an user
+	// ValidPerms list that contains all the valid permissions for a user
 	ValidPerms = []string{PermAny, PermListItems, PermDownload, PermUpload, PermOverwrite, PermRename, PermDelete,
 		PermCreateDirs, PermCreateSymlinks, PermChmod, PermChown, PermChtimes}
 	// ValidSSHLoginMethods list that contains all the valid SSH login methods
@@ -171,7 +171,7 @@ type Config struct {
 	// control of a possibly malicious remote user.
 	//
 	// The program must respond on the standard output with a valid SFTPGo user serialized as JSON if the
-	// authentication succeed or an user with an empty username if the authentication fails.
+	// authentication succeed or a user with an empty username if the authentication fails.
 	// If the authentication succeed the user will be automatically added/updated inside the defined data provider.
 	// Actions defined for user added/updated will not be executed in this case.
 	// The external program should check authentication only, if there are login restrictions such as user
@@ -196,7 +196,7 @@ type Config struct {
 	CredentialsPath string `json:"credentials_path" mapstructure:"credentials_path"`
 	// Absolute path to an external program to start just before the user login.
 	// This program will be started before an existing user try to login and allows to
-	// modify the user.
+	// modify or create the user.
 	// It is useful if you have users with dynamic fields that need to the updated just
 	// before the login.
 	// The external program can read the following environment variables:
@@ -204,14 +204,16 @@ type Config struct {
 	// - SFTPGO_LOGIND_USER, it contains the user trying to login serialized as JSON
 	// - SFTPGO_LOGIND_METHOD, possible values are: "password", "publickey" and "keyboard-interactive"
 	//
-	// The program must respond on the standard output with an empty string if no user
-	// update is needed or with a valid SFTPGo user serialized as JSON.
-	// The JSON response can include only the fields that need to the updated instead
-	// of the full user, for example if you want to disable the user you can return a
-	// response like this:
+	// The program must write on its standard output an empty string if no user update is needed
+	// or a valid SFTPGo user serialized as JSON.
+	// The JSON response can include only the fields to update instead of the full user,
+	// for example if you want to disable the user you can return a response like this:
 	//
 	// {"status":0}
 	//
+	// Please note that if you want to create a new user, the pre-login program response must
+	// include all the mandatory user fields.
+	//
 	// The external program must finish within 60 seconds.
 	//
 	// If an error happens while executing the "PreLoginProgram" then login will be denied.
@@ -245,7 +247,7 @@ func (e *ValidationError) Error() string {
 
 // MethodDisabledError raised if a method is disabled in config file.
 // For example, if user management is disabled, this error is raised
-// every time an user operation is done using the REST API
+// every time a user operation is done using the REST API
 type MethodDisabledError struct {
 	err string
 }
@@ -318,11 +320,11 @@ func Initialize(cnf Config, basePath string) error {
 	}
 	if len(config.PreLoginProgram) > 0 {
 		if !filepath.IsAbs(config.PreLoginProgram) {
-			return fmt.Errorf("invalid pre login program: %#v must be an absolute path", config.PreLoginProgram)
+			return fmt.Errorf("invalid pre-login program: %#v must be an absolute path", config.PreLoginProgram)
 		}
 		_, err := os.Stat(config.PreLoginProgram)
 		if err != nil {
-			providerLog(logger.LevelWarn, "invalid pre login program: %v", err)
+			providerLog(logger.LevelWarn, "invalid pre-login program: %v", err)
 			return err
 		}
 	}
@@ -1182,7 +1184,13 @@ func doKeyboardInteractiveAuth(user User, authProgram string, client ssh.Keyboar
 func executePreLoginProgram(username, loginMethod string) (User, error) {
 	u, err := provider.userExists(username)
 	if err != nil {
-		return u, err
+		if _, ok := err.(*RecordNotFoundError); !ok {
+			return u, err
+		}
+		u = User{
+			ID:       0,
+			Username: username,
+		}
 	}
 	userAsJSON, err := json.Marshal(u)
 	if err != nil {
@@ -1197,10 +1205,14 @@ func executePreLoginProgram(username, loginMethod string) (User, error) {
 		fmt.Sprintf("SFTPGO_LOGIND_METHOD=%v", loginMethod))
 	out, err := cmd.Output()
 	if err != nil {
-		return u, fmt.Errorf("Before login program error: %v", err)
+		return u, fmt.Errorf("Pre-login program error: %v", err)
 	}
 	if len(strings.TrimSpace(string(out))) == 0 {
-		providerLog(logger.LevelDebug, "empty response from before login program, no modification needed for user %#v", username)
+		providerLog(logger.LevelDebug, "empty response from pre-login program, no modification requested for user %#v id: %v",
+			username, u.ID)
+		if u.ID == 0 {
+			return u, &RecordNotFoundError{err: fmt.Sprintf("username %v does not exist", username)}
+		}
 		return u, nil
 	}
 
@@ -1211,18 +1223,22 @@ func executePreLoginProgram(username, loginMethod string) (User, error) {
 	userLastLogin := u.LastLogin
 	err = json.Unmarshal(out, &u)
 	if err != nil {
-		return u, fmt.Errorf("Invalid before login program response %#v, error: %v", string(out), err)
+		return u, fmt.Errorf("Invalid pre-login program response %#v, error: %v", string(out), err)
 	}
 	u.ID = userID
 	u.UsedQuotaSize = userUsedQuotaSize
 	u.UsedQuotaFiles = userUsedQuotaFiles
 	u.LastQuotaUpdate = userLastQuotaUpdate
 	u.LastLogin = userLastLogin
-	err = provider.updateUser(u)
+	if userID == 0 {
+		err = provider.addUser(u)
+	} else {
+		err = provider.updateUser(u)
+	}
 	if err != nil {
 		return u, err
 	}
-	providerLog(logger.LevelDebug, "user %#v updated from before login program response", username)
+	providerLog(logger.LevelDebug, "user %#v added/updated from pre-login program response, id: %v", username, userID)
 	return provider.userExists(username)
 }
 

docs/dynamic-user-mod.md

@@ -1,25 +1,36 @@
-# Dynamic user modification
+# Dynamic user creation or modification
 
-Dynamic user modification is supported via an external program that can be executed just before the user login.
+Dynamic user creation or modification is supported via an external program that can be executed just before the user login.
 To enable dynamic user modification, you must set the absolute path of your program using the `pre_login_program` key in your configuration file.
 
 The external program can read the following environment variables to get info about the user trying to login:
 
-- `SFTPGO_LOGIND_USER`, it contains the user trying to login serialized as JSON
+- `SFTPGO_LOGIND_USER`, it contains the user trying to login serialized as JSON. A JSON serialized user id equal to zero means the user does not exists inside SFTPGo
 - `SFTPGO_LOGIND_METHOD`, possible values are: `password`, `publickey` and `keyboard-interactive`
 
-The program must write, on its the standard output, an empty string (or no response at all) if no user update is needed or the updated SFTPGo user serialized as JSON. Actions defined for users update will not be executed in this case.
-The JSON response can include only the fields that need to the updated instead of the full user. For example, if you want to disable the user, you can return a response like this:
+The program must write, on its the standard output:
+
+- an empty string (or no response at all) if the user should not be created/updated
+- or the SFTPGo user, JSON serialized, if you want create or update the given user
+
+Actions defined for user's updates will not be executed in this case.
+
+The JSON response can include only the fields to update instead of the full user. For example, if you want to disable the user, you can return a response like this:
 
 ```json
 {"status": 0}
 ```
 
+Please note that if you want to create a new user, the pre-login program response must include all the mandatory user fields.
+
 The external program must finish within 60 seconds.
 
-If an error happens while executing your program then login will be denied. "Dynamic user modification" and "External Authentication" are mutally exclusive.
+If an error happens while executing your program then login will be denied.
+
+"Dynamic user creation or modification" and "External Authentication" are mutally exclusive, they are quite similar, the difference is that "External Authentication" returns an already authenticated user while using "Dynamic users modification" you simply create or update a user. The authentication will be checked inside SFTPGo.
+In other words while using "External Authentication" the external program receives the credentials of the user trying to login (for example the clear text password) and it need to validate them. While using "Dynamic users modification" the pre-login program receives the user stored inside the dataprovider (it includes the hashed password if any) and it can modify it, after the modification SFTPGo will check the credentials of the user trying to login.
 
-Let's see a very basic example. Our sample program will grant access to the user `test_user` only in the time range 10:00-18:00. Other users will not be modified since the program will terminate with no output.
+Let's see a very basic example. Our sample program will grant access to the existing user `test_user` only in the time range 10:00-18:00. Other users will not be modified since the program will terminate with no output.
 
 ```
 #!/bin/bash

docs/external-auth.md

@@ -10,7 +10,7 @@ The external program can read the following environment variables to get info ab
 - `SFTPGO_AUTHD_KEYBOARD_INTERACTIVE`, not empty for keyboard interactive authentication
 
 Previous global environment variables aren't cleared when the script is called. The content of these variables is _not_ quoted. They may contain special characters. They are under the control of a possibly malicious remote user.
-The program must write, on its standard output, a valid SFTPGo user serialized as JSON if the authentication succeed or an user with an empty username if the authentication fails.
+The program must write, on its standard output, a valid SFTPGo user serialized as JSON if the authentication succeed or a user with an empty username if the authentication fails.
 If the authentication succeeds, the user will be automatically added/updated inside the defined data provider. Actions defined for users added/updated will not be executed in this case.
 The external program should check authentication only. If there are login restrictions such as user disabled, expired, or login allowed only from specific IP addresses, it is enough to populate the matching user fields, and these conditions will be checked in the same way as for built-in users.
 The external auth program should finish very quickly. It will be killed if it does not exit within 60 seconds.

httpd/api_utils.go

@@ -162,7 +162,7 @@ func RemoveUser(user dataprovider.User, expectedStatusCode int) ([]byte, error)
 	return body, checkResponse(resp.StatusCode, expectedStatusCode)
 }
 
-// GetUserByID gets an user by database id and checks the received HTTP Status code against expectedStatusCode.
+// GetUserByID gets a user by database id and checks the received HTTP Status code against expectedStatusCode.
 func GetUserByID(userID int64, expectedStatusCode int) (dataprovider.User, []byte, error) {
 	var user dataprovider.User
 	var body []byte
@@ -183,7 +183,7 @@ func GetUserByID(userID int64, expectedStatusCode int) (dataprovider.User, []byt
 // GetUsers allows to get a list of users and checks the received HTTP Status code against expectedStatusCode.
 // The number of results can be limited specifying a limit.
 // Some results can be skipped specifying an offset.
-// The results can be filtered specifying an username, the username filter is an exact match
+// The results can be filtered specifying a username, the username filter is an exact match
 func GetUsers(limit int64, offset int64, username string, expectedStatusCode int) ([]dataprovider.User, []byte, error) {
 	var users []dataprovider.User
 	var body []byte

httpd/schema/openapi.yaml

@@ -1028,7 +1028,7 @@ components:
         credentials:
           type: string
           format: byte
-          description: Google Cloud Storage JSON credentials base64 encoded. This field must be populated only when adding/updating an user. It will be always omitted, since there are sensitive data, when you search/get users. The credentials will be stored in the configured "credentials_path"
+          description: Google Cloud Storage JSON credentials base64 encoded. This field must be populated only when adding/updating a user. It will be always omitted, since there are sensitive data, when you search/get users. The credentials will be stored in the configured "credentials_path"
         automatic_credentials:
           type: integer
           nullable: true
@@ -1135,7 +1135,7 @@ components:
         max_sessions:
           type: integer
           format: int32
-          description: Limit the sessions that an user can open. 0 means unlimited
+          description: Limit the sessions that a user can open. 0 means unlimited
         quota_size:
           type: integer
           format: int64

sftpd/sftpd.go

@@ -271,7 +271,7 @@ func GetQuotaScans() []ActiveQuotaScan {
 	return scans
 }
 
-// AddQuotaScan add an user to the ones with active quota scans.
+// AddQuotaScan add a user to the ones with active quota scans.
 // Returns false if the user has a quota scan already running
 func AddQuotaScan(username string) bool {
 	mutex.Lock()
@@ -288,7 +288,7 @@ func AddQuotaScan(username string) bool {
 	return true
 }
 
-// RemoveQuotaScan removes an user from the ones with active quota scans
+// RemoveQuotaScan removes a user from the ones with active quota scans
 func RemoveQuotaScan(username string) error {
 	mutex.Lock()
 	defer mutex.Unlock()

sftpd/sftpd_test.go

@@ -1109,7 +1109,7 @@ func TestLoginInvalidFs(t *testing.T) {
 	if err != nil {
 		t.Errorf("unable to add user: %v", err)
 	}
-	// we update the database using sqlite3 CLI since we cannot add an user with an invalid config
+	// we update the database using sqlite3 CLI since we cannot add a user with an invalid config
 	time.Sleep(200 * time.Millisecond)
 	updateUserQuery := fmt.Sprintf("UPDATE users SET filesystem='{\"provider\":1}' WHERE id=%v", user.ID)
 	cmd := exec.Command("sqlite3", dbPath, updateUserQuery)
@@ -1395,13 +1395,13 @@ func TestPreLoginScript(t *testing.T) {
 	ioutil.WriteFile(preLoginPath, getPreLoginScriptContent(user, true), 0755)
 	_, err = getSftpClient(u, usePubKey)
 	if err == nil {
-		t.Error("pre login script returned a non json response, login must fail")
+		t.Error("pre-login script returned a non json response, login must fail")
 	}
 	user.Status = 0
 	ioutil.WriteFile(preLoginPath, getPreLoginScriptContent(user, false), 0755)
 	_, err = getSftpClient(u, usePubKey)
 	if err == nil {
-		t.Error("pre login script returned a disabled user, login must fail")
+		t.Error("pre-login script returned a disabled user, login must fail")
 	}
 	_, err = httpd.RemoveUser(user, http.StatusOK)
 	if err != nil {
@@ -1421,6 +1421,64 @@ func TestPreLoginScript(t *testing.T) {
 	os.Remove(preLoginPath)
 }
 
+func TestPreLoginUserCreation(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("this test is not available on Windows")
+	}
+	usePubKey := false
+	u := getTestUser(usePubKey)
+	dataProvider := dataprovider.GetProvider()
+	dataprovider.Close(dataProvider)
+	config.LoadConfig(configDir, "")
+	providerConf := config.GetProviderConf()
+	ioutil.WriteFile(preLoginPath, getPreLoginScriptContent(u, false), 0755)
+	providerConf.PreLoginProgram = preLoginPath
+	err := dataprovider.Initialize(providerConf, configDir)
+	if err != nil {
+		t.Errorf("error initializing data provider")
+	}
+	httpd.SetDataProvider(dataprovider.GetProvider())
+	sftpd.SetDataProvider(dataprovider.GetProvider())
+
+	users, out, err := httpd.GetUsers(0, 0, defaultUsername, http.StatusOK)
+	if err != nil {
+		t.Errorf("unable to get users: %v, out: %v", err, string(out))
+	}
+	if len(users) != 0 {
+		t.Errorf("number of users mismatch, expected: 0, actual: %v", len(users))
+	}
+	client, err := getSftpClient(u, usePubKey)
+	if err != nil {
+		t.Errorf("unable to create sftp client: %v", err)
+	} else {
+		defer client.Close()
+		_, err = client.Getwd()
+		if err != nil {
+			t.Errorf("unable to get working dir: %v", err)
+		}
+	}
+	users, out, err = httpd.GetUsers(0, 0, defaultUsername, http.StatusOK)
+	if err != nil {
+		t.Errorf("unable to get users: %v, out: %v", err, string(out))
+	}
+	if len(users) != 1 {
+		t.Errorf("number of users mismatch, expected: 1, actual: %v", len(users))
+	}
+	user := users[0]
+	os.RemoveAll(user.GetHomeDir())
+	dataProvider = dataprovider.GetProvider()
+	dataprovider.Close(dataProvider)
+	config.LoadConfig(configDir, "")
+	providerConf = config.GetProviderConf()
+	err = dataprovider.Initialize(providerConf, configDir)
+	if err != nil {
+		t.Errorf("error initializing data provider")
+	}
+	httpd.SetDataProvider(dataprovider.GetProvider())
+	sftpd.SetDataProvider(dataprovider.GetProvider())
+	os.Remove(preLoginPath)
+}
+
 func TestLoginExternalAuthPwdAndPubKey(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("this test is not available on Windows")