Add support for loading users/folders on startup

Fixes #161

README.md

@@ -54,7 +54,7 @@ SFTPGo is developed and tested on Linux. After each commit, the code is automati
 
 ## Requirements
 
-- Go 1.13 or higher as build only dependency.
+- Go 1.14 or higher as build only dependency.
 - A suitable SQL server to use as data provider: PostgreSQL 9.4+ or MySQL 5.6+ or SQLite 3.x.
 - The SQL server is optional: you can choose to use an embedded bolt database as key/value store or an in memory data provider.
 

cmd/root.go

@@ -13,45 +13,61 @@ import (
 )
 
 const (
-	configDirFlag       = "config-dir"
-	configDirKey        = "config_dir"
-	configFileFlag      = "config-file"
-	configFileKey       = "config_file"
-	logFilePathFlag     = "log-file-path"
-	logFilePathKey      = "log_file_path"
-	logMaxSizeFlag      = "log-max-size"
-	logMaxSizeKey       = "log_max_size"
-	logMaxBackupFlag    = "log-max-backups"
-	logMaxBackupKey     = "log_max_backups"
-	logMaxAgeFlag       = "log-max-age"
-	logMaxAgeKey        = "log_max_age"
-	logCompressFlag     = "log-compress"
-	logCompressKey      = "log_compress"
-	logVerboseFlag      = "log-verbose"
-	logVerboseKey       = "log_verbose"
-	profilerFlag        = "profiler"
-	profilerKey         = "profiler"
-	defaultConfigDir    = "."
-	defaultConfigName   = config.DefaultConfigName
-	defaultLogFile      = "sftpgo.log"
-	defaultLogMaxSize   = 10
-	defaultLogMaxBackup = 5
-	defaultLogMaxAge    = 28
-	defaultLogCompress  = false
-	defaultLogVerbose   = true
-	defaultProfiler     = false
+	configDirFlag            = "config-dir"
+	configDirKey             = "config_dir"
+	configFileFlag           = "config-file"
+	configFileKey            = "config_file"
+	logFilePathFlag          = "log-file-path"
+	logFilePathKey           = "log_file_path"
+	logMaxSizeFlag           = "log-max-size"
+	logMaxSizeKey            = "log_max_size"
+	logMaxBackupFlag         = "log-max-backups"
+	logMaxBackupKey          = "log_max_backups"
+	logMaxAgeFlag            = "log-max-age"
+	logMaxAgeKey             = "log_max_age"
+	logCompressFlag          = "log-compress"
+	logCompressKey           = "log_compress"
+	logVerboseFlag           = "log-verbose"
+	logVerboseKey            = "log_verbose"
+	profilerFlag             = "profiler"
+	profilerKey              = "profiler"
+	loadDataFromFlag         = "loaddata-from"
+	loadDataFromKey          = "loaddata_from"
+	loadDataModeFlag         = "loaddata-mode"
+	loadDataModeKey          = "loaddata_mode"
+	loadDataQuotaScanFlag    = "loaddata-scan"
+	loadDataQuotaScanKey     = "loaddata_scan"
+	loadDataCleanFlag        = "loaddata-clean"
+	loadDataCleanKey         = "loaddata_clean"
+	defaultConfigDir         = "."
+	defaultConfigName        = config.DefaultConfigName
+	defaultLogFile           = "sftpgo.log"
+	defaultLogMaxSize        = 10
+	defaultLogMaxBackup      = 5
+	defaultLogMaxAge         = 28
+	defaultLogCompress       = false
+	defaultLogVerbose        = true
+	defaultProfiler          = false
+	defaultLoadDataFrom      = ""
+	defaultLoadDataMode      = 1
+	defaultLoadDataQuotaScan = 0
+	defaultLoadDataClean     = false
 )
 
 var (
-	configDir     string
-	configFile    string
-	logFilePath   string
-	logMaxSize    int
-	logMaxBackups int
-	logMaxAge     int
-	logCompress   bool
-	logVerbose    bool
-	profiler      bool
+	configDir         string
+	configFile        string
+	logFilePath       string
+	logMaxSize        int
+	logMaxBackups     int
+	logMaxAge         int
+	logCompress       bool
+	logVerbose        bool
+	profiler          bool
+	loadDataFrom      string
+	loadDataMode      int
+	loadDataQuotaScan int
+	loadDataClean     bool
 
 	rootCmd = &cobra.Command{
 		Use:   "sftpgo",
@@ -172,4 +188,51 @@ be accessible via HTTP/HTTPS using the base URL
 This flag can be set using SFTPGO_PROFILER env
 var too.`)
 	viper.BindPFlag(profilerKey, cmd.Flags().Lookup(profilerFlag)) //nolint:errcheck
+
+	viper.SetDefault(loadDataFromKey, defaultLoadDataFrom)
+	viper.BindEnv(loadDataFromKey, "SFTPGO_LOADDATA_FROM") //nolint:errcheck
+	cmd.Flags().StringVar(&loadDataFrom, loadDataFromFlag, viper.GetString(loadDataFromKey),
+		`Load users and folders from this file.
+The file must be specified as absolute path
+and it must contain a backup obtained using
+the "dumpdata" REST API or compatible content.
+This flag can be set using SFTPGO_LOADDATA_FROM
+env var too.
+`)
+	viper.BindPFlag(loadDataFromKey, cmd.Flags().Lookup(loadDataFromFlag)) //nolint:errcheck
+
+	viper.SetDefault(loadDataModeKey, defaultLoadDataMode)
+	viper.BindEnv(loadDataModeKey, "SFTPGO_LOADDATA_MODE") //nolint:errcheck
+	cmd.Flags().IntVar(&loadDataMode, loadDataModeFlag, viper.GetInt(loadDataModeKey),
+		`Restore mode for data to load:
+  0 - new users are added, existing users are
+      updated
+  1 - New users are added, existing users are
+	  not modified
+This flag can be set using SFTPGO_LOADDATA_MODE
+env var too.
+`)
+	viper.BindPFlag(loadDataModeKey, cmd.Flags().Lookup(loadDataModeFlag)) //nolint:errcheck
+
+	viper.SetDefault(loadDataQuotaScanKey, defaultLoadDataQuotaScan)
+	viper.BindEnv(loadDataQuotaScanKey, "SFTPGO_LOADDATA_QUOTA_SCAN") //nolint:errcheck
+	cmd.Flags().IntVar(&loadDataQuotaScan, loadDataQuotaScanFlag, viper.GetInt(loadDataQuotaScanKey),
+		`Quota scan mode after data load:
+  0 - no quota scan
+  1 - scan quota
+  2 - scan quota if the user has quota restrictions
+This flag can be set using SFTPGO_LOADDATA_QUOTA_SCAN
+env var too.
+(default 0)`)
+	viper.BindPFlag(loadDataQuotaScanKey, cmd.Flags().Lookup(loadDataQuotaScanFlag)) //nolint:errcheck
+
+	viper.SetDefault(loadDataCleanKey, defaultLoadDataClean)
+	viper.BindEnv(loadDataCleanKey, "SFTPGO_LOADDATA_CLEAN") //nolint:errcheck
+	cmd.Flags().BoolVar(&loadDataClean, loadDataCleanFlag, viper.GetBool(loadDataCleanKey),
+		`Determine if the loaddata-from file should
+be removed after a successful load. This flag
+can be set using SFTPGO_LOADDATA_CLEAN env var
+too. (default "false")
+`)
+	viper.BindPFlag(logCompressKey, cmd.Flags().Lookup(logCompressFlag)) //nolint:errcheck
 }

cmd/serve.go

@@ -21,16 +21,20 @@ $ sftpgo serve
 Please take a look at the usage below to customize the startup options`,
 		Run: func(cmd *cobra.Command, args []string) {
 			service := service.Service{
-				ConfigDir:     utils.CleanDirInput(configDir),
-				ConfigFile:    configFile,
-				LogFilePath:   logFilePath,
-				LogMaxSize:    logMaxSize,
-				LogMaxBackups: logMaxBackups,
-				LogMaxAge:     logMaxAge,
-				LogCompress:   logCompress,
-				LogVerbose:    logVerbose,
-				Profiler:      profiler,
-				Shutdown:      make(chan bool),
+				ConfigDir:         utils.CleanDirInput(configDir),
+				ConfigFile:        configFile,
+				LogFilePath:       logFilePath,
+				LogMaxSize:        logMaxSize,
+				LogMaxBackups:     logMaxBackups,
+				LogMaxAge:         logMaxAge,
+				LogCompress:       logCompress,
+				LogVerbose:        logVerbose,
+				LoadDataFrom:      loadDataFrom,
+				LoadDataMode:      loadDataMode,
+				LoadDataQuotaScan: loadDataQuotaScan,
+				LoadDataClean:     loadDataClean,
+				Profiler:          profiler,
+				Shutdown:          make(chan bool),
 			}
 			if err := service.Start(); err == nil {
 				service.Wait()

docker/README.md

@@ -23,6 +23,8 @@ docker run --name some-sftpgo -p 127.0.0.1:8080:8080 -p 2022:2022 -d "drakkan/sf
 
 Now visit [http://localhost:8080/](http://localhost:8080/) and create a new SFTPGo user. The SFTP service is available on port 2022.
 
+If you prefer GitHub Container Registry to Docker Hub replace `drakkan/sftpgo:tag` with `ghcr.io/drakkan/sftpgo:tag`.
+
 ### Container shell access and viewing SFTPGo logs
 
 The docker exec command allows you to run commands inside a Docker container. The following command line will give you a bash shell inside your `sftpgo` container:

docs/full-configuration.md

@@ -26,18 +26,23 @@ The `serve` command supports the following flags:
 
 - `--config-dir` string. Location of the config dir. This directory should contain the configuration file and is used as the base directory for any files that use a relative path (eg. the private keys for the SFTP server, the SQLite or bblot database if you use SQLite or bbolt as data provider). The default value is "." or the value of `SFTPGO_CONFIG_DIR` environment variable.
 - `--config-file` string. Name of the configuration file. It must be the name of a file stored in `config-dir`, not the absolute path to the configuration file. The specified file name must have no extension because we automatically append JSON, YAML, TOML, HCL and Java extensions when we search for the file. The default value is "sftpgo" (and therefore `sftpgo.json`, `sftpgo.yaml` and so on are searched) or the value of `SFTPGO_CONFIG_FILE` environment variable.
+- `--loaddata-from` string. Load users and folders from this file. The file must be specified as absolute path and it must contain a backup obtained using the `dumpdata` REST API or compatible content. The default value is empty or the value of `SFTPGO_LOADDATA_FROM` environment variable.
+- `--loaddata-clean` boolean. Determine if the loaddata-from file should be removed after a successful load. Default `false` or the value of `SFTPGO_LOADDATA_CLEAN` environment variable (1 or `true`, 0 or `false`).
+- `--loaddata-mode`, integer. Restore mode for data to load. 0 means new users are added, existing users are updated. 1 means new users are added, existing users are not modified. Default 1 or the value of `SFTPGO_LOADDATA_MODE` environment variable.
+- `--loaddata-scan`, integer. Quota scan mode after data load. 0 means no quota scan. 1 means quota scan. 2 means scan quota if the user has quota restrictions. Default 0 or the value of `SFTPGO_LOADDATA_QUOTA_SCAN` environment variable.
 - `--log-compress` boolean. Determine if the rotated log files should be compressed using gzip. Default `false` or the value of `SFTPGO_LOG_COMPRESS` environment variable (1 or `true`, 0 or `false`). It is unused if `log-file-path` is empty.
 - `--log-file-path` string. Location for the log file, default "sftpgo.log" or the value of `SFTPGO_LOG_FILE_PATH` environment variable. Leave empty to write logs to the standard error.
 - `--log-max-age` int. Maximum number of days to retain old log files. Default 28 or the value of `SFTPGO_LOG_MAX_AGE` environment variable. It is unused if `log-file-path` is empty.
 - `--log-max-backups` int. Maximum number of old log files to retain. Default 5 or the value of `SFTPGO_LOG_MAX_BACKUPS` environment variable. It is unused if `log-file-path` is empty.
 - `--log-max-size` int. Maximum size in megabytes of the log file before it gets rotated. Default 10 or the value of `SFTPGO_LOG_MAX_SIZE` environment variable. It is unused if `log-file-path` is empty.
 - `--log-verbose` boolean. Enable verbose logs. Default `true` or the value of `SFTPGO_LOG_VERBOSE` environment variable (1 or `true`, 0 or `false`).
+- `--profiler` boolean. Enable the built-in profiler. The profiler will be accessible via HTTP/HTTPS using the base URL "/debug/pprof/". Default `false` or the value of `SFTPGO_PROFILER` environment variable (1 or `true`, 0 or `false`).
 
 Log file can be rotated on demand sending a `SIGUSR1` signal on Unix based systems and using the command `sftpgo service rotatelogs` on Windows.
 
-If you don't configure any private host key, the daemon will use `id_rsa`, `id_ecdsa` and `id_ed25519` in the configuration directory. If these files don't exist, the daemon will attempt to autogenerate them (if the user that executes SFTPGo has write access to the `config-dir`). The server supports any private key format supported by [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/keys.go#L33).
+If you don't configure any private host key, the daemon will use `id_rsa`, `id_ecdsa` and `id_ed25519` in the configuration directory. If these files don't exist, the daemon will attempt to autogenerate them. The server supports any private key format supported by [`crypto/ssh`](https://github.com/golang/crypto/blob/master/ssh/keys.go#L33).
 
-The `gen` command allows to generate completion scripts for your shell and man pages. Currently the man pages visual representation is wrong, take a look at this upstream [bug](https://github.com/spf13/cobra/issues/1049) for more details.
+The `gen` command allows to generate completion scripts for your shell and man pages.
 
 ## Configuration file
 
@@ -182,7 +187,7 @@ If you want the default host keys generation in a directory different from the c
 ]
 ```
 
-then SFTPGo will try to create `id_rsa`, `id_ecdsa` and `id_ed25519`, if they are missing, inside the existing directory `/etc/sftpgo/keys`.
+then SFTPGo will try to create `id_rsa`, `id_ecdsa` and `id_ed25519`, if they are missing, inside the directory `/etc/sftpgo/keys`.
 
 The configuration can be read from JSON, TOML, YAML, HCL, envfile and Java properties config files. If your `config-file` flag is set to `sftpgo` (default value), you need to create a configuration file called `sftpgo.json` or `sftpgo.yaml` and so on inside `config-dir`.
 

docs/howto/postgresql-s3.md

@@ -144,7 +144,7 @@ Search for the `data_provider` section and change it as follow.
 }
 ```
 
-This way we set the PostgreSQL connection parameters and a default base directory for new users.
+This way we set the PostgreSQL connection parameters.
 
 If you want to connect to PostgreSQL over a Unix Domain socket you have to set the value `/var/run/postgresql` for the `host` configuration key instead of `127.0.0.1`.
 
@@ -212,4 +212,4 @@ Click `Add` and fill the user details, the minimum required parameters are:
 
 You are done! Now you can connect to you SFTPGo instance using any compatible `sftp` client on port `2022`.
 
-You can mix S3 users with local users but please be aware that we are running the service as the unprivileged `sftpgo` system user so if you set storage as `local` for an SFTPGo virtual user then the home directory for this user must be owned by the `sftpgo` system user. If you don't specify an home directory the default will be `/var/lib/sftpgo/users/<username>` which should be appropriate.
+You can mix S3 users with local users but please be aware that we are running the service as the unprivileged `sftpgo` system user so if you set storage as `local` for an SFTPGo virtual user then the home directory for this user must be owned by the `sftpgo` system user. If you don't specify an home directory the default will be `/srv/sftpgo/data/<username>` which should be appropriate.

httpd/api_maintenance.go

@@ -85,9 +85,9 @@ func loadData(w http.ResponseWriter, r *http.Request) {
 		sendAPIResponse(w, r, err, "", getRespStatus(err))
 		return
 	}
-	if fi.Size() > maxRestoreSize {
+	if fi.Size() > MaxRestoreSize {
 		sendAPIResponse(w, r, err, fmt.Sprintf("Unable to restore input file: %#v size too big: %v/%v bytes",
-			inputFile, fi.Size(), maxRestoreSize), http.StatusBadRequest)
+			inputFile, fi.Size(), MaxRestoreSize), http.StatusBadRequest)
 		return
 	}
 
@@ -103,12 +103,12 @@ func loadData(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err = restoreFolders(dump.Folders, inputFile, scanQuota); err != nil {
+	if err = RestoreFolders(dump.Folders, inputFile, scanQuota); err != nil {
 		sendAPIResponse(w, r, err, "", getRespStatus(err))
 		return
 	}
 
-	if err = restoreUsers(dump.Users, inputFile, mode, scanQuota); err != nil {
+	if err = RestoreUsers(dump.Users, inputFile, mode, scanQuota); err != nil {
 		sendAPIResponse(w, r, err, "", getRespStatus(err))
 		return
 	}
@@ -140,7 +140,8 @@ func getLoaddataOptions(r *http.Request) (string, int, int, error) {
 	return inputFile, scanQuota, restoreMode, err
 }
 
-func restoreFolders(folders []vfs.BaseVirtualFolder, inputFile string, scanQuota int) error {
+// RestoreFolders restores the specified folders
+func RestoreFolders(folders []vfs.BaseVirtualFolder, inputFile string, scanQuota int) error {
 	for _, folder := range folders {
 		_, err := dataprovider.GetFolderByPath(folder.MappedPath)
 		if err == nil {
@@ -163,7 +164,8 @@ func restoreFolders(folders []vfs.BaseVirtualFolder, inputFile string, scanQuota
 	return nil
 }
 
-func restoreUsers(users []dataprovider.User, inputFile string, mode, scanQuota int) error {
+// RestoreUsers restores the specified users
+func RestoreUsers(users []dataprovider.User, inputFile string, mode, scanQuota int) error {
 	for _, user := range users {
 		u, err := dataprovider.UserExists(user.Username)
 		if err == nil {

httpd/httpd.go

@@ -43,8 +43,9 @@ const (
 	webFoldersPath            = "/web/folders"
 	webFolderPath             = "/web/folder"
 	webStaticFilesPath        = "/static"
-	maxRestoreSize            = 10485760 // 10 MB
-	maxRequestSize            = 1048576  // 1MB
+	// MaxRestoreSize defines the max size for the loaddata input file
+	MaxRestoreSize = 10485760 // 10 MB
+	maxRequestSize = 1048576  // 1MB
 )
 
 var (

service/service.go

@@ -2,6 +2,10 @@
 package service
 
 import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
 	"path/filepath"
 
 	"github.com/rs/zerolog"
@@ -9,6 +13,7 @@ import (
 	"github.com/drakkan/sftpgo/common"
 	"github.com/drakkan/sftpgo/config"
 	"github.com/drakkan/sftpgo/dataprovider"
+	"github.com/drakkan/sftpgo/httpd"
 	"github.com/drakkan/sftpgo/logger"
 	"github.com/drakkan/sftpgo/utils"
 	"github.com/drakkan/sftpgo/version"
@@ -24,19 +29,23 @@ var (
 
 // Service defines the SFTPGo service
 type Service struct {
-	ConfigDir     string
-	ConfigFile    string
-	LogFilePath   string
-	LogMaxSize    int
-	LogMaxBackups int
-	LogMaxAge     int
-	PortableMode  int
-	PortableUser  dataprovider.User
-	LogCompress   bool
-	LogVerbose    bool
-	Profiler      bool
-	Shutdown      chan bool
-	Error         error
+	ConfigDir         string
+	ConfigFile        string
+	LogFilePath       string
+	LogMaxSize        int
+	LogMaxBackups     int
+	LogMaxAge         int
+	PortableMode      int
+	PortableUser      dataprovider.User
+	LogCompress       bool
+	LogVerbose        bool
+	Profiler          bool
+	LoadDataClean     bool
+	LoadDataFrom      string
+	LoadDataMode      int
+	LoadDataQuotaScan int
+	Shutdown          chan bool
+	Error             error
 }
 
 // Start initializes the service
@@ -86,6 +95,13 @@ func (s *Service) Start() error {
 		}
 	}
 
+	err = s.loadInitialData()
+	if err != nil {
+		logger.Error(logSender, "", "unable to load initial data: %v", err)
+		logger.ErrorToConsole("unable to load initial data: %v", err)
+		return err
+	}
+
 	httpConfig := config.GetHTTPConfig()
 	httpConfig.Initialize(s.ConfigDir)
 
@@ -165,3 +181,57 @@ func (s *Service) Stop() {
 	close(s.Shutdown)
 	logger.Debug(logSender, "", "Service stopped")
 }
+
+func (s *Service) loadInitialData() error {
+	if s.LoadDataFrom == "" {
+		return nil
+	}
+	if !filepath.IsAbs(s.LoadDataFrom) {
+		return fmt.Errorf("invalid input_file %#v, it must be an absolute path", s.LoadDataFrom)
+	}
+	if s.LoadDataMode < 0 || s.LoadDataMode > 1 {
+		return fmt.Errorf("Invalid loaddata-mode %v", s.LoadDataMode)
+	}
+	if s.LoadDataQuotaScan < 0 || s.LoadDataQuotaScan > 2 {
+		return fmt.Errorf("Invalid loaddata-scan %v", s.LoadDataQuotaScan)
+	}
+	info, err := os.Stat(s.LoadDataFrom)
+	if err != nil {
+		return err
+	}
+	if info.Size() > httpd.MaxRestoreSize {
+		return fmt.Errorf("unable to restore input file %#v size too big: %v/%v bytes",
+			s.LoadDataFrom, info.Size(), httpd.MaxRestoreSize)
+	}
+	content, err := ioutil.ReadFile(s.LoadDataFrom)
+	if err != nil {
+		return fmt.Errorf("unable to read input file %#v: %v", s.LoadDataFrom, err)
+	}
+	var dump dataprovider.BackupData
+
+	err = json.Unmarshal(content, &dump)
+	if err != nil {
+		return fmt.Errorf("unable to parse file to restore %#v: %v", s.LoadDataFrom, err)
+	}
+	err = httpd.RestoreFolders(dump.Folders, s.LoadDataFrom, s.LoadDataQuotaScan)
+	if err != nil {
+		return fmt.Errorf("unable to restore folders from file %#v: %v", s.LoadDataFrom, err)
+	}
+	err = httpd.RestoreUsers(dump.Users, s.LoadDataFrom, s.LoadDataMode, s.LoadDataQuotaScan)
+	if err != nil {
+		return fmt.Errorf("unable to restore users from file %#v: %v", s.LoadDataFrom, err)
+	}
+	logger.Info(logSender, "", "data loaded from file %#v", s.LoadDataFrom)
+	logger.InfoToConsole("data loaded from file %#v", s.LoadDataFrom)
+	if s.LoadDataClean {
+		err = os.Remove(s.LoadDataFrom)
+		if err == nil {
+			logger.Info(logSender, "", "file %#v deleted after successful load", s.LoadDataFrom)
+			logger.InfoToConsole("file %#v deleted after successful load", s.LoadDataFrom)
+		} else {
+			logger.Warn(logSender, "", "unable to delete file %#v after successful load: %v", s.LoadDataFrom, err)
+			logger.WarnToConsole("unable to delete file %#v after successful load: %v", s.LoadDataFrom, err)
+		}
+	}
+	return nil
+}