From 1ea74299211ba313a2fcdf9d25485a2424844f56 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Mon, 28 Feb 2022 17:05:18 +0100 Subject: [PATCH] initprovider: add load data options Fixes #741 Signed-off-by: Nicola Murino --- cmd/initprovider.go | 20 ++++++++-- cmd/root.go | 72 +++++++++++++++++++----------------- dataprovider/dataprovider.go | 5 +++ docker/README.md | 2 +- go.mod | 6 +-- go.sum | 11 +++--- httpd/api_maintenance.go | 2 +- logger/logger.go | 5 ++- service/service.go | 22 ++++------- util/util.go | 2 +- 10 files changed, 84 insertions(+), 63 deletions(-) diff --git a/cmd/initprovider.go b/cmd/initprovider.go index 85d669e5..27876d1f 100644 --- a/cmd/initprovider.go +++ b/cmd/initprovider.go @@ -10,6 +10,7 @@ import ( "github.com/drakkan/sftpgo/v2/config" "github.com/drakkan/sftpgo/v2/dataprovider" "github.com/drakkan/sftpgo/v2/logger" + "github.com/drakkan/sftpgo/v2/service" "github.com/drakkan/sftpgo/v2/util" ) @@ -40,13 +41,13 @@ Please take a look at the usage below to customize the options.`, configDir = util.CleanDirInput(configDir) err := config.LoadConfig(configDir, configFile) if err != nil { - logger.WarnToConsole("Unable to initialize data provider, config load error: %v", err) + logger.ErrorToConsole("Unable to initialize data provider, config load error: %v", err) return } kmsConfig := config.GetKMSConfig() err = kmsConfig.Initialize() if err != nil { - logger.ErrorToConsole("unable to initialize KMS: %v", err) + logger.ErrorToConsole("Unable to initialize KMS: %v", err) os.Exit(1) } providerConf := config.GetProviderConf() @@ -57,9 +58,21 @@ Please take a look at the usage below to customize the options.`, } else if err == dataprovider.ErrNoInitRequired { logger.InfoToConsole("%v", err.Error()) } else { - logger.WarnToConsole("Unable to initialize/update the data provider: %v", err) + logger.ErrorToConsole("Unable to initialize/update the data provider: %v", err) os.Exit(1) } + if providerConf.Driver != dataprovider.MemoryDataProviderName && loadDataFrom != "" { + service := service.Service{ + LoadDataFrom: loadDataFrom, + LoadDataMode: loadDataMode, + LoadDataQuotaScan: loadDataQuotaScan, + LoadDataClean: loadDataClean, + } + if err = service.LoadInitialData(); err != nil { + logger.ErrorToConsole("Cannot load initial data: %v", err) + os.Exit(1) + } + } }, } ) @@ -67,4 +80,5 @@ Please take a look at the usage below to customize the options.`, func init() { rootCmd.AddCommand(initProviderCmd) addConfigFlags(initProviderCmd) + addBaseLoadDataFlags(initProviderCmd) } diff --git a/cmd/root.go b/cmd/root.go index ceb7fe61..16332557 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -126,6 +126,43 @@ env var too.`) viper.BindPFlag(configFileKey, cmd.Flags().Lookup(configFileFlag)) //nolint:errcheck } +func addBaseLoadDataFlags(cmd *cobra.Command) { + viper.SetDefault(loadDataFromKey, defaultLoadDataFrom) + viper.BindEnv(loadDataFromKey, "SFTPGO_LOADDATA_FROM") //nolint:errcheck + cmd.Flags().StringVar(&loadDataFrom, loadDataFromFlag, viper.GetString(loadDataFromKey), + `Load users and folders from this file. +The file must be specified as absolute path +and it must contain a backup obtained using +the "dumpdata" REST API or compatible content. +This flag can be set using SFTPGO_LOADDATA_FROM +env var too. +`) + viper.BindPFlag(loadDataFromKey, cmd.Flags().Lookup(loadDataFromFlag)) //nolint:errcheck + + viper.SetDefault(loadDataModeKey, defaultLoadDataMode) + viper.BindEnv(loadDataModeKey, "SFTPGO_LOADDATA_MODE") //nolint:errcheck + cmd.Flags().IntVar(&loadDataMode, loadDataModeFlag, viper.GetInt(loadDataModeKey), + `Restore mode for data to load: + 0 - new users are added, existing users are + updated + 1 - New users are added, existing users are + not modified +This flag can be set using SFTPGO_LOADDATA_MODE +env var too. +`) + viper.BindPFlag(loadDataModeKey, cmd.Flags().Lookup(loadDataModeFlag)) //nolint:errcheck + + viper.SetDefault(loadDataCleanKey, defaultLoadDataClean) + viper.BindEnv(loadDataCleanKey, "SFTPGO_LOADDATA_CLEAN") //nolint:errcheck + cmd.Flags().BoolVar(&loadDataClean, loadDataCleanFlag, viper.GetBool(loadDataCleanKey), + `Determine if the loaddata-from file should +be removed after a successful load. This flag +can be set using SFTPGO_LOADDATA_CLEAN env var +too. (default "false") +`) + viper.BindPFlag(loadDataCleanKey, cmd.Flags().Lookup(loadDataCleanFlag)) //nolint:errcheck +} + func addServeFlags(cmd *cobra.Command) { addConfigFlags(cmd) @@ -192,30 +229,7 @@ using SFTPGO_LOG_UTC_TIME env var too. `) viper.BindPFlag(logUTCTimeKey, cmd.Flags().Lookup(logUTCTimeFlag)) //nolint:errcheck - viper.SetDefault(loadDataFromKey, defaultLoadDataFrom) - viper.BindEnv(loadDataFromKey, "SFTPGO_LOADDATA_FROM") //nolint:errcheck - cmd.Flags().StringVar(&loadDataFrom, loadDataFromFlag, viper.GetString(loadDataFromKey), - `Load users and folders from this file. -The file must be specified as absolute path -and it must contain a backup obtained using -the "dumpdata" REST API or compatible content. -This flag can be set using SFTPGO_LOADDATA_FROM -env var too. -`) - viper.BindPFlag(loadDataFromKey, cmd.Flags().Lookup(loadDataFromFlag)) //nolint:errcheck - - viper.SetDefault(loadDataModeKey, defaultLoadDataMode) - viper.BindEnv(loadDataModeKey, "SFTPGO_LOADDATA_MODE") //nolint:errcheck - cmd.Flags().IntVar(&loadDataMode, loadDataModeFlag, viper.GetInt(loadDataModeKey), - `Restore mode for data to load: - 0 - new users are added, existing users are - updated - 1 - New users are added, existing users are - not modified -This flag can be set using SFTPGO_LOADDATA_MODE -env var too. -`) - viper.BindPFlag(loadDataModeKey, cmd.Flags().Lookup(loadDataModeFlag)) //nolint:errcheck + addBaseLoadDataFlags(cmd) viper.SetDefault(loadDataQuotaScanKey, defaultLoadDataQuotaScan) viper.BindEnv(loadDataQuotaScanKey, "SFTPGO_LOADDATA_QUOTA_SCAN") //nolint:errcheck @@ -228,14 +242,4 @@ This flag can be set using SFTPGO_LOADDATA_QUOTA_SCAN env var too. (default 0)`) viper.BindPFlag(loadDataQuotaScanKey, cmd.Flags().Lookup(loadDataQuotaScanFlag)) //nolint:errcheck - - viper.SetDefault(loadDataCleanKey, defaultLoadDataClean) - viper.BindEnv(loadDataCleanKey, "SFTPGO_LOADDATA_CLEAN") //nolint:errcheck - cmd.Flags().BoolVar(&loadDataClean, loadDataCleanFlag, viper.GetBool(loadDataCleanKey), - `Determine if the loaddata-from file should -be removed after a successful load. This flag -can be set using SFTPGO_LOADDATA_CLEAN env var -too. (default "false") -`) - viper.BindPFlag(logCompressKey, cmd.Flags().Lookup(logCompressFlag)) //nolint:errcheck } diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go index dddc3d61..656cf491 100644 --- a/dataprovider/dataprovider.go +++ b/dataprovider/dataprovider.go @@ -804,6 +804,11 @@ func InitializeDatabase(cnf Config, basePath string) error { } else { credentialsDirPath = filepath.Join(basePath, config.CredentialsPath) } + vfs.SetCredentialsDirPath(credentialsDirPath) + + if err := initializeHashingAlgo(&cnf); err != nil { + return err + } err := createProvider(basePath) if err != nil { diff --git a/docker/README.md b/docker/README.md index c705b80c..f40f1be0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -133,7 +133,7 @@ Alternately you can mount your custom configuration file to `/var/lib/sftpgo` or Initial data can be loaded in the following ways: -- via the `--loaddata-from` flag or the `SFTPGO_LOADDATA_FROM` environment variable +- via the `--loaddata-from` flag or the `SFTPGO_LOADDATA_FROM` environment variable. This flag is supported for both the `serve` command (load initial data and start the service) and the `initprovider` command (initialize the provider, load initial data and exit) - by providing a dump file to the memory provider Please take a look [here](../docs/full-configuration.md) for more details. diff --git a/go.mod b/go.mod index 5b66c59f..b4536e96 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/prometheus/client_golang v1.12.1 github.com/rs/cors v1.8.2 github.com/rs/xid v1.3.0 - github.com/rs/zerolog v1.26.2-0.20220203140311-fc26014bd4e1 + github.com/rs/zerolog v1.26.2-0.20220227173336-263b0bde3672 github.com/sftpgo/sdk v0.1.1-0.20220225141305-cca7ba31466c github.com/shirou/gopsutil/v3 v3.22.1 github.com/spf13/afero v1.8.1 @@ -58,7 +58,7 @@ require ( golang.org/x/crypto v0.0.0-20220214200702-86341886e292 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b - golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7 + golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 google.golang.org/api v0.70.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 @@ -120,7 +120,7 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect - github.com/tklauser/go-sysconf v0.3.9 // indirect + github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect diff --git a/go.sum b/go.sum index d6981225..741b3743 100644 --- a/go.sum +++ b/go.sum @@ -689,8 +689,8 @@ github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/rs/zerolog v1.26.2-0.20220203140311-fc26014bd4e1 h1:n1Q4XjP7MrFJU2fC5CJqvtWU1HfNNkMiODLS5Of8wEA= -github.com/rs/zerolog v1.26.2-0.20220203140311-fc26014bd4e1/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U= +github.com/rs/zerolog v1.26.2-0.20220227173336-263b0bde3672 h1:8tqGbO3HWm9kqGZxc8YLAND7QGJNppiwq+kMTpn8oOk= +github.com/rs/zerolog v1.26.2-0.20220227173336-263b0bde3672/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -741,8 +741,9 @@ github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 h1:b2nJXyPCa9H github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= +github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= +github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= @@ -961,8 +962,8 @@ golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7 h1:BXxu8t6QN0G1uff4bzZzSkpsax8+ALqTGUtz08QrV00= -golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/httpd/api_maintenance.go b/httpd/api_maintenance.go index c157087a..cf596dc6 100644 --- a/httpd/api_maintenance.go +++ b/httpd/api_maintenance.go @@ -360,7 +360,7 @@ func RestoreUsers(users []dataprovider.User, inputFile string, mode, scanQuota i logger.Debug(logSender, "", "adding new user: %+v, dump file: %#v, error: %v", user, inputFile, err) } if err != nil { - return fmt.Errorf("unable to restoreuser %#v: %w", user.Username, err) + return fmt.Errorf("unable to restore user %#v: %w", user.Username, err) } if scanQuota == 1 || (scanQuota == 2 && user.HasQuotaRestrictions()) { if common.QuotaScans.AddUserQuotaScan(user.Username) { diff --git a/logger/logger.go b/logger/logger.go index 3f3641f5..51800323 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -46,6 +46,10 @@ type StdLoggerWrapper struct { Sender string } +func init() { + zerolog.TimeFieldFormat = dateFormat +} + // Write implements the io.Writer interface. This is useful to set as a writer // for the standard library log. func (l *StdLoggerWrapper) Write(p []byte) (n int, err error) { @@ -138,7 +142,6 @@ func GetLogger() *zerolog.Logger { // SetLogTime sets logging time related setting func SetLogTime(utc bool) { - zerolog.TimeFieldFormat = dateFormat if utc { zerolog.TimestampFunc = func() time.Time { return time.Now().UTC() diff --git a/service/service.go b/service/service.go index 7c0a5f7f..95a01503 100644 --- a/service/service.go +++ b/service/service.go @@ -135,7 +135,7 @@ func (s *Service) Start() error { } } - err = s.loadInitialData() + err = s.LoadInitialData() if err != nil { logger.Error(logSender, "", "unable to load initial data: %v", err) logger.ErrorToConsole("unable to load initial data: %v", err) @@ -248,7 +248,8 @@ func (s *Service) Stop() { logger.Debug(logSender, "", "Service stopped") } -func (s *Service) loadInitialData() error { +// LoadInitialData if a data file is set +func (s *Service) LoadInitialData() error { if s.LoadDataFrom == "" { return nil } @@ -263,12 +264,7 @@ func (s *Service) loadInitialData() error { } info, err := os.Stat(s.LoadDataFrom) if err != nil { - if errors.Is(err, os.ErrNotExist) { - logger.Warn(logSender, "", "unable to load initial data, the file %#v does not exist", s.LoadDataFrom) - logger.WarnToConsole("unable to load initial data, the file %#v does not exist", s.LoadDataFrom) - return nil - } - return err + return fmt.Errorf("unable to stat file %#v: %w", s.LoadDataFrom, err) } if info.Size() > httpd.MaxRestoreSize { return fmt.Errorf("unable to restore input file %#v size too big: %v/%v bytes", @@ -276,20 +272,18 @@ func (s *Service) loadInitialData() error { } content, err := os.ReadFile(s.LoadDataFrom) if err != nil { - return fmt.Errorf("unable to read input file %#v: %v", s.LoadDataFrom, err) + return fmt.Errorf("unable to read input file %#v: %w", s.LoadDataFrom, err) } dump, err := dataprovider.ParseDumpData(content) if err != nil { - return fmt.Errorf("unable to parse file to restore %#v: %v", s.LoadDataFrom, err) + return fmt.Errorf("unable to parse file to restore %#v: %w", s.LoadDataFrom, err) } err = s.restoreDump(&dump) if err != nil { return err } - logger.Info(logSender, "", "data loaded from file %#v mode: %v, quota scan %v", s.LoadDataFrom, - s.LoadDataMode, s.LoadDataQuotaScan) - logger.InfoToConsole("data loaded from file %#v mode: %v, quota scan %v", s.LoadDataFrom, - s.LoadDataMode, s.LoadDataQuotaScan) + logger.Info(logSender, "", "data loaded from file %#v mode: %v", s.LoadDataFrom, s.LoadDataMode) + logger.InfoToConsole("data loaded from file %#v mode: %v", s.LoadDataFrom, s.LoadDataMode) if s.LoadDataClean { err = os.Remove(s.LoadDataFrom) if err == nil { diff --git a/util/util.go b/util/util.go index ede04f40..274e72c5 100644 --- a/util/util.go +++ b/util/util.go @@ -84,7 +84,7 @@ func RemoveDuplicates(obj []string) []string { // GetTimeAsMsSinceEpoch returns unix timestamp as milliseconds from a time struct func GetTimeAsMsSinceEpoch(t time.Time) int64 { - return t.UnixNano() / 1000000 + return t.UnixMilli() } // GetTimeFromMsecSinceEpoch return a time struct from a unix timestamp with millisecond precision