beenull/sftpgo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

initprovider: add load data options

Browse source 
Fixes #741

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2022-02-28 17:05:18 +01:00
parent 816c174036
commit 1ea7429921
No known key found for this signature in database
GPG key ID: 2F1FB59433D5A8CB
10 changed files with 84 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
cmd/initprovider.go
View file

@ -10,6 +10,7 @@ import (
"github.com/drakkan/sftpgo/v2/config"
"github.com/drakkan/sftpgo/v2/dataprovider"
"github.com/drakkan/sftpgo/v2/logger"
"github.com/drakkan/sftpgo/v2/service"
"github.com/drakkan/sftpgo/v2/util"
)
@ -40,13 +41,13 @@ Please take a look at the usage below to customize the options.`,
configDir = util.CleanDirInput(configDir)
err := config.LoadConfig(configDir, configFile)
if err != nil {
logger.WarnToConsole("Unable to initialize data provider, config load error: %v", err)
logger.ErrorToConsole("Unable to initialize data provider, config load error: %v", err)
return
}
kmsConfig := config.GetKMSConfig()
err = kmsConfig.Initialize()
if err != nil {
logger.ErrorToConsole("unable to initialize KMS: %v", err)
logger.ErrorToConsole("Unable to initialize KMS: %v", err)
os.Exit(1)
}
providerConf := config.GetProviderConf()
@ -57,9 +58,21 @@ Please take a look at the usage below to customize the options.`,
} else if err == dataprovider.ErrNoInitRequired {
logger.InfoToConsole("%v", err.Error())
} else {
logger.WarnToConsole("Unable to initialize/update the data provider: %v", err)
logger.ErrorToConsole("Unable to initialize/update the data provider: %v", err)
os.Exit(1)
}
if providerConf.Driver != dataprovider.MemoryDataProviderName && loadDataFrom != "" {
service := service.Service{
LoadDataFrom: loadDataFrom,
LoadDataMode: loadDataMode,
LoadDataQuotaScan: loadDataQuotaScan,
LoadDataClean: loadDataClean,
}
if err = service.LoadInitialData(); err != nil {
logger.ErrorToConsole("Cannot load initial data: %v", err)
os.Exit(1)
}
}
},
}
)
@ -67,4 +80,5 @@ Please take a look at the usage below to customize the options.`,
func init() {
rootCmd.AddCommand(initProviderCmd)
addConfigFlags(initProviderCmd)
addBaseLoadDataFlags(initProviderCmd)
}

72
cmd/root.go
View file

@ -126,6 +126,43 @@ env var too.`)
viper.BindPFlag(configFileKey, cmd.Flags().Lookup(configFileFlag)) //nolint:errcheck
}
func addBaseLoadDataFlags(cmd *cobra.Command) {
viper.SetDefault(loadDataFromKey, defaultLoadDataFrom)
viper.BindEnv(loadDataFromKey, "SFTPGO_LOADDATA_FROM") //nolint:errcheck
cmd.Flags().StringVar(&loadDataFrom, loadDataFromFlag, viper.GetString(loadDataFromKey),
`Load users and folders from this file.
The file must be specified as absolute path
and it must contain a backup obtained using
the "dumpdata" REST API or compatible content.
This flag can be set using SFTPGO_LOADDATA_FROM
env var too.
`)
viper.BindPFlag(loadDataFromKey, cmd.Flags().Lookup(loadDataFromFlag)) //nolint:errcheck
viper.SetDefault(loadDataModeKey, defaultLoadDataMode)
viper.BindEnv(loadDataModeKey, "SFTPGO_LOADDATA_MODE") //nolint:errcheck
cmd.Flags().IntVar(&loadDataMode, loadDataModeFlag, viper.GetInt(loadDataModeKey),
`Restore mode for data to load:
0 - new users are added, existing users are
updated
1 - New users are added, existing users are
not modified
This flag can be set using SFTPGO_LOADDATA_MODE
env var too.
`)
viper.BindPFlag(loadDataModeKey, cmd.Flags().Lookup(loadDataModeFlag)) //nolint:errcheck
viper.SetDefault(loadDataCleanKey, defaultLoadDataClean)
viper.BindEnv(loadDataCleanKey, "SFTPGO_LOADDATA_CLEAN") //nolint:errcheck
cmd.Flags().BoolVar(&loadDataClean, loadDataCleanFlag, viper.GetBool(loadDataCleanKey),
`Determine if the loaddata-from file should
be removed after a successful load. This flag
can be set using SFTPGO_LOADDATA_CLEAN env var
too. (default "false")
`)
viper.BindPFlag(loadDataCleanKey, cmd.Flags().Lookup(loadDataCleanFlag)) //nolint:errcheck
}
func addServeFlags(cmd *cobra.Command) {
addConfigFlags(cmd)
@ -192,30 +229,7 @@ using SFTPGO_LOG_UTC_TIME env var too.
`)
viper.BindPFlag(logUTCTimeKey, cmd.Flags().Lookup(logUTCTimeFlag)) //nolint:errcheck
viper.SetDefault(loadDataFromKey, defaultLoadDataFrom)
viper.BindEnv(loadDataFromKey, "SFTPGO_LOADDATA_FROM") //nolint:errcheck
cmd.Flags().StringVar(&loadDataFrom, loadDataFromFlag, viper.GetString(loadDataFromKey),
`Load users and folders from this file.
The file must be specified as absolute path
and it must contain a backup obtained using
the "dumpdata" REST API or compatible content.
This flag can be set using SFTPGO_LOADDATA_FROM
env var too.
`)
viper.BindPFlag(loadDataFromKey, cmd.Flags().Lookup(loadDataFromFlag)) //nolint:errcheck
viper.SetDefault(loadDataModeKey, defaultLoadDataMode)
viper.BindEnv(loadDataModeKey, "SFTPGO_LOADDATA_MODE") //nolint:errcheck
cmd.Flags().IntVar(&loadDataMode, loadDataModeFlag, viper.GetInt(loadDataModeKey),
`Restore mode for data to load:
0 - new users are added, existing users are
updated
1 - New users are added, existing users are
not modified
This flag can be set using SFTPGO_LOADDATA_MODE
env var too.
`)
viper.BindPFlag(loadDataModeKey, cmd.Flags().Lookup(loadDataModeFlag)) //nolint:errcheck
addBaseLoadDataFlags(cmd)
viper.SetDefault(loadDataQuotaScanKey, defaultLoadDataQuotaScan)
viper.BindEnv(loadDataQuotaScanKey, "SFTPGO_LOADDATA_QUOTA_SCAN") //nolint:errcheck
@ -228,14 +242,4 @@ This flag can be set using SFTPGO_LOADDATA_QUOTA_SCAN
env var too.
(default 0)`)
viper.BindPFlag(loadDataQuotaScanKey, cmd.Flags().Lookup(loadDataQuotaScanFlag)) //nolint:errcheck
viper.SetDefault(loadDataCleanKey, defaultLoadDataClean)
viper.BindEnv(loadDataCleanKey, "SFTPGO_LOADDATA_CLEAN") //nolint:errcheck
cmd.Flags().BoolVar(&loadDataClean, loadDataCleanFlag, viper.GetBool(loadDataCleanKey),
`Determine if the loaddata-from file should
be removed after a successful load. This flag
can be set using SFTPGO_LOADDATA_CLEAN env var
too. (default "false")
`)
viper.BindPFlag(logCompressKey, cmd.Flags().Lookup(logCompressFlag)) //nolint:errcheck
}

5
dataprovider/dataprovider.go
View file

@ -804,6 +804,11 @@ func InitializeDatabase(cnf Config, basePath string) error {
} else {
credentialsDirPath = filepath.Join(basePath, config.CredentialsPath)
}
vfs.SetCredentialsDirPath(credentialsDirPath)
if err := initializeHashingAlgo(&cnf); err != nil {
return err
}
err := createProvider(basePath)
if err != nil {

2
docker/README.md
View file

@ -133,7 +133,7 @@ Alternately you can mount your custom configuration file to `/var/lib/sftpgo` or
Initial data can be loaded in the following ways:
- via the `--loaddata-from` flag or the `SFTPGO_LOADDATA_FROM` environment variable
- via the `--loaddata-from` flag or the `SFTPGO_LOADDATA_FROM` environment variable. This flag is supported for both the `serve` command (load initial data and start the service) and the `initprovider` command (initialize the provider, load initial data and exit)
- by providing a dump file to the memory provider
Please take a look [here](../docs/full-configuration.md) for more details.

6
go.mod
View file

@ -40,7 +40,7 @@ require (
github.com/prometheus/client_golang v1.12.1
github.com/rs/cors v1.8.2
github.com/rs/xid v1.3.0
github.com/rs/zerolog v1.26.2-0.20220203140311-fc26014bd4e1
github.com/rs/zerolog v1.26.2-0.20220227173336-263b0bde3672
github.com/sftpgo/sdk v0.1.1-0.20220225141305-cca7ba31466c
github.com/shirou/gopsutil/v3 v3.22.1
github.com/spf13/afero v1.8.1
@ -58,7 +58,7 @@ require (
golang.org/x/crypto v0.0.0-20220214200702-86341886e292
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b
golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9
golang.org/x/time v0.0.0-20220224211638-0e9765cccd65
google.golang.org/api v0.70.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0
@ -120,7 +120,7 @@ require (
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/tklauser/go-sysconf v0.3.9 // indirect
github.com/tklauser/go-sysconf v0.3.10 // indirect
github.com/tklauser/numcpus v0.4.0 // indirect
github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect

11
go.sum
View file

@ -689,8 +689,8 @@ github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
github.com/rs/zerolog v1.26.2-0.20220203140311-fc26014bd4e1 h1:n1Q4XjP7MrFJU2fC5CJqvtWU1HfNNkMiODLS5Of8wEA=
github.com/rs/zerolog v1.26.2-0.20220203140311-fc26014bd4e1/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U=
github.com/rs/zerolog v1.26.2-0.20220227173336-263b0bde3672 h1:8tqGbO3HWm9kqGZxc8YLAND7QGJNppiwq+kMTpn8oOk=
github.com/rs/zerolog v1.26.2-0.20220227173336-263b0bde3672/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@ -741,8 +741,9 @@ github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 h1:b2nJXyPCa9H
github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo=
github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o=
github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
@ -961,8 +962,8 @@ golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7 h1:BXxu8t6QN0G1uff4bzZzSkpsax8+ALqTGUtz08QrV00=
golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

2
httpd/api_maintenance.go
View file

@ -360,7 +360,7 @@ func RestoreUsers(users []dataprovider.User, inputFile string, mode, scanQuota i
logger.Debug(logSender, "", "adding new user: %+v, dump file: %#v, error: %v", user, inputFile, err)
}
if err != nil {
return fmt.Errorf("unable to restoreuser %#v: %w", user.Username, err)
return fmt.Errorf("unable to restore user %#v: %w", user.Username, err)
}
if scanQuota == 1 || (scanQuota == 2 && user.HasQuotaRestrictions()) {
if common.QuotaScans.AddUserQuotaScan(user.Username) {

5
logger/logger.go
View file

@ -46,6 +46,10 @@ type StdLoggerWrapper struct {
Sender string
}
func init() {
zerolog.TimeFieldFormat = dateFormat
}
// Write implements the io.Writer interface. This is useful to set as a writer
// for the standard library log.
func (l *StdLoggerWrapper) Write(p []byte) (n int, err error) {
@ -138,7 +142,6 @@ func GetLogger() *zerolog.Logger {
// SetLogTime sets logging time related setting
func SetLogTime(utc bool) {
zerolog.TimeFieldFormat = dateFormat
if utc {
zerolog.TimestampFunc = func() time.Time {
return time.Now().UTC()

22
service/service.go
View file

@ -135,7 +135,7 @@ func (s *Service) Start() error {
}
}
err = s.loadInitialData()
err = s.LoadInitialData()
if err != nil {
logger.Error(logSender, "", "unable to load initial data: %v", err)
logger.ErrorToConsole("unable to load initial data: %v", err)
@ -248,7 +248,8 @@ func (s *Service) Stop() {
logger.Debug(logSender, "", "Service stopped")
}
func (s *Service) loadInitialData() error {
// LoadInitialData if a data file is set
func (s *Service) LoadInitialData() error {
if s.LoadDataFrom == "" {
return nil
}
@ -263,12 +264,7 @@ func (s *Service) loadInitialData() error {
}
info, err := os.Stat(s.LoadDataFrom)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
logger.Warn(logSender, "", "unable to load initial data, the file %#v does not exist", s.LoadDataFrom)
logger.WarnToConsole("unable to load initial data, the file %#v does not exist", s.LoadDataFrom)
return nil
}
return err
return fmt.Errorf("unable to stat file %#v: %w", s.LoadDataFrom, err)
}
if info.Size() > httpd.MaxRestoreSize {
return fmt.Errorf("unable to restore input file %#v size too big: %v/%v bytes",
@ -276,20 +272,18 @@ func (s *Service) loadInitialData() error {
}
content, err := os.ReadFile(s.LoadDataFrom)
if err != nil {
return fmt.Errorf("unable to read input file %#v: %v", s.LoadDataFrom, err)
return fmt.Errorf("unable to read input file %#v: %w", s.LoadDataFrom, err)
}
dump, err := dataprovider.ParseDumpData(content)
if err != nil {
return fmt.Errorf("unable to parse file to restore %#v: %v", s.LoadDataFrom, err)
return fmt.Errorf("unable to parse file to restore %#v: %w", s.LoadDataFrom, err)
}
err = s.restoreDump(&dump)
if err != nil {
return err
}
logger.Info(logSender, "", "data loaded from file %#v mode: %v, quota scan %v", s.LoadDataFrom,
s.LoadDataMode, s.LoadDataQuotaScan)
logger.InfoToConsole("data loaded from file %#v mode: %v, quota scan %v", s.LoadDataFrom,
s.LoadDataMode, s.LoadDataQuotaScan)
logger.Info(logSender, "", "data loaded from file %#v mode: %v", s.LoadDataFrom, s.LoadDataMode)
logger.InfoToConsole("data loaded from file %#v mode: %v", s.LoadDataFrom, s.LoadDataMode)
if s.LoadDataClean {
err = os.Remove(s.LoadDataFrom)
if err == nil {

2
util/util.go
View file

@ -84,7 +84,7 @@ func RemoveDuplicates(obj []string) []string {
// GetTimeAsMsSinceEpoch returns unix timestamp as milliseconds from a time struct
func GetTimeAsMsSinceEpoch(t time.Time) int64 {
return t.UnixNano() / 1000000
return t.UnixMilli()
}
// GetTimeFromMsecSinceEpoch return a time struct from a unix timestamp with millisecond precision