From 037d89a3200f345fe4b7e4013c39732f51110c8c Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sat, 2 Jan 2021 14:05:09 +0100 Subject: [PATCH] add support for a basic built-in defender It can help to prevent DoS and brute force password guessing --- README.md | 3 + cmd/startsubsys.go | 5 +- common/common.go | 53 +++- common/common_test.go | 66 ++++- common/defender.go | 439 +++++++++++++++++++++++++++++ common/defender_test.go | 520 +++++++++++++++++++++++++++++++++++ config/config.go | 24 ++ dataprovider/dataprovider.go | 4 +- docs/defender.md | 53 ++++ docs/full-configuration.md | 12 + ftpd/ftpd_test.go | 68 ++++- ftpd/server.go | 25 +- go.mod | 10 +- go.sum | 58 +++- httpd/httpd_test.go | 6 +- pkgs/build.sh | 2 +- service/service.go | 9 +- sftpd/server.go | 70 +++-- sftpd/sftpd_test.go | 106 +++++-- sftpgo.json | 15 +- webdavd/internal_test.go | 33 ++- webdavd/server.go | 31 ++- webdavd/webdavd_test.go | 49 +++- 23 files changed, 1530 insertions(+), 131 deletions(-) create mode 100644 common/defender.go create mode 100644 common/defender_test.go create mode 100644 docs/defender.md diff --git a/README.md b/README.md index c19afa03..6e667e4e 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Several storage backends are supported: local filesystem, encrypted local filesy - Virtual folders are supported: directories outside the user home directory can be exposed as virtual folders. - Configurable custom commands and/or HTTP notifications on file upload, download, pre-delete, delete, rename, on SSH commands and on user add, update and delete. - Automatically terminating idle connections. +- Automatic blocklist management is supported using the built-in [defender](./docs/defender.md). - Atomic uploads are configurable. - Support for Git repositories over SSH. - SCP and rsync are supported. @@ -223,6 +224,8 @@ Anyway, some backends require a pay per use account (or they offer free account The [connection failed logs](./docs/logs.md) can be used for integration in tools such as [Fail2ban](http://www.fail2ban.org/). Example of [jails](./fail2ban/jails) and [filters](./fail2ban/filters) working with `systemd`/`journald` are available in fail2ban directory. +You can also use the built-in [defender](./docs/defender.md). + ## Account's configuration properties Details information about account configuration properties can be found [here](./docs/account.md). diff --git a/cmd/startsubsys.go b/cmd/startsubsys.go index cf6aae9c..7f27a8a6 100644 --- a/cmd/startsubsys.go +++ b/cmd/startsubsys.go @@ -66,7 +66,10 @@ Command-line flags should be specified in the Subsystem declaration. // idle connection are managed externally commonConfig.IdleTimeout = 0 config.SetCommonConfig(commonConfig) - common.Initialize(config.GetCommonConfig()) + if err := common.Initialize(config.GetCommonConfig()); err != nil { + logger.Error(logSender, connectionID, "%v", err) + os.Exit(1) + } kmsConfig := config.GetKMSConfig() if err := kmsConfig.Initialize(); err != nil { logger.Error(logSender, connectionID, "unable to initialize KMS: %v", err) diff --git a/common/common.go b/common/common.go index e65a907a..78e9b66a 100644 --- a/common/common.go +++ b/common/common.go @@ -106,13 +106,41 @@ var ( ) // Initialize sets the common configuration -func Initialize(c Configuration) { +func Initialize(c Configuration) error { Config = c Config.idleLoginTimeout = 2 * time.Minute Config.idleTimeoutAsDuration = time.Duration(Config.IdleTimeout) * time.Minute if Config.IdleTimeout > 0 { startIdleTimeoutTicker(idleTimeoutCheckInterval) } + Config.defender = nil + if c.DefenderConfig.Enabled { + defender, err := newInMemoryDefender(&c.DefenderConfig) + if err != nil { + return fmt.Errorf("defender initialization error: %v", err) + } + logger.Info(logSender, "", "defender initialized with config %+v", c.DefenderConfig) + Config.defender = defender + } + return nil +} + +// IsBanned returns true if the specified IP address is banned +func IsBanned(ip string) bool { + if Config.defender == nil { + return false + } + + return Config.defender.IsBanned(ip) +} + +// AddDefenderEvent adds the specified defender event for the given IP +func AddDefenderEvent(ip string, event HostEvent) { + if Config.defender == nil { + return + } + + Config.defender.AddEvent(ip, event) } func startIdleTimeoutTicker(duration time.Duration) { @@ -250,9 +278,12 @@ type Configuration struct { // ip address. Leave empty do disable. PostConnectHook string `json:"post_connect_hook" mapstructure:"post_connect_hook"` // Maximum number of concurrent client connections. 0 means unlimited - MaxTotalConnections int `json:"max_total_connections" mapstructure:"max_total_connections"` + MaxTotalConnections int `json:"max_total_connections" mapstructure:"max_total_connections"` + // Defender configuration + DefenderConfig DefenderConfig `json:"defender" mapstructure:"defender"` idleTimeoutAsDuration time.Duration idleLoginTimeout time.Duration + defender Defender } // IsAtomicUploadEnabled returns true if atomic upload is enabled @@ -294,51 +325,50 @@ func (c *Configuration) GetProxyListener(listener net.Listener) (*proxyproto.Lis } // ExecutePostConnectHook executes the post connect hook if defined -func (c *Configuration) ExecutePostConnectHook(remoteAddr, protocol string) error { +func (c *Configuration) ExecutePostConnectHook(ipAddr, protocol string) error { if len(c.PostConnectHook) == 0 { return nil } - ip := utils.GetIPFromRemoteAddress(remoteAddr) if strings.HasPrefix(c.PostConnectHook, "http") { var url *url.URL url, err := url.Parse(c.PostConnectHook) if err != nil { logger.Warn(protocol, "", "Login from ip %#v denied, invalid post connect hook %#v: %v", - ip, c.PostConnectHook, err) + ipAddr, c.PostConnectHook, err) return err } httpClient := httpclient.GetHTTPClient() q := url.Query() - q.Add("ip", ip) + q.Add("ip", ipAddr) q.Add("protocol", protocol) url.RawQuery = q.Encode() resp, err := httpClient.Get(url.String()) if err != nil { - logger.Warn(protocol, "", "Login from ip %#v denied, error executing post connect hook: %v", ip, err) + logger.Warn(protocol, "", "Login from ip %#v denied, error executing post connect hook: %v", ipAddr, err) return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logger.Warn(protocol, "", "Login from ip %#v denied, post connect hook response code: %v", ip, resp.StatusCode) + logger.Warn(protocol, "", "Login from ip %#v denied, post connect hook response code: %v", ipAddr, resp.StatusCode) return errUnexpectedHTTResponse } return nil } if !filepath.IsAbs(c.PostConnectHook) { err := fmt.Errorf("invalid post connect hook %#v", c.PostConnectHook) - logger.Warn(protocol, "", "Login from ip %#v denied: %v", ip, err) + logger.Warn(protocol, "", "Login from ip %#v denied: %v", ipAddr, err) return err } ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() cmd := exec.CommandContext(ctx, c.PostConnectHook) cmd.Env = append(os.Environ(), - fmt.Sprintf("SFTPGO_CONNECTION_IP=%v", ip), + fmt.Sprintf("SFTPGO_CONNECTION_IP=%v", ipAddr), fmt.Sprintf("SFTPGO_CONNECTION_PROTOCOL=%v", protocol)) err := cmd.Run() if err != nil { - logger.Warn(protocol, "", "Login from ip %#v denied, connect hook error: %v", ip, err) + logger.Warn(protocol, "", "Login from ip %#v denied, connect hook error: %v", ipAddr, err) } return err } @@ -537,6 +567,7 @@ func (conns *ActiveConnections) checkIdles() { ip := utils.GetIPFromRemoteAddress(c.GetRemoteAddress()) logger.ConnectionFailedLog("", ip, dataprovider.LoginMethodNoAuthTryed, c.GetProtocol(), "client idle") metrics.AddNoAuthTryed() + AddDefenderEvent(ip, HostEventNoLoginTried) dataprovider.ExecutePostLoginHook("", dataprovider.LoginMethodNoAuthTryed, ip, c.GetProtocol(), dataprovider.ErrNoAuthTryed) } diff --git a/common/common_test.go b/common/common_test.go index 44f3708e..a335f11c 100644 --- a/common/common_test.go +++ b/common/common_test.go @@ -99,7 +99,11 @@ func TestMain(m *testing.M) { os.Exit(1) } logger.InfoToConsole("Starting COMMON tests, provider: %v", driver) - Initialize(Configuration{}) + err = Initialize(Configuration{}) + if err != nil { + logger.WarnToConsole("error initializing common: %v", err) + os.Exit(1) + } httpConfig := httpclient.Config{ Timeout: 5, } @@ -225,6 +229,40 @@ func TestSSHConnections(t *testing.T) { assert.NoError(t, sshConn3.Close()) } +func TestDefenderIntegration(t *testing.T) { + // by default defender is nil + configCopy := Config + + ip := "127.1.1.1" + + AddDefenderEvent(ip, HostEventNoLoginTried) + assert.False(t, IsBanned(ip)) + + Config.DefenderConfig = DefenderConfig{ + Enabled: true, + BanTime: 10, + BanTimeIncrement: 50, + Threshold: 0, + ScoreInvalid: 2, + ScoreValid: 1, + ObservationTime: 15, + EntriesSoftLimit: 100, + EntriesHardLimit: 150, + } + err := Initialize(Config) + assert.Error(t, err) + Config.DefenderConfig.Threshold = 3 + err = Initialize(Config) + assert.NoError(t, err) + + AddDefenderEvent(ip, HostEventNoLoginTried) + assert.False(t, IsBanned(ip)) + AddDefenderEvent(ip, HostEventLoginFailed) + assert.True(t, IsBanned(ip)) + + Config = configCopy +} + func TestMaxConnections(t *testing.T) { oldValue := Config.MaxTotalConnections Config.MaxTotalConnections = 1 @@ -249,7 +287,8 @@ func TestIdleConnections(t *testing.T) { configCopy := Config Config.IdleTimeout = 1 - Initialize(Config) + err := Initialize(Config) + assert.NoError(t, err) conn1, conn2 := net.Pipe() customConn1 := &customNetConn{ @@ -520,39 +559,36 @@ func TestProxyProtocol(t *testing.T) { func TestPostConnectHook(t *testing.T) { Config.PostConnectHook = "" - remoteAddr := &net.IPAddr{ - IP: net.ParseIP("127.0.0.1"), - Zone: "", - } + ipAddr := "127.0.0.1" - assert.NoError(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolFTP)) + assert.NoError(t, Config.ExecutePostConnectHook(ipAddr, ProtocolFTP)) Config.PostConnectHook = "http://foo\x7f.com/" - assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolSFTP)) + assert.Error(t, Config.ExecutePostConnectHook(ipAddr, ProtocolSFTP)) Config.PostConnectHook = "http://invalid:1234/" - assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolSFTP)) + assert.Error(t, Config.ExecutePostConnectHook(ipAddr, ProtocolSFTP)) Config.PostConnectHook = fmt.Sprintf("http://%v/404", httpAddr) - assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolFTP)) + assert.Error(t, Config.ExecutePostConnectHook(ipAddr, ProtocolFTP)) Config.PostConnectHook = fmt.Sprintf("http://%v", httpAddr) - assert.NoError(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolFTP)) + assert.NoError(t, Config.ExecutePostConnectHook(ipAddr, ProtocolFTP)) Config.PostConnectHook = "invalid" - assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolFTP)) + assert.Error(t, Config.ExecutePostConnectHook(ipAddr, ProtocolFTP)) if runtime.GOOS == osWindows { Config.PostConnectHook = "C:\\bad\\command" - assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolSFTP)) + assert.Error(t, Config.ExecutePostConnectHook(ipAddr, ProtocolSFTP)) } else { Config.PostConnectHook = "/invalid/path" - assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolSFTP)) + assert.Error(t, Config.ExecutePostConnectHook(ipAddr, ProtocolSFTP)) hookCmd, err := exec.LookPath("true") assert.NoError(t, err) Config.PostConnectHook = hookCmd - assert.NoError(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolSFTP)) + assert.NoError(t, Config.ExecutePostConnectHook(ipAddr, ProtocolSFTP)) } Config.PostConnectHook = "" diff --git a/common/defender.go b/common/defender.go new file mode 100644 index 00000000..e122c88b --- /dev/null +++ b/common/defender.go @@ -0,0 +1,439 @@ +package common + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net" + "os" + "sort" + "sync" + "time" + + "github.com/yl2chen/cidranger" + + "github.com/drakkan/sftpgo/logger" + "github.com/drakkan/sftpgo/utils" +) + +// HostEvent is the enumerable for the support host event +type HostEvent int + +// Supported host events +const ( + HostEventLoginFailed HostEvent = iota + HostEventUserNotFound + HostEventNoLoginTried +) + +// Defender defines the interface that a defender must implements +type Defender interface { + AddEvent(ip string, event HostEvent) + IsBanned(ip string) bool + GetBanTime(ip string) *time.Time + GetScore(ip string) int +} + +// DefenderConfig defines the "defender" configuration +type DefenderConfig struct { + // Set to true to enable the defender + Enabled bool `json:"enabled" mapstructure:"enabled"` + // BanTime is the number of minutes that a host is banned + BanTime int `json:"ban_time" mapstructure:"ban_time"` + // Percentage increase of the ban time if a banned host tries to connect again + BanTimeIncrement int `json:"ban_time_increment" mapstructure:"ban_time_increment"` + // Threshold value for banning a client + Threshold int `json:"threshold" mapstructure:"threshold"` + // Score for invalid login attempts, eg. non-existent user accounts or + // client disconnected for inactivity without authentication attempts + ScoreInvalid int `json:"score_invalid" mapstructure:"score_invalid"` + // Score for valid login attempts, eg. user accounts that exist + ScoreValid int `json:"score_valid" mapstructure:"score_valid"` + // Defines the time window, in minutes, for tracking client errors. + // A host is banned if it has exceeded the defined threshold during + // the last observation time minutes + ObservationTime int `json:"observation_time" mapstructure:"observation_time"` + // The number of banned IPs and host scores kept in memory will vary between the + // soft and hard limit + EntriesSoftLimit int `json:"entries_soft_limit" mapstructure:"entries_soft_limit"` + EntriesHardLimit int `json:"entries_hard_limit" mapstructure:"entries_hard_limit"` + // Path to a file with a list of ip addresses and/or networks to never ban + SafeListFile string `json:"safelist_file" mapstructure:"safelist_file"` + // Path to a file with a list of ip addresses and/or networks to always ban + BlockListFile string `json:"blocklist_file" mapstructure:"blocklist_file"` +} + +type memoryDefender struct { + config *DefenderConfig + sync.RWMutex + // IP addresses of the clients trying to connected are stored inside hosts, + // they are added to banned once the thresold is reached. + // A violation from a banned host will increase the ban time + // based on the configured BanTimeIncrement + hosts map[string]hostScore // the key is the host IP + banned map[string]time.Time // the key is the host IP + safeList *HostList + blockList *HostList +} + +// HostListFile defines the structure expected for safe/block list files +type HostListFile struct { + IPAddresses []string `json:"addresses"` + CIDRNetworks []string `json:"networks"` +} + +// HostList defines the structure used to keep the HostListFile in memory +type HostList struct { + IPAddresses map[string]bool + Ranges cidranger.Ranger +} + +func (h *HostList) isListed(ip string) bool { + if _, ok := h.IPAddresses[ip]; ok { + return true + } + + ok, err := h.Ranges.Contains(net.ParseIP(ip)) + if err != nil { + return false + } + + return ok +} + +type hostEvent struct { + dateTime time.Time + score int +} + +type hostScore struct { + TotalScore int + Events []hostEvent +} + +// validate returns an error if the configuration is invalid +func (c *DefenderConfig) validate() error { + if !c.Enabled { + return nil + } + if c.ScoreInvalid >= c.Threshold { + return fmt.Errorf("score_invalid %v cannot be greater than threshold %v", c.ScoreInvalid, c.Threshold) + } + if c.ScoreValid >= c.Threshold { + return fmt.Errorf("score_valid %v cannot be greater than threshold %v", c.ScoreValid, c.Threshold) + } + if c.BanTime <= 0 { + return fmt.Errorf("invalid ban_time %v", c.BanTime) + } + if c.BanTimeIncrement <= 0 { + return fmt.Errorf("invalid ban_time_increment %v", c.BanTimeIncrement) + } + if c.ObservationTime <= 0 { + return fmt.Errorf("invalid observation_time %v", c.ObservationTime) + } + if c.EntriesSoftLimit <= 0 { + return fmt.Errorf("invalid entries_soft_limit %v", c.EntriesSoftLimit) + } + if c.EntriesHardLimit <= c.EntriesSoftLimit { + return fmt.Errorf("invalid entries_hard_limit %v must be > %v", c.EntriesHardLimit, c.EntriesSoftLimit) + } + + return nil +} + +func newInMemoryDefender(config *DefenderConfig) (Defender, error) { + err := config.validate() + if err != nil { + return nil, err + } + defender := &memoryDefender{ + config: config, + hosts: make(map[string]hostScore), + banned: make(map[string]time.Time), + } + + defender.blockList, err = loadHostListFromFile(config.BlockListFile) + if err != nil { + return nil, err + } + defender.safeList, err = loadHostListFromFile(config.SafeListFile) + if err != nil { + return nil, err + } + + return defender, nil +} + +// IsBanned returns true if the specified IP is banned +// and increase ban time if the IP is found. +// This method must be called as soon as the client connects +func (d *memoryDefender) IsBanned(ip string) bool { + d.RLock() + + if banTime, ok := d.banned[ip]; ok { + if banTime.After(time.Now()) { + increment := d.config.BanTime * d.config.BanTimeIncrement / 100 + if increment == 0 { + increment++ + } + + d.RUnlock() + + // we can save an earlier ban time if there are contemporary updates + // but this should not make much difference. I prefer to hold a read lock + // until possible for performance reasons, this method is called each + // time a new client connects and it must be as fast as possible + d.Lock() + d.banned[ip] = banTime.Add(time.Duration(increment) * time.Minute) + d.Unlock() + + return true + } + } + + defer d.RUnlock() + + if d.blockList != nil && d.blockList.isListed(ip) { + // permanent ban + return true + } + + return false +} + +// AddEvent adds an event for the given IP. +// This method must be called for clients not yet banned +func (d *memoryDefender) AddEvent(ip string, event HostEvent) { + d.Lock() + defer d.Unlock() + + if d.safeList != nil && d.safeList.isListed(ip) { + return + } + + var score int + + switch event { + case HostEventLoginFailed: + score = d.config.ScoreValid + case HostEventUserNotFound, HostEventNoLoginTried: + score = d.config.ScoreInvalid + } + + ev := hostEvent{ + dateTime: time.Now(), + score: score, + } + + if hs, ok := d.hosts[ip]; ok { + hs.Events = append(hs.Events, ev) + hs.TotalScore = 0 + + idx := 0 + for _, event := range hs.Events { + if event.dateTime.Add(time.Duration(d.config.ObservationTime) * time.Minute).After(time.Now()) { + hs.Events[idx] = event + hs.TotalScore += event.score + idx++ + } + } + + hs.Events = hs.Events[:idx] + if hs.TotalScore >= d.config.Threshold { + d.banned[ip] = time.Now().Add(time.Duration(d.config.BanTime) * time.Minute) + delete(d.hosts, ip) + d.cleanupBanned() + } else { + d.hosts[ip] = hs + } + } else { + d.hosts[ip] = hostScore{ + TotalScore: ev.score, + Events: []hostEvent{ev}, + } + d.cleanupHosts() + } +} + +func (d *memoryDefender) countBanned() int { + d.RLock() + defer d.RUnlock() + + return len(d.banned) +} + +func (d *memoryDefender) countHosts() int { + d.RLock() + defer d.RUnlock() + + return len(d.hosts) +} + +// GetBanTime returns the ban time for the given IP or nil if the IP is not banned +func (d *memoryDefender) GetBanTime(ip string) *time.Time { + d.RLock() + defer d.RUnlock() + + if banTime, ok := d.banned[ip]; ok { + return &banTime + } + + return nil +} + +// GetScore returns the score for the given IP +func (d *memoryDefender) GetScore(ip string) int { + d.RLock() + defer d.RUnlock() + + score := 0 + + if hs, ok := d.hosts[ip]; ok { + for _, event := range hs.Events { + if event.dateTime.Add(time.Duration(d.config.ObservationTime) * time.Minute).After(time.Now()) { + score += event.score + } + } + } + + return score +} + +func (d *memoryDefender) cleanupBanned() { + if len(d.banned) > d.config.EntriesHardLimit { + kvList := make(kvList, 0, len(d.banned)) + + for k, v := range d.banned { + if v.Before(time.Now()) { + delete(d.banned, k) + } + + kvList = append(kvList, kv{ + Key: k, + Value: v.UnixNano(), + }) + } + + // we removed expired ip addresses, if any, above, this could be enough + numToRemove := len(d.banned) - d.config.EntriesSoftLimit + + if numToRemove <= 0 { + return + } + + sort.Sort(kvList) + + for idx, kv := range kvList { + if idx >= numToRemove { + break + } + + delete(d.banned, kv.Key) + } + } +} + +func (d *memoryDefender) cleanupHosts() { + if len(d.hosts) > d.config.EntriesHardLimit { + kvList := make(kvList, 0, len(d.hosts)) + + for k, v := range d.hosts { + value := int64(0) + if len(v.Events) > 0 { + value = v.Events[len(v.Events)-1].dateTime.UnixNano() + } + kvList = append(kvList, kv{ + Key: k, + Value: value, + }) + } + + sort.Sort(kvList) + + numToRemove := len(d.hosts) - d.config.EntriesSoftLimit + + for idx, kv := range kvList { + if idx >= numToRemove { + break + } + + delete(d.hosts, kv.Key) + } + } +} + +func loadHostListFromFile(name string) (*HostList, error) { + if name == "" { + return nil, nil + } + if !utils.IsFileInputValid(name) { + return nil, fmt.Errorf("invalid host list file name %#v", name) + } + + info, err := os.Stat(name) + if err != nil { + return nil, err + } + + // opinionated max size, you should avoid big host lists + if info.Size() > 1048576*5 { // 5MB + return nil, fmt.Errorf("host list file %#v is too big: %v bytes", name, info.Size()) + } + + content, err := ioutil.ReadFile(name) + if err != nil { + return nil, fmt.Errorf("unable to read input file %#v: %v", name, err) + } + + var hostList HostListFile + + err = json.Unmarshal(content, &hostList) + if err != nil { + return nil, err + } + + if len(hostList.CIDRNetworks) > 0 || len(hostList.IPAddresses) > 0 { + result := &HostList{ + IPAddresses: make(map[string]bool), + Ranges: cidranger.NewPCTrieRanger(), + } + ipCount := 0 + cdrCount := 0 + for _, ip := range hostList.IPAddresses { + if net.ParseIP(ip) == nil { + logger.Warn(logSender, "", "unable to parse IP %#v", ip) + continue + } + result.IPAddresses[ip] = true + ipCount++ + } + for _, cidrNet := range hostList.CIDRNetworks { + _, network, err := net.ParseCIDR(cidrNet) + if err != nil { + logger.Warn(logSender, "", "unable to parse CIDR network %#v", cidrNet) + continue + } + err = result.Ranges.Insert(cidranger.NewBasicRangerEntry(*network)) + if err == nil { + cdrCount++ + } + } + + logger.Info(logSender, "", "list %#v loaded, ip addresses loaded: %v/%v networks loaded: %v/%v", + name, ipCount, len(hostList.IPAddresses), cdrCount, len(hostList.CIDRNetworks)) + return result, nil + } + + return nil, nil +} + +type kv struct { + Key string + Value int64 +} + +type kvList []kv + +func (p kvList) Len() int { return len(p) } +func (p kvList) Less(i, j int) bool { return p[i].Value < p[j].Value } +func (p kvList) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/common/defender_test.go b/common/defender_test.go new file mode 100644 index 00000000..0cd89643 --- /dev/null +++ b/common/defender_test.go @@ -0,0 +1,520 @@ +package common + +import ( + "crypto/rand" + "encoding/json" + "fmt" + "io/ioutil" + "net" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/yl2chen/cidranger" +) + +func TestBasicDefender(t *testing.T) { + bl := HostListFile{ + IPAddresses: []string{"172.16.1.1", "172.16.1.2"}, + CIDRNetworks: []string{"10.8.0.0/24"}, + } + sl := HostListFile{ + IPAddresses: []string{"172.16.1.3", "172.16.1.4"}, + CIDRNetworks: []string{"192.168.8.0/24"}, + } + blFile := filepath.Join(os.TempDir(), "bl.json") + slFile := filepath.Join(os.TempDir(), "sl.json") + + data, err := json.Marshal(bl) + assert.NoError(t, err) + + err = ioutil.WriteFile(blFile, data, os.ModePerm) + assert.NoError(t, err) + + data, err = json.Marshal(sl) + assert.NoError(t, err) + + err = ioutil.WriteFile(slFile, data, os.ModePerm) + assert.NoError(t, err) + + config := &DefenderConfig{ + Enabled: true, + BanTime: 10, + BanTimeIncrement: 2, + Threshold: 5, + ScoreInvalid: 2, + ScoreValid: 1, + ObservationTime: 15, + EntriesSoftLimit: 1, + EntriesHardLimit: 2, + SafeListFile: "slFile", + BlockListFile: "blFile", + } + + _, err = newInMemoryDefender(config) + assert.Error(t, err) + config.BlockListFile = blFile + _, err = newInMemoryDefender(config) + assert.Error(t, err) + config.SafeListFile = slFile + d, err := newInMemoryDefender(config) + assert.NoError(t, err) + + defender := d.(*memoryDefender) + assert.True(t, defender.IsBanned("172.16.1.1")) + assert.False(t, defender.IsBanned("172.16.1.10")) + assert.False(t, defender.IsBanned("10.8.2.3")) + assert.True(t, defender.IsBanned("10.8.0.3")) + assert.False(t, defender.IsBanned("invalid ip")) + assert.Equal(t, 0, defender.countBanned()) + assert.Equal(t, 0, defender.countHosts()) + + defender.AddEvent("172.16.1.4", HostEventLoginFailed) + defender.AddEvent("192.168.8.4", HostEventUserNotFound) + assert.Equal(t, 0, defender.countHosts()) + + testIP := "12.34.56.78" + defender.AddEvent(testIP, HostEventLoginFailed) + assert.Equal(t, 1, defender.countHosts()) + assert.Equal(t, 0, defender.countBanned()) + assert.Equal(t, 1, defender.GetScore(testIP)) + assert.Nil(t, defender.GetBanTime(testIP)) + defender.AddEvent(testIP, HostEventNoLoginTried) + assert.Equal(t, 1, defender.countHosts()) + assert.Equal(t, 0, defender.countBanned()) + assert.Equal(t, 3, defender.GetScore(testIP)) + defender.AddEvent(testIP, HostEventNoLoginTried) + assert.Equal(t, 0, defender.countHosts()) + assert.Equal(t, 1, defender.countBanned()) + assert.Equal(t, 0, defender.GetScore(testIP)) + assert.NotNil(t, defender.GetBanTime(testIP)) + + // now test cleanup, testIP is already banned + testIP1 := "12.34.56.79" + testIP2 := "12.34.56.80" + testIP3 := "12.34.56.81" + + defender.AddEvent(testIP1, HostEventNoLoginTried) + defender.AddEvent(testIP2, HostEventNoLoginTried) + assert.Equal(t, 2, defender.countHosts()) + time.Sleep(20 * time.Millisecond) + defender.AddEvent(testIP3, HostEventNoLoginTried) + assert.Equal(t, defender.config.EntriesSoftLimit, defender.countHosts()) + // testIP1 and testIP2 should be removed + assert.Equal(t, defender.config.EntriesSoftLimit, defender.countHosts()) + assert.Equal(t, 0, defender.GetScore(testIP1)) + assert.Equal(t, 0, defender.GetScore(testIP2)) + assert.Equal(t, 2, defender.GetScore(testIP3)) + + defender.AddEvent(testIP3, HostEventNoLoginTried) + defender.AddEvent(testIP3, HostEventNoLoginTried) + // IP3 is now banned + assert.NotNil(t, defender.GetBanTime(testIP3)) + assert.Equal(t, 0, defender.countHosts()) + + time.Sleep(20 * time.Millisecond) + for i := 0; i < 3; i++ { + defender.AddEvent(testIP1, HostEventNoLoginTried) + } + assert.Equal(t, 0, defender.countHosts()) + assert.Equal(t, config.EntriesSoftLimit, defender.countBanned()) + assert.Nil(t, defender.GetBanTime(testIP)) + assert.Nil(t, defender.GetBanTime(testIP3)) + assert.NotNil(t, defender.GetBanTime(testIP1)) + + for i := 0; i < 3; i++ { + defender.AddEvent(testIP, HostEventNoLoginTried) + time.Sleep(10 * time.Millisecond) + defender.AddEvent(testIP3, HostEventNoLoginTried) + } + assert.Equal(t, 0, defender.countHosts()) + assert.Equal(t, defender.config.EntriesSoftLimit, defender.countBanned()) + + banTime := defender.GetBanTime(testIP3) + if assert.NotNil(t, banTime) { + assert.True(t, defender.IsBanned(testIP3)) + // ban time should increase + newBanTime := defender.GetBanTime(testIP3) + assert.True(t, newBanTime.After(*banTime)) + } + + err = os.Remove(slFile) + assert.NoError(t, err) + err = os.Remove(blFile) + assert.NoError(t, err) +} + +func TestLoadHostListFromFile(t *testing.T) { + _, err := loadHostListFromFile(".") + assert.Error(t, err) + + hostsFilePath := filepath.Join(os.TempDir(), "hostfile") + content := make([]byte, 1048576*6) + _, err = rand.Read(content) + assert.NoError(t, err) + + err = ioutil.WriteFile(hostsFilePath, content, os.ModePerm) + assert.NoError(t, err) + + _, err = loadHostListFromFile(hostsFilePath) + assert.Error(t, err) + + hl := HostListFile{ + IPAddresses: []string{}, + CIDRNetworks: []string{}, + } + + asJSON, err := json.Marshal(hl) + assert.NoError(t, err) + err = ioutil.WriteFile(hostsFilePath, asJSON, os.ModePerm) + assert.NoError(t, err) + + hostList, err := loadHostListFromFile(hostsFilePath) + assert.NoError(t, err) + assert.Nil(t, hostList) + + hl.IPAddresses = append(hl.IPAddresses, "invalidip") + asJSON, err = json.Marshal(hl) + assert.NoError(t, err) + err = ioutil.WriteFile(hostsFilePath, asJSON, os.ModePerm) + assert.NoError(t, err) + + hostList, err = loadHostListFromFile(hostsFilePath) + assert.NoError(t, err) + assert.Len(t, hostList.IPAddresses, 0) + + hl.IPAddresses = nil + hl.CIDRNetworks = append(hl.CIDRNetworks, "invalid net") + + asJSON, err = json.Marshal(hl) + assert.NoError(t, err) + err = ioutil.WriteFile(hostsFilePath, asJSON, os.ModePerm) + assert.NoError(t, err) + + hostList, err = loadHostListFromFile(hostsFilePath) + assert.NoError(t, err) + assert.NotNil(t, hostList) + assert.Len(t, hostList.IPAddresses, 0) + assert.Equal(t, 0, hostList.Ranges.Len()) + + if runtime.GOOS != "windows" { + err = os.Chmod(hostsFilePath, 0111) + assert.NoError(t, err) + + _, err = loadHostListFromFile(hostsFilePath) + assert.Error(t, err) + + err = os.Chmod(hostsFilePath, 0644) + assert.NoError(t, err) + } + + err = ioutil.WriteFile(hostsFilePath, []byte("non json content"), os.ModePerm) + assert.NoError(t, err) + _, err = loadHostListFromFile(hostsFilePath) + assert.Error(t, err) + + err = os.Remove(hostsFilePath) + assert.NoError(t, err) +} + +func TestDefenderCleanup(t *testing.T) { + d := memoryDefender{ + banned: make(map[string]time.Time), + hosts: make(map[string]hostScore), + config: &DefenderConfig{ + ObservationTime: 1, + EntriesSoftLimit: 2, + EntriesHardLimit: 3, + }, + } + + d.banned["1.1.1.1"] = time.Now().Add(-24 * time.Hour) + d.banned["1.1.1.2"] = time.Now().Add(-24 * time.Hour) + d.banned["1.1.1.3"] = time.Now().Add(-24 * time.Hour) + d.banned["1.1.1.4"] = time.Now().Add(-24 * time.Hour) + + d.cleanupBanned() + assert.Equal(t, 0, d.countBanned()) + + d.banned["2.2.2.2"] = time.Now().Add(2 * time.Minute) + d.banned["2.2.2.3"] = time.Now().Add(1 * time.Minute) + d.banned["2.2.2.4"] = time.Now().Add(3 * time.Minute) + d.banned["2.2.2.5"] = time.Now().Add(4 * time.Minute) + + d.cleanupBanned() + assert.Equal(t, d.config.EntriesSoftLimit, d.countBanned()) + assert.Nil(t, d.GetBanTime("2.2.2.3")) + + d.hosts["3.3.3.3"] = hostScore{ + TotalScore: 0, + Events: []hostEvent{ + { + dateTime: time.Now().Add(-5 * time.Minute), + score: 1, + }, + { + dateTime: time.Now().Add(-3 * time.Minute), + score: 1, + }, + { + dateTime: time.Now(), + score: 1, + }, + }, + } + d.hosts["3.3.3.4"] = hostScore{ + TotalScore: 1, + Events: []hostEvent{ + { + dateTime: time.Now().Add(-3 * time.Minute), + score: 1, + }, + }, + } + d.hosts["3.3.3.5"] = hostScore{ + TotalScore: 1, + Events: []hostEvent{ + { + dateTime: time.Now().Add(-2 * time.Minute), + score: 1, + }, + }, + } + d.hosts["3.3.3.6"] = hostScore{ + TotalScore: 1, + Events: []hostEvent{ + { + dateTime: time.Now().Add(-1 * time.Minute), + score: 1, + }, + }, + } + + assert.Equal(t, 1, d.GetScore("3.3.3.3")) + + d.cleanupHosts() + assert.Equal(t, d.config.EntriesSoftLimit, d.countHosts()) + assert.Equal(t, 0, d.GetScore("3.3.3.4")) +} + +func TestDefenderConfig(t *testing.T) { + c := DefenderConfig{} + err := c.validate() + require.NoError(t, err) + + c.Enabled = true + c.Threshold = 10 + c.ScoreInvalid = 10 + err = c.validate() + require.Error(t, err) + + c.ScoreInvalid = 2 + c.ScoreValid = 10 + err = c.validate() + require.Error(t, err) + + c.ScoreValid = 1 + c.BanTime = 0 + err = c.validate() + require.Error(t, err) + + c.BanTime = 30 + c.BanTimeIncrement = 0 + err = c.validate() + require.Error(t, err) + + c.BanTimeIncrement = 50 + c.ObservationTime = 0 + err = c.validate() + require.Error(t, err) + + c.ObservationTime = 30 + err = c.validate() + require.Error(t, err) + + c.EntriesSoftLimit = 10 + err = c.validate() + require.Error(t, err) + + c.EntriesHardLimit = 10 + err = c.validate() + require.Error(t, err) + + c.EntriesHardLimit = 20 + err = c.validate() + require.NoError(t, err) +} + +func BenchmarkDefenderBannedSearch(b *testing.B) { + d := getDefenderForBench() + + ip, ipnet, err := net.ParseCIDR("10.8.0.0/12") // 1048574 ip addresses + if err != nil { + panic(err) + } + + for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) { + d.banned[ip.String()] = time.Now().Add(10 * time.Minute) + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + d.IsBanned("192.168.1.1") + } +} + +func BenchmarkCleanup(b *testing.B) { + d := getDefenderForBench() + + ip, ipnet, err := net.ParseCIDR("192.168.4.0/24") + if err != nil { + panic(err) + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) { + d.AddEvent(ip.String(), HostEventLoginFailed) + if d.countHosts() > d.config.EntriesHardLimit { + panic("too many hosts") + } + if d.countBanned() > d.config.EntriesSoftLimit { + panic("too many ip banned") + } + } + } +} + +func BenchmarkDefenderBannedSearchWithBlockList(b *testing.B) { + d := getDefenderForBench() + + d.blockList = &HostList{ + IPAddresses: make(map[string]bool), + Ranges: cidranger.NewPCTrieRanger(), + } + + ip, ipnet, err := net.ParseCIDR("129.8.0.0/12") // 1048574 ip addresses + if err != nil { + panic(err) + } + + for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) { + d.banned[ip.String()] = time.Now().Add(10 * time.Minute) + d.blockList.IPAddresses[ip.String()] = true + } + + for i := 0; i < 255; i++ { + cidr := fmt.Sprintf("10.8.%v.1/24", i) + _, network, _ := net.ParseCIDR(cidr) + if err := d.blockList.Ranges.Insert(cidranger.NewBasicRangerEntry(*network)); err != nil { + panic(err) + } + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + d.IsBanned("192.168.1.1") + } +} + +func BenchmarkHostListSearch(b *testing.B) { + hostlist := &HostList{ + IPAddresses: make(map[string]bool), + Ranges: cidranger.NewPCTrieRanger(), + } + + ip, ipnet, _ := net.ParseCIDR("172.16.0.0/16") + + for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) { + hostlist.IPAddresses[ip.String()] = true + } + + for i := 0; i < 255; i++ { + cidr := fmt.Sprintf("10.8.%v.1/24", i) + _, network, _ := net.ParseCIDR(cidr) + if err := hostlist.Ranges.Insert(cidranger.NewBasicRangerEntry(*network)); err != nil { + panic(err) + } + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if hostlist.isListed("192.167.1.2") { + panic("should not be listed") + } + } +} + +func BenchmarkCIDRanger(b *testing.B) { + ranger := cidranger.NewPCTrieRanger() + for i := 0; i < 255; i++ { + cidr := fmt.Sprintf("192.168.%v.1/24", i) + _, network, _ := net.ParseCIDR(cidr) + if err := ranger.Insert(cidranger.NewBasicRangerEntry(*network)); err != nil { + panic(err) + } + } + + ipToMatch := net.ParseIP("192.167.1.2") + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := ranger.Contains(ipToMatch); err != nil { + panic(err) + } + } +} + +func BenchmarkNetContains(b *testing.B) { + var nets []*net.IPNet + for i := 0; i < 255; i++ { + cidr := fmt.Sprintf("192.168.%v.1/24", i) + _, network, _ := net.ParseCIDR(cidr) + nets = append(nets, network) + } + + ipToMatch := net.ParseIP("192.167.1.1") + + b.ResetTimer() + for i := 0; i < b.N; i++ { + for _, n := range nets { + n.Contains(ipToMatch) + } + } +} + +func getDefenderForBench() *memoryDefender { + config := &DefenderConfig{ + Enabled: true, + BanTime: 30, + BanTimeIncrement: 50, + Threshold: 10, + ScoreInvalid: 2, + ScoreValid: 2, + ObservationTime: 30, + EntriesSoftLimit: 50, + EntriesHardLimit: 100, + } + return &memoryDefender{ + config: config, + hosts: make(map[string]hostScore), + banned: make(map[string]time.Time), + } +} + +func inc(ip net.IP) { + for j := len(ip) - 1; j >= 0; j-- { + ip[j]++ + if ip[j] > 0 { + break + } + } +} diff --git a/config/config.go b/config/config.go index 013e2aef..f1fa5d3a 100644 --- a/config/config.go +++ b/config/config.go @@ -93,6 +93,19 @@ func Init() { ProxyAllowed: []string{}, PostConnectHook: "", MaxTotalConnections: 0, + DefenderConfig: common.DefenderConfig{ + Enabled: false, + BanTime: 30, + BanTimeIncrement: 50, + Threshold: 15, + ScoreInvalid: 2, + ScoreValid: 1, + ObservationTime: 30, + EntriesSoftLimit: 100, + EntriesHardLimit: 150, + SafeListFile: "", + BlockListFile: "", + }, }, SFTPD: sftpd.Configuration{ Banner: defaultSFTPDBanner, @@ -666,6 +679,17 @@ func setViperDefaults() { viper.SetDefault("common.proxy_allowed", globalConf.Common.ProxyAllowed) viper.SetDefault("common.post_connect_hook", globalConf.Common.PostConnectHook) viper.SetDefault("common.max_total_connections", globalConf.Common.MaxTotalConnections) + viper.SetDefault("common.defender.enabled", globalConf.Common.DefenderConfig.Enabled) + viper.SetDefault("common.defender.ban_time", globalConf.Common.DefenderConfig.BanTime) + viper.SetDefault("common.defender.ban_time_increment", globalConf.Common.DefenderConfig.BanTimeIncrement) + viper.SetDefault("common.defender.threshold", globalConf.Common.DefenderConfig.Threshold) + viper.SetDefault("common.defender.score_invalid", globalConf.Common.DefenderConfig.ScoreInvalid) + viper.SetDefault("common.defender.score_valid", globalConf.Common.DefenderConfig.ScoreValid) + viper.SetDefault("common.defender.observation_time", globalConf.Common.DefenderConfig.ObservationTime) + viper.SetDefault("common.defender.entries_soft_limit", globalConf.Common.DefenderConfig.EntriesSoftLimit) + viper.SetDefault("common.defender.entries_hard_limit", globalConf.Common.DefenderConfig.EntriesHardLimit) + viper.SetDefault("common.defender.safelist_file", globalConf.Common.DefenderConfig.SafeListFile) + viper.SetDefault("common.defender.blocklist_file", globalConf.Common.DefenderConfig.BlockListFile) viper.SetDefault("sftpd.max_auth_tries", globalConf.SFTPD.MaxAuthTries) viper.SetDefault("sftpd.banner", globalConf.SFTPD.Banner) viper.SetDefault("sftpd.host_keys", globalConf.SFTPD.HostKeys) diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go index 7c29fe12..739eb558 100644 --- a/dataprovider/dataprovider.go +++ b/dataprovider/dataprovider.go @@ -106,7 +106,7 @@ var ( // ErrNoInitRequired defines the error returned by InitProvider if no inizialization/update is required ErrNoInitRequired = errors.New("The data provider is already up to date") // ErrInvalidCredentials defines the error to return if the supplied credentials are invalid - ErrInvalidCredentials = errors.New("Invalid credentials") + ErrInvalidCredentials = errors.New("invalid credentials") webDAVUsersCache sync.Map config Config provider Provider @@ -352,7 +352,7 @@ type RecordNotFoundError struct { } func (e *RecordNotFoundError) Error() string { - return fmt.Sprintf("Not found: %s", e.err) + return fmt.Sprintf("not found: %s", e.err) } // GetQuotaTracking returns the configured mode for user's quota tracking diff --git a/docs/defender.md b/docs/defender.md new file mode 100644 index 00000000..3bceb172 --- /dev/null +++ b/docs/defender.md @@ -0,0 +1,53 @@ +# Defender + +The experimental built-in `defender` allows you to configure an auto-blocking policy for SFTPGo and thus helps to prevent DoS (Denial of Service) and brute force password guessing. + +If enabled it will protect SFTP, FTP and WebDAV services and it will automatically block hosts (IP addresses) that continually fail to log in or attempt to connect. + +You can configure a score for each event type: + +- `score_valid` defines the score for valid login attempts, eg. user accounts that exist. Default `1`. +- `score_invalid` defines the score for invalid login attempts, eg. non-existent user accounts or client disconnected for inactivity without authentication attempts. Default `2`. + +And then you can configure: + +- `observation_time` defines the time window, in minutes, for tracking client errors. +- `threshold` defines the threshold value before banning a host. +- `ban_time` defines the time to ban a client, as minutes + +So a host is banned, for `ban_time` minutes, if it has exceeded the defined threshold during the last observation time minutes. + +If an already banned client tries to log in again its ban time will be incremented based on the `ban_time_increment` configuration. + +The `ban_time_increment` is calculated as percentage of `ban_time`, so if `ban_time` is 30 minutes and `ban_time_increment` is 50 the host will be banned for additionally 15 minutes. You can specify values greater than 100 for `ban_time_increment`. + +The `defender` will keep in memory both the host scores and the banned hosts, you can limit the memory usage using the `entries_soft_limit` and `entries_hard_limit` configuration keys. + +The `defender` can also load a permanent block and/or safe list of ip addresses/networks from a file: + +- `safelist_file`, string. Path to a file with a list of ip addresses and/or networks to never ban. +- `blocklist_file`, string. Path to a file with a list of ip addresses and/or networks to always ban. + +These list must be stored as JSON with the following schema: + +- `addresses`, list of strings. Each string must be a valid IPv4/IPv6 address. +- `networks`, list of strings. Each string must be a valid IPv4/IPv6 CIDR address. + +Here is a small example: + +```json +{ + "addresses":[ + "192.0.2.1", + "2001:db8::68" + ], + "networks":[ + "192.0.2.1/24", + "2001:db8:1234::/48" + ] +} +``` + +These list will be loaded in memory for faster lookups. + +The `defender` is optimized for fast and time constant lookups however as it keeps all the lists and the entries in memory you should carefully measure the memory requirements for your use case. diff --git a/docs/full-configuration.md b/docs/full-configuration.md index a8b3312f..7c4a5f8e 100644 --- a/docs/full-configuration.md +++ b/docs/full-configuration.md @@ -64,6 +64,18 @@ The configuration file contains the following sections: - If `proxy_protocol` is set to 2 and we receive a proxy header from an IP that is not in the list then the connection will be rejected - `post_connect_hook`, string. Absolute path to the command to execute or HTTP URL to notify. See [Post connect hook](./post-connect-hook.md) for more details. Leave empty to disable - `max_total_connections`, integer. Maximum number of concurrent client connections. 0 means unlimited + - `defender`, struct containing the defender configuration. See [Defender](./defender.md) for more details. + - `enabled`, boolean. Default `false`. + - `ban_time`, integer. Ban time in minutes. + - `ban_time_increment`, integer. Ban time increment, as a percentage, if a banned host tries to connect again. + - `threshold`, integer. Threshold value for banning a client. + - `score_invalid`, integer. Score for invalid login attempts, eg. non-existent user accounts or client disconnected for inactivity without authentication attempts. + - `score_valid`, integer. Score for valid login attempts, eg. user accounts that exist. + - `observation_time`, integer. Defines the time window, in minutes, for tracking client errors. A host is banned if it has exceeded the defined threshold during the last observation time minutes. + - `entries_soft_limit`, integer. + - `entries_hard_limit`, integer. The number of banned IPs and host scores kept in memory will vary between the soft and hard limit. + - `safelist_file`, string. Path to a file with a list of ip addresses and/or networks to never ban. + - `blocklist_file`, string. Path to a file with a list of ip addresses and/or networks to always ban. - **"sftpd"**, the configuration for the SFTP server - `bindings`, list of structs. Each struct has the following fields: - `port`, integer. The port used for serving SFTP requests. 0 means disabled. Default: 2022 diff --git a/ftpd/ftpd_test.go b/ftpd/ftpd_test.go index 2838bbd9..cc442e5c 100644 --- a/ftpd/ftpd_test.go +++ b/ftpd/ftpd_test.go @@ -127,8 +127,11 @@ func TestMain(m *testing.M) { os.Exit(1) } - common.Initialize(commonConf) - + err = common.Initialize(commonConf) + if err != nil { + logger.WarnToConsole("error initializing common: %v", err) + os.Exit(1) + } err = dataprovider.Initialize(providerConf, configDir) if err != nil { logger.ErrorToConsole("error initializing data provider: %v", err) @@ -233,12 +236,7 @@ func TestMain(m *testing.M) { }() waitTCPListening(ftpdConf.Bindings[0].GetAddress()) - - // ensure all the initial connections to check if the service is alive are disconnected - time.Sleep(100 * time.Millisecond) - for len(common.Connections.GetStats()) > 0 { - time.Sleep(50 * time.Millisecond) - } + waitNoConnections() exitCode := m.Run() os.Remove(logFilePath) @@ -402,6 +400,12 @@ func TestLoginInvalidPwd(t *testing.T) { assert.NoError(t, err) } +func TestLoginNonExistentUser(t *testing.T) { + user := getTestUser() + _, err := getFTPClient(user, false) + assert.Error(t, err) +} + func TestLoginExternalAuth(t *testing.T) { if runtime.GOOS == osWindows { t.Skip("this test is not available on Windows") @@ -599,6 +603,47 @@ func TestMaxConnections(t *testing.T) { common.Config.MaxTotalConnections = oldValue } +func TestDefender(t *testing.T) { + oldConfig := config.GetCommonConfig() + + cfg := config.GetCommonConfig() + cfg.DefenderConfig.Enabled = true + cfg.DefenderConfig.Threshold = 3 + + err := common.Initialize(cfg) + assert.NoError(t, err) + + user, _, err := httpd.AddUser(getTestUser(), http.StatusOK) + assert.NoError(t, err) + client, err := getFTPClient(user, false) + if assert.NoError(t, err) { + err = checkBasicFTP(client) + assert.NoError(t, err) + err = client.Quit() + assert.NoError(t, err) + } + + for i := 0; i < 3; i++ { + user.Password = "wrong_pwd" + _, err = getFTPClient(user, false) + assert.Error(t, err) + } + + user.Password = defaultPassword + _, err = getFTPClient(user, false) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "Access denied, banned client IP") + } + + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + + err = common.Initialize(oldConfig) + assert.NoError(t, err) +} + func TestMaxSessions(t *testing.T) { u := getTestUser() u.MaxSessions = 1 @@ -2035,6 +2080,13 @@ func waitTCPListening(address string) { } } +func waitNoConnections() { + time.Sleep(50 * time.Millisecond) + for len(common.Connections.GetStats()) > 0 { + time.Sleep(50 * time.Millisecond) + } +} + func getTestUser() dataprovider.User { user := dataprovider.User{ Username: defaultUsername, diff --git a/ftpd/server.go b/ftpd/server.go index 9d58e9f5..6f34d9de 100644 --- a/ftpd/server.go +++ b/ftpd/server.go @@ -103,11 +103,16 @@ func (s *Server) GetSettings() (*ftpserver.Settings, error) { // ClientConnected is called to send the very first welcome message func (s *Server) ClientConnected(cc ftpserver.ClientContext) (string, error) { + ipAddr := utils.GetIPFromRemoteAddress(cc.RemoteAddr().String()) + if common.IsBanned(ipAddr) { + logger.Log(logger.LevelDebug, common.ProtocolFTP, "", "connection refused, ip %#v is banned", ipAddr) + return "Access denied, banned client IP", common.ErrConnectionDenied + } if !common.Connections.IsNewConnectionAllowed() { logger.Log(logger.LevelDebug, common.ProtocolFTP, "", "connection refused, configured limit reached") return "", common.ErrConnectionDenied } - if err := common.Config.ExecutePostConnectHook(cc.RemoteAddr().String(), common.ProtocolFTP); err != nil { + if err := common.Config.ExecutePostConnectHook(ipAddr, common.ProtocolFTP); err != nil { return "", err } connID := fmt.Sprintf("%v_%v", s.ID, cc.ID()) @@ -128,23 +133,23 @@ func (s *Server) ClientDisconnected(cc ftpserver.ClientContext) { // AuthUser authenticates the user and selects an handling driver func (s *Server) AuthUser(cc ftpserver.ClientContext, username, password string) (ftpserver.ClientDriver, error) { - remoteAddr := cc.RemoteAddr().String() - user, err := dataprovider.CheckUserAndPass(username, password, utils.GetIPFromRemoteAddress(remoteAddr), common.ProtocolFTP) + ipAddr := utils.GetIPFromRemoteAddress(cc.RemoteAddr().String()) + user, err := dataprovider.CheckUserAndPass(username, password, ipAddr, common.ProtocolFTP) if err != nil { - updateLoginMetrics(username, remoteAddr, err) + updateLoginMetrics(username, ipAddr, err) return nil, err } connection, err := s.validateUser(user, cc) - defer updateLoginMetrics(username, remoteAddr, err) + defer updateLoginMetrics(username, ipAddr, err) if err != nil { return nil, err } connection.Fs.CheckRootPath(connection.GetUsername(), user.GetUID(), user.GetGID()) connection.Log(logger.LevelInfo, "User id: %d, logged in with FTP, username: %#v, home_dir: %#v remote addr: %#v", - user.ID, user.Username, user.HomeDir, remoteAddr) + user.ID, user.Username, user.HomeDir, cc.RemoteAddr()) dataprovider.UpdateLastLogin(user) //nolint:errcheck return connection, nil } @@ -213,12 +218,16 @@ func (s *Server) validateUser(user dataprovider.User, cc ftpserver.ClientContext return connection, nil } -func updateLoginMetrics(username, remoteAddress string, err error) { +func updateLoginMetrics(username, ip string, err error) { metrics.AddLoginAttempt(dataprovider.LoginMethodPassword) - ip := utils.GetIPFromRemoteAddress(remoteAddress) if err != nil { logger.ConnectionFailedLog(username, ip, dataprovider.LoginMethodPassword, common.ProtocolFTP, err.Error()) + event := common.HostEventLoginFailed + if _, ok := err.(*dataprovider.RecordNotFoundError); ok { + event = common.HostEventUserNotFound + } + common.AddDefenderEvent(ip, event) } metrics.AddLoginResult(dataprovider.LoginMethodPassword, err) dataprovider.ExecutePostLoginHook(username, dataprovider.LoginMethodPassword, ip, common.ProtocolFTP, err) diff --git a/go.mod b/go.mod index 564f530a..066f5dc2 100644 --- a/go.mod +++ b/go.mod @@ -7,16 +7,17 @@ require ( cloud.google.com/go/storage v1.12.0 github.com/Azure/azure-storage-blob-go v0.12.0 github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 - github.com/alexedwards/argon2id v0.0.0-20200802152012-2464efd3196b - github.com/aws/aws-sdk-go v1.36.15 + github.com/alexedwards/argon2id v0.0.0-20201228115903-cf543ebc1f7b + github.com/aws/aws-sdk-go v1.36.18 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d - github.com/fclairamb/ftpserverlib v0.11.1-0.20201228010033-a81903a5fd21 + github.com/fclairamb/ftpserverlib v0.12.0 github.com/frankban/quicktest v1.11.2 // indirect github.com/go-chi/chi v1.5.1 github.com/go-chi/render v1.0.1 github.com/go-sql-driver/mysql v1.5.0 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 + github.com/google/uuid v1.1.3 // indirect github.com/grandcat/zeroconf v1.0.0 github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126 github.com/lib/pq v1.9.0 @@ -42,6 +43,7 @@ require ( github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.6.1 github.com/studio-b12/gowebdav v0.0.0-20200929080739-bdacfab94796 + github.com/yl2chen/cidranger v1.0.2 go.etcd.io/bbolt v1.3.5 go.uber.org/automaxprocs v1.3.0 gocloud.dev v0.21.0 @@ -50,7 +52,7 @@ require ( golang.org/x/net v0.0.0-20201224014010-6772e930b67b golang.org/x/sys v0.0.0-20201223074533-0d417f636930 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect - golang.org/x/tools v0.0.0-20201226215659-b1c90890d22a // indirect + golang.org/x/tools v0.0.0-20201230224404-63754364767c // indirect google.golang.org/api v0.36.0 google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d // indirect gopkg.in/ini.v1 v1.62.0 // indirect diff --git a/go.sum b/go.sum index 4c681423..94717ad3 100644 --- a/go.sum +++ b/go.sum @@ -57,25 +57,32 @@ github.com/Azure/azure-storage-blob-go v0.12.0 h1:7bFXA1QB+lOK2/ASWHhp6/vnxjaeeZ github.com/Azure/azure-storage-blob-go v0.12.0/go.mod h1:A0u4VjtpgZJ7Y7um/+ix2DHBuEKFC6sEIlj0xc13a4Q= github.com/Azure/go-amqp v0.13.0/go.mod h1:qj+o8xPCz9tMSbQ83Vp8boHahuRDl5mkNHyt1xlxUTs= github.com/Azure/go-amqp v0.13.1/go.mod h1:qj+o8xPCz9tMSbQ83Vp8boHahuRDl5mkNHyt1xlxUTs= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.3/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= github.com/Azure/go-autorest/autorest v0.11.7/go.mod h1:V6p3pKZx1KKkJubbxnDWrzNhEIfOy/pTGasLqzHIPHs= github.com/Azure/go-autorest/autorest v0.11.9/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest v0.11.12 h1:gI8ytXbxMfI+IVbI9mP2JGCTXIuhHLgRlvQ9X4PsnHE= github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= github.com/Azure/go-autorest/autorest/adal v0.9.4/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.6 h1:d3pSDwvBWBLqdA91u+keH1zs1cCEzrQdHKY6iqbQNkE= github.com/Azure/go-autorest/autorest/adal v0.9.6/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/azure/auth v0.5.3/go.mod h1:4bJZhUhcq8LB20TruwHbAQsmUs2Xh+QR7utuJpLXX3A= github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.0/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 h1:KeNholpO2xKjgaaSyd+DyQRrsQjhbSeS7qe4nEw8aQw= @@ -85,6 +92,7 @@ github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= @@ -93,8 +101,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alexedwards/argon2id v0.0.0-20200802152012-2464efd3196b h1:rcCpjI1OMGtBY8nnBvExeM1pXNoaM35zqmXBGpgJR2o= -github.com/alexedwards/argon2id v0.0.0-20200802152012-2464efd3196b/go.mod h1:GFtu6vaWaRJV5EvSFaVqgq/3Iq95xyYElBV/aupGzUo= +github.com/alexedwards/argon2id v0.0.0-20201228115903-cf543ebc1f7b h1:jEg+fE+POnmUy40B+aSKEPqZDmsdl55hZU0YKXEzz1k= +github.com/alexedwards/argon2id v0.0.0-20201228115903-cf543ebc1f7b/go.mod h1:Kmn5t2Rb93Q4NTprN4+CCgARGvigKMJyxP0WckpTUp0= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -106,8 +114,8 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.36.1/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.36.15 h1:nGqgPlXegCKPZOKXvWnYCLvLPJPRoSOHHn9d0N0DG7Y= -github.com/aws/aws-sdk-go v1.36.15/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.36.18 h1:PvfZkE0cjM1k1EMQDSb2BrX8LETPx0IFFZ/YKkurmFg= +github.com/aws/aws-sdk-go v1.36.18/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -179,13 +187,15 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/fclairamb/ftpserverlib v0.11.1-0.20201228010033-a81903a5fd21 h1:kEcLQsnpN78+LF9XzCHVLV/olyjuxHTQNvKlIEAx/Gc= -github.com/fclairamb/ftpserverlib v0.11.1-0.20201228010033-a81903a5fd21/go.mod h1:X6sAMSYtN0YDPu+nHfyE9dsKPUOrEZ8O5EMgt1xvPwk= +github.com/fclairamb/ftpserverlib v0.12.0 h1:vud3Q4v/rLZU5CfIDFaXq7ST2+V9BF5cKjzNWPN18c4= +github.com/fclairamb/ftpserverlib v0.12.0/go.mod h1:X6sAMSYtN0YDPu+nHfyE9dsKPUOrEZ8O5EMgt1xvPwk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= +github.com/frankban/quicktest v1.11.2 h1:mjwHjStlXWibxOohM7HYieIViKyh56mmt3+6viyhDDI= github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= @@ -204,11 +214,14 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= @@ -219,6 +232,7 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= @@ -274,12 +288,16 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-replayers/grpcreplay v1.0.0 h1:B5kVOzJ1hBgnevTgIWhSTatQ3608yu/2NnU0Ta1d0kY= github.com/google/go-replayers/grpcreplay v1.0.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE= +github.com/google/go-replayers/httpreplay v0.1.2 h1:HCfx+dQzwN9XbGTHF8qJ+67WN8glL9FTWV5rraCJ/jU= github.com/google/go-replayers/httpreplay v0.1.2/go.mod h1:YKZViNhiGgqdBlUbI2MwGpq4pXxNmhJLPHQ7cv2b5no= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -299,6 +317,8 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.3 h1:twObb+9XcuH5B9V1TBCvvvZoO6iEdILi2a76PYn5rJI= +github.com/google/uuid v1.1.3/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.4.0 h1:kXcsA/rIGzJImVqPdhfnr6q0xsS9gU0515q1EPpJ9fE= github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= @@ -306,6 +326,7 @@ github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -332,6 +353,7 @@ github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVo github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -373,12 +395,14 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -389,7 +413,9 @@ github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -404,8 +430,10 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= @@ -440,6 +468,7 @@ github.com/minio/sio v0.2.1/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebh github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -465,6 +494,7 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= @@ -485,8 +515,10 @@ github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/otiai10/copy v1.4.1 h1:T1Ggae54qC0G+0VA5B/3CJQorvvaNVStzDn3YLZHdLI= github.com/otiai10/copy v1.4.1/go.mod h1:XWfuS3CrI0R6IE0FbgHsEazaXO8G0LpMp9o8tos0x4E= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.2 h1:VYWnrP5fXmz1MXvjuUvcBrXSjGE6xjON+axB/UrpO3E= github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -560,6 +592,7 @@ github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkB github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21R0JnA= github.com/shirou/gopsutil/v3 v3.20.11/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4= @@ -567,7 +600,9 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= @@ -614,6 +649,8 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= +github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -669,6 +706,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -678,6 +716,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -697,6 +736,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -754,6 +794,7 @@ golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201223074533-0d417f636930 h1:vRgIt+nup/B/BwIS0g2oC0haq0iqbV3ZA+u6+0TlNCo= golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -830,7 +871,8 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201203202102-a1a1cbeaa516/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201226215659-b1c90890d22a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201230224404-63754364767c h1:xx3+TTG3yS1I6Ola5Kapxr5vZu85vKkcwKyV6ke9fHA= +golang.org/x/tools v0.0.0-20201230224404-63754364767c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -868,6 +910,7 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -951,6 +994,7 @@ gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUy gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index 05dc733b..941c828e 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -140,7 +140,11 @@ func TestMain(m *testing.M) { os.RemoveAll(credentialsPath) //nolint:errcheck logger.InfoToConsole("Starting HTTPD tests, provider: %v", providerConf.Driver) - common.Initialize(config.GetCommonConfig()) + err = common.Initialize(config.GetCommonConfig()) + if err != nil { + logger.WarnToConsole("error initializing common: %v", err) + os.Exit(1) + } err = dataprovider.Initialize(providerConf, configDir) if err != nil { diff --git a/pkgs/build.sh b/pkgs/build.sh index 070e9429..68a90cb2 100755 --- a/pkgs/build.sh +++ b/pkgs/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -NFPM_VERSION=2.1.1 +NFPM_VERSION=2.2.1 NFPM_ARCH=${NFPM_ARCH:-amd64} if [ -z ${SFTPGO_VERSION} ] then diff --git a/service/service.go b/service/service.go index 69a5678b..da10aac7 100644 --- a/service/service.go +++ b/service/service.go @@ -81,9 +81,14 @@ func (s *Service) Start() error { return errors.New(infoString) } - common.Initialize(config.GetCommonConfig()) + err := common.Initialize(config.GetCommonConfig()) + if err != nil { + logger.Error(logSender, "", "%v", err) + logger.ErrorToConsole("%v", err) + os.Exit(1) + } kmsConfig := config.GetKMSConfig() - err := kmsConfig.Initialize() + err = kmsConfig.Initialize() if err != nil { logger.Error(logSender, "", "unable to initialize KMS: %v", err) logger.ErrorToConsole("unable to initialize KMS: %v", err) diff --git a/sftpd/server.go b/sftpd/server.go index e3815ea7..05f8ca58 100644 --- a/sftpd/server.go +++ b/sftpd/server.go @@ -351,6 +351,18 @@ func (c *Configuration) configureKeyboardInteractiveAuth(serverConfig *ssh.Serve } } +func canAcceptConnection(ip string) bool { + if common.IsBanned(ip) { + logger.Log(logger.LevelDebug, common.ProtocolSSH, "", "connection refused, ip %#v is banned", ip) + return false + } + if !common.Connections.IsNewConnectionAllowed() { + logger.Log(logger.LevelDebug, common.ProtocolSSH, "", "connection refused, configured limit reached") + return false + } + return true +} + // AcceptInboundConnection handles an inbound connection to the server instance and determines if the request should be served or not. func (c *Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.ServerConfig) { defer func() { @@ -358,22 +370,22 @@ func (c *Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.Serve logger.Error(logSender, "", "panic in AcceptInboundConnection: %#v stack strace: %v", r, string(debug.Stack())) } }() - if !common.Connections.IsNewConnectionAllowed() { - logger.Log(logger.LevelDebug, common.ProtocolSSH, "", "connection refused, configured limit reached") + ipAddr := utils.GetIPFromRemoteAddress(conn.RemoteAddr().String()) + if !canAcceptConnection(ipAddr) { conn.Close() return } // Before beginning a handshake must be performed on the incoming net.Conn // we'll set a Deadline for handshake to complete, the default is 2 minutes as OpenSSH conn.SetDeadline(time.Now().Add(handshakeTimeout)) //nolint:errcheck - if err := common.Config.ExecutePostConnectHook(conn.RemoteAddr().String(), common.ProtocolSSH); err != nil { + if err := common.Config.ExecutePostConnectHook(ipAddr, common.ProtocolSSH); err != nil { conn.Close() return } sconn, chans, reqs, err := ssh.NewServerConn(conn, config) if err != nil { logger.Debug(logSender, "", "failed to accept an incoming connection: %v", err) - checkAuthError(conn, err) + checkAuthError(ipAddr, err) return } // handshake completed so remove the deadline, we'll use IdleTimeout configuration from now on @@ -395,7 +407,7 @@ func (c *Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.Serve logger.Log(logger.LevelInfo, common.ProtocolSSH, connectionID, "User id: %d, logged in with: %#v, username: %#v, home_dir: %#v remote addr: %#v", - user.ID, loginType, user.Username, user.HomeDir, conn.RemoteAddr().String()) + user.ID, loginType, user.Username, user.HomeDir, ipAddr) dataprovider.UpdateLastLogin(user) //nolint:errcheck sshConnection := common.NewSSHConnection(connectionID, conn) @@ -500,11 +512,26 @@ func (c *Configuration) createHandler(connection *Connection) sftp.Handlers { } } -func checkAuthError(conn net.Conn, err error) { - if _, ok := err.(*ssh.ServerAuthError); !ok { - ip := utils.GetIPFromRemoteAddress(conn.RemoteAddr().String()) +func checkAuthError(ip string, err error) { + if authErrors, ok := err.(*ssh.ServerAuthError); ok { + // check public key auth errors here + for _, err := range authErrors.Errors { + if err != nil { + // these checks should be improved, we should check for error type and not error strings + if strings.Contains(err.Error(), "public key credentials") { + event := common.HostEventLoginFailed + if strings.Contains(err.Error(), "not found") { + event = common.HostEventUserNotFound + } + common.AddDefenderEvent(ip, event) + break + } + } + } + } else { logger.ConnectionFailedLog("", ip, dataprovider.LoginMethodNoAuthTryed, common.ProtocolSSH, err.Error()) metrics.AddNoAuthTryed() + common.AddDefenderEvent(ip, common.HostEventNoLoginTried) dataprovider.ExecutePostLoginHook("", dataprovider.LoginMethodNoAuthTryed, ip, common.ProtocolSSH, err) } } @@ -757,25 +784,25 @@ func (c *Configuration) validatePublicKeyCredentials(conn ssh.ConnMetadata, pubK connectionID := hex.EncodeToString(conn.SessionID()) method := dataprovider.SSHLoginMethodPublicKey + ipAddr := utils.GetIPFromRemoteAddress(conn.RemoteAddr().String()) cert, ok := pubKey.(*ssh.Certificate) if ok { if cert.CertType != ssh.UserCert { err = fmt.Errorf("ssh: cert has type %d", cert.CertType) - updateLoginMetrics(conn, method, err) + updateLoginMetrics(conn, ipAddr, method, err) return nil, err } if !c.certChecker.IsUserAuthority(cert.SignatureKey) { err = fmt.Errorf("ssh: certificate signed by unrecognized authority") - updateLoginMetrics(conn, method, err) + updateLoginMetrics(conn, ipAddr, method, err) return nil, err } if err := c.certChecker.CheckCert(conn.User(), cert); err != nil { - updateLoginMetrics(conn, method, err) + updateLoginMetrics(conn, ipAddr, method, err) return nil, err } certPerm = &cert.Permissions } - ipAddr := utils.GetIPFromRemoteAddress(conn.RemoteAddr().String()) if user, keyID, err = dataprovider.CheckUserAndPubKey(conn.User(), pubKey.Marshal(), ipAddr, common.ProtocolSSH); err == nil { if user.IsPartialAuth(method) { logger.Debug(logSender, connectionID, "user %#v authenticated with partial success", conn.User()) @@ -793,7 +820,7 @@ func (c *Configuration) validatePublicKeyCredentials(conn ssh.ConnMetadata, pubK } } } - updateLoginMetrics(conn, method, err) + updateLoginMetrics(conn, ipAddr, method, err) return sshPerm, err } @@ -810,7 +837,7 @@ func (c *Configuration) validatePasswordCredentials(conn ssh.ConnMetadata, pass if user, err = dataprovider.CheckUserAndPass(conn.User(), string(pass), ipAddr, common.ProtocolSSH); err == nil { sshPerm, err = loginUser(user, method, "", conn) } - updateLoginMetrics(conn, method, err) + updateLoginMetrics(conn, ipAddr, method, err) return sshPerm, err } @@ -828,15 +855,24 @@ func (c *Configuration) validateKeyboardInteractiveCredentials(conn ssh.ConnMeta ipAddr, common.ProtocolSSH); err == nil { sshPerm, err = loginUser(user, method, "", conn) } - updateLoginMetrics(conn, method, err) + updateLoginMetrics(conn, ipAddr, method, err) return sshPerm, err } -func updateLoginMetrics(conn ssh.ConnMetadata, method string, err error) { +func updateLoginMetrics(conn ssh.ConnMetadata, ip, method string, err error) { metrics.AddLoginAttempt(method) - ip := utils.GetIPFromRemoteAddress(conn.RemoteAddr().String()) if err != nil { logger.ConnectionFailedLog(conn.User(), ip, method, common.ProtocolSSH, err.Error()) + if method != dataprovider.SSHLoginMethodPublicKey { + // some clients try all available public keys for a user, we + // record failed login key auth only once for session if the + // authentication fails in checkAuthError + event := common.HostEventLoginFailed + if _, ok := err.(*dataprovider.RecordNotFoundError); ok { + event = common.HostEventUserNotFound + } + common.AddDefenderEvent(ip, event) + } } metrics.AddLoginResult(method, err) dataprovider.ExecutePostLoginHook(conn.User(), method, ip, common.ProtocolSSH, err) diff --git a/sftpd/sftpd_test.go b/sftpd/sftpd_test.go index d833ee3f..6fa91914 100644 --- a/sftpd/sftpd_test.go +++ b/sftpd/sftpd_test.go @@ -171,7 +171,11 @@ func TestMain(m *testing.M) { scriptArgs = "$@" } - common.Initialize(commonConf) + err = common.Initialize(commonConf) + if err != nil { + logger.WarnToConsole("error initializing common: %v", err) + os.Exit(1) + } err = dataprovider.Initialize(providerConf, configDir) if err != nil { @@ -213,31 +217,7 @@ func TestMain(m *testing.M) { } sftpdConf.KeyboardInteractiveHook = keyIntAuthPath - pubKeyPath = filepath.Join(homeBasePath, "ssh_key.pub") - privateKeyPath = filepath.Join(homeBasePath, "ssh_key") - trustedCAUserKey = filepath.Join(homeBasePath, "ca_user_key") - gitWrapPath = filepath.Join(homeBasePath, "gitwrap.sh") - extAuthPath = filepath.Join(homeBasePath, "extauth.sh") - preLoginPath = filepath.Join(homeBasePath, "prelogin.sh") - postConnectPath = filepath.Join(homeBasePath, "postconnect.sh") - checkPwdPath = filepath.Join(homeBasePath, "checkpwd.sh") - err = ioutil.WriteFile(pubKeyPath, []byte(testPubKey+"\n"), 0600) - if err != nil { - logger.WarnToConsole("unable to save public key to file: %v", err) - } - err = ioutil.WriteFile(privateKeyPath, []byte(testPrivateKey+"\n"), 0600) - if err != nil { - logger.WarnToConsole("unable to save private key to file: %v", err) - } - err = ioutil.WriteFile(gitWrapPath, []byte(fmt.Sprintf("%v -i %v -oStrictHostKeyChecking=no %v\n", - sshPath, privateKeyPath, scriptArgs)), os.ModePerm) - if err != nil { - logger.WarnToConsole("unable to save gitwrap shell script: %v", err) - } - err = ioutil.WriteFile(trustedCAUserKey, []byte(testCAUserKey), 0600) - if err != nil { - logger.WarnToConsole("unable to save trusted CA user key: %v", err) - } + createInitialFiles(scriptArgs) sftpdConf.TrustedUserCAKeys = append(sftpdConf.TrustedUserCAKeys, trustedCAUserKey) go func() { @@ -488,6 +468,52 @@ func TestBasicSFTPFsHandling(t *testing.T) { assert.NoError(t, err) } +func TestLoginNonExistentUser(t *testing.T) { + usePubKey := true + user := getTestUser(usePubKey) + _, err := getSftpClient(user, usePubKey) + assert.Error(t, err) +} + +func TestDefender(t *testing.T) { + oldConfig := config.GetCommonConfig() + + cfg := config.GetCommonConfig() + cfg.DefenderConfig.Enabled = true + cfg.DefenderConfig.Threshold = 3 + + err := common.Initialize(cfg) + assert.NoError(t, err) + + usePubKey := false + user, _, err := httpd.AddUser(getTestUser(usePubKey), http.StatusOK) + assert.NoError(t, err) + client, err := getSftpClient(user, usePubKey) + if assert.NoError(t, err) { + defer client.Close() + err = checkBasicSFTP(client) + assert.NoError(t, err) + } + + for i := 0; i < 3; i++ { + user.Password = "wrong_pwd" + _, err = getSftpClient(user, usePubKey) + assert.Error(t, err) + } + + user.Password = defaultPassword + _, err = getSftpClient(user, usePubKey) + assert.Error(t, err) + + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + + err = common.Initialize(oldConfig) + assert.NoError(t, err) +} + func TestOpenReadWrite(t *testing.T) { usePubKey := false u := getTestUser(usePubKey) @@ -8721,3 +8747,31 @@ func getHostKeysFingerprints(hostKeys []string) { hostKeyFPs = append(hostKeyFPs, fp) } } + +func createInitialFiles(scriptArgs string) { + pubKeyPath = filepath.Join(homeBasePath, "ssh_key.pub") + privateKeyPath = filepath.Join(homeBasePath, "ssh_key") + trustedCAUserKey = filepath.Join(homeBasePath, "ca_user_key") + gitWrapPath = filepath.Join(homeBasePath, "gitwrap.sh") + extAuthPath = filepath.Join(homeBasePath, "extauth.sh") + preLoginPath = filepath.Join(homeBasePath, "prelogin.sh") + postConnectPath = filepath.Join(homeBasePath, "postconnect.sh") + checkPwdPath = filepath.Join(homeBasePath, "checkpwd.sh") + err := ioutil.WriteFile(pubKeyPath, []byte(testPubKey+"\n"), 0600) + if err != nil { + logger.WarnToConsole("unable to save public key to file: %v", err) + } + err = ioutil.WriteFile(privateKeyPath, []byte(testPrivateKey+"\n"), 0600) + if err != nil { + logger.WarnToConsole("unable to save private key to file: %v", err) + } + err = ioutil.WriteFile(gitWrapPath, []byte(fmt.Sprintf("%v -i %v -oStrictHostKeyChecking=no %v\n", + sshPath, privateKeyPath, scriptArgs)), os.ModePerm) + if err != nil { + logger.WarnToConsole("unable to save gitwrap shell script: %v", err) + } + err = ioutil.WriteFile(trustedCAUserKey, []byte(testCAUserKey), 0600) + if err != nil { + logger.WarnToConsole("unable to save trusted CA user key: %v", err) + } +} diff --git a/sftpgo.json b/sftpgo.json index c3c5e0ab..9739a37a 100644 --- a/sftpgo.json +++ b/sftpgo.json @@ -10,7 +10,20 @@ "proxy_protocol": 0, "proxy_allowed": [], "post_connect_hook": "", - "max_total_connections": 0 + "max_total_connections": 0, + "defender": { + "enabled": false, + "ban_time": 30, + "ban_time_increment": 50, + "threshold": 15, + "score_invalid": 2, + "score_valid": 1, + "observation_time": 30, + "entries_soft_limit": 100, + "entries_hard_limit": 150, + "safelist_file": "", + "blocklist_file": "" + } }, "sftpd": { "bindings": [ diff --git a/webdavd/internal_test.go b/webdavd/internal_test.go index 5e8396b6..59570495 100644 --- a/webdavd/internal_test.go +++ b/webdavd/internal_test.go @@ -692,12 +692,14 @@ func TestBasicUsersCache(t *testing.T) { req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user.Username), nil) assert.NoError(t, err) - _, _, _, err = server.authenticate(req) //nolint:dogsled + ipAddr := "127.0.0.1" + + _, _, _, err = server.authenticate(req, ipAddr) //nolint:dogsled assert.Error(t, err) now := time.Now() req.SetBasicAuth(username, password) - _, isCached, _, err := server.authenticate(req) + _, isCached, _, err := server.authenticate(req, ipAddr) assert.NoError(t, err) assert.False(t, isCached) // now the user should be cached @@ -708,14 +710,14 @@ func TestBasicUsersCache(t *testing.T) { assert.False(t, cachedUser.IsExpired()) assert.True(t, cachedUser.Expiration.After(now.Add(time.Duration(c.Cache.Users.ExpirationTime)*time.Minute))) // authenticate must return the cached user now - authUser, isCached, _, err := server.authenticate(req) + authUser, isCached, _, err := server.authenticate(req, ipAddr) assert.NoError(t, err) assert.True(t, isCached) assert.Equal(t, cachedUser.User, authUser) } // a wrong password must fail req.SetBasicAuth(username, "wrong") - _, _, _, err = server.authenticate(req) //nolint:dogsled + _, _, _, err = server.authenticate(req, ipAddr) //nolint:dogsled assert.EqualError(t, err, dataprovider.ErrInvalidCredentials.Error()) req.SetBasicAuth(username, password) @@ -728,7 +730,7 @@ func TestBasicUsersCache(t *testing.T) { assert.True(t, cachedUser.IsExpired()) } // now authenticate should get the user from the data provider and update the cache - _, isCached, _, err = server.authenticate(req) + _, isCached, _, err = server.authenticate(req, ipAddr) assert.NoError(t, err) assert.False(t, isCached) result, ok = dataprovider.GetCachedWebDAVUser(username) @@ -742,7 +744,7 @@ func TestBasicUsersCache(t *testing.T) { _, ok = dataprovider.GetCachedWebDAVUser(username) assert.False(t, ok) - _, isCached, _, err = server.authenticate(req) + _, isCached, _, err = server.authenticate(req, ipAddr) assert.NoError(t, err) assert.False(t, isCached) _, ok = dataprovider.GetCachedWebDAVUser(username) @@ -808,24 +810,25 @@ func TestUsersCacheSizeAndExpiration(t *testing.T) { server, err := newServer(c, configDir) assert.NoError(t, err) + ipAddr := "127.0.1.1" req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user1.Username), nil) assert.NoError(t, err) req.SetBasicAuth(user1.Username, password+"1") - _, isCached, _, err := server.authenticate(req) + _, isCached, _, err := server.authenticate(req, ipAddr) assert.NoError(t, err) assert.False(t, isCached) req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user2.Username), nil) assert.NoError(t, err) req.SetBasicAuth(user2.Username, password+"2") - _, isCached, _, err = server.authenticate(req) + _, isCached, _, err = server.authenticate(req, ipAddr) assert.NoError(t, err) assert.False(t, isCached) req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user3.Username), nil) assert.NoError(t, err) req.SetBasicAuth(user3.Username, password+"3") - _, isCached, _, err = server.authenticate(req) + _, isCached, _, err = server.authenticate(req, ipAddr) assert.NoError(t, err) assert.False(t, isCached) @@ -840,7 +843,7 @@ func TestUsersCacheSizeAndExpiration(t *testing.T) { req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user4.Username), nil) assert.NoError(t, err) req.SetBasicAuth(user4.Username, password+"4") - _, isCached, _, err = server.authenticate(req) + _, isCached, _, err = server.authenticate(req, ipAddr) assert.NoError(t, err) assert.False(t, isCached) // user1, the first cached, should be removed now @@ -857,7 +860,7 @@ func TestUsersCacheSizeAndExpiration(t *testing.T) { req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user1.Username), nil) assert.NoError(t, err) req.SetBasicAuth(user1.Username, password+"1") - _, isCached, _, err = server.authenticate(req) + _, isCached, _, err = server.authenticate(req, ipAddr) assert.NoError(t, err) assert.False(t, isCached) _, ok = dataprovider.GetCachedWebDAVUser(user2.Username) @@ -873,7 +876,7 @@ func TestUsersCacheSizeAndExpiration(t *testing.T) { req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user2.Username), nil) assert.NoError(t, err) req.SetBasicAuth(user2.Username, password+"2") - _, isCached, _, err = server.authenticate(req) + _, isCached, _, err = server.authenticate(req, ipAddr) assert.NoError(t, err) assert.False(t, isCached) _, ok = dataprovider.GetCachedWebDAVUser(user3.Username) @@ -889,7 +892,7 @@ func TestUsersCacheSizeAndExpiration(t *testing.T) { req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user3.Username), nil) assert.NoError(t, err) req.SetBasicAuth(user3.Username, password+"3") - _, isCached, _, err = server.authenticate(req) + _, isCached, _, err = server.authenticate(req, ipAddr) assert.NoError(t, err) assert.False(t, isCached) _, ok = dataprovider.GetCachedWebDAVUser(user4.Username) @@ -910,14 +913,14 @@ func TestUsersCacheSizeAndExpiration(t *testing.T) { req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user4.Username), nil) assert.NoError(t, err) req.SetBasicAuth(user4.Username, password+"4") - _, isCached, _, err = server.authenticate(req) + _, isCached, _, err = server.authenticate(req, ipAddr) assert.NoError(t, err) assert.False(t, isCached) req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user1.Username), nil) assert.NoError(t, err) req.SetBasicAuth(user1.Username, password+"1") - _, isCached, _, err = server.authenticate(req) + _, isCached, _, err = server.authenticate(req, ipAddr) assert.NoError(t, err) assert.False(t, isCached) _, ok = dataprovider.GetCachedWebDAVUser(user2.Username) diff --git a/webdavd/server.go b/webdavd/server.go index 4f36c43b..193a8f92 100644 --- a/webdavd/server.go +++ b/webdavd/server.go @@ -121,11 +121,16 @@ func (s *webDavServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } checkRemoteAddress(r) - if err := common.Config.ExecutePostConnectHook(r.RemoteAddr, common.ProtocolWebDAV); err != nil { + ipAddr := utils.GetIPFromRemoteAddress(r.RemoteAddr) + if common.IsBanned(ipAddr) { http.Error(w, common.ErrConnectionDenied.Error(), http.StatusForbidden) return } - user, _, lockSystem, err := s.authenticate(r) + if err := common.Config.ExecutePostConnectHook(ipAddr, common.ProtocolWebDAV); err != nil { + http.Error(w, common.ErrConnectionDenied.Error(), http.StatusForbidden) + return + } + user, _, lockSystem, err := s.authenticate(r, ipAddr) if err != nil { w.Header().Set("WWW-Authenticate", "Basic realm=\"SFTPGo WebDAV\"") http.Error(w, err401.Error(), http.StatusUnauthorized) @@ -139,19 +144,19 @@ func (s *webDavServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { connectionID, err := s.validateUser(user, r) if err != nil { - updateLoginMetrics(user.Username, r.RemoteAddr, err) + updateLoginMetrics(user.Username, ipAddr, err) http.Error(w, err.Error(), http.StatusForbidden) return } fs, err := user.GetFilesystem(connectionID) if err != nil { - updateLoginMetrics(user.Username, r.RemoteAddr, err) + updateLoginMetrics(user.Username, ipAddr, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } - updateLoginMetrics(user.Username, r.RemoteAddr, err) + updateLoginMetrics(user.Username, ipAddr, err) ctx := context.WithValue(r.Context(), requestIDKey, connectionID) ctx = context.WithValue(ctx, requestStartKey, time.Now()) @@ -177,7 +182,7 @@ func (s *webDavServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { handler.ServeHTTP(w, r.WithContext(ctx)) } -func (s *webDavServer) authenticate(r *http.Request) (dataprovider.User, bool, webdav.LockSystem, error) { +func (s *webDavServer) authenticate(r *http.Request, ip string) (dataprovider.User, bool, webdav.LockSystem, error) { var user dataprovider.User var err error username, password, ok := r.BasicAuth() @@ -193,13 +198,13 @@ func (s *webDavServer) authenticate(r *http.Request) (dataprovider.User, bool, w if len(password) > 0 && cachedUser.Password == password { return cachedUser.User, true, cachedUser.LockSystem, nil } - updateLoginMetrics(username, r.RemoteAddr, dataprovider.ErrInvalidCredentials) + updateLoginMetrics(username, ip, dataprovider.ErrInvalidCredentials) return user, false, nil, dataprovider.ErrInvalidCredentials } } - user, err = dataprovider.CheckUserAndPass(username, password, utils.GetIPFromRemoteAddress(r.RemoteAddr), common.ProtocolWebDAV) + user, err = dataprovider.CheckUserAndPass(username, password, ip, common.ProtocolWebDAV) if err != nil { - updateLoginMetrics(username, r.RemoteAddr, err) + updateLoginMetrics(username, ip, err) return user, false, nil, err } lockSystem := webdav.NewMemLS() @@ -315,11 +320,15 @@ func checkRemoteAddress(r *http.Request) { } } -func updateLoginMetrics(username, remoteAddress string, err error) { +func updateLoginMetrics(username, ip string, err error) { metrics.AddLoginAttempt(dataprovider.LoginMethodPassword) - ip := utils.GetIPFromRemoteAddress(remoteAddress) if err != nil { logger.ConnectionFailedLog(username, ip, dataprovider.LoginMethodPassword, common.ProtocolWebDAV, err.Error()) + event := common.HostEventLoginFailed + if _, ok := err.(*dataprovider.RecordNotFoundError); ok { + event = common.HostEventUserNotFound + } + common.AddDefenderEvent(ip, event) } metrics.AddLoginResult(dataprovider.LoginMethodPassword, err) dataprovider.ExecutePostLoginHook(username, dataprovider.LoginMethodPassword, ip, common.ProtocolWebDAV, err) diff --git a/webdavd/webdavd_test.go b/webdavd/webdavd_test.go index acf519b5..38900279 100644 --- a/webdavd/webdavd_test.go +++ b/webdavd/webdavd_test.go @@ -121,7 +121,11 @@ func TestMain(m *testing.M) { os.Exit(1) } - common.Initialize(commonConf) + err = common.Initialize(commonConf) + if err != nil { + logger.WarnToConsole("error initializing common: %v", err) + os.Exit(1) + } err = dataprovider.Initialize(providerConf, configDir) if err != nil { @@ -502,6 +506,49 @@ func TestLoginInvalidPwd(t *testing.T) { assert.NoError(t, err) } +func TestLoginNonExistentUser(t *testing.T) { + user := getTestUser() + client := getWebDavClient(user) + assert.Error(t, checkBasicFunc(client)) +} + +func TestDefender(t *testing.T) { + oldConfig := config.GetCommonConfig() + + cfg := config.GetCommonConfig() + cfg.DefenderConfig.Enabled = true + cfg.DefenderConfig.Threshold = 3 + + err := common.Initialize(cfg) + assert.NoError(t, err) + + user, _, err := httpd.AddUser(getTestUser(), http.StatusOK) + assert.NoError(t, err) + client := getWebDavClient(user) + assert.NoError(t, checkBasicFunc(client)) + + for i := 0; i < 3; i++ { + user.Password = "wrong_pwd" + client = getWebDavClient(user) + assert.Error(t, checkBasicFunc(client)) + } + + user.Password = defaultPassword + client = getWebDavClient(user) + err = checkBasicFunc(client) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "403") + } + + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + + err = common.Initialize(oldConfig) + assert.NoError(t, err) +} + func TestLoginInvalidURL(t *testing.T) { u := getTestUser() user, _, err := httpd.AddUser(u, http.StatusOK)