add support for a basic built-in defender

It can help to prevent DoS and brute force password guessing

README.md

@@ -33,6 +33,7 @@ Several storage backends are supported: local filesystem, encrypted local filesy
 - Virtual folders are supported: directories outside the user home directory can be exposed as virtual folders.
 - Configurable custom commands and/or HTTP notifications on file upload, download, pre-delete, delete, rename, on SSH commands and on user add, update and delete.
 - Automatically terminating idle connections.
+- Automatic blocklist management is supported using the built-in [defender](./docs/defender.md).
 - Atomic uploads are configurable.
 - Support for Git repositories over SSH.
 - SCP and rsync are supported.
@@ -223,6 +224,8 @@ Anyway, some backends require a pay per use account (or they offer free account
 
 The [connection failed logs](./docs/logs.md) can be used for integration in tools such as [Fail2ban](http://www.fail2ban.org/). Example of [jails](./fail2ban/jails) and [filters](./fail2ban/filters) working with `systemd`/`journald` are available in fail2ban directory.
 
+You can also use the built-in [defender](./docs/defender.md).
+
 ## Account's configuration properties
 
 Details information about account configuration properties can be found [here](./docs/account.md).

cmd/startsubsys.go

@@ -66,7 +66,10 @@ Command-line flags should be specified in the Subsystem declaration.
 			// idle connection are managed externally
 			commonConfig.IdleTimeout = 0
 			config.SetCommonConfig(commonConfig)
-			common.Initialize(config.GetCommonConfig())
+			if err := common.Initialize(config.GetCommonConfig()); err != nil {
+				logger.Error(logSender, connectionID, "%v", err)
+				os.Exit(1)
+			}
 			kmsConfig := config.GetKMSConfig()
 			if err := kmsConfig.Initialize(); err != nil {
 				logger.Error(logSender, connectionID, "unable to initialize KMS: %v", err)

common/common.go

@@ -106,13 +106,41 @@ var (
 )
 
 // Initialize sets the common configuration
-func Initialize(c Configuration) {
+func Initialize(c Configuration) error {
 	Config = c
 	Config.idleLoginTimeout = 2 * time.Minute
 	Config.idleTimeoutAsDuration = time.Duration(Config.IdleTimeout) * time.Minute
 	if Config.IdleTimeout > 0 {
 		startIdleTimeoutTicker(idleTimeoutCheckInterval)
 	}
+	Config.defender = nil
+	if c.DefenderConfig.Enabled {
+		defender, err := newInMemoryDefender(&c.DefenderConfig)
+		if err != nil {
+			return fmt.Errorf("defender initialization error: %v", err)
+		}
+		logger.Info(logSender, "", "defender initialized with config %+v", c.DefenderConfig)
+		Config.defender = defender
+	}
+	return nil
+}
+
+// IsBanned returns true if the specified IP address is banned
+func IsBanned(ip string) bool {
+	if Config.defender == nil {
+		return false
+	}
+
+	return Config.defender.IsBanned(ip)
+}
+
+// AddDefenderEvent adds the specified defender event for the given IP
+func AddDefenderEvent(ip string, event HostEvent) {
+	if Config.defender == nil {
+		return
+	}
+
+	Config.defender.AddEvent(ip, event)
 }
 
 func startIdleTimeoutTicker(duration time.Duration) {
@@ -250,9 +278,12 @@ type Configuration struct {
 	// ip address. Leave empty do disable.
 	PostConnectHook string `json:"post_connect_hook" mapstructure:"post_connect_hook"`
 	// Maximum number of concurrent client connections. 0 means unlimited
-	MaxTotalConnections   int `json:"max_total_connections" mapstructure:"max_total_connections"`
+	MaxTotalConnections int `json:"max_total_connections" mapstructure:"max_total_connections"`
+	// Defender configuration
+	DefenderConfig        DefenderConfig `json:"defender" mapstructure:"defender"`
 	idleTimeoutAsDuration time.Duration
 	idleLoginTimeout      time.Duration
+	defender              Defender
 }
 
 // IsAtomicUploadEnabled returns true if atomic upload is enabled
@@ -294,51 +325,50 @@ func (c *Configuration) GetProxyListener(listener net.Listener) (*proxyproto.Lis
 }
 
 // ExecutePostConnectHook executes the post connect hook if defined
-func (c *Configuration) ExecutePostConnectHook(remoteAddr, protocol string) error {
+func (c *Configuration) ExecutePostConnectHook(ipAddr, protocol string) error {
 	if len(c.PostConnectHook) == 0 {
 		return nil
 	}
-	ip := utils.GetIPFromRemoteAddress(remoteAddr)
 	if strings.HasPrefix(c.PostConnectHook, "http") {
 		var url *url.URL
 		url, err := url.Parse(c.PostConnectHook)
 		if err != nil {
 			logger.Warn(protocol, "", "Login from ip %#v denied, invalid post connect hook %#v: %v",
-				ip, c.PostConnectHook, err)
+				ipAddr, c.PostConnectHook, err)
 			return err
 		}
 		httpClient := httpclient.GetHTTPClient()
 		q := url.Query()
-		q.Add("ip", ip)
+		q.Add("ip", ipAddr)
 		q.Add("protocol", protocol)
 		url.RawQuery = q.Encode()
 
 		resp, err := httpClient.Get(url.String())
 		if err != nil {
-			logger.Warn(protocol, "", "Login from ip %#v denied, error executing post connect hook: %v", ip, err)
+			logger.Warn(protocol, "", "Login from ip %#v denied, error executing post connect hook: %v", ipAddr, err)
 			return err
 		}
 		defer resp.Body.Close()
 		if resp.StatusCode != http.StatusOK {
-			logger.Warn(protocol, "", "Login from ip %#v denied, post connect hook response code: %v", ip, resp.StatusCode)
+			logger.Warn(protocol, "", "Login from ip %#v denied, post connect hook response code: %v", ipAddr, resp.StatusCode)
 			return errUnexpectedHTTResponse
 		}
 		return nil
 	}
 	if !filepath.IsAbs(c.PostConnectHook) {
 		err := fmt.Errorf("invalid post connect hook %#v", c.PostConnectHook)
-		logger.Warn(protocol, "", "Login from ip %#v denied: %v", ip, err)
+		logger.Warn(protocol, "", "Login from ip %#v denied: %v", ipAddr, err)
 		return err
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
 	defer cancel()
 	cmd := exec.CommandContext(ctx, c.PostConnectHook)
 	cmd.Env = append(os.Environ(),
-		fmt.Sprintf("SFTPGO_CONNECTION_IP=%v", ip),
+		fmt.Sprintf("SFTPGO_CONNECTION_IP=%v", ipAddr),
 		fmt.Sprintf("SFTPGO_CONNECTION_PROTOCOL=%v", protocol))
 	err := cmd.Run()
 	if err != nil {
-		logger.Warn(protocol, "", "Login from ip %#v denied, connect hook error: %v", ip, err)
+		logger.Warn(protocol, "", "Login from ip %#v denied, connect hook error: %v", ipAddr, err)
 	}
 	return err
 }
@@ -537,6 +567,7 @@ func (conns *ActiveConnections) checkIdles() {
 					ip := utils.GetIPFromRemoteAddress(c.GetRemoteAddress())
 					logger.ConnectionFailedLog("", ip, dataprovider.LoginMethodNoAuthTryed, c.GetProtocol(), "client idle")
 					metrics.AddNoAuthTryed()
+					AddDefenderEvent(ip, HostEventNoLoginTried)
 					dataprovider.ExecutePostLoginHook("", dataprovider.LoginMethodNoAuthTryed, ip, c.GetProtocol(),
 						dataprovider.ErrNoAuthTryed)
 				}

common/common_test.go

@@ -99,7 +99,11 @@ func TestMain(m *testing.M) {
 		os.Exit(1)
 	}
 	logger.InfoToConsole("Starting COMMON tests, provider: %v", driver)
-	Initialize(Configuration{})
+	err = Initialize(Configuration{})
+	if err != nil {
+		logger.WarnToConsole("error initializing common: %v", err)
+		os.Exit(1)
+	}
 	httpConfig := httpclient.Config{
 		Timeout: 5,
 	}
@@ -225,6 +229,40 @@ func TestSSHConnections(t *testing.T) {
 	assert.NoError(t, sshConn3.Close())
 }
 
+func TestDefenderIntegration(t *testing.T) {
+	// by default defender is nil
+	configCopy := Config
+
+	ip := "127.1.1.1"
+
+	AddDefenderEvent(ip, HostEventNoLoginTried)
+	assert.False(t, IsBanned(ip))
+
+	Config.DefenderConfig = DefenderConfig{
+		Enabled:          true,
+		BanTime:          10,
+		BanTimeIncrement: 50,
+		Threshold:        0,
+		ScoreInvalid:     2,
+		ScoreValid:       1,
+		ObservationTime:  15,
+		EntriesSoftLimit: 100,
+		EntriesHardLimit: 150,
+	}
+	err := Initialize(Config)
+	assert.Error(t, err)
+	Config.DefenderConfig.Threshold = 3
+	err = Initialize(Config)
+	assert.NoError(t, err)
+
+	AddDefenderEvent(ip, HostEventNoLoginTried)
+	assert.False(t, IsBanned(ip))
+	AddDefenderEvent(ip, HostEventLoginFailed)
+	assert.True(t, IsBanned(ip))
+
+	Config = configCopy
+}
+
 func TestMaxConnections(t *testing.T) {
 	oldValue := Config.MaxTotalConnections
 	Config.MaxTotalConnections = 1
@@ -249,7 +287,8 @@ func TestIdleConnections(t *testing.T) {
 	configCopy := Config
 
 	Config.IdleTimeout = 1
-	Initialize(Config)
+	err := Initialize(Config)
+	assert.NoError(t, err)
 
 	conn1, conn2 := net.Pipe()
 	customConn1 := &customNetConn{
@@ -520,39 +559,36 @@ func TestProxyProtocol(t *testing.T) {
 func TestPostConnectHook(t *testing.T) {
 	Config.PostConnectHook = ""
 
-	remoteAddr := &net.IPAddr{
-		IP:   net.ParseIP("127.0.0.1"),
-		Zone: "",
-	}
+	ipAddr := "127.0.0.1"
 
-	assert.NoError(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolFTP))
+	assert.NoError(t, Config.ExecutePostConnectHook(ipAddr, ProtocolFTP))
 
 	Config.PostConnectHook = "http://foo\x7f.com/"
-	assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolSFTP))
+	assert.Error(t, Config.ExecutePostConnectHook(ipAddr, ProtocolSFTP))
 
 	Config.PostConnectHook = "http://invalid:1234/"
-	assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolSFTP))
+	assert.Error(t, Config.ExecutePostConnectHook(ipAddr, ProtocolSFTP))
 
 	Config.PostConnectHook = fmt.Sprintf("http://%v/404", httpAddr)
-	assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolFTP))
+	assert.Error(t, Config.ExecutePostConnectHook(ipAddr, ProtocolFTP))
 
 	Config.PostConnectHook = fmt.Sprintf("http://%v", httpAddr)
-	assert.NoError(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolFTP))
+	assert.NoError(t, Config.ExecutePostConnectHook(ipAddr, ProtocolFTP))
 
 	Config.PostConnectHook = "invalid"
-	assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolFTP))
+	assert.Error(t, Config.ExecutePostConnectHook(ipAddr, ProtocolFTP))
 
 	if runtime.GOOS == osWindows {
 		Config.PostConnectHook = "C:\\bad\\command"
-		assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolSFTP))
+		assert.Error(t, Config.ExecutePostConnectHook(ipAddr, ProtocolSFTP))
 	} else {
 		Config.PostConnectHook = "/invalid/path"
-		assert.Error(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolSFTP))
+		assert.Error(t, Config.ExecutePostConnectHook(ipAddr, ProtocolSFTP))
 
 		hookCmd, err := exec.LookPath("true")
 		assert.NoError(t, err)
 		Config.PostConnectHook = hookCmd
-		assert.NoError(t, Config.ExecutePostConnectHook(remoteAddr.String(), ProtocolSFTP))
+		assert.NoError(t, Config.ExecutePostConnectHook(ipAddr, ProtocolSFTP))
 	}
 
 	Config.PostConnectHook = ""

common/defender.go

@@ -0,0 +1,439 @@
+package common
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/yl2chen/cidranger"
+
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/utils"
+)
+
+// HostEvent is the enumerable for the support host event
+type HostEvent int
+
+// Supported host events
+const (
+	HostEventLoginFailed HostEvent = iota
+	HostEventUserNotFound
+	HostEventNoLoginTried
+)
+
+// Defender defines the interface that a defender must implements
+type Defender interface {
+	AddEvent(ip string, event HostEvent)
+	IsBanned(ip string) bool
+	GetBanTime(ip string) *time.Time
+	GetScore(ip string) int
+}
+
+// DefenderConfig defines the "defender" configuration
+type DefenderConfig struct {
+	// Set to true to enable the defender
+	Enabled bool `json:"enabled" mapstructure:"enabled"`
+	// BanTime is the number of minutes that a host is banned
+	BanTime int `json:"ban_time" mapstructure:"ban_time"`
+	// Percentage increase of the ban time if a banned host tries to connect again
+	BanTimeIncrement int `json:"ban_time_increment" mapstructure:"ban_time_increment"`
+	// Threshold value for banning a client
+	Threshold int `json:"threshold" mapstructure:"threshold"`
+	// Score for invalid login attempts, eg. non-existent user accounts or
+	// client disconnected for inactivity without authentication attempts
+	ScoreInvalid int `json:"score_invalid" mapstructure:"score_invalid"`
+	// Score for valid login attempts, eg. user accounts that exist
+	ScoreValid int `json:"score_valid" mapstructure:"score_valid"`
+	// Defines the time window, in minutes, for tracking client errors.
+	// A host is banned if it has exceeded the defined threshold during
+	// the last observation time minutes
+	ObservationTime int `json:"observation_time" mapstructure:"observation_time"`
+	// The number of banned IPs and host scores kept in memory will vary between the
+	// soft and hard limit
+	EntriesSoftLimit int `json:"entries_soft_limit" mapstructure:"entries_soft_limit"`
+	EntriesHardLimit int `json:"entries_hard_limit" mapstructure:"entries_hard_limit"`
+	// Path to a file with a list of ip addresses and/or networks to never ban
+	SafeListFile string `json:"safelist_file" mapstructure:"safelist_file"`
+	// Path to a file with a list of ip addresses and/or networks to always ban
+	BlockListFile string `json:"blocklist_file" mapstructure:"blocklist_file"`
+}
+
+type memoryDefender struct {
+	config *DefenderConfig
+	sync.RWMutex
+	// IP addresses of the clients trying to connected are stored inside hosts,
+	// they are added to banned once the thresold is reached.
+	// A violation from a banned host will increase the ban time
+	// based on the configured BanTimeIncrement
+	hosts     map[string]hostScore // the key is the host IP
+	banned    map[string]time.Time // the key is the host IP
+	safeList  *HostList
+	blockList *HostList
+}
+
+// HostListFile defines the structure expected for safe/block list files
+type HostListFile struct {
+	IPAddresses  []string `json:"addresses"`
+	CIDRNetworks []string `json:"networks"`
+}
+
+// HostList defines the structure used to keep the HostListFile in memory
+type HostList struct {
+	IPAddresses map[string]bool
+	Ranges      cidranger.Ranger
+}
+
+func (h *HostList) isListed(ip string) bool {
+	if _, ok := h.IPAddresses[ip]; ok {
+		return true
+	}
+
+	ok, err := h.Ranges.Contains(net.ParseIP(ip))
+	if err != nil {
+		return false
+	}
+
+	return ok
+}
+
+type hostEvent struct {
+	dateTime time.Time
+	score    int
+}
+
+type hostScore struct {
+	TotalScore int
+	Events     []hostEvent
+}
+
+// validate returns an error if the configuration is invalid
+func (c *DefenderConfig) validate() error {
+	if !c.Enabled {
+		return nil
+	}
+	if c.ScoreInvalid >= c.Threshold {
+		return fmt.Errorf("score_invalid %v cannot be greater than threshold %v", c.ScoreInvalid, c.Threshold)
+	}
+	if c.ScoreValid >= c.Threshold {
+		return fmt.Errorf("score_valid %v cannot be greater than threshold %v", c.ScoreValid, c.Threshold)
+	}
+	if c.BanTime <= 0 {
+		return fmt.Errorf("invalid ban_time %v", c.BanTime)
+	}
+	if c.BanTimeIncrement <= 0 {
+		return fmt.Errorf("invalid ban_time_increment %v", c.BanTimeIncrement)
+	}
+	if c.ObservationTime <= 0 {
+		return fmt.Errorf("invalid observation_time %v", c.ObservationTime)
+	}
+	if c.EntriesSoftLimit <= 0 {
+		return fmt.Errorf("invalid entries_soft_limit %v", c.EntriesSoftLimit)
+	}
+	if c.EntriesHardLimit <= c.EntriesSoftLimit {
+		return fmt.Errorf("invalid entries_hard_limit %v must be > %v", c.EntriesHardLimit, c.EntriesSoftLimit)
+	}
+
+	return nil
+}
+
+func newInMemoryDefender(config *DefenderConfig) (Defender, error) {
+	err := config.validate()
+	if err != nil {
+		return nil, err
+	}
+	defender := &memoryDefender{
+		config: config,
+		hosts:  make(map[string]hostScore),
+		banned: make(map[string]time.Time),
+	}
+
+	defender.blockList, err = loadHostListFromFile(config.BlockListFile)
+	if err != nil {
+		return nil, err
+	}
+	defender.safeList, err = loadHostListFromFile(config.SafeListFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return defender, nil
+}
+
+// IsBanned returns true if the specified IP is banned
+// and increase ban time if the IP is found.
+// This method must be called as soon as the client connects
+func (d *memoryDefender) IsBanned(ip string) bool {
+	d.RLock()
+
+	if banTime, ok := d.banned[ip]; ok {
+		if banTime.After(time.Now()) {
+			increment := d.config.BanTime * d.config.BanTimeIncrement / 100
+			if increment == 0 {
+				increment++
+			}
+
+			d.RUnlock()
+
+			// we can save an earlier ban time if there are contemporary updates
+			// but this should not make much difference. I prefer to hold a read lock
+			// until possible for performance reasons, this method is called each
+			// time a new client connects and it must be as fast as possible
+			d.Lock()
+			d.banned[ip] = banTime.Add(time.Duration(increment) * time.Minute)
+			d.Unlock()
+
+			return true
+		}
+	}
+
+	defer d.RUnlock()
+
+	if d.blockList != nil && d.blockList.isListed(ip) {
+		// permanent ban
+		return true
+	}
+
+	return false
+}
+
+// AddEvent adds an event for the given IP.
+// This method must be called for clients not yet banned
+func (d *memoryDefender) AddEvent(ip string, event HostEvent) {
+	d.Lock()
+	defer d.Unlock()
+
+	if d.safeList != nil && d.safeList.isListed(ip) {
+		return
+	}
+
+	var score int
+
+	switch event {
+	case HostEventLoginFailed:
+		score = d.config.ScoreValid
+	case HostEventUserNotFound, HostEventNoLoginTried:
+		score = d.config.ScoreInvalid
+	}
+
+	ev := hostEvent{
+		dateTime: time.Now(),
+		score:    score,
+	}
+
+	if hs, ok := d.hosts[ip]; ok {
+		hs.Events = append(hs.Events, ev)
+		hs.TotalScore = 0
+
+		idx := 0
+		for _, event := range hs.Events {
+			if event.dateTime.Add(time.Duration(d.config.ObservationTime) * time.Minute).After(time.Now()) {
+				hs.Events[idx] = event
+				hs.TotalScore += event.score
+				idx++
+			}
+		}
+
+		hs.Events = hs.Events[:idx]
+		if hs.TotalScore >= d.config.Threshold {
+			d.banned[ip] = time.Now().Add(time.Duration(d.config.BanTime) * time.Minute)
+			delete(d.hosts, ip)
+			d.cleanupBanned()
+		} else {
+			d.hosts[ip] = hs
+		}
+	} else {
+		d.hosts[ip] = hostScore{
+			TotalScore: ev.score,
+			Events:     []hostEvent{ev},
+		}
+		d.cleanupHosts()
+	}
+}
+
+func (d *memoryDefender) countBanned() int {
+	d.RLock()
+	defer d.RUnlock()
+
+	return len(d.banned)
+}
+
+func (d *memoryDefender) countHosts() int {
+	d.RLock()
+	defer d.RUnlock()
+
+	return len(d.hosts)
+}
+
+// GetBanTime returns the ban time for the given IP or nil if the IP is not banned
+func (d *memoryDefender) GetBanTime(ip string) *time.Time {
+	d.RLock()
+	defer d.RUnlock()
+
+	if banTime, ok := d.banned[ip]; ok {
+		return &banTime
+	}
+
+	return nil
+}
+
+// GetScore returns the score for the given IP
+func (d *memoryDefender) GetScore(ip string) int {
+	d.RLock()
+	defer d.RUnlock()
+
+	score := 0
+
+	if hs, ok := d.hosts[ip]; ok {
+		for _, event := range hs.Events {
+			if event.dateTime.Add(time.Duration(d.config.ObservationTime) * time.Minute).After(time.Now()) {
+				score += event.score
+			}
+		}
+	}
+
+	return score
+}
+
+func (d *memoryDefender) cleanupBanned() {
+	if len(d.banned) > d.config.EntriesHardLimit {
+		kvList := make(kvList, 0, len(d.banned))
+
+		for k, v := range d.banned {
+			if v.Before(time.Now()) {
+				delete(d.banned, k)
+			}
+
+			kvList = append(kvList, kv{
+				Key:   k,
+				Value: v.UnixNano(),
+			})
+		}
+
+		// we removed expired ip addresses, if any, above, this could be enough
+		numToRemove := len(d.banned) - d.config.EntriesSoftLimit
+
+		if numToRemove <= 0 {
+			return
+		}
+
+		sort.Sort(kvList)
+
+		for idx, kv := range kvList {
+			if idx >= numToRemove {
+				break
+			}
+
+			delete(d.banned, kv.Key)
+		}
+	}
+}
+
+func (d *memoryDefender) cleanupHosts() {
+	if len(d.hosts) > d.config.EntriesHardLimit {
+		kvList := make(kvList, 0, len(d.hosts))
+
+		for k, v := range d.hosts {
+			value := int64(0)
+			if len(v.Events) > 0 {
+				value = v.Events[len(v.Events)-1].dateTime.UnixNano()
+			}
+			kvList = append(kvList, kv{
+				Key:   k,
+				Value: value,
+			})
+		}
+
+		sort.Sort(kvList)
+
+		numToRemove := len(d.hosts) - d.config.EntriesSoftLimit
+
+		for idx, kv := range kvList {
+			if idx >= numToRemove {
+				break
+			}
+
+			delete(d.hosts, kv.Key)
+		}
+	}
+}
+
+func loadHostListFromFile(name string) (*HostList, error) {
+	if name == "" {
+		return nil, nil
+	}
+	if !utils.IsFileInputValid(name) {
+		return nil, fmt.Errorf("invalid host list file name %#v", name)
+	}
+
+	info, err := os.Stat(name)
+	if err != nil {
+		return nil, err
+	}
+
+	// opinionated max size, you should avoid big host lists
+	if info.Size() > 1048576*5 { // 5MB
+		return nil, fmt.Errorf("host list file %#v is too big: %v bytes", name, info.Size())
+	}
+
+	content, err := ioutil.ReadFile(name)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read input file %#v: %v", name, err)
+	}
+
+	var hostList HostListFile
+
+	err = json.Unmarshal(content, &hostList)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(hostList.CIDRNetworks) > 0 || len(hostList.IPAddresses) > 0 {
+		result := &HostList{
+			IPAddresses: make(map[string]bool),
+			Ranges:      cidranger.NewPCTrieRanger(),
+		}
+		ipCount := 0
+		cdrCount := 0
+		for _, ip := range hostList.IPAddresses {
+			if net.ParseIP(ip) == nil {
+				logger.Warn(logSender, "", "unable to parse IP %#v", ip)
+				continue
+			}
+			result.IPAddresses[ip] = true
+			ipCount++
+		}
+		for _, cidrNet := range hostList.CIDRNetworks {
+			_, network, err := net.ParseCIDR(cidrNet)
+			if err != nil {
+				logger.Warn(logSender, "", "unable to parse CIDR network %#v", cidrNet)
+				continue
+			}
+			err = result.Ranges.Insert(cidranger.NewBasicRangerEntry(*network))
+			if err == nil {
+				cdrCount++
+			}
+		}
+
+		logger.Info(logSender, "", "list %#v loaded, ip addresses loaded: %v/%v networks loaded: %v/%v",
+			name, ipCount, len(hostList.IPAddresses), cdrCount, len(hostList.CIDRNetworks))
+		return result, nil
+	}
+
+	return nil, nil
+}
+
+type kv struct {
+	Key   string
+	Value int64
+}
+
+type kvList []kv
+
+func (p kvList) Len() int           { return len(p) }
+func (p kvList) Less(i, j int) bool { return p[i].Value < p[j].Value }
+func (p kvList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

common/defender_test.go

@@ -0,0 +1,520 @@
+package common
+
+import (
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/yl2chen/cidranger"
+)
+
+func TestBasicDefender(t *testing.T) {
+	bl := HostListFile{
+		IPAddresses:  []string{"172.16.1.1", "172.16.1.2"},
+		CIDRNetworks: []string{"10.8.0.0/24"},
+	}
+	sl := HostListFile{
+		IPAddresses:  []string{"172.16.1.3", "172.16.1.4"},
+		CIDRNetworks: []string{"192.168.8.0/24"},
+	}
+	blFile := filepath.Join(os.TempDir(), "bl.json")
+	slFile := filepath.Join(os.TempDir(), "sl.json")
+
+	data, err := json.Marshal(bl)
+	assert.NoError(t, err)
+
+	err = ioutil.WriteFile(blFile, data, os.ModePerm)
+	assert.NoError(t, err)
+
+	data, err = json.Marshal(sl)
+	assert.NoError(t, err)
+
+	err = ioutil.WriteFile(slFile, data, os.ModePerm)
+	assert.NoError(t, err)
+
+	config := &DefenderConfig{
+		Enabled:          true,
+		BanTime:          10,
+		BanTimeIncrement: 2,
+		Threshold:        5,
+		ScoreInvalid:     2,
+		ScoreValid:       1,
+		ObservationTime:  15,
+		EntriesSoftLimit: 1,
+		EntriesHardLimit: 2,
+		SafeListFile:     "slFile",
+		BlockListFile:    "blFile",
+	}
+
+	_, err = newInMemoryDefender(config)
+	assert.Error(t, err)
+	config.BlockListFile = blFile
+	_, err = newInMemoryDefender(config)
+	assert.Error(t, err)
+	config.SafeListFile = slFile
+	d, err := newInMemoryDefender(config)
+	assert.NoError(t, err)
+
+	defender := d.(*memoryDefender)
+	assert.True(t, defender.IsBanned("172.16.1.1"))
+	assert.False(t, defender.IsBanned("172.16.1.10"))
+	assert.False(t, defender.IsBanned("10.8.2.3"))
+	assert.True(t, defender.IsBanned("10.8.0.3"))
+	assert.False(t, defender.IsBanned("invalid ip"))
+	assert.Equal(t, 0, defender.countBanned())
+	assert.Equal(t, 0, defender.countHosts())
+
+	defender.AddEvent("172.16.1.4", HostEventLoginFailed)
+	defender.AddEvent("192.168.8.4", HostEventUserNotFound)
+	assert.Equal(t, 0, defender.countHosts())
+
+	testIP := "12.34.56.78"
+	defender.AddEvent(testIP, HostEventLoginFailed)
+	assert.Equal(t, 1, defender.countHosts())
+	assert.Equal(t, 0, defender.countBanned())
+	assert.Equal(t, 1, defender.GetScore(testIP))
+	assert.Nil(t, defender.GetBanTime(testIP))
+	defender.AddEvent(testIP, HostEventNoLoginTried)
+	assert.Equal(t, 1, defender.countHosts())
+	assert.Equal(t, 0, defender.countBanned())
+	assert.Equal(t, 3, defender.GetScore(testIP))
+	defender.AddEvent(testIP, HostEventNoLoginTried)
+	assert.Equal(t, 0, defender.countHosts())
+	assert.Equal(t, 1, defender.countBanned())
+	assert.Equal(t, 0, defender.GetScore(testIP))
+	assert.NotNil(t, defender.GetBanTime(testIP))
+
+	// now test cleanup, testIP is already banned
+	testIP1 := "12.34.56.79"
+	testIP2 := "12.34.56.80"
+	testIP3 := "12.34.56.81"
+
+	defender.AddEvent(testIP1, HostEventNoLoginTried)
+	defender.AddEvent(testIP2, HostEventNoLoginTried)
+	assert.Equal(t, 2, defender.countHosts())
+	time.Sleep(20 * time.Millisecond)
+	defender.AddEvent(testIP3, HostEventNoLoginTried)
+	assert.Equal(t, defender.config.EntriesSoftLimit, defender.countHosts())
+	// testIP1 and testIP2 should be removed
+	assert.Equal(t, defender.config.EntriesSoftLimit, defender.countHosts())
+	assert.Equal(t, 0, defender.GetScore(testIP1))
+	assert.Equal(t, 0, defender.GetScore(testIP2))
+	assert.Equal(t, 2, defender.GetScore(testIP3))
+
+	defender.AddEvent(testIP3, HostEventNoLoginTried)
+	defender.AddEvent(testIP3, HostEventNoLoginTried)
+	// IP3 is now banned
+	assert.NotNil(t, defender.GetBanTime(testIP3))
+	assert.Equal(t, 0, defender.countHosts())
+
+	time.Sleep(20 * time.Millisecond)
+	for i := 0; i < 3; i++ {
+		defender.AddEvent(testIP1, HostEventNoLoginTried)
+	}
+	assert.Equal(t, 0, defender.countHosts())
+	assert.Equal(t, config.EntriesSoftLimit, defender.countBanned())
+	assert.Nil(t, defender.GetBanTime(testIP))
+	assert.Nil(t, defender.GetBanTime(testIP3))
+	assert.NotNil(t, defender.GetBanTime(testIP1))
+
+	for i := 0; i < 3; i++ {
+		defender.AddEvent(testIP, HostEventNoLoginTried)
+		time.Sleep(10 * time.Millisecond)
+		defender.AddEvent(testIP3, HostEventNoLoginTried)
+	}
+	assert.Equal(t, 0, defender.countHosts())
+	assert.Equal(t, defender.config.EntriesSoftLimit, defender.countBanned())
+
+	banTime := defender.GetBanTime(testIP3)
+	if assert.NotNil(t, banTime) {
+		assert.True(t, defender.IsBanned(testIP3))
+		// ban time should increase
+		newBanTime := defender.GetBanTime(testIP3)
+		assert.True(t, newBanTime.After(*banTime))
+	}
+
+	err = os.Remove(slFile)
+	assert.NoError(t, err)
+	err = os.Remove(blFile)
+	assert.NoError(t, err)
+}
+
+func TestLoadHostListFromFile(t *testing.T) {
+	_, err := loadHostListFromFile(".")
+	assert.Error(t, err)
+
+	hostsFilePath := filepath.Join(os.TempDir(), "hostfile")
+	content := make([]byte, 1048576*6)
+	_, err = rand.Read(content)
+	assert.NoError(t, err)
+
+	err = ioutil.WriteFile(hostsFilePath, content, os.ModePerm)
+	assert.NoError(t, err)
+
+	_, err = loadHostListFromFile(hostsFilePath)
+	assert.Error(t, err)
+
+	hl := HostListFile{
+		IPAddresses:  []string{},
+		CIDRNetworks: []string{},
+	}
+
+	asJSON, err := json.Marshal(hl)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(hostsFilePath, asJSON, os.ModePerm)
+	assert.NoError(t, err)
+
+	hostList, err := loadHostListFromFile(hostsFilePath)
+	assert.NoError(t, err)
+	assert.Nil(t, hostList)
+
+	hl.IPAddresses = append(hl.IPAddresses, "invalidip")
+	asJSON, err = json.Marshal(hl)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(hostsFilePath, asJSON, os.ModePerm)
+	assert.NoError(t, err)
+
+	hostList, err = loadHostListFromFile(hostsFilePath)
+	assert.NoError(t, err)
+	assert.Len(t, hostList.IPAddresses, 0)
+
+	hl.IPAddresses = nil
+	hl.CIDRNetworks = append(hl.CIDRNetworks, "invalid net")
+
+	asJSON, err = json.Marshal(hl)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(hostsFilePath, asJSON, os.ModePerm)
+	assert.NoError(t, err)
+
+	hostList, err = loadHostListFromFile(hostsFilePath)
+	assert.NoError(t, err)
+	assert.NotNil(t, hostList)
+	assert.Len(t, hostList.IPAddresses, 0)
+	assert.Equal(t, 0, hostList.Ranges.Len())
+
+	if runtime.GOOS != "windows" {
+		err = os.Chmod(hostsFilePath, 0111)
+		assert.NoError(t, err)
+
+		_, err = loadHostListFromFile(hostsFilePath)
+		assert.Error(t, err)
+
+		err = os.Chmod(hostsFilePath, 0644)
+		assert.NoError(t, err)
+	}
+
+	err = ioutil.WriteFile(hostsFilePath, []byte("non json content"), os.ModePerm)
+	assert.NoError(t, err)
+	_, err = loadHostListFromFile(hostsFilePath)
+	assert.Error(t, err)
+
+	err = os.Remove(hostsFilePath)
+	assert.NoError(t, err)
+}
+
+func TestDefenderCleanup(t *testing.T) {
+	d := memoryDefender{
+		banned: make(map[string]time.Time),
+		hosts:  make(map[string]hostScore),
+		config: &DefenderConfig{
+			ObservationTime:  1,
+			EntriesSoftLimit: 2,
+			EntriesHardLimit: 3,
+		},
+	}
+
+	d.banned["1.1.1.1"] = time.Now().Add(-24 * time.Hour)
+	d.banned["1.1.1.2"] = time.Now().Add(-24 * time.Hour)
+	d.banned["1.1.1.3"] = time.Now().Add(-24 * time.Hour)
+	d.banned["1.1.1.4"] = time.Now().Add(-24 * time.Hour)
+
+	d.cleanupBanned()
+	assert.Equal(t, 0, d.countBanned())
+
+	d.banned["2.2.2.2"] = time.Now().Add(2 * time.Minute)
+	d.banned["2.2.2.3"] = time.Now().Add(1 * time.Minute)
+	d.banned["2.2.2.4"] = time.Now().Add(3 * time.Minute)
+	d.banned["2.2.2.5"] = time.Now().Add(4 * time.Minute)
+
+	d.cleanupBanned()
+	assert.Equal(t, d.config.EntriesSoftLimit, d.countBanned())
+	assert.Nil(t, d.GetBanTime("2.2.2.3"))
+
+	d.hosts["3.3.3.3"] = hostScore{
+		TotalScore: 0,
+		Events: []hostEvent{
+			{
+				dateTime: time.Now().Add(-5 * time.Minute),
+				score:    1,
+			},
+			{
+				dateTime: time.Now().Add(-3 * time.Minute),
+				score:    1,
+			},
+			{
+				dateTime: time.Now(),
+				score:    1,
+			},
+		},
+	}
+	d.hosts["3.3.3.4"] = hostScore{
+		TotalScore: 1,
+		Events: []hostEvent{
+			{
+				dateTime: time.Now().Add(-3 * time.Minute),
+				score:    1,
+			},
+		},
+	}
+	d.hosts["3.3.3.5"] = hostScore{
+		TotalScore: 1,
+		Events: []hostEvent{
+			{
+				dateTime: time.Now().Add(-2 * time.Minute),
+				score:    1,
+			},
+		},
+	}
+	d.hosts["3.3.3.6"] = hostScore{
+		TotalScore: 1,
+		Events: []hostEvent{
+			{
+				dateTime: time.Now().Add(-1 * time.Minute),
+				score:    1,
+			},
+		},
+	}
+
+	assert.Equal(t, 1, d.GetScore("3.3.3.3"))
+
+	d.cleanupHosts()
+	assert.Equal(t, d.config.EntriesSoftLimit, d.countHosts())
+	assert.Equal(t, 0, d.GetScore("3.3.3.4"))
+}
+
+func TestDefenderConfig(t *testing.T) {
+	c := DefenderConfig{}
+	err := c.validate()
+	require.NoError(t, err)
+
+	c.Enabled = true
+	c.Threshold = 10
+	c.ScoreInvalid = 10
+	err = c.validate()
+	require.Error(t, err)
+
+	c.ScoreInvalid = 2
+	c.ScoreValid = 10
+	err = c.validate()
+	require.Error(t, err)
+
+	c.ScoreValid = 1
+	c.BanTime = 0
+	err = c.validate()
+	require.Error(t, err)
+
+	c.BanTime = 30
+	c.BanTimeIncrement = 0
+	err = c.validate()
+	require.Error(t, err)
+
+	c.BanTimeIncrement = 50
+	c.ObservationTime = 0
+	err = c.validate()
+	require.Error(t, err)
+
+	c.ObservationTime = 30
+	err = c.validate()
+	require.Error(t, err)
+
+	c.EntriesSoftLimit = 10
+	err = c.validate()
+	require.Error(t, err)
+
+	c.EntriesHardLimit = 10
+	err = c.validate()
+	require.Error(t, err)
+
+	c.EntriesHardLimit = 20
+	err = c.validate()
+	require.NoError(t, err)
+}
+
+func BenchmarkDefenderBannedSearch(b *testing.B) {
+	d := getDefenderForBench()
+
+	ip, ipnet, err := net.ParseCIDR("10.8.0.0/12") // 1048574 ip addresses
+	if err != nil {
+		panic(err)
+	}
+
+	for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) {
+		d.banned[ip.String()] = time.Now().Add(10 * time.Minute)
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		d.IsBanned("192.168.1.1")
+	}
+}
+
+func BenchmarkCleanup(b *testing.B) {
+	d := getDefenderForBench()
+
+	ip, ipnet, err := net.ParseCIDR("192.168.4.0/24")
+	if err != nil {
+		panic(err)
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) {
+			d.AddEvent(ip.String(), HostEventLoginFailed)
+			if d.countHosts() > d.config.EntriesHardLimit {
+				panic("too many hosts")
+			}
+			if d.countBanned() > d.config.EntriesSoftLimit {
+				panic("too many ip banned")
+			}
+		}
+	}
+}
+
+func BenchmarkDefenderBannedSearchWithBlockList(b *testing.B) {
+	d := getDefenderForBench()
+
+	d.blockList = &HostList{
+		IPAddresses: make(map[string]bool),
+		Ranges:      cidranger.NewPCTrieRanger(),
+	}
+
+	ip, ipnet, err := net.ParseCIDR("129.8.0.0/12") // 1048574 ip addresses
+	if err != nil {
+		panic(err)
+	}
+
+	for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) {
+		d.banned[ip.String()] = time.Now().Add(10 * time.Minute)
+		d.blockList.IPAddresses[ip.String()] = true
+	}
+
+	for i := 0; i < 255; i++ {
+		cidr := fmt.Sprintf("10.8.%v.1/24", i)
+		_, network, _ := net.ParseCIDR(cidr)
+		if err := d.blockList.Ranges.Insert(cidranger.NewBasicRangerEntry(*network)); err != nil {
+			panic(err)
+		}
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		d.IsBanned("192.168.1.1")
+	}
+}
+
+func BenchmarkHostListSearch(b *testing.B) {
+	hostlist := &HostList{
+		IPAddresses: make(map[string]bool),
+		Ranges:      cidranger.NewPCTrieRanger(),
+	}
+
+	ip, ipnet, _ := net.ParseCIDR("172.16.0.0/16")
+
+	for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) {
+		hostlist.IPAddresses[ip.String()] = true
+	}
+
+	for i := 0; i < 255; i++ {
+		cidr := fmt.Sprintf("10.8.%v.1/24", i)
+		_, network, _ := net.ParseCIDR(cidr)
+		if err := hostlist.Ranges.Insert(cidranger.NewBasicRangerEntry(*network)); err != nil {
+			panic(err)
+		}
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		if hostlist.isListed("192.167.1.2") {
+			panic("should not be listed")
+		}
+	}
+}
+
+func BenchmarkCIDRanger(b *testing.B) {
+	ranger := cidranger.NewPCTrieRanger()
+	for i := 0; i < 255; i++ {
+		cidr := fmt.Sprintf("192.168.%v.1/24", i)
+		_, network, _ := net.ParseCIDR(cidr)
+		if err := ranger.Insert(cidranger.NewBasicRangerEntry(*network)); err != nil {
+			panic(err)
+		}
+	}
+
+	ipToMatch := net.ParseIP("192.167.1.2")
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, err := ranger.Contains(ipToMatch); err != nil {
+			panic(err)
+		}
+	}
+}
+
+func BenchmarkNetContains(b *testing.B) {
+	var nets []*net.IPNet
+	for i := 0; i < 255; i++ {
+		cidr := fmt.Sprintf("192.168.%v.1/24", i)
+		_, network, _ := net.ParseCIDR(cidr)
+		nets = append(nets, network)
+	}
+
+	ipToMatch := net.ParseIP("192.167.1.1")
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for _, n := range nets {
+			n.Contains(ipToMatch)
+		}
+	}
+}
+
+func getDefenderForBench() *memoryDefender {
+	config := &DefenderConfig{
+		Enabled:          true,
+		BanTime:          30,
+		BanTimeIncrement: 50,
+		Threshold:        10,
+		ScoreInvalid:     2,
+		ScoreValid:       2,
+		ObservationTime:  30,
+		EntriesSoftLimit: 50,
+		EntriesHardLimit: 100,
+	}
+	return &memoryDefender{
+		config: config,
+		hosts:  make(map[string]hostScore),
+		banned: make(map[string]time.Time),
+	}
+}
+
+func inc(ip net.IP) {
+	for j := len(ip) - 1; j >= 0; j-- {
+		ip[j]++
+		if ip[j] > 0 {
+			break
+		}
+	}
+}

config/config.go

@@ -93,6 +93,19 @@ func Init() {
 			ProxyAllowed:        []string{},
 			PostConnectHook:     "",
 			MaxTotalConnections: 0,
+			DefenderConfig: common.DefenderConfig{
+				Enabled:          false,
+				BanTime:          30,
+				BanTimeIncrement: 50,
+				Threshold:        15,
+				ScoreInvalid:     2,
+				ScoreValid:       1,
+				ObservationTime:  30,
+				EntriesSoftLimit: 100,
+				EntriesHardLimit: 150,
+				SafeListFile:     "",
+				BlockListFile:    "",
+			},
 		},
 		SFTPD: sftpd.Configuration{
 			Banner:                  defaultSFTPDBanner,
@@ -666,6 +679,17 @@ func setViperDefaults() {
 	viper.SetDefault("common.proxy_allowed", globalConf.Common.ProxyAllowed)
 	viper.SetDefault("common.post_connect_hook", globalConf.Common.PostConnectHook)
 	viper.SetDefault("common.max_total_connections", globalConf.Common.MaxTotalConnections)
+	viper.SetDefault("common.defender.enabled", globalConf.Common.DefenderConfig.Enabled)
+	viper.SetDefault("common.defender.ban_time", globalConf.Common.DefenderConfig.BanTime)
+	viper.SetDefault("common.defender.ban_time_increment", globalConf.Common.DefenderConfig.BanTimeIncrement)
+	viper.SetDefault("common.defender.threshold", globalConf.Common.DefenderConfig.Threshold)
+	viper.SetDefault("common.defender.score_invalid", globalConf.Common.DefenderConfig.ScoreInvalid)
+	viper.SetDefault("common.defender.score_valid", globalConf.Common.DefenderConfig.ScoreValid)
+	viper.SetDefault("common.defender.observation_time", globalConf.Common.DefenderConfig.ObservationTime)
+	viper.SetDefault("common.defender.entries_soft_limit", globalConf.Common.DefenderConfig.EntriesSoftLimit)
+	viper.SetDefault("common.defender.entries_hard_limit", globalConf.Common.DefenderConfig.EntriesHardLimit)
+	viper.SetDefault("common.defender.safelist_file", globalConf.Common.DefenderConfig.SafeListFile)
+	viper.SetDefault("common.defender.blocklist_file", globalConf.Common.DefenderConfig.BlockListFile)
 	viper.SetDefault("sftpd.max_auth_tries", globalConf.SFTPD.MaxAuthTries)
 	viper.SetDefault("sftpd.banner", globalConf.SFTPD.Banner)
 	viper.SetDefault("sftpd.host_keys", globalConf.SFTPD.HostKeys)

dataprovider/dataprovider.go

@@ -106,7 +106,7 @@ var (
 	// ErrNoInitRequired defines the error returned by InitProvider if no inizialization/update is required
 	ErrNoInitRequired = errors.New("The data provider is already up to date")
 	// ErrInvalidCredentials defines the error to return if the supplied credentials are invalid
-	ErrInvalidCredentials = errors.New("Invalid credentials")
+	ErrInvalidCredentials = errors.New("invalid credentials")
 	webDAVUsersCache      sync.Map
 	config                Config
 	provider              Provider
@@ -352,7 +352,7 @@ type RecordNotFoundError struct {
 }
 
 func (e *RecordNotFoundError) Error() string {
-	return fmt.Sprintf("Not found: %s", e.err)
+	return fmt.Sprintf("not found: %s", e.err)
 }
 
 // GetQuotaTracking returns the configured mode for user's quota tracking

docs/defender.md

@@ -0,0 +1,53 @@
+# Defender
+
+The experimental built-in `defender` allows you to configure an auto-blocking policy for SFTPGo and thus helps to prevent DoS (Denial of Service) and brute force password guessing.
+
+If enabled it will protect SFTP, FTP and WebDAV services and it will automatically block hosts (IP addresses) that continually fail to log in or attempt to connect.
+
+You can configure a score for each event type:
+
+- `score_valid` defines the score for valid login attempts, eg. user accounts that exist. Default `1`.
+- `score_invalid` defines the score for invalid login attempts, eg. non-existent user accounts or client disconnected for inactivity without authentication attempts. Default `2`.
+
+And then you can configure:
+
+- `observation_time` defines the time window, in minutes, for tracking client errors.
+- `threshold` defines the threshold value before banning a host.
+- `ban_time` defines the time to ban a client, as minutes
+
+So a host is banned, for `ban_time` minutes, if it has exceeded the defined threshold during the last observation time minutes.
+
+If an already banned client tries to log in again its ban time will be incremented based on the `ban_time_increment` configuration.
+
+The `ban_time_increment` is calculated as percentage of `ban_time`, so if `ban_time` is 30 minutes and `ban_time_increment` is 50 the host will be banned for additionally 15 minutes. You can specify values greater than 100 for `ban_time_increment`.
+
+The `defender` will keep in memory both the host scores and the banned hosts, you can limit the memory usage using the `entries_soft_limit` and `entries_hard_limit` configuration keys.
+
+The `defender` can also load a permanent block and/or safe list of ip addresses/networks from a file:
+
+- `safelist_file`, string. Path to a file with a list of ip addresses and/or networks to never ban.
+- `blocklist_file`, string. Path to a file with a list of ip addresses and/or networks to always ban.
+
+These list must be stored as JSON with the following schema:
+
+- `addresses`, list of strings. Each string must be a valid IPv4/IPv6 address.
+- `networks`, list of strings. Each string must be a valid IPv4/IPv6 CIDR address.
+
+Here is a small example:
+
+```json
+{
+    "addresses":[
+        "192.0.2.1",
+        "2001:db8::68"
+    ],
+    "networks":[
+        "192.0.2.1/24",
+        "2001:db8:1234::/48"
+    ]
+}
+```
+
+These list will be loaded in memory for faster lookups.
+
+The `defender` is optimized for fast and time constant lookups however as it keeps all the lists and the entries in memory you should carefully measure the memory requirements for your use case.

docs/full-configuration.md

@@ -64,6 +64,18 @@ The configuration file contains the following sections:
     - If `proxy_protocol` is set to 2 and we receive a proxy header from an IP that is not in the list then the connection will be rejected
   - `post_connect_hook`, string. Absolute path to the command to execute or HTTP URL to notify. See [Post connect hook](./post-connect-hook.md) for more details. Leave empty to disable
   - `max_total_connections`, integer. Maximum number of concurrent client connections. 0 means unlimited
+  - `defender`, struct containing the defender configuration. See [Defender](./defender.md) for more details.
+    - `enabled`, boolean. Default `false`.
+    - `ban_time`, integer. Ban time in minutes.
+    - `ban_time_increment`, integer. Ban time increment, as a percentage, if a banned host tries to connect again.
+    - `threshold`, integer. Threshold value for banning a client.
+    - `score_invalid`, integer. Score for invalid login attempts, eg. non-existent user accounts or client disconnected for inactivity without authentication attempts.
+    - `score_valid`, integer. Score for valid login attempts, eg. user accounts that exist.
+    - `observation_time`, integer. Defines the time window, in minutes, for tracking client errors. A host is banned if it has exceeded the defined threshold during the last observation time minutes.
+    - `entries_soft_limit`, integer.
+    - `entries_hard_limit`, integer. The number of banned IPs and host scores kept in memory will vary between the soft and hard limit.
+    - `safelist_file`, string. Path to a file with a list of ip addresses and/or networks to never ban.
+    - `blocklist_file`, string. Path to a file with a list of ip addresses and/or networks to always ban.
 - **"sftpd"**, the configuration for the SFTP server
   - `bindings`, list of structs. Each struct has the following fields:
     - `port`, integer. The port used for serving SFTP requests. 0 means disabled. Default: 2022

ftpd/ftpd_test.go

@@ -127,8 +127,11 @@ func TestMain(m *testing.M) {
 		os.Exit(1)
 	}
 
-	common.Initialize(commonConf)
-
+	err = common.Initialize(commonConf)
+	if err != nil {
+		logger.WarnToConsole("error initializing common: %v", err)
+		os.Exit(1)
+	}
 	err = dataprovider.Initialize(providerConf, configDir)
 	if err != nil {
 		logger.ErrorToConsole("error initializing data provider: %v", err)
@@ -233,12 +236,7 @@ func TestMain(m *testing.M) {
 	}()
 
 	waitTCPListening(ftpdConf.Bindings[0].GetAddress())
-
-	// ensure all the initial connections to check if the service is alive are disconnected
-	time.Sleep(100 * time.Millisecond)
-	for len(common.Connections.GetStats()) > 0 {
-		time.Sleep(50 * time.Millisecond)
-	}
+	waitNoConnections()
 
 	exitCode := m.Run()
 	os.Remove(logFilePath)
@@ -402,6 +400,12 @@ func TestLoginInvalidPwd(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestLoginNonExistentUser(t *testing.T) {
+	user := getTestUser()
+	_, err := getFTPClient(user, false)
+	assert.Error(t, err)
+}
+
 func TestLoginExternalAuth(t *testing.T) {
 	if runtime.GOOS == osWindows {
 		t.Skip("this test is not available on Windows")
@@ -599,6 +603,47 @@ func TestMaxConnections(t *testing.T) {
 	common.Config.MaxTotalConnections = oldValue
 }
 
+func TestDefender(t *testing.T) {
+	oldConfig := config.GetCommonConfig()
+
+	cfg := config.GetCommonConfig()
+	cfg.DefenderConfig.Enabled = true
+	cfg.DefenderConfig.Threshold = 3
+
+	err := common.Initialize(cfg)
+	assert.NoError(t, err)
+
+	user, _, err := httpd.AddUser(getTestUser(), http.StatusOK)
+	assert.NoError(t, err)
+	client, err := getFTPClient(user, false)
+	if assert.NoError(t, err) {
+		err = checkBasicFTP(client)
+		assert.NoError(t, err)
+		err = client.Quit()
+		assert.NoError(t, err)
+	}
+
+	for i := 0; i < 3; i++ {
+		user.Password = "wrong_pwd"
+		_, err = getFTPClient(user, false)
+		assert.Error(t, err)
+	}
+
+	user.Password = defaultPassword
+	_, err = getFTPClient(user, false)
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "Access denied, banned client IP")
+	}
+
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+
+	err = common.Initialize(oldConfig)
+	assert.NoError(t, err)
+}
+
 func TestMaxSessions(t *testing.T) {
 	u := getTestUser()
 	u.MaxSessions = 1
@@ -2035,6 +2080,13 @@ func waitTCPListening(address string) {
 	}
 }
 
+func waitNoConnections() {
+	time.Sleep(50 * time.Millisecond)
+	for len(common.Connections.GetStats()) > 0 {
+		time.Sleep(50 * time.Millisecond)
+	}
+}
+
 func getTestUser() dataprovider.User {
 	user := dataprovider.User{
 		Username:       defaultUsername,

ftpd/server.go

@@ -103,11 +103,16 @@ func (s *Server) GetSettings() (*ftpserver.Settings, error) {
 
 // ClientConnected is called to send the very first welcome message
 func (s *Server) ClientConnected(cc ftpserver.ClientContext) (string, error) {
+	ipAddr := utils.GetIPFromRemoteAddress(cc.RemoteAddr().String())
+	if common.IsBanned(ipAddr) {
+		logger.Log(logger.LevelDebug, common.ProtocolFTP, "", "connection refused, ip %#v is banned", ipAddr)
+		return "Access denied, banned client IP", common.ErrConnectionDenied
+	}
 	if !common.Connections.IsNewConnectionAllowed() {
 		logger.Log(logger.LevelDebug, common.ProtocolFTP, "", "connection refused, configured limit reached")
 		return "", common.ErrConnectionDenied
 	}
-	if err := common.Config.ExecutePostConnectHook(cc.RemoteAddr().String(), common.ProtocolFTP); err != nil {
+	if err := common.Config.ExecutePostConnectHook(ipAddr, common.ProtocolFTP); err != nil {
 		return "", err
 	}
 	connID := fmt.Sprintf("%v_%v", s.ID, cc.ID())
@@ -128,23 +133,23 @@ func (s *Server) ClientDisconnected(cc ftpserver.ClientContext) {
 
 // AuthUser authenticates the user and selects an handling driver
 func (s *Server) AuthUser(cc ftpserver.ClientContext, username, password string) (ftpserver.ClientDriver, error) {
-	remoteAddr := cc.RemoteAddr().String()
-	user, err := dataprovider.CheckUserAndPass(username, password, utils.GetIPFromRemoteAddress(remoteAddr), common.ProtocolFTP)
+	ipAddr := utils.GetIPFromRemoteAddress(cc.RemoteAddr().String())
+	user, err := dataprovider.CheckUserAndPass(username, password, ipAddr, common.ProtocolFTP)
 	if err != nil {
-		updateLoginMetrics(username, remoteAddr, err)
+		updateLoginMetrics(username, ipAddr, err)
 		return nil, err
 	}
 
 	connection, err := s.validateUser(user, cc)
 
-	defer updateLoginMetrics(username, remoteAddr, err)
+	defer updateLoginMetrics(username, ipAddr, err)
 
 	if err != nil {
 		return nil, err
 	}
 	connection.Fs.CheckRootPath(connection.GetUsername(), user.GetUID(), user.GetGID())
 	connection.Log(logger.LevelInfo, "User id: %d, logged in with FTP, username: %#v, home_dir: %#v remote addr: %#v",
-		user.ID, user.Username, user.HomeDir, remoteAddr)
+		user.ID, user.Username, user.HomeDir, cc.RemoteAddr())
 	dataprovider.UpdateLastLogin(user) //nolint:errcheck
 	return connection, nil
 }
@@ -213,12 +218,16 @@ func (s *Server) validateUser(user dataprovider.User, cc ftpserver.ClientContext
 	return connection, nil
 }
 
-func updateLoginMetrics(username, remoteAddress string, err error) {
+func updateLoginMetrics(username, ip string, err error) {
 	metrics.AddLoginAttempt(dataprovider.LoginMethodPassword)
-	ip := utils.GetIPFromRemoteAddress(remoteAddress)
 	if err != nil {
 		logger.ConnectionFailedLog(username, ip, dataprovider.LoginMethodPassword,
 			common.ProtocolFTP, err.Error())
+		event := common.HostEventLoginFailed
+		if _, ok := err.(*dataprovider.RecordNotFoundError); ok {
+			event = common.HostEventUserNotFound
+		}
+		common.AddDefenderEvent(ip, event)
 	}
 	metrics.AddLoginResult(dataprovider.LoginMethodPassword, err)
 	dataprovider.ExecutePostLoginHook(username, dataprovider.LoginMethodPassword, ip, common.ProtocolFTP, err)

go.mod

@@ -7,16 +7,17 @@ require (
 	cloud.google.com/go/storage v1.12.0
 	github.com/Azure/azure-storage-blob-go v0.12.0
 	github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
-	github.com/alexedwards/argon2id v0.0.0-20200802152012-2464efd3196b
-	github.com/aws/aws-sdk-go v1.36.15
+	github.com/alexedwards/argon2id v0.0.0-20201228115903-cf543ebc1f7b
+	github.com/aws/aws-sdk-go v1.36.18
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
 	github.com/eikenb/pipeat v0.0.0-20200430215831-470df5986b6d
-	github.com/fclairamb/ftpserverlib v0.11.1-0.20201228010033-a81903a5fd21
+	github.com/fclairamb/ftpserverlib v0.12.0
 	github.com/frankban/quicktest v1.11.2 // indirect
 	github.com/go-chi/chi v1.5.1
 	github.com/go-chi/render v1.0.1
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/google/uuid v1.1.3 // indirect
 	github.com/grandcat/zeroconf v1.0.0
 	github.com/jlaffaye/ftp v0.0.0-20201112195030-9aae4d151126
 	github.com/lib/pq v1.9.0
@@ -42,6 +43,7 @@ require (
 	github.com/spf13/viper v1.7.1
 	github.com/stretchr/testify v1.6.1
 	github.com/studio-b12/gowebdav v0.0.0-20200929080739-bdacfab94796
+	github.com/yl2chen/cidranger v1.0.2
 	go.etcd.io/bbolt v1.3.5
 	go.uber.org/automaxprocs v1.3.0
 	gocloud.dev v0.21.0
@@ -50,7 +52,7 @@ require (
 	golang.org/x/net v0.0.0-20201224014010-6772e930b67b
 	golang.org/x/sys v0.0.0-20201223074533-0d417f636930
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
-	golang.org/x/tools v0.0.0-20201226215659-b1c90890d22a // indirect
+	golang.org/x/tools v0.0.0-20201230224404-63754364767c // indirect
 	google.golang.org/api v0.36.0
 	google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect

go.sum

@@ -57,25 +57,32 @@ github.com/Azure/azure-storage-blob-go v0.12.0 h1:7bFXA1QB+lOK2/ASWHhp6/vnxjaeeZ
 github.com/Azure/azure-storage-blob-go v0.12.0/go.mod h1:A0u4VjtpgZJ7Y7um/+ix2DHBuEKFC6sEIlj0xc13a4Q=
 github.com/Azure/go-amqp v0.13.0/go.mod h1:qj+o8xPCz9tMSbQ83Vp8boHahuRDl5mkNHyt1xlxUTs=
 github.com/Azure/go-amqp v0.13.1/go.mod h1:qj+o8xPCz9tMSbQ83Vp8boHahuRDl5mkNHyt1xlxUTs=
+github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.3/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
 github.com/Azure/go-autorest/autorest v0.11.7/go.mod h1:V6p3pKZx1KKkJubbxnDWrzNhEIfOy/pTGasLqzHIPHs=
 github.com/Azure/go-autorest/autorest v0.11.9/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
+github.com/Azure/go-autorest/autorest v0.11.12 h1:gI8ytXbxMfI+IVbI9mP2JGCTXIuhHLgRlvQ9X4PsnHE=
 github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
 github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
 github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE=
 github.com/Azure/go-autorest/autorest/adal v0.9.4/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE=
 github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
+github.com/Azure/go-autorest/autorest/adal v0.9.6 h1:d3pSDwvBWBLqdA91u+keH1zs1cCEzrQdHKY6iqbQNkE=
 github.com/Azure/go-autorest/autorest/adal v0.9.6/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
 github.com/Azure/go-autorest/autorest/azure/auth v0.5.3/go.mod h1:4bJZhUhcq8LB20TruwHbAQsmUs2Xh+QR7utuJpLXX3A=
 github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM=
+github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
 github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
 github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE=
 github.com/Azure/go-autorest/autorest/validation v0.3.0/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E=
 github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 h1:KeNholpO2xKjgaaSyd+DyQRrsQjhbSeS7qe4nEw8aQw=
@@ -85,6 +92,7 @@ github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
@@ -93,8 +101,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/alexedwards/argon2id v0.0.0-20200802152012-2464efd3196b h1:rcCpjI1OMGtBY8nnBvExeM1pXNoaM35zqmXBGpgJR2o=
-github.com/alexedwards/argon2id v0.0.0-20200802152012-2464efd3196b/go.mod h1:GFtu6vaWaRJV5EvSFaVqgq/3Iq95xyYElBV/aupGzUo=
+github.com/alexedwards/argon2id v0.0.0-20201228115903-cf543ebc1f7b h1:jEg+fE+POnmUy40B+aSKEPqZDmsdl55hZU0YKXEzz1k=
+github.com/alexedwards/argon2id v0.0.0-20201228115903-cf543ebc1f7b/go.mod h1:Kmn5t2Rb93Q4NTprN4+CCgARGvigKMJyxP0WckpTUp0=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@@ -106,8 +114,8 @@ github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZo
 github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.36.1/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
-github.com/aws/aws-sdk-go v1.36.15 h1:nGqgPlXegCKPZOKXvWnYCLvLPJPRoSOHHn9d0N0DG7Y=
-github.com/aws/aws-sdk-go v1.36.15/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
+github.com/aws/aws-sdk-go v1.36.18 h1:PvfZkE0cjM1k1EMQDSb2BrX8LETPx0IFFZ/YKkurmFg=
+github.com/aws/aws-sdk-go v1.36.18/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -179,13 +187,15 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
-github.com/fclairamb/ftpserverlib v0.11.1-0.20201228010033-a81903a5fd21 h1:kEcLQsnpN78+LF9XzCHVLV/olyjuxHTQNvKlIEAx/Gc=
-github.com/fclairamb/ftpserverlib v0.11.1-0.20201228010033-a81903a5fd21/go.mod h1:X6sAMSYtN0YDPu+nHfyE9dsKPUOrEZ8O5EMgt1xvPwk=
+github.com/fclairamb/ftpserverlib v0.12.0 h1:vud3Q4v/rLZU5CfIDFaXq7ST2+V9BF5cKjzNWPN18c4=
+github.com/fclairamb/ftpserverlib v0.12.0/go.mod h1:X6sAMSYtN0YDPu+nHfyE9dsKPUOrEZ8O5EMgt1xvPwk=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
 github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y=
+github.com/frankban/quicktest v1.11.2 h1:mjwHjStlXWibxOohM7HYieIViKyh56mmt3+6viyhDDI=
 github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
@@ -204,11 +214,14 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2
 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo=
 github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o=
 github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
@@ -219,6 +232,7 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
@@ -274,12 +288,16 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-replayers/grpcreplay v1.0.0 h1:B5kVOzJ1hBgnevTgIWhSTatQ3608yu/2NnU0Ta1d0kY=
 github.com/google/go-replayers/grpcreplay v1.0.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE=
+github.com/google/go-replayers/httpreplay v0.1.2 h1:HCfx+dQzwN9XbGTHF8qJ+67WN8glL9FTWV5rraCJ/jU=
 github.com/google/go-replayers/httpreplay v0.1.2/go.mod h1:YKZViNhiGgqdBlUbI2MwGpq4pXxNmhJLPHQ7cv2b5no=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE=
 github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -299,6 +317,8 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.3 h1:twObb+9XcuH5B9V1TBCvvvZoO6iEdILi2a76PYn5rJI=
+github.com/google/uuid v1.1.3/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/wire v0.4.0 h1:kXcsA/rIGzJImVqPdhfnr6q0xsS9gU0515q1EPpJ9fE=
 github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
 github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww=
@@ -306,6 +326,7 @@ github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
@@ -332,6 +353,7 @@ github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVo
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI=
 github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
 github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
@@ -373,12 +395,14 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
 github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -389,7 +413,9 @@ github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
@@ -404,8 +430,10 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
@@ -440,6 +468,7 @@ github.com/minio/sio v0.2.1/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebh
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
@@ -465,6 +494,7 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE
 github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
 github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
@@ -485,8 +515,10 @@ github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh
 github.com/otiai10/copy v1.4.1 h1:T1Ggae54qC0G+0VA5B/3CJQorvvaNVStzDn3YLZHdLI=
 github.com/otiai10/copy v1.4.1/go.mod h1:XWfuS3CrI0R6IE0FbgHsEazaXO8G0LpMp9o8tos0x4E=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI=
 github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/otiai10/mint v1.3.2 h1:VYWnrP5fXmz1MXvjuUvcBrXSjGE6xjON+axB/UrpO3E=
 github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
 github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -560,6 +592,7 @@ github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkB
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo=
 github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY=
 github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21R0JnA=
 github.com/shirou/gopsutil/v3 v3.20.11/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4=
@@ -567,7 +600,9 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
@@ -614,6 +649,8 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
+github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -669,6 +706,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI=
 golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
@@ -678,6 +716,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -697,6 +736,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -754,6 +794,7 @@ golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201223074533-0d417f636930 h1:vRgIt+nup/B/BwIS0g2oC0haq0iqbV3ZA+u6+0TlNCo=
 golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -830,7 +871,8 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201203202102-a1a1cbeaa516/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201226215659-b1c90890d22a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201230224404-63754364767c h1:xx3+TTG3yS1I6Ola5Kapxr5vZu85vKkcwKyV6ke9fHA=
+golang.org/x/tools v0.0.0-20201230224404-63754364767c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -868,6 +910,7 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww
 google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -951,6 +994,7 @@ gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUy
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=

httpd/httpd_test.go

@@ -140,7 +140,11 @@ func TestMain(m *testing.M) {
 	os.RemoveAll(credentialsPath) //nolint:errcheck
 	logger.InfoToConsole("Starting HTTPD tests, provider: %v", providerConf.Driver)
 
-	common.Initialize(config.GetCommonConfig())
+	err = common.Initialize(config.GetCommonConfig())
+	if err != nil {
+		logger.WarnToConsole("error initializing common: %v", err)
+		os.Exit(1)
+	}
 
 	err = dataprovider.Initialize(providerConf, configDir)
 	if err != nil {

pkgs/build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-NFPM_VERSION=2.1.1
+NFPM_VERSION=2.2.1
 NFPM_ARCH=${NFPM_ARCH:-amd64}
 if [ -z ${SFTPGO_VERSION} ]
 then

service/service.go

@@ -81,9 +81,14 @@ func (s *Service) Start() error {
 		return errors.New(infoString)
 	}
 
-	common.Initialize(config.GetCommonConfig())
+	err := common.Initialize(config.GetCommonConfig())
+	if err != nil {
+		logger.Error(logSender, "", "%v", err)
+		logger.ErrorToConsole("%v", err)
+		os.Exit(1)
+	}
 	kmsConfig := config.GetKMSConfig()
-	err := kmsConfig.Initialize()
+	err = kmsConfig.Initialize()
 	if err != nil {
 		logger.Error(logSender, "", "unable to initialize KMS: %v", err)
 		logger.ErrorToConsole("unable to initialize KMS: %v", err)

sftpd/server.go

@@ -351,6 +351,18 @@ func (c *Configuration) configureKeyboardInteractiveAuth(serverConfig *ssh.Serve
 	}
 }
 
+func canAcceptConnection(ip string) bool {
+	if common.IsBanned(ip) {
+		logger.Log(logger.LevelDebug, common.ProtocolSSH, "", "connection refused, ip %#v is banned", ip)
+		return false
+	}
+	if !common.Connections.IsNewConnectionAllowed() {
+		logger.Log(logger.LevelDebug, common.ProtocolSSH, "", "connection refused, configured limit reached")
+		return false
+	}
+	return true
+}
+
 // AcceptInboundConnection handles an inbound connection to the server instance and determines if the request should be served or not.
 func (c *Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.ServerConfig) {
 	defer func() {
@@ -358,22 +370,22 @@ func (c *Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.Serve
 			logger.Error(logSender, "", "panic in AcceptInboundConnection: %#v stack strace: %v", r, string(debug.Stack()))
 		}
 	}()
-	if !common.Connections.IsNewConnectionAllowed() {
-		logger.Log(logger.LevelDebug, common.ProtocolSSH, "", "connection refused, configured limit reached")
+	ipAddr := utils.GetIPFromRemoteAddress(conn.RemoteAddr().String())
+	if !canAcceptConnection(ipAddr) {
 		conn.Close()
 		return
 	}
 	// Before beginning a handshake must be performed on the incoming net.Conn
 	// we'll set a Deadline for handshake to complete, the default is 2 minutes as OpenSSH
 	conn.SetDeadline(time.Now().Add(handshakeTimeout)) //nolint:errcheck
-	if err := common.Config.ExecutePostConnectHook(conn.RemoteAddr().String(), common.ProtocolSSH); err != nil {
+	if err := common.Config.ExecutePostConnectHook(ipAddr, common.ProtocolSSH); err != nil {
 		conn.Close()
 		return
 	}
 	sconn, chans, reqs, err := ssh.NewServerConn(conn, config)
 	if err != nil {
 		logger.Debug(logSender, "", "failed to accept an incoming connection: %v", err)
-		checkAuthError(conn, err)
+		checkAuthError(ipAddr, err)
 		return
 	}
 	// handshake completed so remove the deadline, we'll use IdleTimeout configuration from now on
@@ -395,7 +407,7 @@ func (c *Configuration) AcceptInboundConnection(conn net.Conn, config *ssh.Serve
 
 	logger.Log(logger.LevelInfo, common.ProtocolSSH, connectionID,
 		"User id: %d, logged in with: %#v, username: %#v, home_dir: %#v remote addr: %#v",
-		user.ID, loginType, user.Username, user.HomeDir, conn.RemoteAddr().String())
+		user.ID, loginType, user.Username, user.HomeDir, ipAddr)
 	dataprovider.UpdateLastLogin(user) //nolint:errcheck
 
 	sshConnection := common.NewSSHConnection(connectionID, conn)
@@ -500,11 +512,26 @@ func (c *Configuration) createHandler(connection *Connection) sftp.Handlers {
 	}
 }
 
-func checkAuthError(conn net.Conn, err error) {
-	if _, ok := err.(*ssh.ServerAuthError); !ok {
-		ip := utils.GetIPFromRemoteAddress(conn.RemoteAddr().String())
+func checkAuthError(ip string, err error) {
+	if authErrors, ok := err.(*ssh.ServerAuthError); ok {
+		// check public key auth errors here
+		for _, err := range authErrors.Errors {
+			if err != nil {
+				// these checks should be improved, we should check for error type and not error strings
+				if strings.Contains(err.Error(), "public key credentials") {
+					event := common.HostEventLoginFailed
+					if strings.Contains(err.Error(), "not found") {
+						event = common.HostEventUserNotFound
+					}
+					common.AddDefenderEvent(ip, event)
+					break
+				}
+			}
+		}
+	} else {
 		logger.ConnectionFailedLog("", ip, dataprovider.LoginMethodNoAuthTryed, common.ProtocolSSH, err.Error())
 		metrics.AddNoAuthTryed()
+		common.AddDefenderEvent(ip, common.HostEventNoLoginTried)
 		dataprovider.ExecutePostLoginHook("", dataprovider.LoginMethodNoAuthTryed, ip, common.ProtocolSSH, err)
 	}
 }
@@ -757,25 +784,25 @@ func (c *Configuration) validatePublicKeyCredentials(conn ssh.ConnMetadata, pubK
 
 	connectionID := hex.EncodeToString(conn.SessionID())
 	method := dataprovider.SSHLoginMethodPublicKey
+	ipAddr := utils.GetIPFromRemoteAddress(conn.RemoteAddr().String())
 	cert, ok := pubKey.(*ssh.Certificate)
 	if ok {
 		if cert.CertType != ssh.UserCert {
 			err = fmt.Errorf("ssh: cert has type %d", cert.CertType)
-			updateLoginMetrics(conn, method, err)
+			updateLoginMetrics(conn, ipAddr, method, err)
 			return nil, err
 		}
 		if !c.certChecker.IsUserAuthority(cert.SignatureKey) {
 			err = fmt.Errorf("ssh: certificate signed by unrecognized authority")
-			updateLoginMetrics(conn, method, err)
+			updateLoginMetrics(conn, ipAddr, method, err)
 			return nil, err
 		}
 		if err := c.certChecker.CheckCert(conn.User(), cert); err != nil {
-			updateLoginMetrics(conn, method, err)
+			updateLoginMetrics(conn, ipAddr, method, err)
 			return nil, err
 		}
 		certPerm = &cert.Permissions
 	}
-	ipAddr := utils.GetIPFromRemoteAddress(conn.RemoteAddr().String())
 	if user, keyID, err = dataprovider.CheckUserAndPubKey(conn.User(), pubKey.Marshal(), ipAddr, common.ProtocolSSH); err == nil {
 		if user.IsPartialAuth(method) {
 			logger.Debug(logSender, connectionID, "user %#v authenticated with partial success", conn.User())
@@ -793,7 +820,7 @@ func (c *Configuration) validatePublicKeyCredentials(conn ssh.ConnMetadata, pubK
 			}
 		}
 	}
-	updateLoginMetrics(conn, method, err)
+	updateLoginMetrics(conn, ipAddr, method, err)
 	return sshPerm, err
 }
 
@@ -810,7 +837,7 @@ func (c *Configuration) validatePasswordCredentials(conn ssh.ConnMetadata, pass
 	if user, err = dataprovider.CheckUserAndPass(conn.User(), string(pass), ipAddr, common.ProtocolSSH); err == nil {
 		sshPerm, err = loginUser(user, method, "", conn)
 	}
-	updateLoginMetrics(conn, method, err)
+	updateLoginMetrics(conn, ipAddr, method, err)
 	return sshPerm, err
 }
 
@@ -828,15 +855,24 @@ func (c *Configuration) validateKeyboardInteractiveCredentials(conn ssh.ConnMeta
 		ipAddr, common.ProtocolSSH); err == nil {
 		sshPerm, err = loginUser(user, method, "", conn)
 	}
-	updateLoginMetrics(conn, method, err)
+	updateLoginMetrics(conn, ipAddr, method, err)
 	return sshPerm, err
 }
 
-func updateLoginMetrics(conn ssh.ConnMetadata, method string, err error) {
+func updateLoginMetrics(conn ssh.ConnMetadata, ip, method string, err error) {
 	metrics.AddLoginAttempt(method)
-	ip := utils.GetIPFromRemoteAddress(conn.RemoteAddr().String())
 	if err != nil {
 		logger.ConnectionFailedLog(conn.User(), ip, method, common.ProtocolSSH, err.Error())
+		if method != dataprovider.SSHLoginMethodPublicKey {
+			// some clients try all available public keys for a user, we
+			// record failed login key auth only once for session if the
+			// authentication fails in checkAuthError
+			event := common.HostEventLoginFailed
+			if _, ok := err.(*dataprovider.RecordNotFoundError); ok {
+				event = common.HostEventUserNotFound
+			}
+			common.AddDefenderEvent(ip, event)
+		}
 	}
 	metrics.AddLoginResult(method, err)
 	dataprovider.ExecutePostLoginHook(conn.User(), method, ip, common.ProtocolSSH, err)

sftpd/sftpd_test.go

@@ -171,7 +171,11 @@ func TestMain(m *testing.M) {
 		scriptArgs = "$@"
 	}
 
-	common.Initialize(commonConf)
+	err = common.Initialize(commonConf)
+	if err != nil {
+		logger.WarnToConsole("error initializing common: %v", err)
+		os.Exit(1)
+	}
 
 	err = dataprovider.Initialize(providerConf, configDir)
 	if err != nil {
@@ -213,31 +217,7 @@ func TestMain(m *testing.M) {
 	}
 	sftpdConf.KeyboardInteractiveHook = keyIntAuthPath
 
-	pubKeyPath = filepath.Join(homeBasePath, "ssh_key.pub")
-	privateKeyPath = filepath.Join(homeBasePath, "ssh_key")
-	trustedCAUserKey = filepath.Join(homeBasePath, "ca_user_key")
-	gitWrapPath = filepath.Join(homeBasePath, "gitwrap.sh")
-	extAuthPath = filepath.Join(homeBasePath, "extauth.sh")
-	preLoginPath = filepath.Join(homeBasePath, "prelogin.sh")
-	postConnectPath = filepath.Join(homeBasePath, "postconnect.sh")
-	checkPwdPath = filepath.Join(homeBasePath, "checkpwd.sh")
-	err = ioutil.WriteFile(pubKeyPath, []byte(testPubKey+"\n"), 0600)
-	if err != nil {
-		logger.WarnToConsole("unable to save public key to file: %v", err)
-	}
-	err = ioutil.WriteFile(privateKeyPath, []byte(testPrivateKey+"\n"), 0600)
-	if err != nil {
-		logger.WarnToConsole("unable to save private key to file: %v", err)
-	}
-	err = ioutil.WriteFile(gitWrapPath, []byte(fmt.Sprintf("%v -i %v -oStrictHostKeyChecking=no %v\n",
-		sshPath, privateKeyPath, scriptArgs)), os.ModePerm)
-	if err != nil {
-		logger.WarnToConsole("unable to save gitwrap shell script: %v", err)
-	}
-	err = ioutil.WriteFile(trustedCAUserKey, []byte(testCAUserKey), 0600)
-	if err != nil {
-		logger.WarnToConsole("unable to save trusted CA user key: %v", err)
-	}
+	createInitialFiles(scriptArgs)
 	sftpdConf.TrustedUserCAKeys = append(sftpdConf.TrustedUserCAKeys, trustedCAUserKey)
 
 	go func() {
@@ -488,6 +468,52 @@ func TestBasicSFTPFsHandling(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestLoginNonExistentUser(t *testing.T) {
+	usePubKey := true
+	user := getTestUser(usePubKey)
+	_, err := getSftpClient(user, usePubKey)
+	assert.Error(t, err)
+}
+
+func TestDefender(t *testing.T) {
+	oldConfig := config.GetCommonConfig()
+
+	cfg := config.GetCommonConfig()
+	cfg.DefenderConfig.Enabled = true
+	cfg.DefenderConfig.Threshold = 3
+
+	err := common.Initialize(cfg)
+	assert.NoError(t, err)
+
+	usePubKey := false
+	user, _, err := httpd.AddUser(getTestUser(usePubKey), http.StatusOK)
+	assert.NoError(t, err)
+	client, err := getSftpClient(user, usePubKey)
+	if assert.NoError(t, err) {
+		defer client.Close()
+		err = checkBasicSFTP(client)
+		assert.NoError(t, err)
+	}
+
+	for i := 0; i < 3; i++ {
+		user.Password = "wrong_pwd"
+		_, err = getSftpClient(user, usePubKey)
+		assert.Error(t, err)
+	}
+
+	user.Password = defaultPassword
+	_, err = getSftpClient(user, usePubKey)
+	assert.Error(t, err)
+
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+
+	err = common.Initialize(oldConfig)
+	assert.NoError(t, err)
+}
+
 func TestOpenReadWrite(t *testing.T) {
 	usePubKey := false
 	u := getTestUser(usePubKey)
@@ -8721,3 +8747,31 @@ func getHostKeysFingerprints(hostKeys []string) {
 		hostKeyFPs = append(hostKeyFPs, fp)
 	}
 }
+
+func createInitialFiles(scriptArgs string) {
+	pubKeyPath = filepath.Join(homeBasePath, "ssh_key.pub")
+	privateKeyPath = filepath.Join(homeBasePath, "ssh_key")
+	trustedCAUserKey = filepath.Join(homeBasePath, "ca_user_key")
+	gitWrapPath = filepath.Join(homeBasePath, "gitwrap.sh")
+	extAuthPath = filepath.Join(homeBasePath, "extauth.sh")
+	preLoginPath = filepath.Join(homeBasePath, "prelogin.sh")
+	postConnectPath = filepath.Join(homeBasePath, "postconnect.sh")
+	checkPwdPath = filepath.Join(homeBasePath, "checkpwd.sh")
+	err := ioutil.WriteFile(pubKeyPath, []byte(testPubKey+"\n"), 0600)
+	if err != nil {
+		logger.WarnToConsole("unable to save public key to file: %v", err)
+	}
+	err = ioutil.WriteFile(privateKeyPath, []byte(testPrivateKey+"\n"), 0600)
+	if err != nil {
+		logger.WarnToConsole("unable to save private key to file: %v", err)
+	}
+	err = ioutil.WriteFile(gitWrapPath, []byte(fmt.Sprintf("%v -i %v -oStrictHostKeyChecking=no %v\n",
+		sshPath, privateKeyPath, scriptArgs)), os.ModePerm)
+	if err != nil {
+		logger.WarnToConsole("unable to save gitwrap shell script: %v", err)
+	}
+	err = ioutil.WriteFile(trustedCAUserKey, []byte(testCAUserKey), 0600)
+	if err != nil {
+		logger.WarnToConsole("unable to save trusted CA user key: %v", err)
+	}
+}

sftpgo.json

@@ -10,7 +10,20 @@
     "proxy_protocol": 0,
     "proxy_allowed": [],
     "post_connect_hook": "",
-    "max_total_connections": 0
+    "max_total_connections": 0,
+    "defender": {
+      "enabled": false,
+      "ban_time": 30,
+      "ban_time_increment": 50,
+      "threshold": 15,
+      "score_invalid": 2,
+      "score_valid": 1,
+      "observation_time": 30,
+      "entries_soft_limit": 100,
+      "entries_hard_limit": 150,
+      "safelist_file": "",
+      "blocklist_file": ""
+    }
   },
   "sftpd": {
     "bindings": [

webdavd/internal_test.go

@@ -692,12 +692,14 @@ func TestBasicUsersCache(t *testing.T) {
 	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user.Username), nil)
 	assert.NoError(t, err)
 
-	_, _, _, err = server.authenticate(req) //nolint:dogsled
+	ipAddr := "127.0.0.1"
+
+	_, _, _, err = server.authenticate(req, ipAddr) //nolint:dogsled
 	assert.Error(t, err)
 
 	now := time.Now()
 	req.SetBasicAuth(username, password)
-	_, isCached, _, err := server.authenticate(req)
+	_, isCached, _, err := server.authenticate(req, ipAddr)
 	assert.NoError(t, err)
 	assert.False(t, isCached)
 	// now the user should be cached
@@ -708,14 +710,14 @@ func TestBasicUsersCache(t *testing.T) {
 		assert.False(t, cachedUser.IsExpired())
 		assert.True(t, cachedUser.Expiration.After(now.Add(time.Duration(c.Cache.Users.ExpirationTime)*time.Minute)))
 		// authenticate must return the cached user now
-		authUser, isCached, _, err := server.authenticate(req)
+		authUser, isCached, _, err := server.authenticate(req, ipAddr)
 		assert.NoError(t, err)
 		assert.True(t, isCached)
 		assert.Equal(t, cachedUser.User, authUser)
 	}
 	// a wrong password must fail
 	req.SetBasicAuth(username, "wrong")
-	_, _, _, err = server.authenticate(req) //nolint:dogsled
+	_, _, _, err = server.authenticate(req, ipAddr) //nolint:dogsled
 	assert.EqualError(t, err, dataprovider.ErrInvalidCredentials.Error())
 	req.SetBasicAuth(username, password)
 
@@ -728,7 +730,7 @@ func TestBasicUsersCache(t *testing.T) {
 		assert.True(t, cachedUser.IsExpired())
 	}
 	// now authenticate should get the user from the data provider and update the cache
-	_, isCached, _, err = server.authenticate(req)
+	_, isCached, _, err = server.authenticate(req, ipAddr)
 	assert.NoError(t, err)
 	assert.False(t, isCached)
 	result, ok = dataprovider.GetCachedWebDAVUser(username)
@@ -742,7 +744,7 @@ func TestBasicUsersCache(t *testing.T) {
 	_, ok = dataprovider.GetCachedWebDAVUser(username)
 	assert.False(t, ok)
 
-	_, isCached, _, err = server.authenticate(req)
+	_, isCached, _, err = server.authenticate(req, ipAddr)
 	assert.NoError(t, err)
 	assert.False(t, isCached)
 	_, ok = dataprovider.GetCachedWebDAVUser(username)
@@ -808,24 +810,25 @@ func TestUsersCacheSizeAndExpiration(t *testing.T) {
 	server, err := newServer(c, configDir)
 	assert.NoError(t, err)
 
+	ipAddr := "127.0.1.1"
 	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user1.Username), nil)
 	assert.NoError(t, err)
 	req.SetBasicAuth(user1.Username, password+"1")
-	_, isCached, _, err := server.authenticate(req)
+	_, isCached, _, err := server.authenticate(req, ipAddr)
 	assert.NoError(t, err)
 	assert.False(t, isCached)
 
 	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user2.Username), nil)
 	assert.NoError(t, err)
 	req.SetBasicAuth(user2.Username, password+"2")
-	_, isCached, _, err = server.authenticate(req)
+	_, isCached, _, err = server.authenticate(req, ipAddr)
 	assert.NoError(t, err)
 	assert.False(t, isCached)
 
 	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user3.Username), nil)
 	assert.NoError(t, err)
 	req.SetBasicAuth(user3.Username, password+"3")
-	_, isCached, _, err = server.authenticate(req)
+	_, isCached, _, err = server.authenticate(req, ipAddr)
 	assert.NoError(t, err)
 	assert.False(t, isCached)
 
@@ -840,7 +843,7 @@ func TestUsersCacheSizeAndExpiration(t *testing.T) {
 	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user4.Username), nil)
 	assert.NoError(t, err)
 	req.SetBasicAuth(user4.Username, password+"4")
-	_, isCached, _, err = server.authenticate(req)
+	_, isCached, _, err = server.authenticate(req, ipAddr)
 	assert.NoError(t, err)
 	assert.False(t, isCached)
 	// user1, the first cached, should be removed now
@@ -857,7 +860,7 @@ func TestUsersCacheSizeAndExpiration(t *testing.T) {
 	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user1.Username), nil)
 	assert.NoError(t, err)
 	req.SetBasicAuth(user1.Username, password+"1")
-	_, isCached, _, err = server.authenticate(req)
+	_, isCached, _, err = server.authenticate(req, ipAddr)
 	assert.NoError(t, err)
 	assert.False(t, isCached)
 	_, ok = dataprovider.GetCachedWebDAVUser(user2.Username)
@@ -873,7 +876,7 @@ func TestUsersCacheSizeAndExpiration(t *testing.T) {
 	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user2.Username), nil)
 	assert.NoError(t, err)
 	req.SetBasicAuth(user2.Username, password+"2")
-	_, isCached, _, err = server.authenticate(req)
+	_, isCached, _, err = server.authenticate(req, ipAddr)
 	assert.NoError(t, err)
 	assert.False(t, isCached)
 	_, ok = dataprovider.GetCachedWebDAVUser(user3.Username)
@@ -889,7 +892,7 @@ func TestUsersCacheSizeAndExpiration(t *testing.T) {
 	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user3.Username), nil)
 	assert.NoError(t, err)
 	req.SetBasicAuth(user3.Username, password+"3")
-	_, isCached, _, err = server.authenticate(req)
+	_, isCached, _, err = server.authenticate(req, ipAddr)
 	assert.NoError(t, err)
 	assert.False(t, isCached)
 	_, ok = dataprovider.GetCachedWebDAVUser(user4.Username)
@@ -910,14 +913,14 @@ func TestUsersCacheSizeAndExpiration(t *testing.T) {
 	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user4.Username), nil)
 	assert.NoError(t, err)
 	req.SetBasicAuth(user4.Username, password+"4")
-	_, isCached, _, err = server.authenticate(req)
+	_, isCached, _, err = server.authenticate(req, ipAddr)
 	assert.NoError(t, err)
 	assert.False(t, isCached)
 
 	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("/%v", user1.Username), nil)
 	assert.NoError(t, err)
 	req.SetBasicAuth(user1.Username, password+"1")
-	_, isCached, _, err = server.authenticate(req)
+	_, isCached, _, err = server.authenticate(req, ipAddr)
 	assert.NoError(t, err)
 	assert.False(t, isCached)
 	_, ok = dataprovider.GetCachedWebDAVUser(user2.Username)

webdavd/server.go

@@ -121,11 +121,16 @@ func (s *webDavServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	checkRemoteAddress(r)
-	if err := common.Config.ExecutePostConnectHook(r.RemoteAddr, common.ProtocolWebDAV); err != nil {
+	ipAddr := utils.GetIPFromRemoteAddress(r.RemoteAddr)
+	if common.IsBanned(ipAddr) {
 		http.Error(w, common.ErrConnectionDenied.Error(), http.StatusForbidden)
 		return
 	}
-	user, _, lockSystem, err := s.authenticate(r)
+	if err := common.Config.ExecutePostConnectHook(ipAddr, common.ProtocolWebDAV); err != nil {
+		http.Error(w, common.ErrConnectionDenied.Error(), http.StatusForbidden)
+		return
+	}
+	user, _, lockSystem, err := s.authenticate(r, ipAddr)
 	if err != nil {
 		w.Header().Set("WWW-Authenticate", "Basic realm=\"SFTPGo WebDAV\"")
 		http.Error(w, err401.Error(), http.StatusUnauthorized)
@@ -139,19 +144,19 @@ func (s *webDavServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	connectionID, err := s.validateUser(user, r)
 	if err != nil {
-		updateLoginMetrics(user.Username, r.RemoteAddr, err)
+		updateLoginMetrics(user.Username, ipAddr, err)
 		http.Error(w, err.Error(), http.StatusForbidden)
 		return
 	}
 
 	fs, err := user.GetFilesystem(connectionID)
 	if err != nil {
-		updateLoginMetrics(user.Username, r.RemoteAddr, err)
+		updateLoginMetrics(user.Username, ipAddr, err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
-	updateLoginMetrics(user.Username, r.RemoteAddr, err)
+	updateLoginMetrics(user.Username, ipAddr, err)
 
 	ctx := context.WithValue(r.Context(), requestIDKey, connectionID)
 	ctx = context.WithValue(ctx, requestStartKey, time.Now())
@@ -177,7 +182,7 @@ func (s *webDavServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	handler.ServeHTTP(w, r.WithContext(ctx))
 }
 
-func (s *webDavServer) authenticate(r *http.Request) (dataprovider.User, bool, webdav.LockSystem, error) {
+func (s *webDavServer) authenticate(r *http.Request, ip string) (dataprovider.User, bool, webdav.LockSystem, error) {
 	var user dataprovider.User
 	var err error
 	username, password, ok := r.BasicAuth()
@@ -193,13 +198,13 @@ func (s *webDavServer) authenticate(r *http.Request) (dataprovider.User, bool, w
 			if len(password) > 0 && cachedUser.Password == password {
 				return cachedUser.User, true, cachedUser.LockSystem, nil
 			}
-			updateLoginMetrics(username, r.RemoteAddr, dataprovider.ErrInvalidCredentials)
+			updateLoginMetrics(username, ip, dataprovider.ErrInvalidCredentials)
 			return user, false, nil, dataprovider.ErrInvalidCredentials
 		}
 	}
-	user, err = dataprovider.CheckUserAndPass(username, password, utils.GetIPFromRemoteAddress(r.RemoteAddr), common.ProtocolWebDAV)
+	user, err = dataprovider.CheckUserAndPass(username, password, ip, common.ProtocolWebDAV)
 	if err != nil {
-		updateLoginMetrics(username, r.RemoteAddr, err)
+		updateLoginMetrics(username, ip, err)
 		return user, false, nil, err
 	}
 	lockSystem := webdav.NewMemLS()
@@ -315,11 +320,15 @@ func checkRemoteAddress(r *http.Request) {
 	}
 }
 
-func updateLoginMetrics(username, remoteAddress string, err error) {
+func updateLoginMetrics(username, ip string, err error) {
 	metrics.AddLoginAttempt(dataprovider.LoginMethodPassword)
-	ip := utils.GetIPFromRemoteAddress(remoteAddress)
 	if err != nil {
 		logger.ConnectionFailedLog(username, ip, dataprovider.LoginMethodPassword, common.ProtocolWebDAV, err.Error())
+		event := common.HostEventLoginFailed
+		if _, ok := err.(*dataprovider.RecordNotFoundError); ok {
+			event = common.HostEventUserNotFound
+		}
+		common.AddDefenderEvent(ip, event)
 	}
 	metrics.AddLoginResult(dataprovider.LoginMethodPassword, err)
 	dataprovider.ExecutePostLoginHook(username, dataprovider.LoginMethodPassword, ip, common.ProtocolWebDAV, err)

webdavd/webdavd_test.go

@@ -121,7 +121,11 @@ func TestMain(m *testing.M) {
 		os.Exit(1)
 	}
 
-	common.Initialize(commonConf)
+	err = common.Initialize(commonConf)
+	if err != nil {
+		logger.WarnToConsole("error initializing common: %v", err)
+		os.Exit(1)
+	}
 
 	err = dataprovider.Initialize(providerConf, configDir)
 	if err != nil {
@@ -502,6 +506,49 @@ func TestLoginInvalidPwd(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestLoginNonExistentUser(t *testing.T) {
+	user := getTestUser()
+	client := getWebDavClient(user)
+	assert.Error(t, checkBasicFunc(client))
+}
+
+func TestDefender(t *testing.T) {
+	oldConfig := config.GetCommonConfig()
+
+	cfg := config.GetCommonConfig()
+	cfg.DefenderConfig.Enabled = true
+	cfg.DefenderConfig.Threshold = 3
+
+	err := common.Initialize(cfg)
+	assert.NoError(t, err)
+
+	user, _, err := httpd.AddUser(getTestUser(), http.StatusOK)
+	assert.NoError(t, err)
+	client := getWebDavClient(user)
+	assert.NoError(t, checkBasicFunc(client))
+
+	for i := 0; i < 3; i++ {
+		user.Password = "wrong_pwd"
+		client = getWebDavClient(user)
+		assert.Error(t, checkBasicFunc(client))
+	}
+
+	user.Password = defaultPassword
+	client = getWebDavClient(user)
+	err = checkBasicFunc(client)
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "403")
+	}
+
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+
+	err = common.Initialize(oldConfig)
+	assert.NoError(t, err)
+}
+
 func TestLoginInvalidURL(t *testing.T) {
 	u := getTestUser()
 	user, _, err := httpd.AddUser(u, http.StatusOK)