From ea01c3a1252f9dec9360bf191350c62b5c8d655f Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sun, 3 Oct 2021 20:50:05 +0200 Subject: [PATCH] rate limiting: allow to exclude IP addresses/ranges Fixes #563 --- common/common.go | 5 +++++ common/common_test.go | 13 +++++++++++++ common/ratelimiter.go | 14 ++++++++++++++ common/ratelimiter_test.go | 13 +++++++++++++ config/config.go | 7 +++++++ config/config_test.go | 9 +++++++++ docs/defender.md | 2 +- docs/full-configuration.md | 1 + docs/rate-limiting.md | 14 ++++++++++++++ sftpgo.json | 1 + util/util.go | 3 ++- 11 files changed, 80 insertions(+), 2 deletions(-) diff --git a/common/common.go b/common/common.go index a4fa8ba6..4abb611b 100644 --- a/common/common.go +++ b/common/common.go @@ -149,7 +149,12 @@ func Initialize(c Configuration) error { if err := rlCfg.validate(); err != nil { return fmt.Errorf("rate limiters initialization error: %v", err) } + allowList, err := util.ParseAllowedIPAndRanges(rlCfg.AllowList) + if err != nil { + return fmt.Errorf("unable to parse rate limiter allow list %v: %v", rlCfg.AllowList, err) + } rateLimiter := rlCfg.getLimiter() + rateLimiter.allowList = allowList for _, protocol := range rlCfg.Protocols { rateLimiters[protocol] = append(rateLimiters[protocol], rateLimiter) } diff --git a/common/common_test.go b/common/common_test.go index 75bbfe33..0d452067 100644 --- a/common/common_test.go +++ b/common/common_test.go @@ -213,6 +213,14 @@ func TestRateLimitersIntegration(t *testing.T) { err := Initialize(Config) assert.Error(t, err) Config.RateLimitersConfig[0].Period = 1000 + Config.RateLimitersConfig[0].AllowList = []string{"1.1.1", "1.1.1.2"} + err = Initialize(Config) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "unable to parse rate limiter allow list") + } + Config.RateLimitersConfig[0].AllowList = []string{"172.16.24.7"} + Config.RateLimitersConfig[1].AllowList = []string{"172.16.0.0/16"} + err = Initialize(Config) assert.NoError(t, err) @@ -224,6 +232,7 @@ func TestRateLimitersIntegration(t *testing.T) { source1 := "127.1.1.1" source2 := "127.1.1.2" + source3 := "172.16.24.7" // whitelisted _, err = LimitRate(ProtocolSSH, source1) assert.NoError(t, err) @@ -242,6 +251,10 @@ func TestRateLimitersIntegration(t *testing.T) { assert.NoError(t, err) _, err = LimitRate(ProtocolSSH, source2) assert.NoError(t, err) + for i := 0; i < 10; i++ { + _, err = LimitRate(ProtocolWebDAV, source3) + assert.NoError(t, err) + } Config = configCopy } diff --git a/common/ratelimiter.go b/common/ratelimiter.go index 5f2920ac..d72266e6 100644 --- a/common/ratelimiter.go +++ b/common/ratelimiter.go @@ -3,6 +3,7 @@ package common import ( "errors" "fmt" + "net" "sort" "sync" "sync/atomic" @@ -47,6 +48,8 @@ type RateLimiterConfig struct { // Available protocols are: "SFTP", "FTP", "DAV". // A rate limiter with no protocols defined is disabled Protocols []string `json:"protocols" mapstructure:"protocols"` + // AllowList defines a list of IP addresses and IP ranges excluded from rate limiting + AllowList []string `json:"allow_list" mapstructure:"mapstructure"` // If the rate limit is exceeded, the defender is enabled, and this is a per-source limiter, // a new defender event will be generated GenerateDefenderEvents bool `json:"generate_defender_events" mapstructure:"generate_defender_events"` @@ -125,12 +128,23 @@ type rateLimiter struct { globalBucket *rate.Limiter buckets sourceBuckets generateDefenderEvents bool + allowList []func(net.IP) bool } // Wait blocks until the limit allows one event to happen // or returns an error if the time to wait exceeds the max // allowed delay func (rl *rateLimiter) Wait(source string) (time.Duration, error) { + if len(rl.allowList) > 0 { + ip := net.ParseIP(source) + if ip != nil { + for idx := range rl.allowList { + if rl.allowList[idx](ip) { + return 0, nil + } + } + } + } var res *rate.Reservation if rl.globalBucket != nil { res = rl.globalBucket.Reserve() diff --git a/common/ratelimiter_test.go b/common/ratelimiter_test.go index 7955c0fd..82b029eb 100644 --- a/common/ratelimiter_test.go +++ b/common/ratelimiter_test.go @@ -6,6 +6,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/drakkan/sftpgo/v2/util" ) func TestRateLimiterConfig(t *testing.T) { @@ -83,6 +85,17 @@ func TestRateLimiter(t *testing.T) { _, err = limiter.Wait(source + "1") require.NoError(t, err) + allowList := []string{"192.168.1.0/24"} + allowFuncs, err := util.ParseAllowedIPAndRanges(allowList) + assert.NoError(t, err) + limiter.allowList = allowFuncs + for i := 0; i < 5; i++ { + _, err = limiter.Wait(source) + require.NoError(t, err) + } + _, err = limiter.Wait("not an ip") + require.NoError(t, err) + config.Burst = 0 limiter = config.getLimiter() _, err = limiter.Wait(source) diff --git a/config/config.go b/config/config.go index 7abfbfb2..76531075 100644 --- a/config/config.go +++ b/config/config.go @@ -85,6 +85,7 @@ var ( Burst: 1, Type: 2, Protocols: []string{common.ProtocolSSH, common.ProtocolFTP, common.ProtocolWebDAV, common.ProtocolHTTP}, + AllowList: []string{}, GenerateDefenderEvents: false, EntriesSoftLimit: 100, EntriesHardLimit: 150, @@ -637,6 +638,12 @@ func getRateLimitersFromEnv(idx int) { isSet = true } + allowList, ok := lookupStringListFromEnv(fmt.Sprintf("SFTPGO_COMMON__RATE_LIMITERS__%v__ALLOW_LIST", idx)) + if ok { + rtlConfig.AllowList = allowList + isSet = true + } + generateEvents, ok := lookupBoolFromEnv(fmt.Sprintf("SFTPGO_COMMON__RATE_LIMITERS__%v__GENERATE_DEFENDER_EVENTS", idx)) if ok { rtlConfig.GenerateDefenderEvents = generateEvents diff --git a/config/config_test.go b/config/config_test.go index 1df51158..e9023832 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -515,7 +515,9 @@ func TestRateLimitersFromEnv(t *testing.T) { os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__0__GENERATE_DEFENDER_EVENTS", "1") os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__0__ENTRIES_SOFT_LIMIT", "50") os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__0__ENTRIES_HARD_LIMIT", "100") + os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__0__ALLOW_LIST", ", 172.16.2.4, ") os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__8__AVERAGE", "50") + os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__8__ALLOW_LIST", "192.168.1.1, 192.168.2.0/24") t.Cleanup(func() { os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__AVERAGE") os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__PERIOD") @@ -525,7 +527,9 @@ func TestRateLimitersFromEnv(t *testing.T) { os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__GENERATE_DEFENDER_EVENTS") os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__ENTRIES_SOFT_LIMIT") os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__ENTRIES_HARD_LIMIT") + os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__ALLOW_LIST") os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__8__AVERAGE") + os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__8__ALLOW_LIST") }) configDir := ".." @@ -544,7 +548,12 @@ func TestRateLimitersFromEnv(t *testing.T) { require.True(t, limiters[0].GenerateDefenderEvents) require.Equal(t, 50, limiters[0].EntriesSoftLimit) require.Equal(t, 100, limiters[0].EntriesHardLimit) + require.Len(t, limiters[0].AllowList, 1) + require.Equal(t, "172.16.2.4", limiters[0].AllowList[0]) require.Equal(t, int64(50), limiters[1].Average) + require.Len(t, limiters[1].AllowList, 2) + require.Equal(t, "192.168.1.1", limiters[1].AllowList[0]) + require.Equal(t, "192.168.2.0/24", limiters[1].AllowList[1]) // we check the default values here require.Equal(t, int64(1000), limiters[1].Period) require.Equal(t, 1, limiters[1].Burst) diff --git a/docs/defender.md b/docs/defender.md index e44b2c51..886f382e 100644 --- a/docs/defender.md +++ b/docs/defender.md @@ -52,7 +52,7 @@ Here is a small example: "2001:db8::68" ], "networks":[ - "192.0.2.0/24", + "192.0.3.0/24", "2001:db8:1234::/48" ] } diff --git a/docs/full-configuration.md b/docs/full-configuration.md index 6728e61b..fde2fef6 100644 --- a/docs/full-configuration.md +++ b/docs/full-configuration.md @@ -91,6 +91,7 @@ The configuration file contains the following sections: - `burst`, integer. Burst defines the maximum number of requests allowed to go through in the same arbitrarily small period of time. Default: 1 - `type`, integer. 1 means a global rate limiter, independent from the source host. 2 means a per-ip rate limiter. Default: 2 - `protocols`, list of strings. Available protocols are `SSH`, `FTP`, `DAV`, `HTTP`. By default all supported protocols are enabled + - `allow_list`, list of IP addresses and IP ranges excluded from rate limiting. Default: empty - `generate_defender_events`, boolean. If `true`, the defender is enabled, and this is not a global rate limiter, a new defender event will be generated each time the configured limit is exceeded. Default `false` - `entries_soft_limit`, integer. - `entries_hard_limit`, integer. The number of per-ip rate limiters kept in memory will vary between the soft and hard limit diff --git a/docs/rate-limiting.md b/docs/rate-limiting.md index 6c1631fa..3609c0dc 100644 --- a/docs/rate-limiting.md +++ b/docs/rate-limiting.md @@ -22,6 +22,20 @@ You can also define two types of rate limiters: If you configure a per-host rate limiter, SFTPGo will keep a rate limiter in memory for each host that connects to the service, you can limit the memory usage using the `entries_soft_limit` and `entries_hard_limit` configuration keys. +For each rate limiter you can exclude a list of IP addresses and IP ranges by defining an `allow_list`. +The allow list supports IPv4/IPv6 address and CIDR networks, for example: + +```json +... +"allow_list": [ + "192.0.2.1", + "192.168.1.0/24", + "2001:db8::68", + "2001:db8:1234::/48" +], +... +``` + You can defines how many rate limiters as you want, but keep in mind that if you defines multiple rate limiters each request will be checked against all the configured limiters and so it can potentially be delayed multiple times. Let's clarify with an example, here is a configuration that defines a global rate limiter and a per-host rate limiter for the FTP protocol: ```json diff --git a/sftpgo.json b/sftpgo.json index bea386a2..a9e9b9f2 100644 --- a/sftpgo.json +++ b/sftpgo.json @@ -42,6 +42,7 @@ "DAV", "HTTP" ], + "allow_list": [], "generate_defender_events": false, "entries_soft_limit": 100, "entries_hard_limit": 150 diff --git a/util/util.go b/util/util.go index bf5e4181..d04110fd 100644 --- a/util/util.go +++ b/util/util.go @@ -576,7 +576,8 @@ func GetHTTPLocalAddress(r *http.Request) string { return "" } -// ParseAllowedIPAndRanges returns a list of functions that allow to find if a +// ParseAllowedIPAndRanges returns a list of functions that allow to find if an +// IP is equal or is contained within the allowed list func ParseAllowedIPAndRanges(allowed []string) ([]func(net.IP) bool, error) { res := make([]func(net.IP) bool, len(allowed)) for i, allowFrom := range allowed {