rate limiting: allow to exclude IP addresses/ranges

Fixes #563

common/common.go

@@ -149,7 +149,12 @@ func Initialize(c Configuration) error {
 			if err := rlCfg.validate(); err != nil {
 				return fmt.Errorf("rate limiters initialization error: %v", err)
 			}
+			allowList, err := util.ParseAllowedIPAndRanges(rlCfg.AllowList)
+			if err != nil {
+				return fmt.Errorf("unable to parse rate limiter allow list %v: %v", rlCfg.AllowList, err)
+			}
 			rateLimiter := rlCfg.getLimiter()
+			rateLimiter.allowList = allowList
 			for _, protocol := range rlCfg.Protocols {
 				rateLimiters[protocol] = append(rateLimiters[protocol], rateLimiter)
 			}

common/common_test.go

@@ -213,6 +213,14 @@ func TestRateLimitersIntegration(t *testing.T) {
 	err := Initialize(Config)
 	assert.Error(t, err)
 	Config.RateLimitersConfig[0].Period = 1000
+	Config.RateLimitersConfig[0].AllowList = []string{"1.1.1", "1.1.1.2"}
+	err = Initialize(Config)
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "unable to parse rate limiter allow list")
+	}
+	Config.RateLimitersConfig[0].AllowList = []string{"172.16.24.7"}
+	Config.RateLimitersConfig[1].AllowList = []string{"172.16.0.0/16"}
+
 	err = Initialize(Config)
 	assert.NoError(t, err)
 
@@ -224,6 +232,7 @@ func TestRateLimitersIntegration(t *testing.T) {
 
 	source1 := "127.1.1.1"
 	source2 := "127.1.1.2"
+	source3 := "172.16.24.7" // whitelisted
 
 	_, err = LimitRate(ProtocolSSH, source1)
 	assert.NoError(t, err)
@@ -242,6 +251,10 @@ func TestRateLimitersIntegration(t *testing.T) {
 	assert.NoError(t, err)
 	_, err = LimitRate(ProtocolSSH, source2)
 	assert.NoError(t, err)
+	for i := 0; i < 10; i++ {
+		_, err = LimitRate(ProtocolWebDAV, source3)
+		assert.NoError(t, err)
+	}
 
 	Config = configCopy
 }

common/ratelimiter.go

@@ -3,6 +3,7 @@ package common
 import (
 	"errors"
 	"fmt"
+	"net"
 	"sort"
 	"sync"
 	"sync/atomic"
@@ -47,6 +48,8 @@ type RateLimiterConfig struct {
 	// Available protocols are: "SFTP", "FTP", "DAV".
 	// A rate limiter with no protocols defined is disabled
 	Protocols []string `json:"protocols" mapstructure:"protocols"`
+	// AllowList defines a list of IP addresses and IP ranges excluded from rate limiting
+	AllowList []string `json:"allow_list" mapstructure:"mapstructure"`
 	// If the rate limit is exceeded, the defender is enabled, and this is a per-source limiter,
 	// a new defender event will be generated
 	GenerateDefenderEvents bool `json:"generate_defender_events" mapstructure:"generate_defender_events"`
@@ -125,12 +128,23 @@ type rateLimiter struct {
 	globalBucket           *rate.Limiter
 	buckets                sourceBuckets
 	generateDefenderEvents bool
+	allowList              []func(net.IP) bool
 }
 
 // Wait blocks until the limit allows one event to happen
 // or returns an error if the time to wait exceeds the max
 // allowed delay
 func (rl *rateLimiter) Wait(source string) (time.Duration, error) {
+	if len(rl.allowList) > 0 {
+		ip := net.ParseIP(source)
+		if ip != nil {
+			for idx := range rl.allowList {
+				if rl.allowList[idx](ip) {
+					return 0, nil
+				}
+			}
+		}
+	}
 	var res *rate.Reservation
 	if rl.globalBucket != nil {
 		res = rl.globalBucket.Reserve()

common/ratelimiter_test.go

@@ -6,6 +6,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/drakkan/sftpgo/v2/util"
 )
 
 func TestRateLimiterConfig(t *testing.T) {
@@ -83,6 +85,17 @@ func TestRateLimiter(t *testing.T) {
 	_, err = limiter.Wait(source + "1")
 	require.NoError(t, err)
 
+	allowList := []string{"192.168.1.0/24"}
+	allowFuncs, err := util.ParseAllowedIPAndRanges(allowList)
+	assert.NoError(t, err)
+	limiter.allowList = allowFuncs
+	for i := 0; i < 5; i++ {
+		_, err = limiter.Wait(source)
+		require.NoError(t, err)
+	}
+	_, err = limiter.Wait("not an ip")
+	require.NoError(t, err)
+
 	config.Burst = 0
 	limiter = config.getLimiter()
 	_, err = limiter.Wait(source)

config/config.go

@@ -85,6 +85,7 @@ var (
 		Burst:                  1,
 		Type:                   2,
 		Protocols:              []string{common.ProtocolSSH, common.ProtocolFTP, common.ProtocolWebDAV, common.ProtocolHTTP},
+		AllowList:              []string{},
 		GenerateDefenderEvents: false,
 		EntriesSoftLimit:       100,
 		EntriesHardLimit:       150,
@@ -637,6 +638,12 @@ func getRateLimitersFromEnv(idx int) {
 		isSet = true
 	}
 
+	allowList, ok := lookupStringListFromEnv(fmt.Sprintf("SFTPGO_COMMON__RATE_LIMITERS__%v__ALLOW_LIST", idx))
+	if ok {
+		rtlConfig.AllowList = allowList
+		isSet = true
+	}
+
 	generateEvents, ok := lookupBoolFromEnv(fmt.Sprintf("SFTPGO_COMMON__RATE_LIMITERS__%v__GENERATE_DEFENDER_EVENTS", idx))
 	if ok {
 		rtlConfig.GenerateDefenderEvents = generateEvents

config/config_test.go

@@ -515,7 +515,9 @@ func TestRateLimitersFromEnv(t *testing.T) {
 	os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__0__GENERATE_DEFENDER_EVENTS", "1")
 	os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__0__ENTRIES_SOFT_LIMIT", "50")
 	os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__0__ENTRIES_HARD_LIMIT", "100")
+	os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__0__ALLOW_LIST", ", 172.16.2.4, ")
 	os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__8__AVERAGE", "50")
+	os.Setenv("SFTPGO_COMMON__RATE_LIMITERS__8__ALLOW_LIST", "192.168.1.1, 192.168.2.0/24")
 	t.Cleanup(func() {
 		os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__AVERAGE")
 		os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__PERIOD")
@@ -525,7 +527,9 @@ func TestRateLimitersFromEnv(t *testing.T) {
 		os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__GENERATE_DEFENDER_EVENTS")
 		os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__ENTRIES_SOFT_LIMIT")
 		os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__ENTRIES_HARD_LIMIT")
+		os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__0__ALLOW_LIST")
 		os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__8__AVERAGE")
+		os.Unsetenv("SFTPGO_COMMON__RATE_LIMITERS__8__ALLOW_LIST")
 	})
 
 	configDir := ".."
@@ -544,7 +548,12 @@ func TestRateLimitersFromEnv(t *testing.T) {
 	require.True(t, limiters[0].GenerateDefenderEvents)
 	require.Equal(t, 50, limiters[0].EntriesSoftLimit)
 	require.Equal(t, 100, limiters[0].EntriesHardLimit)
+	require.Len(t, limiters[0].AllowList, 1)
+	require.Equal(t, "172.16.2.4", limiters[0].AllowList[0])
 	require.Equal(t, int64(50), limiters[1].Average)
+	require.Len(t, limiters[1].AllowList, 2)
+	require.Equal(t, "192.168.1.1", limiters[1].AllowList[0])
+	require.Equal(t, "192.168.2.0/24", limiters[1].AllowList[1])
 	// we check the default values here
 	require.Equal(t, int64(1000), limiters[1].Period)
 	require.Equal(t, 1, limiters[1].Burst)

docs/defender.md

@@ -52,7 +52,7 @@ Here is a small example:
         "2001:db8::68"
     ],
     "networks":[
-        "192.0.2.0/24",
+        "192.0.3.0/24",
         "2001:db8:1234::/48"
     ]
 }

docs/full-configuration.md

@@ -91,6 +91,7 @@ The configuration file contains the following sections:
     - `burst`, integer. Burst defines the maximum number of requests allowed to go through in the same arbitrarily small period of time. Default: 1
     - `type`, integer. 1 means a global rate limiter, independent from the source host. 2 means a per-ip rate limiter. Default: 2
     - `protocols`, list of strings. Available protocols are `SSH`, `FTP`, `DAV`, `HTTP`. By default all supported protocols are enabled
+    - `allow_list`, list of IP addresses and IP ranges excluded from rate limiting. Default: empty
     - `generate_defender_events`, boolean. If `true`, the defender is enabled, and this is not a global rate limiter, a new defender event will be generated each time the configured limit is exceeded. Default `false`
     - `entries_soft_limit`, integer.
     - `entries_hard_limit`, integer. The number of per-ip rate limiters kept in memory will vary between the soft and hard limit

docs/rate-limiting.md

@@ -22,6 +22,20 @@ You can also define two types of rate limiters:
 
 If you configure a per-host rate limiter, SFTPGo will keep a rate limiter in memory for each host that connects to the service, you can limit the memory usage using the `entries_soft_limit` and `entries_hard_limit` configuration keys.
 
+For each rate limiter you can exclude a list of IP addresses and IP ranges by defining an `allow_list`.
+The allow list supports IPv4/IPv6 address and CIDR networks, for example:
+
+```json
+...
+"allow_list": [
+  "192.0.2.1",
+  "192.168.1.0/24",
+  "2001:db8::68",
+  "2001:db8:1234::/48"
+],
+...
+```
+
 You can defines how many rate limiters as you want, but keep in mind that if you defines multiple rate limiters each request will be checked against all the configured limiters and so it can potentially be delayed multiple times. Let's clarify with an example, here is a configuration that defines a global rate limiter and a per-host rate limiter for the FTP protocol:
 
 ```json

sftpgo.json

@@ -42,6 +42,7 @@
           "DAV",
           "HTTP"
         ],
+        "allow_list": [],
         "generate_defender_events": false,
         "entries_soft_limit": 100,
         "entries_hard_limit": 150

util/util.go

@@ -576,7 +576,8 @@ func GetHTTPLocalAddress(r *http.Request) string {
 	return ""
 }
 
-// ParseAllowedIPAndRanges returns a list of functions that allow to find if a
+// ParseAllowedIPAndRanges returns a list of functions that allow to find if an
+// IP is equal or is contained within the allowed list
 func ParseAllowedIPAndRanges(allowed []string) ([]func(net.IP) bool, error) {
 	res := make([]func(net.IP) bool, len(allowed))
 	for i, allowFrom := range allowed {