support ip and cidr based whitelists for capi and 3rd party blocklists (#2132)

* support ip and cidr based whitelists for capi and 3rd party blocklist

cmd/crowdsec-cli/papi.go

@@ -51,7 +51,7 @@ func NewPapiStatusCmd() *cobra.Command {
 				log.Fatalf("unable to initialize database client : %s", err)
 			}
 
-			apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig)
+			apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig, csConfig.API.Server.CapiWhitelists)
 
 			if err != nil {
 				log.Fatalf("unable to initialize API client : %s", err)
@@ -101,7 +101,7 @@ func NewPapiSyncCmd() *cobra.Command {
 				log.Fatalf("unable to initialize database client : %s", err)
 			}
 
-			apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig)
+			apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig, csConfig.API.Server.CapiWhitelists)
 
 			if err != nil {
 				log.Fatalf("unable to initialize API client : %s", err)

pkg/apiserver/apic.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"math/rand"
+	"net"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -60,6 +61,7 @@ type apic struct {
 	credentials   *csconfig.ApiCredentialsCfg
 	scenarioList  []string
 	consoleConfig *csconfig.ConsoleConfig
+	whitelists    *csconfig.CapiWhitelist
 }
 
 // randomDuration returns a duration value between d-delta and d+delta
@@ -149,7 +151,7 @@ func alertToSignal(alert *models.Alert, scenarioTrust string, shareContext bool)
 	return signal
 }
 
-func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, consoleConfig *csconfig.ConsoleConfig) (*apic, error) {
+func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, consoleConfig *csconfig.ConsoleConfig, apicWhitelist *csconfig.CapiWhitelist) (*apic, error) {
 	var err error
 	ret := &apic{
 
@@ -169,6 +171,7 @@ func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, con
 		pushIntervalFirst:    randomDuration(pushIntervalDefault, pushIntervalDelta),
 		metricsInterval:      metricsIntervalDefault,
 		metricsIntervalFirst: randomDuration(metricsIntervalDefault, metricsIntervalDelta),
+		whitelists:           apicWhitelist,
 	}
 
 	password := strfmt.Password(config.Credentials.Password)
@@ -573,6 +576,9 @@ func (a *apic) PullTop() error {
 
 	// create one alert for community blocklist using the first decision
 	decisions := a.apiClient.Decisions.GetDecisionsFromGroups(data.New)
+	//apply APIC specific whitelists
+	decisions = a.ApplyApicWhitelists(decisions)
+
 	alert := createAlertForDecision(decisions[0])
 	alertsFromCapi := []*models.Alert{alert}
 	alertsFromCapi = fillAlertsWithDecisions(alertsFromCapi, decisions, add_counters)
@@ -589,6 +595,47 @@ func (a *apic) PullTop() error {
 	return nil
 }
 
+func (a *apic) ApplyApicWhitelists(decisions []*models.Decision) []*models.Decision {
+	if a.whitelists == nil {
+		return decisions
+	}
+	//deal with CAPI whitelists for fire. We want to avoid having a second list, so we shrink in place
+	outIdx := 0
+	for _, decision := range decisions {
+		if decision.Value == nil {
+			continue
+		}
+		skip := false
+		ipval := net.ParseIP(*decision.Value)
+		for _, cidr := range a.whitelists.Cidrs {
+			if skip {
+				break
+			}
+			if cidr.Contains(ipval) {
+				log.Infof("%s from %s is whitelisted by %s", *decision.Value, *decision.Scenario, cidr.String())
+				skip = true
+			}
+		}
+		for _, ip := range a.whitelists.Ips {
+			if skip {
+				break
+			}
+			if ip != nil && ip.Equal(ipval) {
+				log.Infof("%s from %s is whitelisted by %s", *decision.Value, *decision.Scenario, ip.String())
+				skip = true
+			}
+		}
+		if !skip {
+			decisions[outIdx] = decision
+			outIdx++
+		}
+
+	}
+	//shrink the list, those are deleted items
+	decisions = decisions[:outIdx]
+	return decisions
+}
+
 func (a *apic) SaveAlerts(alertsFromCapi []*models.Alert, add_counters map[string]map[string]int, delete_counters map[string]map[string]int) error {
 	for idx, alert := range alertsFromCapi {
 		alertsFromCapi[idx] = setAlertScenario(add_counters, delete_counters, alert)
@@ -690,6 +737,8 @@ func (a *apic) UpdateBlocklists(links *modelscapi.GetDecisionsStreamResponseLink
 			log.Infof("blocklist %s has no decisions", *blocklist.Name)
 			continue
 		}
+		//apply APIC specific whitelists
+		decisions = a.ApplyApicWhitelists(decisions)
 		alert := createAlertForDecision(decisions[0])
 		alertsFromCapi := []*models.Alert{alert}
 		alertsFromCapi = fillAlertsWithDecisions(alertsFromCapi, decisions, add_counters)

pkg/apiserver/apic_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -232,7 +233,7 @@ func TestNewAPIC(t *testing.T) {
 				),
 			))
 			tc.action()
-			_, err := NewAPIC(testConfig, tc.args.dbClient, tc.args.consoleConfig)
+			_, err := NewAPIC(testConfig, tc.args.dbClient, tc.args.consoleConfig, nil)
 			cstest.RequireErrorContains(t, err, tc.expectedErr)
 		})
 	}
@@ -527,6 +528,188 @@ func TestFillAlertsWithDecisions(t *testing.T) {
 	}
 }
 
+func TestAPICWhitelists(t *testing.T) {
+	api := getAPIC(t)
+	//one whitelist on IP, one on CIDR
+	api.whitelists = &csconfig.CapiWhitelist{}
+	ipwl1 := "9.2.3.4"
+	ip := net.ParseIP(ipwl1)
+	api.whitelists.Ips = append(api.whitelists.Ips, ip)
+	ipwl1 = "7.2.3.4"
+	ip = net.ParseIP(ipwl1)
+	api.whitelists.Ips = append(api.whitelists.Ips, ip)
+	cidrwl1 := "13.2.3.0/24"
+	_, tnet, err := net.ParseCIDR(cidrwl1)
+	if err != nil {
+		t.Fatalf("unable to parse cidr : %s", err)
+	}
+	api.whitelists.Cidrs = append(api.whitelists.Cidrs, tnet)
+	cidrwl1 = "11.2.3.0/24"
+	_, tnet, err = net.ParseCIDR(cidrwl1)
+	if err != nil {
+		t.Fatalf("unable to parse cidr : %s", err)
+	}
+	api.whitelists.Cidrs = append(api.whitelists.Cidrs, tnet)
+	api.dbClient.Ent.Decision.Create().
+		SetOrigin(types.CAPIOrigin).
+		SetType("ban").
+		SetValue("9.9.9.9").
+		SetScope("Ip").
+		SetScenario("crowdsecurity/ssh-bf").
+		SetUntil(time.Now().Add(time.Hour)).
+		ExecX(context.Background())
+	assertTotalDecisionCount(t, api.dbClient, 1)
+	assertTotalValidDecisionCount(t, api.dbClient, 1)
+	httpmock.Activate()
+	defer httpmock.DeactivateAndReset()
+	httpmock.RegisterResponder("GET", "http://api.crowdsec.net/api/decisions/stream", httpmock.NewBytesResponder(
+		200, jsonMarshalX(
+			modelscapi.GetDecisionsStreamResponse{
+				Deleted: modelscapi.GetDecisionsStreamResponseDeleted{
+					&modelscapi.GetDecisionsStreamResponseDeletedItem{
+						Decisions: []string{
+							"9.9.9.9", // This is already present in DB
+							"9.1.9.9", // This not present in DB
+						},
+						Scope: types.StrPtr("Ip"),
+					}, // This is already present in DB
+				},
+				New: modelscapi.GetDecisionsStreamResponseNew{
+					&modelscapi.GetDecisionsStreamResponseNewItem{
+						Scenario: types.StrPtr("crowdsecurity/test1"),
+						Scope:    types.StrPtr("Ip"),
+						Decisions: []*modelscapi.GetDecisionsStreamResponseNewItemDecisionsItems0{
+							{
+								Value:    types.StrPtr("13.2.3.4"), //wl by cidr
+								Duration: types.StrPtr("24h"),
+							},
+						},
+					},
+
+					&modelscapi.GetDecisionsStreamResponseNewItem{
+						Scenario: types.StrPtr("crowdsecurity/test1"),
+						Scope:    types.StrPtr("Ip"),
+						Decisions: []*modelscapi.GetDecisionsStreamResponseNewItemDecisionsItems0{
+							{
+								Value:    types.StrPtr("2.2.3.4"),
+								Duration: types.StrPtr("24h"),
+							},
+						},
+					},
+					&modelscapi.GetDecisionsStreamResponseNewItem{
+						Scenario: types.StrPtr("crowdsecurity/test2"),
+						Scope:    types.StrPtr("Ip"),
+						Decisions: []*modelscapi.GetDecisionsStreamResponseNewItemDecisionsItems0{
+							{
+								Value:    types.StrPtr("13.2.3.5"), //wl by cidr
+								Duration: types.StrPtr("24h"),
+							},
+						},
+					}, // These two are from community list.
+					&modelscapi.GetDecisionsStreamResponseNewItem{
+						Scenario: types.StrPtr("crowdsecurity/test1"),
+						Scope:    types.StrPtr("Ip"),
+						Decisions: []*modelscapi.GetDecisionsStreamResponseNewItemDecisionsItems0{
+							{
+								Value:    types.StrPtr("6.2.3.4"),
+								Duration: types.StrPtr("24h"),
+							},
+						},
+					},
+					&modelscapi.GetDecisionsStreamResponseNewItem{
+						Scenario: types.StrPtr("crowdsecurity/test1"),
+						Scope:    types.StrPtr("Ip"),
+						Decisions: []*modelscapi.GetDecisionsStreamResponseNewItemDecisionsItems0{
+							{
+								Value:    types.StrPtr("9.2.3.4"), //wl by ip
+								Duration: types.StrPtr("24h"),
+							},
+						},
+					},
+				},
+				Links: &modelscapi.GetDecisionsStreamResponseLinks{
+					Blocklists: []*modelscapi.BlocklistLink{
+						{
+							URL:         types.StrPtr("http://api.crowdsec.net/blocklist1"),
+							Name:        types.StrPtr("blocklist1"),
+							Scope:       types.StrPtr("Ip"),
+							Remediation: types.StrPtr("ban"),
+							Duration:    types.StrPtr("24h"),
+						},
+						{
+							URL:         types.StrPtr("http://api.crowdsec.net/blocklist2"),
+							Name:        types.StrPtr("blocklist2"),
+							Scope:       types.StrPtr("Ip"),
+							Remediation: types.StrPtr("ban"),
+							Duration:    types.StrPtr("24h"),
+						},
+					},
+				},
+			},
+		),
+	))
+	httpmock.RegisterResponder("GET", "http://api.crowdsec.net/blocklist1", httpmock.NewStringResponder(
+		200, "1.2.3.6",
+	))
+	httpmock.RegisterResponder("GET", "http://api.crowdsec.net/blocklist2", httpmock.NewStringResponder(
+		200, "1.2.3.7",
+	))
+	url, err := url.ParseRequestURI("http://api.crowdsec.net/")
+	require.NoError(t, err)
+
+	apic, err := apiclient.NewDefaultClient(
+		url,
+		"/api",
+		fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
+		nil,
+	)
+	require.NoError(t, err)
+
+	api.apiClient = apic
+	err = api.PullTop()
+	require.NoError(t, err)
+
+	assertTotalDecisionCount(t, api.dbClient, 5) //2 from FIRE + 2 from bl + 1 existing
+	assertTotalValidDecisionCount(t, api.dbClient, 4)
+	assertTotalAlertCount(t, api.dbClient, 3) // 2 for list sub , 1 for community list.
+	alerts := api.dbClient.Ent.Alert.Query().AllX(context.Background())
+	validDecisions := api.dbClient.Ent.Decision.Query().Where(
+		decision.UntilGT(time.Now())).
+		AllX(context.Background())
+
+	decisionScenarioFreq := make(map[string]int)
+	decisionIp := make(map[string]int)
+
+	alertScenario := make(map[string]int)
+
+	for _, alert := range alerts {
+		alertScenario[alert.SourceScope]++
+	}
+	assert.Equal(t, 3, len(alertScenario))
+	assert.Equal(t, 1, alertScenario[SCOPE_CAPI_ALIAS_ALIAS])
+	assert.Equal(t, 1, alertScenario["lists:blocklist1"])
+	assert.Equal(t, 1, alertScenario["lists:blocklist2"])
+
+	for _, decisions := range validDecisions {
+		decisionScenarioFreq[decisions.Scenario]++
+		decisionIp[decisions.Value]++
+	}
+	assert.Equal(t, 1, decisionIp["2.2.3.4"], 1)
+	assert.Equal(t, 1, decisionIp["6.2.3.4"], 1)
+	if _, ok := decisionIp["13.2.3.4"]; ok {
+		t.Errorf("13.2.3.4 is whitelisted")
+	}
+	if _, ok := decisionIp["13.2.3.5"]; ok {
+		t.Errorf("13.2.3.5 is whitelisted")
+	}
+	if _, ok := decisionIp["9.2.3.4"]; ok {
+		t.Errorf("9.2.3.4 is whitelisted")
+	}
+	assert.Equal(t, 1, decisionScenarioFreq["blocklist1"], 1)
+	assert.Equal(t, 1, decisionScenarioFreq["blocklist2"], 1)
+	assert.Equal(t, 2, decisionScenarioFreq["crowdsecurity/test1"], 2)
+}
+
 func TestAPICPullTop(t *testing.T) {
 	api := getAPIC(t)
 	api.dbClient.Ent.Decision.Create().

pkg/apiserver/apiserver.go

@@ -217,7 +217,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
 
 	if config.OnlineClient != nil && config.OnlineClient.Credentials != nil {
 		log.Printf("Loading CAPI manager")
-		apiClient, err = NewAPIC(config.OnlineClient, dbClient, config.ConsoleConfig)
+		apiClient, err = NewAPIC(config.OnlineClient, dbClient, config.ConsoleConfig, config.CapiWhitelists)
 		if err != nil {
 			return &APIServer{}, err
 		}

pkg/csconfig/api.go

@@ -173,6 +173,11 @@ func toValidCIDR(ip string) string {
 	return ip + "/32"
 }
 
+type CapiWhitelist struct {
+	Ips   []net.IP     `yaml:"ips,omitempty"`
+	Cidrs []*net.IPNet `yaml:"cidrs,omitempty"`
+}
+
 /*local api service configuration*/
 type LocalApiServerCfg struct {
 	Enable                        *bool               `yaml:"enable"`
@@ -196,6 +201,8 @@ type LocalApiServerCfg struct {
 	TrustedIPs                    []string            `yaml:"trusted_ips,omitempty"`
 	PapiLogLevel                  *log.Level          `yaml:"papi_log_level"`
 	DisableRemoteLapiRegistration bool                `yaml:"disable_remote_lapi_registration,omitempty"`
+	CapiWhitelistsPath            string              `yaml:"capi_whitelists_path,omitempty"`
+	CapiWhitelists                *CapiWhitelist      `yaml:"-"`
 }
 
 type TLSCfg struct {
@@ -242,6 +249,11 @@ func (c *Config) LoadAPIServer() error {
 		if err := c.LoadDBConfig(); err != nil {
 			return err
 		}
+
+		if err := c.API.Server.LoadCapiWhitelists(); err != nil {
+			return err
+		}
+
 	} else {
 		log.Warning("crowdsec local API is disabled")
 		c.DisableAPI = true
@@ -306,6 +318,49 @@ func (c *Config) LoadAPIServer() error {
 	return nil
 }
 
+// we cannot unmarshal to type net.IPNet, so we need to do it manually
+type capiWhitelists struct {
+	Ips   []string `yaml:"ips"`
+	Cidrs []string `yaml:"cidrs"`
+}
+
+func (s *LocalApiServerCfg) LoadCapiWhitelists() error {
+	if s.CapiWhitelistsPath == "" {
+		return nil
+	}
+	if _, err := os.Stat(s.CapiWhitelistsPath); os.IsNotExist(err) {
+		return fmt.Errorf("capi whitelist file '%s' does not exist", s.CapiWhitelistsPath)
+	}
+	fd, err := os.Open(s.CapiWhitelistsPath)
+	if err != nil {
+		return fmt.Errorf("unable to open capi whitelist file '%s': %s", s.CapiWhitelistsPath, err)
+	}
+
+	var fromCfg capiWhitelists
+	s.CapiWhitelists = &CapiWhitelist{}
+
+	defer fd.Close()
+	decoder := yaml.NewDecoder(fd)
+	if err := decoder.Decode(&fromCfg); err != nil {
+		return fmt.Errorf("while parsing capi whitelist file '%s': %s", s.CapiWhitelistsPath, err)
+	}
+	for _, v := range fromCfg.Ips {
+		ip := net.ParseIP(v)
+		if ip == nil {
+			return fmt.Errorf("unable to parse ip whitelist '%s'", v)
+		}
+		s.CapiWhitelists.Ips = append(s.CapiWhitelists.Ips, ip)
+	}
+	for _, v := range fromCfg.Cidrs {
+		_, tnet, err := net.ParseCIDR(v)
+		if err != nil {
+			return fmt.Errorf("unable to parse cidr whitelist '%s' : %v.", v, err)
+		}
+		s.CapiWhitelists.Cidrs = append(s.CapiWhitelists.Cidrs, tnet)
+	}
+	return nil
+}
+
 func (c *Config) LoadAPIClient() error {
 	if c.API == nil || c.API.Client == nil || c.API.Client.CredentialsFilePath == "" || c.DisableAgent {
 		return fmt.Errorf("no API client section in configuration")