0ct0pu5/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

support ip and cidr based whitelists for capi and 3rd party blocklists (#2132)

Browse source 
* support ip and cidr based whitelists for capi and 3rd party blocklist
This commit is contained in:
Thibault "bui" Koechlin 2023-03-21 11:50:10 +01:00 • committed by GitHub
parent d87f088b8f
commit a74e424d53
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 292 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cmd/crowdsec-cli/papi.go
View file

@ -51,7 +51,7 @@ func NewPapiStatusCmd() *cobra.Command {
log.Fatalf("unable to initialize database client : %s", err)
}
apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig)
apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig, csConfig.API.Server.CapiWhitelists)
if err != nil {
log.Fatalf("unable to initialize API client : %s", err)
@ -101,7 +101,7 @@ func NewPapiSyncCmd() *cobra.Command {
log.Fatalf("unable to initialize database client : %s", err)
}
apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig)
apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig, csConfig.API.Server.CapiWhitelists)
if err != nil {
log.Fatalf("unable to initialize API client : %s", err)

51
pkg/apiserver/apic.go
View file

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"math/rand"
"net"
"net/http"
"net/url"
"strconv"
@ -60,6 +61,7 @@ type apic struct {
credentials *csconfig.ApiCredentialsCfg
scenarioList []string
consoleConfig *csconfig.ConsoleConfig
whitelists *csconfig.CapiWhitelist
}
// randomDuration returns a duration value between d-delta and d+delta
@ -149,7 +151,7 @@ func alertToSignal(alert *models.Alert, scenarioTrust string, shareContext bool)
return signal
}
func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, consoleConfig *csconfig.ConsoleConfig) (*apic, error) {
func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, consoleConfig *csconfig.ConsoleConfig, apicWhitelist *csconfig.CapiWhitelist) (*apic, error) {
var err error
ret := &apic{
@ -169,6 +171,7 @@ func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, con
pushIntervalFirst: randomDuration(pushIntervalDefault, pushIntervalDelta),
metricsInterval: metricsIntervalDefault,
metricsIntervalFirst: randomDuration(metricsIntervalDefault, metricsIntervalDelta),
whitelists: apicWhitelist,
}
password := strfmt.Password(config.Credentials.Password)
@ -573,6 +576,9 @@ func (a *apic) PullTop() error {
// create one alert for community blocklist using the first decision
decisions := a.apiClient.Decisions.GetDecisionsFromGroups(data.New)
//apply APIC specific whitelists
decisions = a.ApplyApicWhitelists(decisions)
alert := createAlertForDecision(decisions[0])
alertsFromCapi := []*models.Alert{alert}
alertsFromCapi = fillAlertsWithDecisions(alertsFromCapi, decisions, add_counters)
@ -589,6 +595,47 @@ func (a *apic) PullTop() error {
return nil
}
func (a *apic) ApplyApicWhitelists(decisions []*models.Decision) []*models.Decision {
if a.whitelists == nil {
return decisions
}
//deal with CAPI whitelists for fire. We want to avoid having a second list, so we shrink in place
outIdx := 0
for _, decision := range decisions {
if decision.Value == nil {
continue
}
skip := false
ipval := net.ParseIP(*decision.Value)
for _, cidr := range a.whitelists.Cidrs {
if skip {
break
}
if cidr.Contains(ipval) {
log.Infof("%s from %s is whitelisted by %s", *decision.Value, *decision.Scenario, cidr.String())
skip = true
}
}
for _, ip := range a.whitelists.Ips {
if skip {
break
}
if ip != nil && ip.Equal(ipval) {
log.Infof("%s from %s is whitelisted by %s", *decision.Value, *decision.Scenario, ip.String())
skip = true
}
}
if !skip {
decisions[outIdx] = decision
outIdx++
}
}
//shrink the list, those are deleted items
decisions = decisions[:outIdx]
return decisions
}
func (a *apic) SaveAlerts(alertsFromCapi []*models.Alert, add_counters map[string]map[string]int, delete_counters map[string]map[string]int) error {
for idx, alert := range alertsFromCapi {
alertsFromCapi[idx] = setAlertScenario(add_counters, delete_counters, alert)
@ -690,6 +737,8 @@ func (a *apic) UpdateBlocklists(links *modelscapi.GetDecisionsStreamResponseLink
log.Infof("blocklist %s has no decisions", *blocklist.Name)
continue
}
//apply APIC specific whitelists
decisions = a.ApplyApicWhitelists(decisions)
alert := createAlertForDecision(decisions[0])
alertsFromCapi := []*models.Alert{alert}
alertsFromCapi = fillAlertsWithDecisions(alertsFromCapi, decisions, add_counters)

185
pkg/apiserver/apic_test.go
View file

@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"net/url"
"os"
@ -232,7 +233,7 @@ func TestNewAPIC(t *testing.T) {
),
))
tc.action()
_, err := NewAPIC(testConfig, tc.args.dbClient, tc.args.consoleConfig)
_, err := NewAPIC(testConfig, tc.args.dbClient, tc.args.consoleConfig, nil)
cstest.RequireErrorContains(t, err, tc.expectedErr)
})
}
@ -527,6 +528,188 @@ func TestFillAlertsWithDecisions(t *testing.T) {
}
}
func TestAPICWhitelists(t *testing.T) {
api := getAPIC(t)
//one whitelist on IP, one on CIDR
api.whitelists = &csconfig.CapiWhitelist{}
ipwl1 := "9.2.3.4"
ip := net.ParseIP(ipwl1)
api.whitelists.Ips = append(api.whitelists.Ips, ip)
ipwl1 = "7.2.3.4"
ip = net.ParseIP(ipwl1)
api.whitelists.Ips = append(api.whitelists.Ips, ip)
cidrwl1 := "13.2.3.0/24"
_, tnet, err := net.ParseCIDR(cidrwl1)
if err != nil {
t.Fatalf("unable to parse cidr : %s", err)
}
api.whitelists.Cidrs = append(api.whitelists.Cidrs, tnet)
cidrwl1 = "11.2.3.0/24"
_, tnet, err = net.ParseCIDR(cidrwl1)
if err != nil {
t.Fatalf("unable to parse cidr : %s", err)
}
api.whitelists.Cidrs = append(api.whitelists.Cidrs, tnet)
api.dbClient.Ent.Decision.Create().
SetOrigin(types.CAPIOrigin).
SetType("ban").
SetValue("9.9.9.9").
SetScope("Ip").
SetScenario("crowdsecurity/ssh-bf").
SetUntil(time.Now().Add(time.Hour)).
ExecX(context.Background())
assertTotalDecisionCount(t, api.dbClient, 1)
assertTotalValidDecisionCount(t, api.dbClient, 1)
httpmock.Activate()
defer httpmock.DeactivateAndReset()
httpmock.RegisterResponder("GET", "http://api.crowdsec.net/api/decisions/stream", httpmock.NewBytesResponder(
200, jsonMarshalX(
modelscapi.GetDecisionsStreamResponse{
Deleted: modelscapi.GetDecisionsStreamResponseDeleted{
&modelscapi.GetDecisionsStreamResponseDeletedItem{
Decisions: []string{
"9.9.9.9", // This is already present in DB
"9.1.9.9", // This not present in DB
},
Scope: types.StrPtr("Ip"),
}, // This is already present in DB
},
New: modelscapi.GetDecisionsStreamResponseNew{
&modelscapi.GetDecisionsStreamResponseNewItem{
Scenario: types.StrPtr("crowdsecurity/test1"),
Scope: types.StrPtr("Ip"),
Decisions: []*modelscapi.GetDecisionsStreamResponseNewItemDecisionsItems0{
{
Value: types.StrPtr("13.2.3.4"), //wl by cidr
Duration: types.StrPtr("24h"),
},
},
},
&modelscapi.GetDecisionsStreamResponseNewItem{
Scenario: types.StrPtr("crowdsecurity/test1"),
Scope: types.StrPtr("Ip"),
Decisions: []*modelscapi.GetDecisionsStreamResponseNewItemDecisionsItems0{
{
Value: types.StrPtr("2.2.3.4"),
Duration: types.StrPtr("24h"),
},
},
},
&modelscapi.GetDecisionsStreamResponseNewItem{
Scenario: types.StrPtr("crowdsecurity/test2"),
Scope: types.StrPtr("Ip"),
Decisions: []*modelscapi.GetDecisionsStreamResponseNewItemDecisionsItems0{
{
Value: types.StrPtr("13.2.3.5"), //wl by cidr
Duration: types.StrPtr("24h"),
},
},
}, // These two are from community list.
&modelscapi.GetDecisionsStreamResponseNewItem{
Scenario: types.StrPtr("crowdsecurity/test1"),
Scope: types.StrPtr("Ip"),
Decisions: []*modelscapi.GetDecisionsStreamResponseNewItemDecisionsItems0{
{
Value: types.StrPtr("6.2.3.4"),
Duration: types.StrPtr("24h"),
},
},
},
&modelscapi.GetDecisionsStreamResponseNewItem{
Scenario: types.StrPtr("crowdsecurity/test1"),
Scope: types.StrPtr("Ip"),
Decisions: []*modelscapi.GetDecisionsStreamResponseNewItemDecisionsItems0{
{
Value: types.StrPtr("9.2.3.4"), //wl by ip
Duration: types.StrPtr("24h"),
},
},
},
},
Links: &modelscapi.GetDecisionsStreamResponseLinks{
Blocklists: []*modelscapi.BlocklistLink{
{
URL: types.StrPtr("http://api.crowdsec.net/blocklist1"),
Name: types.StrPtr("blocklist1"),
Scope: types.StrPtr("Ip"),
Remediation: types.StrPtr("ban"),
Duration: types.StrPtr("24h"),
},
{
URL: types.StrPtr("http://api.crowdsec.net/blocklist2"),
Name: types.StrPtr("blocklist2"),
Scope: types.StrPtr("Ip"),
Remediation: types.StrPtr("ban"),
Duration: types.StrPtr("24h"),
},
},
},
},
),
))
httpmock.RegisterResponder("GET", "http://api.crowdsec.net/blocklist1", httpmock.NewStringResponder(
200, "1.2.3.6",
))
httpmock.RegisterResponder("GET", "http://api.crowdsec.net/blocklist2", httpmock.NewStringResponder(
200, "1.2.3.7",
))
url, err := url.ParseRequestURI("http://api.crowdsec.net/")
require.NoError(t, err)
apic, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
nil,
)
require.NoError(t, err)
api.apiClient = apic
err = api.PullTop()
require.NoError(t, err)
assertTotalDecisionCount(t, api.dbClient, 5) //2 from FIRE + 2 from bl + 1 existing
assertTotalValidDecisionCount(t, api.dbClient, 4)
assertTotalAlertCount(t, api.dbClient, 3) // 2 for list sub , 1 for community list.
alerts := api.dbClient.Ent.Alert.Query().AllX(context.Background())
validDecisions := api.dbClient.Ent.Decision.Query().Where(
decision.UntilGT(time.Now())).
AllX(context.Background())
decisionScenarioFreq := make(map[string]int)
decisionIp := make(map[string]int)
alertScenario := make(map[string]int)
for _, alert := range alerts {
alertScenario[alert.SourceScope]++
}
assert.Equal(t, 3, len(alertScenario))
assert.Equal(t, 1, alertScenario[SCOPE_CAPI_ALIAS_ALIAS])
assert.Equal(t, 1, alertScenario["lists:blocklist1"])
assert.Equal(t, 1, alertScenario["lists:blocklist2"])
for _, decisions := range validDecisions {
decisionScenarioFreq[decisions.Scenario]++
decisionIp[decisions.Value]++
}
assert.Equal(t, 1, decisionIp["2.2.3.4"], 1)
assert.Equal(t, 1, decisionIp["6.2.3.4"], 1)
if _, ok := decisionIp["13.2.3.4"]; ok {
t.Errorf("13.2.3.4 is whitelisted")
}
if _, ok := decisionIp["13.2.3.5"]; ok {
t.Errorf("13.2.3.5 is whitelisted")
}
if _, ok := decisionIp["9.2.3.4"]; ok {
t.Errorf("9.2.3.4 is whitelisted")
}
assert.Equal(t, 1, decisionScenarioFreq["blocklist1"], 1)
assert.Equal(t, 1, decisionScenarioFreq["blocklist2"], 1)
assert.Equal(t, 2, decisionScenarioFreq["crowdsecurity/test1"], 2)
}
func TestAPICPullTop(t *testing.T) {
api := getAPIC(t)
api.dbClient.Ent.Decision.Create().

2
pkg/apiserver/apiserver.go
View file

@ -217,7 +217,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
if config.OnlineClient != nil && config.OnlineClient.Credentials != nil {
log.Printf("Loading CAPI manager")
apiClient, err = NewAPIC(config.OnlineClient, dbClient, config.ConsoleConfig)
apiClient, err = NewAPIC(config.OnlineClient, dbClient, config.ConsoleConfig, config.CapiWhitelists)
if err != nil {
return &APIServer{}, err
}

55
pkg/csconfig/api.go
View file

@ -173,6 +173,11 @@ func toValidCIDR(ip string) string {
return ip + "/32"
}
type CapiWhitelist struct {
Ips []net.IP `yaml:"ips,omitempty"`
Cidrs []*net.IPNet `yaml:"cidrs,omitempty"`
}
/*local api service configuration*/
type LocalApiServerCfg struct {
Enable *bool `yaml:"enable"`
@ -196,6 +201,8 @@ type LocalApiServerCfg struct {
TrustedIPs []string `yaml:"trusted_ips,omitempty"`
PapiLogLevel *log.Level `yaml:"papi_log_level"`
DisableRemoteLapiRegistration bool `yaml:"disable_remote_lapi_registration,omitempty"`
CapiWhitelistsPath string `yaml:"capi_whitelists_path,omitempty"`
CapiWhitelists *CapiWhitelist `yaml:"-"`
}
type TLSCfg struct {
@ -242,6 +249,11 @@ func (c *Config) LoadAPIServer() error {
if err := c.LoadDBConfig(); err != nil {
return err
}
if err := c.API.Server.LoadCapiWhitelists(); err != nil {
return err
}
} else {
log.Warning("crowdsec local API is disabled")
c.DisableAPI = true
@ -306,6 +318,49 @@ func (c *Config) LoadAPIServer() error {
return nil
}
// we cannot unmarshal to type net.IPNet, so we need to do it manually
type capiWhitelists struct {
Ips []string `yaml:"ips"`
Cidrs []string `yaml:"cidrs"`
}
func (s *LocalApiServerCfg) LoadCapiWhitelists() error {
if s.CapiWhitelistsPath == "" {
return nil
}
if _, err := os.Stat(s.CapiWhitelistsPath); os.IsNotExist(err) {
return fmt.Errorf("capi whitelist file '%s' does not exist", s.CapiWhitelistsPath)
}
fd, err := os.Open(s.CapiWhitelistsPath)
if err != nil {
return fmt.Errorf("unable to open capi whitelist file '%s': %s", s.CapiWhitelistsPath, err)
}
var fromCfg capiWhitelists
s.CapiWhitelists = &CapiWhitelist{}
defer fd.Close()
decoder := yaml.NewDecoder(fd)
if err := decoder.Decode(&fromCfg); err != nil {
return fmt.Errorf("while parsing capi whitelist file '%s': %s", s.CapiWhitelistsPath, err)
}
for _, v := range fromCfg.Ips {
ip := net.ParseIP(v)
if ip == nil {
return fmt.Errorf("unable to parse ip whitelist '%s'", v)
}
s.CapiWhitelists.Ips = append(s.CapiWhitelists.Ips, ip)
}
for _, v := range fromCfg.Cidrs {
_, tnet, err := net.ParseCIDR(v)
if err != nil {
return fmt.Errorf("unable to parse cidr whitelist '%s' : %v.", v, err)
}
s.CapiWhitelists.Cidrs = append(s.CapiWhitelists.Cidrs, tnet)
}
return nil
}
func (c *Config) LoadAPIClient() error {
if c.API == nil || c.API.Client == nil || c.API.Client.CredentialsFilePath == "" || c.DisableAgent {
return fmt.Errorf("no API client section in configuration")