diff --git a/cmd/crowdsec-cli/decisions_import.go b/cmd/crowdsec-cli/decisions_import.go index fb134c32e..2d7ee485b 100644 --- a/cmd/crowdsec-cli/decisions_import.go +++ b/cmd/crowdsec-cli/decisions_import.go @@ -243,6 +243,7 @@ func (cli cliDecisions) NewImportCmd() *cobra.Command { Long: "expected format:\n" + "csv : any of duration,reason,scope,type,value, with a header line\n" + "json :" + "`{" + `"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"` + "}`", + Args: cobra.NoArgs, DisableAutoGenTag: true, Example: `decisions.csv: duration,scope,value diff --git a/pkg/database/alerts.go b/pkg/database/alerts.go index 83af76464..8524884d7 100644 --- a/pkg/database/alerts.go +++ b/pkg/database/alerts.go @@ -209,9 +209,9 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert) //add missing decisions log.Debugf("Adding %d missing decisions to alert %s", len(missingDecisions), foundAlert.UUID) - decisionBuilders := make([]*ent.DecisionCreate, len(missingDecisions)) + decisionBuilders := []*ent.DecisionCreate{} - for i, decisionItem := range missingDecisions { + for _, decisionItem := range missingDecisions { var start_ip, start_sfx, end_ip, end_sfx int64 var sz int @@ -219,7 +219,8 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert) if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" { sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value) if err != nil { - return "", errors.Wrapf(InvalidIPOrRange, "invalid addr/range %s : %s", *decisionItem.Value, err) + log.Errorf("invalid addr/range '%s': %s", *decisionItem.Value, err) + continue } } @@ -254,7 +255,7 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert) SetSimulated(*alertItem.Simulated). SetUUID(decisionItem.UUID) - decisionBuilders[i] = decisionBuilder + decisionBuilders = append(decisionBuilders, decisionBuilder) } decisions := []*ent.Decision{} @@ -486,9 +487,9 @@ func (c *Client) UpdateCommunityBlocklist(alertItem *models.Alert) (int, int, in } func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decisions []*models.Decision) ([]*ent.Decision, error) { - decisionCreate := make([]*ent.DecisionCreate, len(decisions)) + decisionCreate := []*ent.DecisionCreate{} - for i, decisionItem := range decisions { + for _, decisionItem := range decisions { var start_ip, start_sfx, end_ip, end_sfx int64 var sz int @@ -501,7 +502,8 @@ func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decis if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" { sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value) if err != nil { - return nil, fmt.Errorf("%s: %w", *decisionItem.Value, InvalidIPOrRange) + log.Errorf("invalid addr/range '%s': %s", *decisionItem.Value, err) + continue } } @@ -520,7 +522,11 @@ func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decis SetSimulated(simulated). SetUUID(decisionItem.UUID) - decisionCreate[i] = newDecision + decisionCreate = append(decisionCreate, newDecision) + } + + if len(decisionCreate) == 0 { + return nil, nil } ret, err := c.Ent.Decision.CreateBulk(decisionCreate...).Save(c.CTX) @@ -532,10 +538,10 @@ func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decis } func (c *Client) createAlertChunk(machineID string, owner *ent.Machine, alerts []*models.Alert) ([]string, error) { - alertBuilders := make([]*ent.AlertCreate, len(alerts)) - alertDecisions := make([][]*ent.Decision, len(alerts)) + alertBuilders := []*ent.AlertCreate{} + alertDecisions := [][]*ent.Decision{} - for i, alertItem := range alerts { + for _, alertItem := range alerts { var metas []*ent.Meta var events []*ent.Event @@ -656,6 +662,17 @@ func (c *Client) createAlertChunk(machineID string, owner *ent.Machine, alerts [ decisions = append(decisions, decisionRet...) } + discarded := len(alertItem.Decisions) - len(decisions) + if discarded > 0 { + c.Log.Warningf("discarded %d decisions for %s", discarded, alertItem.UUID) + } + + // if all decisions were discarded, discard the alert too + if discarded > 0 && len(decisions) == 0 { + c.Log.Warningf("dropping alert %s with invalid decisions", alertItem.UUID) + continue + } + alertBuilder := c.Ent.Alert. Create(). SetScenario(*alertItem.Scenario). @@ -685,8 +702,13 @@ func (c *Client) createAlertChunk(machineID string, owner *ent.Machine, alerts [ alertBuilder.SetOwner(owner) } - alertBuilders[i] = alertBuilder - alertDecisions[i] = decisions + alertBuilders = append(alertBuilders, alertBuilder) + alertDecisions = append(alertDecisions, decisions) + } + + if len(alertBuilders) == 0 { + log.Warningf("no alerts to create, discarded?") + return nil, nil } alertsCreateBulk, err := c.Ent.Alert.CreateBulk(alertBuilders...).Save(c.CTX) diff --git a/test/bats/90_decisions.bats b/test/bats/90_decisions.bats index 5870eb36b..8a2b9d3ae 100644 --- a/test/bats/90_decisions.bats +++ b/test/bats/90_decisions.bats @@ -16,7 +16,10 @@ teardown_file() { setup() { load "../lib/setup.sh" + load "../lib/bats-file/load.bash" ./instance-data load + LOGFILE=$(config_get '.common.log_dir')/crowdsec.log + export LOGFILE ./instance-crowdsec start } @@ -151,6 +154,7 @@ teardown() { assert_stderr --partial 'Parsing values' assert_stderr --partial 'Imported 3 decisions' + # leading or trailing spaces are ignored rune -0 cscli decisions import -i - --format values <<-EOT 10.2.3.4 10.2.3.5 @@ -159,11 +163,39 @@ teardown() { assert_stderr --partial 'Parsing values' assert_stderr --partial 'Imported 3 decisions' - rune -1 cscli decisions import -i - --format values <<-EOT + # silently discarding (but logging) invalid decisions + + rune -0 cscli alerts delete --all + truncate -s 0 "${LOGFILE}" + + rune -0 cscli decisions import -i - --format values <<-EOT whatever EOT assert_stderr --partial 'Parsing values' - assert_stderr --partial 'creating alert decisions: whatever: invalid ip address / range' + assert_stderr --partial 'Imported 1 decisions' + assert_file_contains "$LOGFILE" "invalid addr/range 'whatever': invalid address" + + rune -0 cscli decisions list -a -o json + assert_json '[]' + + # disarding only some invalid decisions + + + rune -0 cscli alerts delete --all + truncate -s 0 "${LOGFILE}" + + rune -0 cscli decisions import -i - --format values <<-EOT + 1.2.3.4 + bad-apple + 1.2.3.5 + EOT + assert_stderr --partial 'Parsing values' + assert_stderr --partial 'Imported 3 decisions' + assert_file_contains "$LOGFILE" "invalid addr/range 'bad-apple': invalid address" + + rune -0 cscli decisions list -a -o json + rune -0 jq -r '.[0].decisions | length' <(output) + assert_output 2 #---------- # Batch