diff --git a/cmd/crowdsec-cli/console.go b/cmd/crowdsec-cli/console.go index ad17a1316..dcd6fb37f 100644 --- a/cmd/crowdsec-cli/console.go +++ b/cmd/crowdsec-cli/console.go @@ -7,6 +7,7 @@ import ( "fmt" "net/url" "os" + "strings" "github.com/fatih/color" "github.com/go-openapi/strfmt" @@ -47,6 +48,7 @@ func NewConsoleCmd() *cobra.Command { name := "" overwrite := false tags := []string{} + opts := []string{} cmdEnroll := &cobra.Command{ Use: "enroll [enroll-key]", @@ -56,10 +58,12 @@ Enroll this instance to https://app.crowdsec.net You can get your enrollment key by creating an account on https://app.crowdsec.net. After running this command your will need to validate the enrollment in the webapp.`, - Example: `cscli console enroll YOUR-ENROLL-KEY + Example: fmt.Sprintf(`cscli console enroll YOUR-ENROLL-KEY cscli console enroll --name [instance_name] YOUR-ENROLL-KEY cscli console enroll --name [instance_name] --tags [tag_1] --tags [tag_2] YOUR-ENROLL-KEY -`, + cscli console enroll --enable context,manual YOUR-ENROLL-KEY + + valid options are : %s,all (see 'cscli console status' for details)`, strings.Join(csconfig.CONSOLE_CONFIGS, ",")), Args: cobra.ExactArgs(1), DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -83,6 +87,37 @@ After running this command your will need to validate the enrollment in the weba scenarios = make([]string, 0) } + enable_opts := []string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS} + if len(opts) != 0 { + for _, opt := range opts { + valid := false + if opt == "all" { + enable_opts = csconfig.CONSOLE_CONFIGS + break + } + for _, available_opt := range csconfig.CONSOLE_CONFIGS { + if opt == available_opt { + valid = true + enable := true + for _, enabled_opt := range enable_opts { + if opt == enabled_opt { + enable = false + continue + } + } + if enable { + enable_opts = append(enable_opts, opt) + } + break + } + } + if !valid { + return fmt.Errorf("option %s doesn't exist", opt) + + } + } + } + c, _ := apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Server.OnlineClient.Credentials.Login, Password: password, @@ -100,11 +135,13 @@ After running this command your will need to validate the enrollment in the weba return nil } - if err := SetConsoleOpts([]string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}, true); err != nil { + if err := SetConsoleOpts(enable_opts, true); err != nil { return err } - log.Info("Enabled tainted&manual alerts sharing, see 'cscli console status'.") + for _, opt := range enable_opts { + log.Infof("Enabled %s : %s", opt, csconfig.CONSOLE_CONFIGS_HELP[opt]) + } log.Info("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.") log.Info("Please restart crowdsec after accepting the enrollment.") return nil @@ -113,6 +150,7 @@ After running this command your will need to validate the enrollment in the weba cmdEnroll.Flags().StringVarP(&name, "name", "n", "", "Name to display in the console") cmdEnroll.Flags().BoolVarP(&overwrite, "overwrite", "", false, "Force enroll the instance") cmdEnroll.Flags().StringSliceVarP(&tags, "tags", "t", tags, "Tags to display in the console") + cmdEnroll.Flags().StringSliceVarP(&opts, "enable", "e", opts, "Enable console options") cmdConsole.AddCommand(cmdEnroll) var enableAll, disableAll bool diff --git a/cmd/crowdsec-cli/console_table.go b/cmd/crowdsec-cli/console_table.go index fa2559daa..2a221e36f 100644 --- a/cmd/crowdsec-cli/console_table.go +++ b/cmd/crowdsec-cli/console_table.go @@ -17,45 +17,30 @@ func cmdConsoleStatusTable(out io.Writer, csConfig csconfig.Config) { t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) for _, option := range csconfig.CONSOLE_CONFIGS { + activated := string(emoji.CrossMark) switch option { case csconfig.SEND_CUSTOM_SCENARIOS: - activated := string(emoji.CrossMark) if *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios { activated = string(emoji.CheckMarkButton) } - - t.AddRow(option, activated, "Send alerts from custom scenarios to the console") - case csconfig.SEND_MANUAL_SCENARIOS: - activated := string(emoji.CrossMark) if *csConfig.API.Server.ConsoleConfig.ShareManualDecisions { activated = string(emoji.CheckMarkButton) } - - t.AddRow(option, activated, "Send manual decisions to the console") - case csconfig.SEND_TAINTED_SCENARIOS: - activated := string(emoji.CrossMark) if *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios { activated = string(emoji.CheckMarkButton) } - - t.AddRow(option, activated, "Send alerts from tainted scenarios to the console") case csconfig.SEND_CONTEXT: - activated := string(emoji.CrossMark) if *csConfig.API.Server.ConsoleConfig.ShareContext { activated = string(emoji.CheckMarkButton) } - - t.AddRow(option, activated, "Send context with alerts to the console") case csconfig.CONSOLE_MANAGEMENT: - activated := string(emoji.CrossMark) if *csConfig.API.Server.ConsoleConfig.ConsoleManagement { activated = string(emoji.CheckMarkButton) } - - t.AddRow(option, activated, "Receive decisions from console") } + t.AddRow(option, activated, csconfig.CONSOLE_CONFIGS_HELP[option]) } t.Render() diff --git a/cmd/crowdsec-cli/decisions_import.go b/cmd/crowdsec-cli/decisions_import.go index fb134c32e..2d7ee485b 100644 --- a/cmd/crowdsec-cli/decisions_import.go +++ b/cmd/crowdsec-cli/decisions_import.go @@ -243,6 +243,7 @@ func (cli cliDecisions) NewImportCmd() *cobra.Command { Long: "expected format:\n" + "csv : any of duration,reason,scope,type,value, with a header line\n" + "json :" + "`{" + `"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"` + "}`", + Args: cobra.NoArgs, DisableAutoGenTag: true, Example: `decisions.csv: duration,scope,value diff --git a/config/dev.yaml b/config/dev.yaml index 2123dc858..ca1f35f32 100644 --- a/config/dev.yaml +++ b/config/dev.yaml @@ -33,6 +33,7 @@ api: client: credentials_path: ./config/local_api_credentials.yaml server: + console_path: ./config/console.yaml #insecure_skip_verify: true listen_uri: 127.0.0.1:8081 profiles_path: ./config/profiles.yaml diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index b1724784c..19a0085d2 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -243,9 +243,9 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { controller.AlertsAddChan = apiClient.AlertsAddChan if apiClient.apiClient.IsEnrolled() { - log.Infof("Machine is enrolled in the console, Loading PAPI Client") - if config.ConsoleConfig.IsPAPIEnabled() { + log.Info("Machine is enrolled in the console, Loading PAPI Client") + papiClient, err = NewPAPI(apiClient, dbClient, config.ConsoleConfig, *config.PapiLogLevel) if err != nil { return nil, err diff --git a/pkg/appsec/appsec.go b/pkg/appsec/appsec.go index cbf9b5876..ec7e7bef3 100644 --- a/pkg/appsec/appsec.go +++ b/pkg/appsec/appsec.go @@ -130,9 +130,9 @@ type AppsecConfig struct { } func (w *AppsecRuntimeConfig) ClearResponse() { - log.Debugf("#-> %p", w) + w.Logger.Debugf("#-> %p", w) w.Response = AppsecTempResponse{} - log.Debugf("-> %p", w.Config) + w.Logger.Debugf("-> %p", w.Config) w.Response.Action = w.Config.DefaultPassAction w.Response.HTTPResponseCode = w.Config.PassedHTTPCode w.Response.SendEvent = true @@ -290,20 +290,26 @@ func (w *AppsecRuntimeConfig) ProcessOnLoadRules() error { switch t := output.(type) { case bool: if !t { - log.Debugf("filter didnt match") + w.Logger.Debugf("filter didnt match") continue } default: - log.Errorf("Filter must return a boolean, can't filter") + w.Logger.Errorf("Filter must return a boolean, can't filter") continue } } for _, applyExpr := range rule.ApplyExpr { - _, err := exprhelpers.Run(applyExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel) + o, err := exprhelpers.Run(applyExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { - log.Errorf("unable to apply appsec on_load expr: %s", err) + w.Logger.Errorf("unable to apply appsec on_load expr: %s", err) continue } + switch t := o.(type) { + case error: + w.Logger.Errorf("unable to apply appsec on_load expr: %s", t) + continue + default: + } } } return nil @@ -320,27 +326,33 @@ func (w *AppsecRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt ty switch t := output.(type) { case bool: if !t { - log.Debugf("filter didnt match") + w.Logger.Debugf("filter didnt match") continue } default: - log.Errorf("Filter must return a boolean, can't filter") + w.Logger.Errorf("Filter must return a boolean, can't filter") continue } } for _, applyExpr := range rule.ApplyExpr { - _, err := exprhelpers.Run(applyExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel) + o, err := exprhelpers.Run(applyExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { - log.Errorf("unable to apply appsec on_match expr: %s", err) + w.Logger.Errorf("unable to apply appsec on_match expr: %s", err) continue } + switch t := o.(type) { + case error: + w.Logger.Errorf("unable to apply appsec on_match expr: %s", t) + continue + default: + } } } return nil } func (w *AppsecRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { - log.Debugf("processing %d pre_eval rules", len(w.CompiledPreEval)) + w.Logger.Debugf("processing %d pre_eval rules", len(w.CompiledPreEval)) for _, rule := range w.CompiledPreEval { if rule.FilterExpr != nil { output, err := exprhelpers.Run(rule.FilterExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) @@ -350,21 +362,27 @@ func (w *AppsecRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error switch t := output.(type) { case bool: if !t { - log.Debugf("filter didnt match") + w.Logger.Debugf("filter didnt match") continue } default: - log.Errorf("Filter must return a boolean, can't filter") + w.Logger.Errorf("Filter must return a boolean, can't filter") continue } } // here means there is no filter or the filter matched for _, applyExpr := range rule.ApplyExpr { - _, err := exprhelpers.Run(applyExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) + o, err := exprhelpers.Run(applyExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) if err != nil { - log.Errorf("unable to apply appsec pre_eval expr: %s", err) + w.Logger.Errorf("unable to apply appsec pre_eval expr: %s", err) continue } + switch t := o.(type) { + case error: + w.Logger.Errorf("unable to apply appsec pre_eval expr: %s", t) + continue + default: + } } } @@ -381,21 +399,29 @@ func (w *AppsecRuntimeConfig) ProcessPostEvalRules(request *ParsedRequest) error switch t := output.(type) { case bool: if !t { - log.Debugf("filter didnt match") + w.Logger.Debugf("filter didnt match") continue } default: - log.Errorf("Filter must return a boolean, can't filter") + w.Logger.Errorf("Filter must return a boolean, can't filter") continue } } // here means there is no filter or the filter matched for _, applyExpr := range rule.ApplyExpr { - _, err := exprhelpers.Run(applyExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) + o, err := exprhelpers.Run(applyExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) + if err != nil { - log.Errorf("unable to apply appsec post_eval expr: %s", err) + w.Logger.Errorf("unable to apply appsec post_eval expr: %s", err) continue } + + switch t := o.(type) { + case error: + w.Logger.Errorf("unable to apply appsec post_eval expr: %s", t) + continue + default: + } } } diff --git a/pkg/appsec/request.go b/pkg/appsec/request.go index e2378277a..f244cee9c 100644 --- a/pkg/appsec/request.go +++ b/pkg/appsec/request.go @@ -38,7 +38,7 @@ type ParsedRequest struct { Body []byte `json:"body,omitempty"` TransferEncoding []string `json:"transfer_encoding,omitempty"` UUID string `json:"uuid,omitempty"` - Tx ExtendedTransaction `json:"transaction,omitempty"` + Tx ExtendedTransaction `json:"-"` ResponseChannel chan AppsecTempResponse `json:"-"` IsInBand bool `json:"-"` IsOutBand bool `json:"-"` @@ -260,12 +260,17 @@ func (r *ReqDumpFilter) ToJSON() error { req := r.GetFilteredRequest() - log.Warningf("dumping : %+v", req) + log.Tracef("dumping : %+v", req) if err := enc.Encode(req); err != nil { + //Don't clobber the temp directory with empty files + err2 := os.Remove(fd.Name()) + if err2 != nil { + log.Errorf("while removing temp file %s: %s", fd.Name(), err) + } return fmt.Errorf("while encoding request: %w", err) } - log.Warningf("request dumped to %s", fd.Name()) + log.Infof("request dumped to %s", fd.Name()) return nil } diff --git a/pkg/csconfig/console.go b/pkg/csconfig/console.go index 9e3b1e5ac..1e8974154 100644 --- a/pkg/csconfig/console.go +++ b/pkg/csconfig/console.go @@ -19,6 +19,13 @@ const ( ) var CONSOLE_CONFIGS = []string{SEND_CUSTOM_SCENARIOS, SEND_MANUAL_SCENARIOS, SEND_TAINTED_SCENARIOS, SEND_CONTEXT, CONSOLE_MANAGEMENT} +var CONSOLE_CONFIGS_HELP = map[string]string{ + SEND_CUSTOM_SCENARIOS: "Forward alerts from custom scenarios to the console", + SEND_MANUAL_SCENARIOS: "Forward manual decisions to the console", + SEND_TAINTED_SCENARIOS: "Forward alerts from tainted scenarios to the console", + SEND_CONTEXT: "Forward context with alerts to the console", + CONSOLE_MANAGEMENT: "Receive decisions from console", +} var DefaultConsoleConfigFilePath = DefaultConfigPath("console.yaml") diff --git a/pkg/database/alerts.go b/pkg/database/alerts.go index 83af76464..8524884d7 100644 --- a/pkg/database/alerts.go +++ b/pkg/database/alerts.go @@ -209,9 +209,9 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert) //add missing decisions log.Debugf("Adding %d missing decisions to alert %s", len(missingDecisions), foundAlert.UUID) - decisionBuilders := make([]*ent.DecisionCreate, len(missingDecisions)) + decisionBuilders := []*ent.DecisionCreate{} - for i, decisionItem := range missingDecisions { + for _, decisionItem := range missingDecisions { var start_ip, start_sfx, end_ip, end_sfx int64 var sz int @@ -219,7 +219,8 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert) if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" { sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value) if err != nil { - return "", errors.Wrapf(InvalidIPOrRange, "invalid addr/range %s : %s", *decisionItem.Value, err) + log.Errorf("invalid addr/range '%s': %s", *decisionItem.Value, err) + continue } } @@ -254,7 +255,7 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert) SetSimulated(*alertItem.Simulated). SetUUID(decisionItem.UUID) - decisionBuilders[i] = decisionBuilder + decisionBuilders = append(decisionBuilders, decisionBuilder) } decisions := []*ent.Decision{} @@ -486,9 +487,9 @@ func (c *Client) UpdateCommunityBlocklist(alertItem *models.Alert) (int, int, in } func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decisions []*models.Decision) ([]*ent.Decision, error) { - decisionCreate := make([]*ent.DecisionCreate, len(decisions)) + decisionCreate := []*ent.DecisionCreate{} - for i, decisionItem := range decisions { + for _, decisionItem := range decisions { var start_ip, start_sfx, end_ip, end_sfx int64 var sz int @@ -501,7 +502,8 @@ func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decis if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" { sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value) if err != nil { - return nil, fmt.Errorf("%s: %w", *decisionItem.Value, InvalidIPOrRange) + log.Errorf("invalid addr/range '%s': %s", *decisionItem.Value, err) + continue } } @@ -520,7 +522,11 @@ func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decis SetSimulated(simulated). SetUUID(decisionItem.UUID) - decisionCreate[i] = newDecision + decisionCreate = append(decisionCreate, newDecision) + } + + if len(decisionCreate) == 0 { + return nil, nil } ret, err := c.Ent.Decision.CreateBulk(decisionCreate...).Save(c.CTX) @@ -532,10 +538,10 @@ func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decis } func (c *Client) createAlertChunk(machineID string, owner *ent.Machine, alerts []*models.Alert) ([]string, error) { - alertBuilders := make([]*ent.AlertCreate, len(alerts)) - alertDecisions := make([][]*ent.Decision, len(alerts)) + alertBuilders := []*ent.AlertCreate{} + alertDecisions := [][]*ent.Decision{} - for i, alertItem := range alerts { + for _, alertItem := range alerts { var metas []*ent.Meta var events []*ent.Event @@ -656,6 +662,17 @@ func (c *Client) createAlertChunk(machineID string, owner *ent.Machine, alerts [ decisions = append(decisions, decisionRet...) } + discarded := len(alertItem.Decisions) - len(decisions) + if discarded > 0 { + c.Log.Warningf("discarded %d decisions for %s", discarded, alertItem.UUID) + } + + // if all decisions were discarded, discard the alert too + if discarded > 0 && len(decisions) == 0 { + c.Log.Warningf("dropping alert %s with invalid decisions", alertItem.UUID) + continue + } + alertBuilder := c.Ent.Alert. Create(). SetScenario(*alertItem.Scenario). @@ -685,8 +702,13 @@ func (c *Client) createAlertChunk(machineID string, owner *ent.Machine, alerts [ alertBuilder.SetOwner(owner) } - alertBuilders[i] = alertBuilder - alertDecisions[i] = decisions + alertBuilders = append(alertBuilders, alertBuilder) + alertDecisions = append(alertDecisions, decisions) + } + + if len(alertBuilders) == 0 { + log.Warningf("no alerts to create, discarded?") + return nil, nil } alertsCreateBulk, err := c.Ent.Alert.CreateBulk(alertBuilders...).Save(c.CTX) diff --git a/test/bats/01_cscli.bats b/test/bats/01_cscli.bats index f843137a5..3a5b4aad0 100644 --- a/test/bats/01_cscli.bats +++ b/test/bats/01_cscli.bats @@ -115,16 +115,16 @@ teardown() { assert_output "&false" # complex type - rune -0 cscli config show -o json --key Config.Prometheus + rune -0 cscli config show --key Config.Prometheus assert_output - <<-EOT - &csconfig.PrometheusCfg{ - Enabled: true, - Level: "full", - ListenAddr: "127.0.0.1", - ListenPort: 6060, - } - EOT - } + &csconfig.PrometheusCfg{ + Enabled: true, + Level: "full", + ListenAddr: "127.0.0.1", + ListenPort: 6060, + } + EOT +} @test "cscli - required configuration paths" { diff --git a/test/bats/90_decisions.bats b/test/bats/90_decisions.bats index 5870eb36b..8a2b9d3ae 100644 --- a/test/bats/90_decisions.bats +++ b/test/bats/90_decisions.bats @@ -16,7 +16,10 @@ teardown_file() { setup() { load "../lib/setup.sh" + load "../lib/bats-file/load.bash" ./instance-data load + LOGFILE=$(config_get '.common.log_dir')/crowdsec.log + export LOGFILE ./instance-crowdsec start } @@ -151,6 +154,7 @@ teardown() { assert_stderr --partial 'Parsing values' assert_stderr --partial 'Imported 3 decisions' + # leading or trailing spaces are ignored rune -0 cscli decisions import -i - --format values <<-EOT 10.2.3.4 10.2.3.5 @@ -159,11 +163,39 @@ teardown() { assert_stderr --partial 'Parsing values' assert_stderr --partial 'Imported 3 decisions' - rune -1 cscli decisions import -i - --format values <<-EOT + # silently discarding (but logging) invalid decisions + + rune -0 cscli alerts delete --all + truncate -s 0 "${LOGFILE}" + + rune -0 cscli decisions import -i - --format values <<-EOT whatever EOT assert_stderr --partial 'Parsing values' - assert_stderr --partial 'creating alert decisions: whatever: invalid ip address / range' + assert_stderr --partial 'Imported 1 decisions' + assert_file_contains "$LOGFILE" "invalid addr/range 'whatever': invalid address" + + rune -0 cscli decisions list -a -o json + assert_json '[]' + + # disarding only some invalid decisions + + + rune -0 cscli alerts delete --all + truncate -s 0 "${LOGFILE}" + + rune -0 cscli decisions import -i - --format values <<-EOT + 1.2.3.4 + bad-apple + 1.2.3.5 + EOT + assert_stderr --partial 'Parsing values' + assert_stderr --partial 'Imported 3 decisions' + assert_file_contains "$LOGFILE" "invalid addr/range 'bad-apple': invalid address" + + rune -0 cscli decisions list -a -o json + rune -0 jq -r '.[0].decisions | length' <(output) + assert_output 2 #---------- # Batch