Compare commits

...

10 commits

Author SHA1 Message Date
Thibault "bui" Koechlin
9e1cfb18b9 Support console options in console enroll (#2760)
* make dev.yaml has a valid/default console path

* simplify and make more consistent help message about console opts

* allow enroll to specify options to enable

* allow 'all' shortcut for --enable

lapi/papi: when receiving alerts, log and discard invalid addr/range (#2708)

https://github.com/crowdsecurity/crowdsec/issues/2687

log "loading papi client" only if papi is enabled (#2762)

Appsec hooks fixes (#2769)
2024-01-23 17:52:37 +01:00
sabban
977d904b94 use Prometheus COnfig instead of Plugin 2024-01-22 18:44:59 +01:00
sabban
6508965192 typos 2024-01-19 23:22:47 +01:00
sabban
eeb3aade32 typo 2024-01-19 18:13:58 +01:00
sabban
98239eff31 fix for centos9 2024-01-19 18:00:07 +01:00
sabban
b0c0b5ee00 typo 2024-01-19 16:30:52 +01:00
sabban
cbb6359e10 typo 2024-01-19 16:19:27 +01:00
sabban
2eb513aef9 typo 2024-01-19 15:45:12 +01:00
sabban
89fd883466 try fix 2024-01-19 15:40:13 +01:00
sabban
094d845e11 fix 04_capi.bats to avoid // 2024-01-19 15:02:19 +01:00
12 changed files with 185 additions and 65 deletions

View file

@ -7,6 +7,7 @@ import (
"fmt"
"net/url"
"os"
"strings"
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
@ -47,6 +48,7 @@ func NewConsoleCmd() *cobra.Command {
name := ""
overwrite := false
tags := []string{}
opts := []string{}
cmdEnroll := &cobra.Command{
Use: "enroll [enroll-key]",
@ -56,10 +58,12 @@ Enroll this instance to https://app.crowdsec.net
You can get your enrollment key by creating an account on https://app.crowdsec.net.
After running this command your will need to validate the enrollment in the webapp.`,
Example: `cscli console enroll YOUR-ENROLL-KEY
Example: fmt.Sprintf(`cscli console enroll YOUR-ENROLL-KEY
cscli console enroll --name [instance_name] YOUR-ENROLL-KEY
cscli console enroll --name [instance_name] --tags [tag_1] --tags [tag_2] YOUR-ENROLL-KEY
`,
cscli console enroll --enable context,manual YOUR-ENROLL-KEY
valid options are : %s,all (see 'cscli console status' for details)`, strings.Join(csconfig.CONSOLE_CONFIGS, ",")),
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
@ -83,6 +87,37 @@ After running this command your will need to validate the enrollment in the weba
scenarios = make([]string, 0)
}
enable_opts := []string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}
if len(opts) != 0 {
for _, opt := range opts {
valid := false
if opt == "all" {
enable_opts = csconfig.CONSOLE_CONFIGS
break
}
for _, available_opt := range csconfig.CONSOLE_CONFIGS {
if opt == available_opt {
valid = true
enable := true
for _, enabled_opt := range enable_opts {
if opt == enabled_opt {
enable = false
continue
}
}
if enable {
enable_opts = append(enable_opts, opt)
}
break
}
}
if !valid {
return fmt.Errorf("option %s doesn't exist", opt)
}
}
}
c, _ := apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Server.OnlineClient.Credentials.Login,
Password: password,
@ -100,11 +135,13 @@ After running this command your will need to validate the enrollment in the weba
return nil
}
if err := SetConsoleOpts([]string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}, true); err != nil {
if err := SetConsoleOpts(enable_opts, true); err != nil {
return err
}
log.Info("Enabled tainted&manual alerts sharing, see 'cscli console status'.")
for _, opt := range enable_opts {
log.Infof("Enabled %s : %s", opt, csconfig.CONSOLE_CONFIGS_HELP[opt])
}
log.Info("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
log.Info("Please restart crowdsec after accepting the enrollment.")
return nil
@ -113,6 +150,7 @@ After running this command your will need to validate the enrollment in the weba
cmdEnroll.Flags().StringVarP(&name, "name", "n", "", "Name to display in the console")
cmdEnroll.Flags().BoolVarP(&overwrite, "overwrite", "", false, "Force enroll the instance")
cmdEnroll.Flags().StringSliceVarP(&tags, "tags", "t", tags, "Tags to display in the console")
cmdEnroll.Flags().StringSliceVarP(&opts, "enable", "e", opts, "Enable console options")
cmdConsole.AddCommand(cmdEnroll)
var enableAll, disableAll bool

View file

@ -17,45 +17,30 @@ func cmdConsoleStatusTable(out io.Writer, csConfig csconfig.Config) {
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, option := range csconfig.CONSOLE_CONFIGS {
activated := string(emoji.CrossMark)
switch option {
case csconfig.SEND_CUSTOM_SCENARIOS:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send alerts from custom scenarios to the console")
case csconfig.SEND_MANUAL_SCENARIOS:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareManualDecisions {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send manual decisions to the console")
case csconfig.SEND_TAINTED_SCENARIOS:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send alerts from tainted scenarios to the console")
case csconfig.SEND_CONTEXT:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareContext {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send context with alerts to the console")
case csconfig.CONSOLE_MANAGEMENT:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ConsoleManagement {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Receive decisions from console")
}
t.AddRow(option, activated, csconfig.CONSOLE_CONFIGS_HELP[option])
}
t.Render()

View file

@ -243,6 +243,7 @@ func (cli cliDecisions) NewImportCmd() *cobra.Command {
Long: "expected format:\n" +
"csv : any of duration,reason,scope,type,value, with a header line\n" +
"json :" + "`{" + `"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"` + "}`",
Args: cobra.NoArgs,
DisableAutoGenTag: true,
Example: `decisions.csv:
duration,scope,value

View file

@ -33,6 +33,7 @@ api:
client:
credentials_path: ./config/local_api_credentials.yaml
server:
console_path: ./config/console.yaml
#insecure_skip_verify: true
listen_uri: 127.0.0.1:8081
profiles_path: ./config/profiles.yaml

View file

@ -243,9 +243,9 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
controller.AlertsAddChan = apiClient.AlertsAddChan
if apiClient.apiClient.IsEnrolled() {
log.Infof("Machine is enrolled in the console, Loading PAPI Client")
if config.ConsoleConfig.IsPAPIEnabled() {
log.Info("Machine is enrolled in the console, Loading PAPI Client")
papiClient, err = NewPAPI(apiClient, dbClient, config.ConsoleConfig, *config.PapiLogLevel)
if err != nil {
return nil, err

View file

@ -130,9 +130,9 @@ type AppsecConfig struct {
}
func (w *AppsecRuntimeConfig) ClearResponse() {
log.Debugf("#-> %p", w)
w.Logger.Debugf("#-> %p", w)
w.Response = AppsecTempResponse{}
log.Debugf("-> %p", w.Config)
w.Logger.Debugf("-> %p", w.Config)
w.Response.Action = w.Config.DefaultPassAction
w.Response.HTTPResponseCode = w.Config.PassedHTTPCode
w.Response.SendEvent = true
@ -290,20 +290,26 @@ func (w *AppsecRuntimeConfig) ProcessOnLoadRules() error {
switch t := output.(type) {
case bool:
if !t {
log.Debugf("filter didnt match")
w.Logger.Debugf("filter didnt match")
continue
}
default:
log.Errorf("Filter must return a boolean, can't filter")
w.Logger.Errorf("Filter must return a boolean, can't filter")
continue
}
}
for _, applyExpr := range rule.ApplyExpr {
_, err := exprhelpers.Run(applyExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel)
o, err := exprhelpers.Run(applyExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel)
if err != nil {
log.Errorf("unable to apply appsec on_load expr: %s", err)
w.Logger.Errorf("unable to apply appsec on_load expr: %s", err)
continue
}
switch t := o.(type) {
case error:
w.Logger.Errorf("unable to apply appsec on_load expr: %s", t)
continue
default:
}
}
}
return nil
@ -320,27 +326,33 @@ func (w *AppsecRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt ty
switch t := output.(type) {
case bool:
if !t {
log.Debugf("filter didnt match")
w.Logger.Debugf("filter didnt match")
continue
}
default:
log.Errorf("Filter must return a boolean, can't filter")
w.Logger.Errorf("Filter must return a boolean, can't filter")
continue
}
}
for _, applyExpr := range rule.ApplyExpr {
_, err := exprhelpers.Run(applyExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel)
o, err := exprhelpers.Run(applyExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel)
if err != nil {
log.Errorf("unable to apply appsec on_match expr: %s", err)
w.Logger.Errorf("unable to apply appsec on_match expr: %s", err)
continue
}
switch t := o.(type) {
case error:
w.Logger.Errorf("unable to apply appsec on_match expr: %s", t)
continue
default:
}
}
}
return nil
}
func (w *AppsecRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error {
log.Debugf("processing %d pre_eval rules", len(w.CompiledPreEval))
w.Logger.Debugf("processing %d pre_eval rules", len(w.CompiledPreEval))
for _, rule := range w.CompiledPreEval {
if rule.FilterExpr != nil {
output, err := exprhelpers.Run(rule.FilterExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel)
@ -350,21 +362,27 @@ func (w *AppsecRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error
switch t := output.(type) {
case bool:
if !t {
log.Debugf("filter didnt match")
w.Logger.Debugf("filter didnt match")
continue
}
default:
log.Errorf("Filter must return a boolean, can't filter")
w.Logger.Errorf("Filter must return a boolean, can't filter")
continue
}
}
// here means there is no filter or the filter matched
for _, applyExpr := range rule.ApplyExpr {
_, err := exprhelpers.Run(applyExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel)
o, err := exprhelpers.Run(applyExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel)
if err != nil {
log.Errorf("unable to apply appsec pre_eval expr: %s", err)
w.Logger.Errorf("unable to apply appsec pre_eval expr: %s", err)
continue
}
switch t := o.(type) {
case error:
w.Logger.Errorf("unable to apply appsec pre_eval expr: %s", t)
continue
default:
}
}
}
@ -381,21 +399,29 @@ func (w *AppsecRuntimeConfig) ProcessPostEvalRules(request *ParsedRequest) error
switch t := output.(type) {
case bool:
if !t {
log.Debugf("filter didnt match")
w.Logger.Debugf("filter didnt match")
continue
}
default:
log.Errorf("Filter must return a boolean, can't filter")
w.Logger.Errorf("Filter must return a boolean, can't filter")
continue
}
}
// here means there is no filter or the filter matched
for _, applyExpr := range rule.ApplyExpr {
_, err := exprhelpers.Run(applyExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel)
o, err := exprhelpers.Run(applyExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel)
if err != nil {
log.Errorf("unable to apply appsec post_eval expr: %s", err)
w.Logger.Errorf("unable to apply appsec post_eval expr: %s", err)
continue
}
switch t := o.(type) {
case error:
w.Logger.Errorf("unable to apply appsec post_eval expr: %s", t)
continue
default:
}
}
}

View file

@ -38,7 +38,7 @@ type ParsedRequest struct {
Body []byte `json:"body,omitempty"`
TransferEncoding []string `json:"transfer_encoding,omitempty"`
UUID string `json:"uuid,omitempty"`
Tx ExtendedTransaction `json:"transaction,omitempty"`
Tx ExtendedTransaction `json:"-"`
ResponseChannel chan AppsecTempResponse `json:"-"`
IsInBand bool `json:"-"`
IsOutBand bool `json:"-"`
@ -260,12 +260,17 @@ func (r *ReqDumpFilter) ToJSON() error {
req := r.GetFilteredRequest()
log.Warningf("dumping : %+v", req)
log.Tracef("dumping : %+v", req)
if err := enc.Encode(req); err != nil {
//Don't clobber the temp directory with empty files
err2 := os.Remove(fd.Name())
if err2 != nil {
log.Errorf("while removing temp file %s: %s", fd.Name(), err)
}
return fmt.Errorf("while encoding request: %w", err)
}
log.Warningf("request dumped to %s", fd.Name())
log.Infof("request dumped to %s", fd.Name())
return nil
}

View file

@ -19,6 +19,13 @@ const (
)
var CONSOLE_CONFIGS = []string{SEND_CUSTOM_SCENARIOS, SEND_MANUAL_SCENARIOS, SEND_TAINTED_SCENARIOS, SEND_CONTEXT, CONSOLE_MANAGEMENT}
var CONSOLE_CONFIGS_HELP = map[string]string{
SEND_CUSTOM_SCENARIOS: "Forward alerts from custom scenarios to the console",
SEND_MANUAL_SCENARIOS: "Forward manual decisions to the console",
SEND_TAINTED_SCENARIOS: "Forward alerts from tainted scenarios to the console",
SEND_CONTEXT: "Forward context with alerts to the console",
CONSOLE_MANAGEMENT: "Receive decisions from console",
}
var DefaultConsoleConfigFilePath = DefaultConfigPath("console.yaml")

View file

@ -209,9 +209,9 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert)
//add missing decisions
log.Debugf("Adding %d missing decisions to alert %s", len(missingDecisions), foundAlert.UUID)
decisionBuilders := make([]*ent.DecisionCreate, len(missingDecisions))
decisionBuilders := []*ent.DecisionCreate{}
for i, decisionItem := range missingDecisions {
for _, decisionItem := range missingDecisions {
var start_ip, start_sfx, end_ip, end_sfx int64
var sz int
@ -219,7 +219,8 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert)
if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" {
sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value)
if err != nil {
return "", errors.Wrapf(InvalidIPOrRange, "invalid addr/range %s : %s", *decisionItem.Value, err)
log.Errorf("invalid addr/range '%s': %s", *decisionItem.Value, err)
continue
}
}
@ -254,7 +255,7 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert)
SetSimulated(*alertItem.Simulated).
SetUUID(decisionItem.UUID)
decisionBuilders[i] = decisionBuilder
decisionBuilders = append(decisionBuilders, decisionBuilder)
}
decisions := []*ent.Decision{}
@ -486,9 +487,9 @@ func (c *Client) UpdateCommunityBlocklist(alertItem *models.Alert) (int, int, in
}
func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decisions []*models.Decision) ([]*ent.Decision, error) {
decisionCreate := make([]*ent.DecisionCreate, len(decisions))
decisionCreate := []*ent.DecisionCreate{}
for i, decisionItem := range decisions {
for _, decisionItem := range decisions {
var start_ip, start_sfx, end_ip, end_sfx int64
var sz int
@ -501,7 +502,8 @@ func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decis
if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" {
sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value)
if err != nil {
return nil, fmt.Errorf("%s: %w", *decisionItem.Value, InvalidIPOrRange)
log.Errorf("invalid addr/range '%s': %s", *decisionItem.Value, err)
continue
}
}
@ -520,7 +522,11 @@ func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decis
SetSimulated(simulated).
SetUUID(decisionItem.UUID)
decisionCreate[i] = newDecision
decisionCreate = append(decisionCreate, newDecision)
}
if len(decisionCreate) == 0 {
return nil, nil
}
ret, err := c.Ent.Decision.CreateBulk(decisionCreate...).Save(c.CTX)
@ -532,10 +538,10 @@ func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decis
}
func (c *Client) createAlertChunk(machineID string, owner *ent.Machine, alerts []*models.Alert) ([]string, error) {
alertBuilders := make([]*ent.AlertCreate, len(alerts))
alertDecisions := make([][]*ent.Decision, len(alerts))
alertBuilders := []*ent.AlertCreate{}
alertDecisions := [][]*ent.Decision{}
for i, alertItem := range alerts {
for _, alertItem := range alerts {
var metas []*ent.Meta
var events []*ent.Event
@ -656,6 +662,17 @@ func (c *Client) createAlertChunk(machineID string, owner *ent.Machine, alerts [
decisions = append(decisions, decisionRet...)
}
discarded := len(alertItem.Decisions) - len(decisions)
if discarded > 0 {
c.Log.Warningf("discarded %d decisions for %s", discarded, alertItem.UUID)
}
// if all decisions were discarded, discard the alert too
if discarded > 0 && len(decisions) == 0 {
c.Log.Warningf("dropping alert %s with invalid decisions", alertItem.UUID)
continue
}
alertBuilder := c.Ent.Alert.
Create().
SetScenario(*alertItem.Scenario).
@ -685,8 +702,13 @@ func (c *Client) createAlertChunk(machineID string, owner *ent.Machine, alerts [
alertBuilder.SetOwner(owner)
}
alertBuilders[i] = alertBuilder
alertDecisions[i] = decisions
alertBuilders = append(alertBuilders, alertBuilder)
alertDecisions = append(alertDecisions, decisions)
}
if len(alertBuilders) == 0 {
log.Warningf("no alerts to create, discarded?")
return nil, nil
}
alertsCreateBulk, err := c.Ent.Alert.CreateBulk(alertBuilders...).Save(c.CTX)

View file

@ -115,15 +115,18 @@ teardown() {
assert_output "&false"
# complex type
rune -0 cscli config show --key Config.PluginConfig
rune -0 cscli config show --key Config.Prometheus
assert_output - <<-EOT
&csconfig.PluginCfg{
User: "nobody",
Group: "nogroup",
&csconfig.PrometheusCfg{
Enabled: true,
Level: "full",
ListenAddr: "127.0.0.1",
ListenPort: 6060,
}
EOT
}
@test "cscli - required configuration paths" {
config=$(cat "${CONFIG_YAML}")
configdir=$(config_get '.config_paths.config_dir')

View file

@ -43,7 +43,7 @@ setup() {
config_set 'del(.api.server.online_client)'
rune -1 cscli capi status
assert_stderr --partial "no configuration for Central API (CAPI) in '$CONFIG_YAML'"
assert_stderr --regexp "no configuration for Central API \(CAPI\) in '$(echo $CONFIG_YAML|sed s#//#/#g)'"
}
@test "cscli capi status" {

View file

@ -16,7 +16,10 @@ teardown_file() {
setup() {
load "../lib/setup.sh"
load "../lib/bats-file/load.bash"
./instance-data load
LOGFILE=$(config_get '.common.log_dir')/crowdsec.log
export LOGFILE
./instance-crowdsec start
}
@ -151,6 +154,7 @@ teardown() {
assert_stderr --partial 'Parsing values'
assert_stderr --partial 'Imported 3 decisions'
# leading or trailing spaces are ignored
rune -0 cscli decisions import -i - --format values <<-EOT
10.2.3.4
10.2.3.5
@ -159,11 +163,39 @@ teardown() {
assert_stderr --partial 'Parsing values'
assert_stderr --partial 'Imported 3 decisions'
rune -1 cscli decisions import -i - --format values <<-EOT
# silently discarding (but logging) invalid decisions
rune -0 cscli alerts delete --all
truncate -s 0 "${LOGFILE}"
rune -0 cscli decisions import -i - --format values <<-EOT
whatever
EOT
assert_stderr --partial 'Parsing values'
assert_stderr --partial 'creating alert decisions: whatever: invalid ip address / range'
assert_stderr --partial 'Imported 1 decisions'
assert_file_contains "$LOGFILE" "invalid addr/range 'whatever': invalid address"
rune -0 cscli decisions list -a -o json
assert_json '[]'
# disarding only some invalid decisions
rune -0 cscli alerts delete --all
truncate -s 0 "${LOGFILE}"
rune -0 cscli decisions import -i - --format values <<-EOT
1.2.3.4
bad-apple
1.2.3.5
EOT
assert_stderr --partial 'Parsing values'
assert_stderr --partial 'Imported 3 decisions'
assert_file_contains "$LOGFILE" "invalid addr/range 'bad-apple': invalid address"
rune -0 cscli decisions list -a -o json
rune -0 jq -r '.[0].decisions | length' <(output)
assert_output 2
#----------
# Batch