beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Support console options in console enroll (#2760)

Browse source 
* make dev.yaml has a valid/default console path

* simplify and make more consistent help message about console opts

* allow enroll to specify options to enable

* allow 'all' shortcut for --enable

lapi/papi: when receiving alerts, log and discard invalid addr/range (#2708)

https://github.com/crowdsecurity/crowdsec/issues/2687

log "loading papi client" only if papi is enabled (#2762)

Appsec hooks fixes (#2769)
This commit is contained in:
Thibault "bui" Koechlin 2024-01-19 15:49:00 +01:00 committed by sabban
parent 977d904b94
commit 9e1cfb18b9
11 changed files with 186 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
cmd/crowdsec-cli/console.go
View file

@ -7,6 +7,7 @@ import (
"fmt"
"net/url"
"os"
"strings"
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
@ -47,6 +48,7 @@ func NewConsoleCmd() *cobra.Command {
name := ""
overwrite := false
tags := []string{}
opts := []string{}
cmdEnroll := &cobra.Command{
Use: "enroll [enroll-key]",
@ -56,10 +58,12 @@ Enroll this instance to https://app.crowdsec.net
You can get your enrollment key by creating an account on https://app.crowdsec.net.
After running this command your will need to validate the enrollment in the webapp.`,
Example: `cscli console enroll YOUR-ENROLL-KEY
Example: fmt.Sprintf(`cscli console enroll YOUR-ENROLL-KEY
cscli console enroll --name [instance_name] YOUR-ENROLL-KEY
cscli console enroll --name [instance_name] --tags [tag_1] --tags [tag_2] YOUR-ENROLL-KEY
`,
cscli console enroll --enable context,manual YOUR-ENROLL-KEY
valid options are : %s,all (see 'cscli console status' for details)`, strings.Join(csconfig.CONSOLE_CONFIGS, ",")),
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
@ -83,6 +87,37 @@ After running this command your will need to validate the enrollment in the weba
scenarios = make([]string, 0)
}
enable_opts := []string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}
if len(opts) != 0 {
for _, opt := range opts {
valid := false
if opt == "all" {
enable_opts = csconfig.CONSOLE_CONFIGS
break
}
for _, available_opt := range csconfig.CONSOLE_CONFIGS {
if opt == available_opt {
valid = true
enable := true
for _, enabled_opt := range enable_opts {
if opt == enabled_opt {
enable = false
continue
}
}
if enable {
enable_opts = append(enable_opts, opt)
}
break
}
}
if !valid {
return fmt.Errorf("option %s doesn't exist", opt)
}
}
}
c, _ := apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Server.OnlineClient.Credentials.Login,
Password: password,
@ -100,11 +135,13 @@ After running this command your will need to validate the enrollment in the weba
return nil
}
if err := SetConsoleOpts([]string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}, true); err != nil {
if err := SetConsoleOpts(enable_opts, true); err != nil {
return err
}
log.Info("Enabled tainted&manual alerts sharing, see 'cscli console status'.")
for _, opt := range enable_opts {
log.Infof("Enabled %s : %s", opt, csconfig.CONSOLE_CONFIGS_HELP[opt])
}
log.Info("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
log.Info("Please restart crowdsec after accepting the enrollment.")
return nil
@ -113,6 +150,7 @@ After running this command your will need to validate the enrollment in the weba
cmdEnroll.Flags().StringVarP(&name, "name", "n", "", "Name to display in the console")
cmdEnroll.Flags().BoolVarP(&overwrite, "overwrite", "", false, "Force enroll the instance")
cmdEnroll.Flags().StringSliceVarP(&tags, "tags", "t", tags, "Tags to display in the console")
cmdEnroll.Flags().StringSliceVarP(&opts, "enable", "e", opts, "Enable console options")
cmdConsole.AddCommand(cmdEnroll)
var enableAll, disableAll bool

19
cmd/crowdsec-cli/console_table.go
View file

@ -17,45 +17,30 @@ func cmdConsoleStatusTable(out io.Writer, csConfig csconfig.Config) {
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, option := range csconfig.CONSOLE_CONFIGS {
activated := string(emoji.CrossMark)
switch option {
case csconfig.SEND_CUSTOM_SCENARIOS:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send alerts from custom scenarios to the console")
case csconfig.SEND_MANUAL_SCENARIOS:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareManualDecisions {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send manual decisions to the console")
case csconfig.SEND_TAINTED_SCENARIOS:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send alerts from tainted scenarios to the console")
case csconfig.SEND_CONTEXT:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareContext {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send context with alerts to the console")
case csconfig.CONSOLE_MANAGEMENT:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ConsoleManagement {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Receive decisions from console")
}
t.AddRow(option, activated, csconfig.CONSOLE_CONFIGS_HELP[option])
}
t.Render()

1
cmd/crowdsec-cli/decisions_import.go
View file

@ -243,6 +243,7 @@ func (cli cliDecisions) NewImportCmd() *cobra.Command {
Long: "expected format:\n" +
"csv : any of duration,reason,scope,type,value, with a header line\n" +
"json :" + "`{" + `"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"` + "}`",
Args: cobra.NoArgs,
DisableAutoGenTag: true,
Example: `decisions.csv:
duration,scope,value

1
config/dev.yaml
View file

@ -33,6 +33,7 @@ api:
client:
credentials_path: ./config/local_api_credentials.yaml
server:
console_path: ./config/console.yaml
#insecure_skip_verify: true
listen_uri: 127.0.0.1:8081
profiles_path: ./config/profiles.yaml

4
pkg/apiserver/apiserver.go
View file

@ -243,9 +243,9 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) {
controller.AlertsAddChan = apiClient.AlertsAddChan
if apiClient.apiClient.IsEnrolled() {
log.Infof("Machine is enrolled in the console, Loading PAPI Client")
if config.ConsoleConfig.IsPAPIEnabled() {
log.Info("Machine is enrolled in the console, Loading PAPI Client")
papiClient, err = NewPAPI(apiClient, dbClient, config.ConsoleConfig, *config.PapiLogLevel)
if err != nil {
return nil, err

64
pkg/appsec/appsec.go
View file

@ -130,9 +130,9 @@ type AppsecConfig struct {
}
func (w *AppsecRuntimeConfig) ClearResponse() {
log.Debugf("#-> %p", w)
w.Logger.Debugf("#-> %p", w)
w.Response = AppsecTempResponse{}
log.Debugf("-> %p", w.Config)
w.Logger.Debugf("-> %p", w.Config)
w.Response.Action = w.Config.DefaultPassAction
w.Response.HTTPResponseCode = w.Config.PassedHTTPCode
w.Response.SendEvent = true
@ -290,20 +290,26 @@ func (w *AppsecRuntimeConfig) ProcessOnLoadRules() error {
switch t := output.(type) {
case bool:
if !t {
log.Debugf("filter didnt match")
w.Logger.Debugf("filter didnt match")
continue
}
default:
log.Errorf("Filter must return a boolean, can't filter")
w.Logger.Errorf("Filter must return a boolean, can't filter")
continue
}
}
for _, applyExpr := range rule.ApplyExpr {
_, err := exprhelpers.Run(applyExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel)
o, err := exprhelpers.Run(applyExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel)
if err != nil {
log.Errorf("unable to apply appsec on_load expr: %s", err)
w.Logger.Errorf("unable to apply appsec on_load expr: %s", err)
continue
}
switch t := o.(type) {
case error:
w.Logger.Errorf("unable to apply appsec on_load expr: %s", t)
continue
default:
}
}
}
return nil
@ -320,27 +326,33 @@ func (w *AppsecRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt ty
switch t := output.(type) {
case bool:
if !t {
log.Debugf("filter didnt match")
w.Logger.Debugf("filter didnt match")
continue
}
default:
log.Errorf("Filter must return a boolean, can't filter")
w.Logger.Errorf("Filter must return a boolean, can't filter")
continue
}
}
for _, applyExpr := range rule.ApplyExpr {
_, err := exprhelpers.Run(applyExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel)
o, err := exprhelpers.Run(applyExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel)
if err != nil {
log.Errorf("unable to apply appsec on_match expr: %s", err)
w.Logger.Errorf("unable to apply appsec on_match expr: %s", err)
continue
}
switch t := o.(type) {
case error:
w.Logger.Errorf("unable to apply appsec on_match expr: %s", t)
continue
default:
}
}
}
return nil
}
func (w *AppsecRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error {
log.Debugf("processing %d pre_eval rules", len(w.CompiledPreEval))
w.Logger.Debugf("processing %d pre_eval rules", len(w.CompiledPreEval))
for _, rule := range w.CompiledPreEval {
if rule.FilterExpr != nil {
output, err := exprhelpers.Run(rule.FilterExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel)
@ -350,21 +362,27 @@ func (w *AppsecRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error
switch t := output.(type) {
case bool:
if !t {
log.Debugf("filter didnt match")
w.Logger.Debugf("filter didnt match")
continue
}
default:
log.Errorf("Filter must return a boolean, can't filter")
w.Logger.Errorf("Filter must return a boolean, can't filter")
continue
}
}
// here means there is no filter or the filter matched
for _, applyExpr := range rule.ApplyExpr {
_, err := exprhelpers.Run(applyExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel)
o, err := exprhelpers.Run(applyExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel)
if err != nil {
log.Errorf("unable to apply appsec pre_eval expr: %s", err)
w.Logger.Errorf("unable to apply appsec pre_eval expr: %s", err)
continue
}
switch t := o.(type) {
case error:
w.Logger.Errorf("unable to apply appsec pre_eval expr: %s", t)
continue
default:
}
}
}
@ -381,21 +399,29 @@ func (w *AppsecRuntimeConfig) ProcessPostEvalRules(request *ParsedRequest) error
switch t := output.(type) {
case bool:
if !t {
log.Debugf("filter didnt match")
w.Logger.Debugf("filter didnt match")
continue
}
default:
log.Errorf("Filter must return a boolean, can't filter")
w.Logger.Errorf("Filter must return a boolean, can't filter")
continue
}
}
// here means there is no filter or the filter matched
for _, applyExpr := range rule.ApplyExpr {
_, err := exprhelpers.Run(applyExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel)
o, err := exprhelpers.Run(applyExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel)
if err != nil {
log.Errorf("unable to apply appsec post_eval expr: %s", err)
w.Logger.Errorf("unable to apply appsec post_eval expr: %s", err)
continue
}
switch t := o.(type) {
case error:
w.Logger.Errorf("unable to apply appsec post_eval expr: %s", t)
continue
default:
}
}
}

11
pkg/appsec/request.go
View file

@ -38,7 +38,7 @@ type ParsedRequest struct {
Body []byte `json:"body,omitempty"`
TransferEncoding []string `json:"transfer_encoding,omitempty"`
UUID string `json:"uuid,omitempty"`
Tx ExtendedTransaction `json:"transaction,omitempty"`
Tx ExtendedTransaction `json:"-"`
ResponseChannel chan AppsecTempResponse `json:"-"`
IsInBand bool `json:"-"`
IsOutBand bool `json:"-"`
@ -260,12 +260,17 @@ func (r *ReqDumpFilter) ToJSON() error {
req := r.GetFilteredRequest()
log.Warningf("dumping : %+v", req)
log.Tracef("dumping : %+v", req)
if err := enc.Encode(req); err != nil {
//Don't clobber the temp directory with empty files
err2 := os.Remove(fd.Name())
if err2 != nil {
log.Errorf("while removing temp file %s: %s", fd.Name(), err)
}
return fmt.Errorf("while encoding request: %w", err)
}
log.Warningf("request dumped to %s", fd.Name())
log.Infof("request dumped to %s", fd.Name())
return nil
}

7
pkg/csconfig/console.go
View file

@ -19,6 +19,13 @@ const (
)
var CONSOLE_CONFIGS = []string{SEND_CUSTOM_SCENARIOS, SEND_MANUAL_SCENARIOS, SEND_TAINTED_SCENARIOS, SEND_CONTEXT, CONSOLE_MANAGEMENT}
var CONSOLE_CONFIGS_HELP = map[string]string{
SEND_CUSTOM_SCENARIOS: "Forward alerts from custom scenarios to the console",
SEND_MANUAL_SCENARIOS: "Forward manual decisions to the console",
SEND_TAINTED_SCENARIOS: "Forward alerts from tainted scenarios to the console",
SEND_CONTEXT: "Forward context with alerts to the console",
CONSOLE_MANAGEMENT: "Receive decisions from console",
}
var DefaultConsoleConfigFilePath = DefaultConfigPath("console.yaml")

48
pkg/database/alerts.go
View file

@ -209,9 +209,9 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert)
//add missing decisions
log.Debugf("Adding %d missing decisions to alert %s", len(missingDecisions), foundAlert.UUID)
decisionBuilders := make([]*ent.DecisionCreate, len(missingDecisions))
decisionBuilders := []*ent.DecisionCreate{}
for i, decisionItem := range missingDecisions {
for _, decisionItem := range missingDecisions {
var start_ip, start_sfx, end_ip, end_sfx int64
var sz int
@ -219,7 +219,8 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert)
if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" {
sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value)
if err != nil {
return "", errors.Wrapf(InvalidIPOrRange, "invalid addr/range %s : %s", *decisionItem.Value, err)
log.Errorf("invalid addr/range '%s': %s", *decisionItem.Value, err)
continue
}
}
@ -254,7 +255,7 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert)
SetSimulated(*alertItem.Simulated).
SetUUID(decisionItem.UUID)
decisionBuilders[i] = decisionBuilder
decisionBuilders = append(decisionBuilders, decisionBuilder)
}
decisions := []*ent.Decision{}
@ -486,9 +487,9 @@ func (c *Client) UpdateCommunityBlocklist(alertItem *models.Alert) (int, int, in
}
func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decisions []*models.Decision) ([]*ent.Decision, error) {
decisionCreate := make([]*ent.DecisionCreate, len(decisions))
decisionCreate := []*ent.DecisionCreate{}
for i, decisionItem := range decisions {
for _, decisionItem := range decisions {
var start_ip, start_sfx, end_ip, end_sfx int64
var sz int
@ -501,7 +502,8 @@ func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decis
if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" {
sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value)
if err != nil {
return nil, fmt.Errorf("%s: %w", *decisionItem.Value, InvalidIPOrRange)
log.Errorf("invalid addr/range '%s': %s", *decisionItem.Value, err)
continue
}
}
@ -520,7 +522,11 @@ func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decis
SetSimulated(simulated).
SetUUID(decisionItem.UUID)
decisionCreate[i] = newDecision
decisionCreate = append(decisionCreate, newDecision)
}
if len(decisionCreate) == 0 {
return nil, nil
}
ret, err := c.Ent.Decision.CreateBulk(decisionCreate...).Save(c.CTX)
@ -532,10 +538,10 @@ func (c *Client) createDecisionChunk(simulated bool, stopAtTime time.Time, decis
}
func (c *Client) createAlertChunk(machineID string, owner *ent.Machine, alerts []*models.Alert) ([]string, error) {
alertBuilders := make([]*ent.AlertCreate, len(alerts))
alertDecisions := make([][]*ent.Decision, len(alerts))
alertBuilders := []*ent.AlertCreate{}
alertDecisions := [][]*ent.Decision{}
for i, alertItem := range alerts {
for _, alertItem := range alerts {
var metas []*ent.Meta
var events []*ent.Event
@ -656,6 +662,17 @@ func (c *Client) createAlertChunk(machineID string, owner *ent.Machine, alerts [
decisions = append(decisions, decisionRet...)
}
discarded := len(alertItem.Decisions) - len(decisions)
if discarded > 0 {
c.Log.Warningf("discarded %d decisions for %s", discarded, alertItem.UUID)
}
// if all decisions were discarded, discard the alert too
if discarded > 0 && len(decisions) == 0 {
c.Log.Warningf("dropping alert %s with invalid decisions", alertItem.UUID)
continue
}
alertBuilder := c.Ent.Alert.
Create().
SetScenario(*alertItem.Scenario).
@ -685,8 +702,13 @@ func (c *Client) createAlertChunk(machineID string, owner *ent.Machine, alerts [
alertBuilder.SetOwner(owner)
}
alertBuilders[i] = alertBuilder
alertDecisions[i] = decisions
alertBuilders = append(alertBuilders, alertBuilder)
alertDecisions = append(alertDecisions, decisions)
}
if len(alertBuilders) == 0 {
log.Warningf("no alerts to create, discarded?")
return nil, nil
}
alertsCreateBulk, err := c.Ent.Alert.CreateBulk(alertBuilders...).Save(c.CTX)

18
test/bats/01_cscli.bats
View file

@ -115,16 +115,16 @@ teardown() {
assert_output "&false"
# complex type
rune -0 cscli config show -o json --key Config.Prometheus
rune -0 cscli config show --key Config.Prometheus
assert_output - <<-EOT
&csconfig.PrometheusCfg{
Enabled: true,
Level: "full",
ListenAddr: "127.0.0.1",
ListenPort: 6060,
}
EOT
}
&csconfig.PrometheusCfg{
Enabled: true,
Level: "full",
ListenAddr: "127.0.0.1",
ListenPort: 6060,
}
EOT
}
@test "cscli - required configuration paths" {

36
test/bats/90_decisions.bats
View file

@ -16,7 +16,10 @@ teardown_file() {
setup() {
load "../lib/setup.sh"
load "../lib/bats-file/load.bash"
./instance-data load
LOGFILE=$(config_get '.common.log_dir')/crowdsec.log
export LOGFILE
./instance-crowdsec start
}
@ -151,6 +154,7 @@ teardown() {
assert_stderr --partial 'Parsing values'
assert_stderr --partial 'Imported 3 decisions'
# leading or trailing spaces are ignored
rune -0 cscli decisions import -i - --format values <<-EOT
10.2.3.4
10.2.3.5
@ -159,11 +163,39 @@ teardown() {
assert_stderr --partial 'Parsing values'
assert_stderr --partial 'Imported 3 decisions'
rune -1 cscli decisions import -i - --format values <<-EOT
# silently discarding (but logging) invalid decisions
rune -0 cscli alerts delete --all
truncate -s 0 "${LOGFILE}"
rune -0 cscli decisions import -i - --format values <<-EOT
whatever
EOT
assert_stderr --partial 'Parsing values'
assert_stderr --partial 'creating alert decisions: whatever: invalid ip address / range'
assert_stderr --partial 'Imported 1 decisions'
assert_file_contains "$LOGFILE" "invalid addr/range 'whatever': invalid address"
rune -0 cscli decisions list -a -o json
assert_json '[]'
# disarding only some invalid decisions
rune -0 cscli alerts delete --all
truncate -s 0 "${LOGFILE}"
rune -0 cscli decisions import -i - --format values <<-EOT
1.2.3.4
bad-apple
1.2.3.5
EOT
assert_stderr --partial 'Parsing values'
assert_stderr --partial 'Imported 3 decisions'
assert_file_contains "$LOGFILE" "invalid addr/range 'bad-apple': invalid address"
rune -0 cscli decisions list -a -o json
rune -0 jq -r '.[0].decisions | length' <(output)
assert_output 2
#----------
# Batch