diff --git a/cmd/crowdsec-cli/alerts.go b/cmd/crowdsec-cli/alerts.go index 8f96525fa..0ac3b88b9 100644 --- a/cmd/crowdsec-cli/alerts.go +++ b/cmd/crowdsec-cli/alerts.go @@ -7,6 +7,7 @@ import ( "fmt" "net/url" "os" + "sort" "strconv" "strings" @@ -112,6 +113,29 @@ func DisplayOneAlert(alert *models.Alert, withDetail bool) error { alertDecisionsTable(color.Output, alert) + if len(alert.Meta) > 0 { + fmt.Printf("\n - Context :\n") + sort.Slice(alert.Meta, func(i, j int) bool { + return alert.Meta[i].Key < alert.Meta[j].Key + }) + table := newTable(color.Output) + table.SetRowLines(false) + table.SetHeaders("Key", "Value") + for _, meta := range alert.Meta { + var valSlice []string + if err := json.Unmarshal([]byte(meta.Value), &valSlice); err != nil { + return fmt.Errorf("unknown context value type '%s' : %s", meta.Value, err) + } + for _, value := range valSlice { + table.AddRow( + meta.Key, + value, + ) + } + } + table.Render() + } + if withDetail { fmt.Printf("\n - Events :\n") for _, event := range alert.Events { diff --git a/cmd/crowdsec-cli/console.go b/cmd/crowdsec-cli/console.go index b4ad9db7a..2653300fa 100644 --- a/cmd/crowdsec-cli/console.go +++ b/cmd/crowdsec-cli/console.go @@ -46,7 +46,7 @@ func NewConsoleCmd() *cobra.Command { log.Fatalf("No configuration for Central API (CAPI) in '%s'", *csConfig.FilePath) } if csConfig.API.Server.OnlineClient.Credentials == nil { - log.Fatal("You must configure Central API (CAPI) with `cscli capi register` before enrolling your instance") + log.Fatal("You must configure Central API (CAPI) with `cscli capi register` before accessing console features.") } return nil }, @@ -129,9 +129,9 @@ After running this command your will need to validate the enrollment in the weba var enableAll, disableAll bool cmdEnable := &cobra.Command{ - Use: "enable [feature-flag]", - Short: "Enable a feature flag", - Example: "enable tainted", + Use: "enable [option]", + Short: "Enable a console option", + Example: "sudo cscli console enable tainted", Long: ` Enable given information push to the central API. Allows to empower the console`, ValidArgs: csconfig.CONSOLE_CONFIGS, @@ -153,13 +153,13 @@ Enable given information push to the central API. Allows to empower the console` log.Infof(ReloadMessage()) }, } - cmdEnable.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all feature flags") + cmdEnable.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all console options") cmdConsole.AddCommand(cmdEnable) cmdDisable := &cobra.Command{ - Use: "disable [feature-flag]", - Short: "Disable a feature flag", - Example: "disable tainted", + Use: "disable [option]", + Short: "Disable a console option", + Example: "sudo cscli console disable tainted", Long: ` Disable given information push to the central API.`, ValidArgs: csconfig.CONSOLE_CONFIGS, @@ -183,13 +183,13 @@ Disable given information push to the central API.`, log.Infof(ReloadMessage()) }, } - cmdDisable.Flags().BoolVarP(&disableAll, "all", "a", false, "Enable all feature flags") + cmdDisable.Flags().BoolVarP(&disableAll, "all", "a", false, "Disable all console options") cmdConsole.AddCommand(cmdDisable) cmdConsoleStatus := &cobra.Command{ - Use: "status [feature-flag]", - Short: "Shows status of one or all feature flags", - Example: "status tainted", + Use: "status [option]", + Short: "Shows status of one or all console options", + Example: `sudo cscli console status`, DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { switch csConfig.Cscli.Output { @@ -212,6 +212,7 @@ Disable given information push to the central API.`, {"share_manual_decisions", fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareManualDecisions)}, {"share_custom", fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios)}, {"share_tainted", fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios)}, + {"share_context", fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareContext)}, } for _, row := range rows { err = csvwriter.Write(row) @@ -223,8 +224,8 @@ Disable given information push to the central API.`, } }, } - cmdConsole.AddCommand(cmdConsoleStatus) + return cmdConsole } @@ -270,6 +271,19 @@ func SetConsoleOpts(args []string, wanted bool) { log.Infof("%s set to %t", csconfig.SEND_MANUAL_SCENARIOS, wanted) csConfig.API.Server.ConsoleConfig.ShareManualDecisions = types.BoolPtr(wanted) } + case csconfig.SEND_CONTEXT: + /*for each flag check if it's already set before setting it*/ + if csConfig.API.Server.ConsoleConfig.ShareContext != nil { + if *csConfig.API.Server.ConsoleConfig.ShareContext == wanted { + log.Infof("%s already set to %t", csconfig.SEND_CONTEXT, wanted) + } else { + log.Infof("%s set to %t", csconfig.SEND_CONTEXT, wanted) + *csConfig.API.Server.ConsoleConfig.ShareContext = wanted + } + } else { + log.Infof("%s set to %t", csconfig.SEND_CONTEXT, wanted) + csConfig.API.Server.ConsoleConfig.ShareContext = types.BoolPtr(wanted) + } default: log.Fatalf("unknown flag %s", arg) } diff --git a/cmd/crowdsec-cli/console_table.go b/cmd/crowdsec-cli/console_table.go index 014ffc9ad..2093ad7a1 100644 --- a/cmd/crowdsec-cli/console_table.go +++ b/cmd/crowdsec-cli/console_table.go @@ -41,6 +41,12 @@ func cmdConsoleStatusTable(out io.Writer, csConfig csconfig.Config) { } t.AddRow(option, activated, "Send alerts from tainted scenarios to the console") + case csconfig.SEND_CONTEXT: + activated := string(emoji.CrossMark) + if *csConfig.API.Server.ConsoleConfig.ShareContext { + activated = string(emoji.CheckMarkButton) + } + t.AddRow(option, activated, "Send context with alerts to the console") } } diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index c588c50a4..97ecfed88 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -5,6 +5,7 @@ import ( "fmt" "net/url" "os" + "sort" "strings" "github.com/go-openapi/strfmt" @@ -13,16 +14,20 @@ import ( "github.com/spf13/cobra" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/crowdsec/pkg/alertcontext" "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwversion" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/crowdsecurity/crowdsec/pkg/parser" + "github.com/crowdsecurity/crowdsec/pkg/types" ) var LAPIURLPrefix string = "v1" -func runLapiStatus (cmd *cobra.Command, args []string) error { +func runLapiStatus(cmd *cobra.Command, args []string) error { var err error password := strfmt.Password(csConfig.API.Client.Credentials.Password) @@ -68,7 +73,6 @@ func runLapiStatus (cmd *cobra.Command, args []string) error { return nil } - func runLapiRegister(cmd *cobra.Command, args []string) error { var err error @@ -160,7 +164,6 @@ func runLapiRegister(cmd *cobra.Command, args []string) error { return nil } - func NewLapiStatusCmd() *cobra.Command { cmdLapiStatus := &cobra.Command{ Use: "status", @@ -173,7 +176,6 @@ func NewLapiStatusCmd() *cobra.Command { return cmdLapiStatus } - func NewLapiRegisterCmd() *cobra.Command { cmdLapiRegister := &cobra.Command{ Use: "register", @@ -182,7 +184,7 @@ func NewLapiRegisterCmd() *cobra.Command { Keep in mind the machine needs to be validated by an administrator on LAPI side to be effective.`, Args: cobra.MinimumNArgs(0), DisableAutoGenTag: true, - RunE: runLapiRegister, + RunE: runLapiRegister, } flags := cmdLapiRegister.Flags() @@ -193,7 +195,6 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side return cmdLapiRegister } - func NewLapiCmd() *cobra.Command { var cmdLapi = &cobra.Command{ Use: "lapi [action]", @@ -210,6 +211,350 @@ func NewLapiCmd() *cobra.Command { cmdLapi.AddCommand(NewLapiRegisterCmd()) cmdLapi.AddCommand(NewLapiStatusCmd()) + cmdLapi.AddCommand(NewLapiContextCmd()) return cmdLapi } + +func NewLapiContextCmd() *cobra.Command { + cmdContext := &cobra.Command{ + Use: "context [command]", + Short: "Manage context to send with alerts", + DisableAutoGenTag: true, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if err := csConfig.LoadCrowdsec(); err != nil { + fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", csConfig.Crowdsec.ConsoleContextPath) + if err.Error() != fileNotFoundMessage { + log.Fatalf("Unable to load CrowdSec Agent: %s", err) + } + } + if csConfig.DisableAgent { + log.Fatalf("Agent is disabled and lapi context can only be used on the agent") + } + + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + printHelp(cmd) + }, + } + + var keyToAdd string + var valuesToAdd []string + cmdContextAdd := &cobra.Command{ + Use: "add", + Short: "Add context to send with alerts. You must specify the output key with the expr value you want", + Example: `cscli lapi context add --key source_ip --value evt.Meta.source_ip +cscli lapi context add --key file_source --value evt.Line.Src + `, + DisableAutoGenTag: true, + Run: func(cmd *cobra.Command, args []string) { + if err := alertcontext.ValidateContextExpr(keyToAdd, valuesToAdd); err != nil { + log.Fatalf("invalid context configuration :%s", err) + } + if _, ok := csConfig.Crowdsec.ContextToSend[keyToAdd]; !ok { + csConfig.Crowdsec.ContextToSend[keyToAdd] = make([]string, 0) + log.Infof("key '%s' added", keyToAdd) + } + data := csConfig.Crowdsec.ContextToSend[keyToAdd] + for _, val := range valuesToAdd { + if !inSlice(val, data) { + log.Infof("value '%s' added to key '%s'", val, keyToAdd) + data = append(data, val) + } + csConfig.Crowdsec.ContextToSend[keyToAdd] = data + } + if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil { + log.Fatalf(err.Error()) + } + }, + } + cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send") + cmdContextAdd.Flags().StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key") + cmdContextAdd.MarkFlagRequired("key") + cmdContextAdd.MarkFlagRequired("value") + cmdContext.AddCommand(cmdContextAdd) + + cmdContextStatus := &cobra.Command{ + Use: "status", + Short: "List context to send with alerts", + DisableAutoGenTag: true, + Run: func(cmd *cobra.Command, args []string) { + if len(csConfig.Crowdsec.ContextToSend) == 0 { + fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.") + return + } + + dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend) + if err != nil { + log.Fatalf("unable to show context status: %s", err) + } + + fmt.Println(string(dump)) + + }, + } + cmdContext.AddCommand(cmdContextStatus) + + var detectAll bool + cmdContextDetect := &cobra.Command{ + Use: "detect", + Short: "Detect available fields from the installed parsers", + Example: `cscli lapi context detect --all +cscli lapi context detect crowdsecurity/sshd-logs + `, + DisableAutoGenTag: true, + Run: func(cmd *cobra.Command, args []string) { + var err error + + if !detectAll && len(args) == 0 { + log.Infof("Please provide parsers to detect or --all flag.") + printHelp(cmd) + } + + // to avoid all the log.Info from the loaders functions + log.SetLevel(log.ErrorLevel) + + err = exprhelpers.Init(nil) + if err != nil { + log.Fatalf("Failed to init expr helpers : %s", err) + } + + // Populate cwhub package tools + if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { + log.Fatalf("Failed to load hub index : %s", err) + } + + csParsers := parser.NewParsers() + if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil { + log.Fatalf("unable to load parsers: %s", err) + } + + fieldByParsers := make(map[string][]string) + for _, node := range csParsers.Nodes { + if !detectAll && !inSlice(node.Name, args) { + continue + } + if !detectAll { + args = removeFromSlice(node.Name, args) + } + fieldByParsers[node.Name] = make([]string, 0) + fieldByParsers[node.Name] = detectNode(node, *csParsers.Ctx) + + subNodeFields := detectSubNode(node, *csParsers.Ctx) + for _, field := range subNodeFields { + if !inSlice(field, fieldByParsers[node.Name]) { + fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field) + } + } + + } + + fmt.Printf("Acquisition :\n\n") + fmt.Printf(" - evt.Line.Module\n") + fmt.Printf(" - evt.Line.Raw\n") + fmt.Printf(" - evt.Line.Src\n") + fmt.Println() + + parsersKey := make([]string, 0) + for k := range fieldByParsers { + parsersKey = append(parsersKey, k) + } + sort.Strings(parsersKey) + + for _, k := range parsersKey { + if len(fieldByParsers[k]) == 0 { + continue + } + fmt.Printf("%s :\n\n", k) + values := fieldByParsers[k] + sort.Strings(values) + for _, value := range values { + fmt.Printf(" - %s\n", value) + } + fmt.Println() + } + + if len(args) > 0 { + for _, parserNotFound := range args { + log.Errorf("parser '%s' not found, can't detect fields", parserNotFound) + } + } + }, + } + cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser") + cmdContext.AddCommand(cmdContextDetect) + + var keysToDelete []string + var valuesToDelete []string + cmdContextDelete := &cobra.Command{ + Use: "delete", + Short: "Delete context to send with alerts", + Example: `cscli lapi context delete --key source_ip +cscli lapi context delete --value evt.Line.Src + `, + DisableAutoGenTag: true, + Run: func(cmd *cobra.Command, args []string) { + if len(keysToDelete) == 0 && len(valuesToDelete) == 0 { + log.Fatalf("please provide at least a key or a value to delete") + } + + for _, key := range keysToDelete { + if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok { + delete(csConfig.Crowdsec.ContextToSend, key) + log.Infof("key '%s' has been removed", key) + } else { + log.Warningf("key '%s' doesn't exist", key) + } + } + + for _, value := range valuesToDelete { + valueFound := false + for key, context := range csConfig.Crowdsec.ContextToSend { + if inSlice(value, context) { + valueFound = true + csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, context) + log.Infof("value '%s' has been removed from key '%s'", value, key) + } + if len(csConfig.Crowdsec.ContextToSend[key]) == 0 { + delete(csConfig.Crowdsec.ContextToSend, key) + } + } + if !valueFound { + log.Warningf("value '%s' not found", value) + } + } + + if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil { + log.Fatalf(err.Error()) + } + + }, + } + cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete") + cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete") + cmdContext.AddCommand(cmdContextDelete) + + return cmdContext +} + +func detectStaticField(GrokStatics []types.ExtraField) []string { + ret := make([]string, 0) + for _, static := range GrokStatics { + if static.Parsed != "" { + fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed) + if !inSlice(fieldName, ret) { + ret = append(ret, fieldName) + } + } + if static.Meta != "" { + fieldName := fmt.Sprintf("evt.Meta.%s", static.Meta) + if !inSlice(fieldName, ret) { + ret = append(ret, fieldName) + } + } + if static.TargetByName != "" { + fieldName := static.TargetByName + if !strings.HasPrefix(fieldName, "evt.") { + fieldName = "evt." + fieldName + } + if !inSlice(fieldName, ret) { + ret = append(ret, fieldName) + } + } + } + + return ret +} + +func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string { + var ret = make([]string, 0) + if node.Grok.RunTimeRegexp != nil { + for _, capturedField := range node.Grok.RunTimeRegexp.Names() { + fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField) + if !inSlice(fieldName, ret) { + ret = append(ret, fieldName) + } + } + } + + if node.Grok.RegexpName != "" { + grokCompiled, err := parserCTX.Grok.Get(node.Grok.RegexpName) + if err != nil { + log.Warningf("Can't get subgrok: %s", err) + } + for _, capturedField := range grokCompiled.Names() { + fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField) + if !inSlice(fieldName, ret) { + ret = append(ret, fieldName) + } + } + } + + if len(node.Grok.Statics) > 0 { + staticsField := detectStaticField(node.Grok.Statics) + for _, staticField := range staticsField { + if !inSlice(staticField, ret) { + ret = append(ret, staticField) + } + } + } + + if len(node.Statics) > 0 { + staticsField := detectStaticField(node.Statics) + for _, staticField := range staticsField { + if !inSlice(staticField, ret) { + ret = append(ret, staticField) + } + } + } + + return ret +} + +func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string { + var ret = make([]string, 0) + + for _, subnode := range node.LeavesNodes { + if subnode.Grok.RunTimeRegexp != nil { + for _, capturedField := range subnode.Grok.RunTimeRegexp.Names() { + fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField) + if !inSlice(fieldName, ret) { + ret = append(ret, fieldName) + } + } + } + if subnode.Grok.RegexpName != "" { + grokCompiled, err := parserCTX.Grok.Get(subnode.Grok.RegexpName) + if err != nil { + log.Warningf("Can't get subgrok: %s", err) + } + for _, capturedField := range grokCompiled.Names() { + fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField) + if !inSlice(fieldName, ret) { + ret = append(ret, fieldName) + } + } + } + + if len(subnode.Grok.Statics) > 0 { + staticsField := detectStaticField(subnode.Grok.Statics) + for _, staticField := range staticsField { + if !inSlice(staticField, ret) { + ret = append(ret, staticField) + } + } + } + + if len(subnode.Statics) > 0 { + staticsField := detectStaticField(subnode.Statics) + for _, staticField := range staticsField { + if !inSlice(staticField, ret) { + ret = append(ret, staticField) + } + } + } + } + + return ret +} diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index 00a7b21f7..b2f14f45c 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -748,3 +748,28 @@ func getDBClient() (*database.Client, error) { } return ret, nil } + + +func removeFromSlice(val string, slice []string) []string { + var i int + var value string + + valueFound := false + + // get the index + for i, value = range slice { + if value == val { + valueFound = true + break + } + } + + if valueFound { + slice[i] = slice[len(slice)-1] + slice[len(slice)-1] = "" + slice = slice[:len(slice)-1] + } + + return slice + +} diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 84cf0838b..940faff70 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -28,7 +28,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { } // Start loading configs - csParsers := newParsers() + csParsers := parser.NewParsers() if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err) } diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index c799265d8..1aec6b211 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -7,7 +7,6 @@ import ( "os" "path/filepath" "runtime" - "sort" "strings" "time" @@ -72,45 +71,6 @@ type Flags struct { type labelsMap map[string]string -// Return new parsers -// nodes and povfwnodes are already initialized in parser.LoadStages -func newParsers() *parser.Parsers { - parsers := &parser.Parsers{ - Ctx: &parser.UnixParserCtx{}, - Povfwctx: &parser.UnixParserCtx{}, - StageFiles: make([]parser.Stagefile, 0), - PovfwStageFiles: make([]parser.Stagefile, 0), - } - for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} { - for _, hubParserItem := range cwhub.GetItemMap(itemType) { - if hubParserItem.Installed { - stagefile := parser.Stagefile{ - Filename: hubParserItem.LocalPath, - Stage: hubParserItem.Stage, - } - if itemType == cwhub.PARSERS { - parsers.StageFiles = append(parsers.StageFiles, stagefile) - } - if itemType == cwhub.PARSERS_OVFLW { - parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile) - } - } - } - } - if parsers.StageFiles != nil { - sort.Slice(parsers.StageFiles, func(i, j int) bool { - return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename - }) - } - if parsers.PovfwStageFiles != nil { - sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool { - return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename - }) - } - - return parsers -} - func LoadBuckets(cConfig *csconfig.Config) error { var ( err error diff --git a/config/config.yaml b/config/config.yaml index fcba937a0..232b0bc43 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -16,6 +16,7 @@ config_paths: notification_dir: /etc/crowdsec/notifications/ plugin_dir: /usr/local/lib/crowdsec/plugins/ crowdsec_service: + #console_context_path: /etc/crowdsec/console/context.yaml acquisition_path: /etc/crowdsec/acquis.yaml acquisition_dir: /etc/crowdsec/acquis.d parser_routines: 1 diff --git a/config/config_win.yaml b/config/config_win.yaml index 021f9836c..7863f4fdd 100644 --- a/config/config_win.yaml +++ b/config/config_win.yaml @@ -13,6 +13,7 @@ config_paths: plugin_dir: C:\ProgramData\CrowdSec\plugins\ notification_dir: C:\ProgramData\CrowdSec\config\notifications\ crowdsec_service: + #console_context_path: C:\ProgramData\CrowdSec\console\context.yaml acquisition_path: C:\ProgramData\CrowdSec\config\acquis.yaml parser_routines: 1 cscli: diff --git a/config/console.yaml b/config/console.yaml index e83658d7b..aa0cc30ac 100644 --- a/config/console.yaml +++ b/config/console.yaml @@ -1,3 +1,4 @@ share_manual_decisions: false share_custom: true share_tainted: true +share_context: false \ No newline at end of file diff --git a/config/context.yaml b/config/context.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/debian/rules b/debian/rules index f496a9e80..9f9258a2f 100755 --- a/debian/rules +++ b/debian/rules @@ -28,7 +28,7 @@ override_dh_auto_install: mkdir -p debian/crowdsec/usr/share/crowdsec mkdir -p debian/crowdsec/etc/crowdsec/hub/ mkdir -p debian/crowdsec/usr/share/crowdsec/config - + mkdir -p debian/crowdsec/etc/crowdsec/console/ mkdir -p debian/crowdsec/usr/lib/crowdsec/plugins/ mkdir -p debian/crowdsec/etc/crowdsec/notifications/ @@ -44,6 +44,7 @@ override_dh_auto_install: install -m 600 config/config.yaml debian/crowdsec/etc/crowdsec/config.yaml cp config/simulation.yaml debian/crowdsec/etc/crowdsec/simulation.yaml cp config/profiles.yaml debian/crowdsec/etc/crowdsec/profiles.yaml + cp config/context.yaml debian/crowdsec/etc/crowdsec/console/context.yaml cp config/console.yaml debian/crowdsec/etc/crowdsec/console.yaml cp -a config/patterns debian/crowdsec/etc/crowdsec diff --git a/pkg/alertcontext/alertcontext.go b/pkg/alertcontext/alertcontext.go new file mode 100644 index 000000000..9cf6a586c --- /dev/null +++ b/pkg/alertcontext/alertcontext.go @@ -0,0 +1,157 @@ +package alertcontext + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/antonmedv/expr" + "github.com/antonmedv/expr/vm" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/crowdsecurity/crowdsec/pkg/types" + log "github.com/sirupsen/logrus" +) + +const ( + maxContextValueLen = 4000 +) + +var ( + alertContext = Context{} +) + +type Context struct { + ContextToSend map[string][]string + ContextValueLen int + ContextToSendCompiled map[string][]*vm.Program + Log *log.Logger +} + +func ValidateContextExpr(key string, expressions []string) error { + for _, expression := range expressions { + _, err := expr.Compile(expression, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) + if err != nil { + return fmt.Errorf("compilation of '%s' failed: %v", expression, err) + } + } + return nil +} + +func NewAlertContext(contextToSend map[string][]string, valueLength int) error { + var clog = log.New() + if err := types.ConfigureLogger(clog); err != nil { + return fmt.Errorf("couldn't create logger for alert context: %s", err) + } + + if valueLength == 0 { + clog.Debugf("No console context value length provided, using default: %d", maxContextValueLen) + valueLength = maxContextValueLen + } + if valueLength > maxContextValueLen { + clog.Debugf("Provided console context value length (%d) is higher than the maximum, using default: %d", valueLength, maxContextValueLen) + valueLength = maxContextValueLen + } + + alertContext = Context{ + ContextToSend: contextToSend, + ContextValueLen: valueLength, + Log: clog, + ContextToSendCompiled: make(map[string][]*vm.Program), + } + + for key, values := range contextToSend { + alertContext.ContextToSendCompiled[key] = make([]*vm.Program, 0) + for _, value := range values { + valueCompiled, err := expr.Compile(value, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) + if err != nil { + return fmt.Errorf("compilation of '%s' context value failed: %v", value, err) + } + alertContext.ContextToSendCompiled[key] = append(alertContext.ContextToSendCompiled[key], valueCompiled) + } + } + + return nil +} + +func truncate(values []string, contextValueLen int) (string, error) { + var ret string + valueByte, err := json.Marshal(values) + if err != nil { + return "", fmt.Errorf("unable to dump metas: %s", err) + } + ret = string(valueByte) + for { + if len(ret) <= contextValueLen { + break + } + // if there is only 1 value left and that the size is too big, truncate it + if len(values) == 1 { + valueToTruncate := values[0] + half := len(valueToTruncate) / 2 + lastValueTruncated := valueToTruncate[:half] + "..." + values = values[:len(values)-1] + values = append(values, lastValueTruncated) + } else { + // if there is multiple value inside, just remove the last one + values = values[:len(values)-1] + } + valueByte, err = json.Marshal(values) + if err != nil { + return "", fmt.Errorf("unable to dump metas: %s", err) + } + ret = string(valueByte) + } + return ret, nil +} + +func EventToContext(events []types.Event) (models.Meta, []error) { + var errors []error + + metas := make([]*models.MetaItems0, 0) + tmpContext := make(map[string][]string) + for _, evt := range events { + for key, values := range alertContext.ContextToSendCompiled { + if _, ok := tmpContext[key]; !ok { + tmpContext[key] = make([]string, 0) + } + for _, value := range values { + var val string + output, err := expr.Run(value, exprhelpers.GetExprEnv(map[string]interface{}{"evt": evt})) + if err != nil { + errors = append(errors, fmt.Errorf("failed to get value for %s : %v", key, err)) + continue + } + switch out := output.(type) { + case string: + val = out + case int: + val = strconv.Itoa(out) + default: + errors = append(errors, fmt.Errorf("unexpected return type for %s : %T", key, output)) + continue + } + if val != "" && !types.InSlice(val, tmpContext[key]) { + tmpContext[key] = append(tmpContext[key], val) + } + } + } + } + for key, values := range tmpContext { + if len(values) == 0 { + continue + } + valueStr, err := truncate(values, alertContext.ContextValueLen) + if err != nil { + log.Warningf(err.Error()) + } + meta := models.MetaItems0{ + Key: key, + Value: valueStr, + } + metas = append(metas, &meta) + } + + ret := models.Meta(metas) + return ret, errors +} diff --git a/pkg/alertcontext/alertcontext_test.go b/pkg/alertcontext/alertcontext_test.go new file mode 100644 index 000000000..8e6ca849a --- /dev/null +++ b/pkg/alertcontext/alertcontext_test.go @@ -0,0 +1,201 @@ +package alertcontext + +import ( + "fmt" + "testing" + + "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/stretchr/testify/assert" +) + +func TestNewAlertContext(t *testing.T) { + tests := []struct { + name string + contextToSend map[string][]string + valueLength int + expectedErr error + }{ + { + name: "basic config test", + contextToSend: map[string][]string{ + "test": []string{"evt.Parsed.source_ip"}, + }, + valueLength: 100, + expectedErr: nil, + }, + } + + for _, test := range tests { + fmt.Printf("Running test '%s'\n", test.name) + err := NewAlertContext(test.contextToSend, test.valueLength) + assert.ErrorIs(t, err, test.expectedErr) + + } +} + +func TestEventToContext(t *testing.T) { + tests := []struct { + name string + contextToSend map[string][]string + valueLength int + events []types.Event + expectedResult models.Meta + }{ + { + name: "basic test", + contextToSend: map[string][]string{ + "source_ip": []string{"evt.Parsed.source_ip"}, + "nonexistent_field": []string{"evt.Parsed.nonexist"}, + }, + valueLength: 100, + events: []types.Event{ + { + Parsed: map[string]string{ + "source_ip": "1.2.3.4", + "source_machine": "mymachine", + }, + }, + }, + expectedResult: []*models.MetaItems0{ + { + Key: "source_ip", + Value: "[\"1.2.3.4\"]", + }, + }, + }, + { + name: "test many events", + contextToSend: map[string][]string{ + "source_ip": []string{"evt.Parsed.source_ip"}, + "source_machine": []string{"evt.Parsed.source_machine"}, + "cve": []string{"evt.Parsed.cve"}, + }, + valueLength: 100, + events: []types.Event{ + { + Parsed: map[string]string{ + "source_ip": "1.2.3.4", + "source_machine": "mymachine", + "cve": "CVE-2022-1234", + }, + }, + { + Parsed: map[string]string{ + "source_ip": "1.2.3.4", + "source_machine": "mymachine", + "cve": "CVE-2022-1235", + }, + }, + { + Parsed: map[string]string{ + "source_ip": "1.2.3.4", + "source_machine": "mymachine", + "cve": "CVE-2022-125", + }, + }, + }, + expectedResult: []*models.MetaItems0{ + { + Key: "source_ip", + Value: "[\"1.2.3.4\"]", + }, + { + Key: "source_machine", + Value: "[\"mymachine\"]", + }, + { + Key: "cve", + Value: "[\"CVE-2022-1234\",\"CVE-2022-1235\",\"CVE-2022-125\"]", + }, + }, + }, + { + name: "test many events with result above max length (need truncate, keep only 2 on 3 elements)", + contextToSend: map[string][]string{ + "source_ip": []string{"evt.Parsed.source_ip"}, + "source_machine": []string{"evt.Parsed.source_machine"}, + "uri": []string{"evt.Parsed.uri"}, + }, + valueLength: 100, + events: []types.Event{ + { + Parsed: map[string]string{ + "source_ip": "1.2.3.4", + "source_machine": "mymachine", + "uri": "/test/test/test/../../../../../../../../", + }, + }, + { + Parsed: map[string]string{ + "source_ip": "1.2.3.4", + "source_machine": "mymachine", + "uri": "/admin/admin/admin/../../../../../../../../", + }, + }, + { + Parsed: map[string]string{ + "source_ip": "1.2.3.4", + "source_machine": "mymachine", + "uri": "/login/login/login/../../../../../../../../../../../", + }, + }, + }, + expectedResult: []*models.MetaItems0{ + { + Key: "source_ip", + Value: "[\"1.2.3.4\"]", + }, + { + Key: "source_machine", + Value: "[\"mymachine\"]", + }, + { + Key: "uri", + Value: "[\"/test/test/test/../../../../../../../../\",\"/admin/admin/admin/../../../../../../../../\"]", + }, + }, + }, + { + name: "test one events with result above max length (need truncate on one element)", + contextToSend: map[string][]string{ + "source_ip": []string{"evt.Parsed.source_ip"}, + "source_machine": []string{"evt.Parsed.source_machine"}, + "uri": []string{"evt.Parsed.uri"}, + }, + valueLength: 100, + events: []types.Event{ + { + Parsed: map[string]string{ + "source_ip": "1.2.3.4", + "source_machine": "mymachine", + "uri": "/test/test/test/../../../../.should_truncate_just_after_this/../../../..../../../../../../../../../../../../../../../end", + }, + }, + }, + expectedResult: []*models.MetaItems0{ + { + Key: "source_machine", + Value: "[\"mymachine\"]", + }, + { + Key: "uri", + Value: "[\"/test/test/test/../../../../.should_truncate_just_after_this...\"]", + }, + { + Key: "source_ip", + Value: "[\"1.2.3.4\"]", + }, + }, + }, + } + + for _, test := range tests { + fmt.Printf("Running test '%s'\n", test.name) + err := NewAlertContext(test.contextToSend, test.valueLength) + assert.ErrorIs(t, err, nil) + + metas, _ := EventToContext(test.events) + assert.ElementsMatch(t, test.expectedResult, metas) + } +} diff --git a/pkg/apiserver/apic.go b/pkg/apiserver/apic.go index db33aca8e..6b9dd6216 100644 --- a/pkg/apiserver/apic.go +++ b/pkg/apiserver/apic.go @@ -85,8 +85,8 @@ func (a *apic) FetchScenariosListFromDB() ([]string, error) { return scenarios, nil } -func alertToSignal(alert *models.Alert, scenarioTrust string) *models.AddSignalsRequestItem { - return &models.AddSignalsRequestItem{ +func alertToSignal(alert *models.Alert, scenarioTrust string, shareContext bool) *models.AddSignalsRequestItem { + signal := &models.AddSignalsRequestItem{ Message: alert.Message, Scenario: alert.Scenario, ScenarioHash: alert.ScenarioHash, @@ -96,8 +96,19 @@ func alertToSignal(alert *models.Alert, scenarioTrust string) *models.AddSignals StopAt: alert.StopAt, CreatedAt: alert.CreatedAt, MachineID: alert.MachineID, - ScenarioTrust: &scenarioTrust, + ScenarioTrust: scenarioTrust, } + if shareContext { + signal.Context = make([]*models.AddSignalsRequestItemContextItems0, 0) + for _, meta := range alert.Meta { + contextItem := models.AddSignalsRequestItemContextItems0{ + Key: meta.Key, + Value: meta.Value, + } + signal.Context = append(signal.Context, &contextItem) + } + } + return signal } func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, consoleConfig *csconfig.ConsoleConfig) (*apic, error) { @@ -176,7 +187,7 @@ func (a *apic) Push() error { var signals []*models.AddSignalsRequestItem for _, alert := range alerts { if ok := shouldShareAlert(alert, a.consoleConfig); ok { - signals = append(signals, alertToSignal(alert, getScenarioTrustOfAlert(alert))) + signals = append(signals, alertToSignal(alert, getScenarioTrustOfAlert(alert), *a.consoleConfig.ShareContext)) } } a.mu.Lock() diff --git a/pkg/apiserver/apic_test.go b/pkg/apiserver/apic_test.go index a06a750d8..7b0d27c2c 100644 --- a/pkg/apiserver/apic_test.go +++ b/pkg/apiserver/apic_test.go @@ -58,6 +58,7 @@ func getAPIC(t *testing.T) *apic { ShareManualDecisions: types.BoolPtr(false), ShareTaintedScenarios: types.BoolPtr(false), ShareCustomScenarios: types.BoolPtr(false), + ShareContext: types.BoolPtr(false), }, } } diff --git a/pkg/csconfig/api_test.go b/pkg/csconfig/api_test.go index 6014b9824..2caddc204 100644 --- a/pkg/csconfig/api_test.go +++ b/pkg/csconfig/api_test.go @@ -213,6 +213,7 @@ func TestLoadAPIServer(t *testing.T) { ShareManualDecisions: types.BoolPtr(false), ShareTaintedScenarios: types.BoolPtr(true), ShareCustomScenarios: types.BoolPtr(true), + ShareContext: types.BoolPtr(false), }, LogDir: LogDirFullPath, LogMedia: "stdout", diff --git a/pkg/csconfig/console.go b/pkg/csconfig/console.go index e00d0146c..31adfaf31 100644 --- a/pkg/csconfig/console.go +++ b/pkg/csconfig/console.go @@ -14,9 +14,10 @@ const ( SEND_CUSTOM_SCENARIOS = "custom" SEND_TAINTED_SCENARIOS = "tainted" SEND_MANUAL_SCENARIOS = "manual" + SEND_CONTEXT = "context" ) -var CONSOLE_CONFIGS = []string{SEND_CUSTOM_SCENARIOS, SEND_MANUAL_SCENARIOS, SEND_TAINTED_SCENARIOS} +var CONSOLE_CONFIGS = []string{SEND_CUSTOM_SCENARIOS, SEND_MANUAL_SCENARIOS, SEND_TAINTED_SCENARIOS, SEND_CONTEXT} var DefaultConsoleConfigFilePath = DefaultConfigPath("console.yaml") @@ -24,6 +25,7 @@ type ConsoleConfig struct { ShareManualDecisions *bool `yaml:"share_manual_decisions"` ShareTaintedScenarios *bool `yaml:"share_tainted"` ShareCustomScenarios *bool `yaml:"share_custom"` + ShareContext *bool `yaml:"share_context"` } func (c *LocalApiServerCfg) LoadConsoleConfig() error { @@ -33,6 +35,7 @@ func (c *LocalApiServerCfg) LoadConsoleConfig() error { c.ConsoleConfig.ShareCustomScenarios = types.BoolPtr(true) c.ConsoleConfig.ShareTaintedScenarios = types.BoolPtr(true) c.ConsoleConfig.ShareManualDecisions = types.BoolPtr(false) + c.ConsoleConfig.ShareContext = types.BoolPtr(false) return nil } @@ -57,6 +60,12 @@ func (c *LocalApiServerCfg) LoadConsoleConfig() error { log.Debugf("no share_manual scenarios found, setting to false") c.ConsoleConfig.ShareManualDecisions = types.BoolPtr(false) } + + if c.ConsoleConfig.ShareContext == nil { + log.Debugf("no 'context' found, setting to false") + c.ConsoleConfig.ShareContext = types.BoolPtr(false) + } + log.Debugf("Console configuration '%s' loaded successfully", c.ConsoleConfigPath) return nil diff --git a/pkg/csconfig/crowdsec_service.go b/pkg/csconfig/crowdsec_service.go index 91b56504e..ee6f9d38b 100644 --- a/pkg/csconfig/crowdsec_service.go +++ b/pkg/csconfig/crowdsec_service.go @@ -7,31 +7,34 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" "github.com/crowdsecurity/crowdsec/pkg/types" ) // CrowdsecServiceCfg contains the location of parsers/scenarios/... and acquisition files type CrowdsecServiceCfg struct { - Enable *bool `yaml:"enable"` - AcquisitionFilePath string `yaml:"acquisition_path,omitempty"` - AcquisitionDirPath string `yaml:"acquisition_dir,omitempty"` + Enable *bool `yaml:"enable"` + AcquisitionFilePath string `yaml:"acquisition_path,omitempty"` + AcquisitionDirPath string `yaml:"acquisition_dir,omitempty"` + ConsoleContextPath string `yaml:"console_context_path"` + ConsoleContextValueLength int `yaml:"console_context_value_length"` + AcquisitionFiles []string `yaml:"-"` + ParserRoutinesCount int `yaml:"parser_routines"` + BucketsRoutinesCount int `yaml:"buckets_routines"` + OutputRoutinesCount int `yaml:"output_routines"` + SimulationConfig *SimulationConfig `yaml:"-"` + LintOnly bool `yaml:"-"` // if set to true, exit after loading configs + BucketStateFile string `yaml:"state_input_file,omitempty"` // if we need to unserialize buckets at start + BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown + BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode - AcquisitionFiles []string `yaml:"-"` - ParserRoutinesCount int `yaml:"parser_routines"` - BucketsRoutinesCount int `yaml:"buckets_routines"` - OutputRoutinesCount int `yaml:"output_routines"` - SimulationConfig *SimulationConfig `yaml:"-"` - LintOnly bool `yaml:"-"` // if set to true, exit after loading configs - BucketStateFile string `yaml:"state_input_file,omitempty"` // if we need to unserialize buckets at start - BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown - BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode - - HubDir string `yaml:"-"` - DataDir string `yaml:"-"` - ConfigDir string `yaml:"-"` - HubIndexFile string `yaml:"-"` - SimulationFilePath string `yaml:"-"` + HubDir string `yaml:"-"` + DataDir string `yaml:"-"` + ConfigDir string `yaml:"-"` + HubIndexFile string `yaml:"-"` + SimulationFilePath string `yaml:"-"` + ContextToSend map[string][]string `yaml:"-"` } func (c *Config) LoadCrowdsec() error { @@ -152,5 +155,50 @@ func (c *Config) LoadCrowdsec() error { return errors.Wrap(err, "while loading hub") } + c.Crowdsec.ContextToSend = make(map[string][]string, 0) + fallback := false + if c.Crowdsec.ConsoleContextPath == "" { + // fallback to default config file + c.Crowdsec.ConsoleContextPath = filepath.Join(c.Crowdsec.ConfigDir, "console", "context.yaml") + fallback = true + } + + f, err := filepath.Abs(c.Crowdsec.ConsoleContextPath) + if err != nil { + return fmt.Errorf("fail to get absolute path of %s: %s", c.Crowdsec.ConsoleContextPath, err) + } + + c.Crowdsec.ConsoleContextPath = f + yamlFile, err := os.ReadFile(c.Crowdsec.ConsoleContextPath) + if err != nil { + if fallback { + log.Debugf("Default context config file doesn't exist, will not use it") + } else { + return fmt.Errorf("failed to open context file: %s", err) + } + } else { + err = yaml.Unmarshal(yamlFile, c.Crowdsec.ContextToSend) + if err != nil { + return fmt.Errorf("unmarshaling labels console config file '%s': %s", c.Crowdsec.ConsoleContextPath, err) + } + } + + return nil +} + +func (c *CrowdsecServiceCfg) DumpContextConfigFile() error { + var out []byte + var err error + + if out, err = yaml.Marshal(c.ContextToSend); err != nil { + return errors.Wrapf(err, "while marshaling ConsoleConfig (for %s)", c.ConsoleContextPath) + } + + if err := os.WriteFile(c.ConsoleContextPath, out, 0600); err != nil { + return errors.Wrapf(err, "while dumping console config to %s", c.ConsoleContextPath) + } + + log.Infof("%s file saved", c.ConsoleContextPath) + return nil } diff --git a/pkg/csconfig/crowdsec_service_test.go b/pkg/csconfig/crowdsec_service_test.go index b835d5627..b9701ee52 100644 --- a/pkg/csconfig/crowdsec_service_test.go +++ b/pkg/csconfig/crowdsec_service_test.go @@ -33,6 +33,9 @@ func TestLoadCrowdsec(t *testing.T) { hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json") require.NoError(t, err) + contextFileFullPath, err := filepath.Abs("./tests/context.yaml") + require.NoError(t, err) + tests := []struct { name string input *Config @@ -53,23 +56,30 @@ func TestLoadCrowdsec(t *testing.T) { }, }, Crowdsec: &CrowdsecServiceCfg{ - AcquisitionFilePath: "./tests/acquis.yaml", - SimulationFilePath: "./tests/simulation.yaml", + AcquisitionFilePath: "./tests/acquis.yaml", + SimulationFilePath: "./tests/simulation.yaml", + ConsoleContextPath: "./tests/context.yaml", + ConsoleContextValueLength: 2500, }, }, expectedResult: &CrowdsecServiceCfg{ - Enable: types.BoolPtr(true), - AcquisitionDirPath: "", - AcquisitionFilePath: acquisFullPath, - ConfigDir: configDirFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, - HubIndexFile: hubIndexFileFullPath, - BucketsRoutinesCount: 1, - ParserRoutinesCount: 1, - OutputRoutinesCount: 1, - AcquisitionFiles: []string{acquisFullPath}, - SimulationFilePath: "./tests/simulation.yaml", + Enable: types.BoolPtr(true), + AcquisitionDirPath: "", + ConsoleContextPath: contextFileFullPath, + AcquisitionFilePath: acquisFullPath, + ConfigDir: configDirFullPath, + DataDir: dataFullPath, + HubDir: hubFullPath, + HubIndexFile: hubIndexFileFullPath, + BucketsRoutinesCount: 1, + ParserRoutinesCount: 1, + OutputRoutinesCount: 1, + ConsoleContextValueLength: 2500, + AcquisitionFiles: []string{acquisFullPath}, + SimulationFilePath: "./tests/simulation.yaml", + ContextToSend: map[string][]string{ + "source_ip": {"evt.Parsed.source_ip"}, + }, SimulationConfig: &SimulationConfig{ Simulation: &falseBoolPtr, }, @@ -92,21 +102,27 @@ func TestLoadCrowdsec(t *testing.T) { AcquisitionFilePath: "./tests/acquis.yaml", AcquisitionDirPath: "./tests/acquis/", SimulationFilePath: "./tests/simulation.yaml", + ConsoleContextPath: "./tests/context.yaml", }, }, expectedResult: &CrowdsecServiceCfg{ - Enable: types.BoolPtr(true), - AcquisitionDirPath: acquisDirFullPath, - AcquisitionFilePath: acquisFullPath, - ConfigDir: configDirFullPath, - HubIndexFile: hubIndexFileFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, - BucketsRoutinesCount: 1, - ParserRoutinesCount: 1, - OutputRoutinesCount: 1, - AcquisitionFiles: []string{acquisFullPath, acquisInDirFullPath}, - SimulationFilePath: "./tests/simulation.yaml", + Enable: types.BoolPtr(true), + AcquisitionDirPath: acquisDirFullPath, + AcquisitionFilePath: acquisFullPath, + ConsoleContextPath: contextFileFullPath, + ConfigDir: configDirFullPath, + HubIndexFile: hubIndexFileFullPath, + DataDir: dataFullPath, + HubDir: hubFullPath, + BucketsRoutinesCount: 1, + ParserRoutinesCount: 1, + OutputRoutinesCount: 1, + ConsoleContextValueLength: 0, + AcquisitionFiles: []string{acquisFullPath, acquisInDirFullPath}, + ContextToSend: map[string][]string{ + "source_ip": {"evt.Parsed.source_ip"}, + }, + SimulationFilePath: "./tests/simulation.yaml", SimulationConfig: &SimulationConfig{ Simulation: &falseBoolPtr, }, @@ -125,21 +141,29 @@ func TestLoadCrowdsec(t *testing.T) { CredentialsFilePath: "./tests/lapi-secrets.yaml", }, }, - Crowdsec: &CrowdsecServiceCfg{}, + Crowdsec: &CrowdsecServiceCfg{ + ConsoleContextPath: contextFileFullPath, + ConsoleContextValueLength: 10, + }, }, expectedResult: &CrowdsecServiceCfg{ - Enable: types.BoolPtr(true), - AcquisitionDirPath: "", - AcquisitionFilePath: "", - ConfigDir: configDirFullPath, - HubIndexFile: hubIndexFileFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, - BucketsRoutinesCount: 1, - ParserRoutinesCount: 1, - OutputRoutinesCount: 1, - AcquisitionFiles: []string{}, - SimulationFilePath: "", + Enable: types.BoolPtr(true), + AcquisitionDirPath: "", + AcquisitionFilePath: "", + ConfigDir: configDirFullPath, + HubIndexFile: hubIndexFileFullPath, + DataDir: dataFullPath, + HubDir: hubFullPath, + ConsoleContextPath: contextFileFullPath, + BucketsRoutinesCount: 1, + ParserRoutinesCount: 1, + OutputRoutinesCount: 1, + ConsoleContextValueLength: 10, + AcquisitionFiles: []string{}, + SimulationFilePath: "", + ContextToSend: map[string][]string{ + "source_ip": {"evt.Parsed.source_ip"}, + }, SimulationConfig: &SimulationConfig{ Simulation: &falseBoolPtr, }, @@ -159,6 +183,7 @@ func TestLoadCrowdsec(t *testing.T) { }, }, Crowdsec: &CrowdsecServiceCfg{ + ConsoleContextPath: "", AcquisitionFilePath: "./tests/acquis_not_exist.yaml", }, }, diff --git a/pkg/csconfig/tests/context.yaml b/pkg/csconfig/tests/context.yaml new file mode 100644 index 000000000..f8d41911c --- /dev/null +++ b/pkg/csconfig/tests/context.yaml @@ -0,0 +1,2 @@ +source_ip: + - evt.Parsed.source_ip \ No newline at end of file diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index e7bd35014..a786e2f6b 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -11,6 +11,8 @@ import ( "sync" "time" + "github.com/crowdsecurity/crowdsec/pkg/alertcontext" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwversion" @@ -225,6 +227,11 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb. ret = append(ret, bucketFactory) } } + + if err := alertcontext.NewAlertContext(cscfg.ContextToSend, cscfg.ConsoleContextValueLength); err != nil { + return nil, nil, fmt.Errorf("unable to load alert context: %s", err) + } + log.Warningf("Loaded %d scenarios", len(ret)) return ret, response, nil } @@ -349,6 +356,7 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error { return fmt.Errorf("invalid bucket from %s : %v", bucketFactory.Filename, err) } bucketFactory.tomb = tomb + return nil } diff --git a/pkg/leakybucket/overflows.go b/pkg/leakybucket/overflows.go index 3a7732aa6..3c1f3ebc5 100644 --- a/pkg/leakybucket/overflows.go +++ b/pkg/leakybucket/overflows.go @@ -6,6 +6,7 @@ import ( "sort" "strconv" + "github.com/crowdsecurity/crowdsec/pkg/alertcontext" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/davecgh/go-spew/spew" @@ -17,7 +18,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" ) -//SourceFromEvent extracts and formats a valid models.Source object from an Event +// SourceFromEvent extracts and formats a valid models.Source object from an Event func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, error) { srcs := make(map[string]models.Source) /*if it's already an overflow, we have properly formatted sources. @@ -160,7 +161,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e return srcs, nil } -//EventsFromQueue iterates the queue to collect & prepare meta-datas from alert +// EventsFromQueue iterates the queue to collect & prepare meta-datas from alert func EventsFromQueue(queue *Queue) []*models.Event { events := []*models.Event{} @@ -207,7 +208,7 @@ func EventsFromQueue(queue *Queue) []*models.Event { return events } -//alertFormatSource iterates over the queue to collect sources +// alertFormatSource iterates over the queue to collect sources func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, string, error) { var sources map[string]models.Source = make(map[string]models.Source) var source_type string @@ -233,7 +234,7 @@ func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, st return sources, source_type, nil } -//NewAlert will generate a RuntimeAlert and its APIAlert(s) from a bucket that overflowed +// NewAlert will generate a RuntimeAlert and its APIAlert(s) from a bucket that overflowed func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) { var runtimeAlert types.RuntimeAlert @@ -293,6 +294,11 @@ func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) { *apiAlert.Message = fmt.Sprintf("%s %s performed '%s' (%d events over %s) at %s", source_scope, sourceStr, leaky.Name, leaky.Total_count, leaky.Ovflw_ts.Sub(leaky.First_ts), leaky.Last_ts) //Get the events from Leaky/Queue apiAlert.Events = EventsFromQueue(queue) + var warnings []error + apiAlert.Meta, warnings = alertcontext.EventToContext(leaky.Queue.GetQueue()) + for _, w := range warnings { + log.Warningf("while extracting context from bucket %s : %s", leaky.Name, w) + } //Loop over the Sources and generate appropriate number of ApiAlerts for _, srcValue := range sources { diff --git a/pkg/models/add_signals_request_item.go b/pkg/models/add_signals_request_item.go index 2a5acdd23..ec6d056df 100644 --- a/pkg/models/add_signals_request_item.go +++ b/pkg/models/add_signals_request_item.go @@ -7,6 +7,7 @@ package models import ( "context" + "strconv" "github.com/go-openapi/errors" "github.com/go-openapi/strfmt" @@ -19,6 +20,12 @@ import ( // swagger:model AddSignalsRequestItem type AddSignalsRequestItem struct { + // alert id + AlertID int64 `json:"alert_id,omitempty"` + + // context + Context []*AddSignalsRequestItemContextItems0 `json:"context"` + // created at CreatedAt string `json:"created_at,omitempty"` @@ -38,8 +45,7 @@ type AddSignalsRequestItem struct { ScenarioHash *string `json:"scenario_hash"` // scenario trust - // Required: true - ScenarioTrust *string `json:"scenario_trust"` + ScenarioTrust string `json:"scenario_trust,omitempty"` // scenario version // Required: true @@ -62,6 +68,10 @@ type AddSignalsRequestItem struct { func (m *AddSignalsRequestItem) Validate(formats strfmt.Registry) error { var res []error + if err := m.validateContext(formats); err != nil { + res = append(res, err) + } + if err := m.validateMessage(formats); err != nil { res = append(res, err) } @@ -74,10 +84,6 @@ func (m *AddSignalsRequestItem) Validate(formats strfmt.Registry) error { res = append(res, err) } - if err := m.validateScenarioTrust(formats); err != nil { - res = append(res, err) - } - if err := m.validateScenarioVersion(formats); err != nil { res = append(res, err) } @@ -100,6 +106,32 @@ func (m *AddSignalsRequestItem) Validate(formats strfmt.Registry) error { return nil } +func (m *AddSignalsRequestItem) validateContext(formats strfmt.Registry) error { + if swag.IsZero(m.Context) { // not required + return nil + } + + for i := 0; i < len(m.Context); i++ { + if swag.IsZero(m.Context[i]) { // not required + continue + } + + if m.Context[i] != nil { + if err := m.Context[i].Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("context" + "." + strconv.Itoa(i)) + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("context" + "." + strconv.Itoa(i)) + } + return err + } + } + + } + + return nil +} + func (m *AddSignalsRequestItem) validateMessage(formats strfmt.Registry) error { if err := validate.Required("message", "body", m.Message); err != nil { @@ -127,15 +159,6 @@ func (m *AddSignalsRequestItem) validateScenarioHash(formats strfmt.Registry) er return nil } -func (m *AddSignalsRequestItem) validateScenarioTrust(formats strfmt.Registry) error { - - if err := validate.Required("scenario_trust", "body", m.ScenarioTrust); err != nil { - return err - } - - return nil -} - func (m *AddSignalsRequestItem) validateScenarioVersion(formats strfmt.Registry) error { if err := validate.Required("scenario_version", "body", m.ScenarioVersion); err != nil { @@ -187,6 +210,10 @@ func (m *AddSignalsRequestItem) validateStopAt(formats strfmt.Registry) error { func (m *AddSignalsRequestItem) ContextValidate(ctx context.Context, formats strfmt.Registry) error { var res []error + if err := m.contextValidateContext(ctx, formats); err != nil { + res = append(res, err) + } + if err := m.contextValidateSource(ctx, formats); err != nil { res = append(res, err) } @@ -197,6 +224,26 @@ func (m *AddSignalsRequestItem) ContextValidate(ctx context.Context, formats str return nil } +func (m *AddSignalsRequestItem) contextValidateContext(ctx context.Context, formats strfmt.Registry) error { + + for i := 0; i < len(m.Context); i++ { + + if m.Context[i] != nil { + if err := m.Context[i].ContextValidate(ctx, formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("context" + "." + strconv.Itoa(i)) + } else if ce, ok := err.(*errors.CompositeError); ok { + return ce.ValidateName("context" + "." + strconv.Itoa(i)) + } + return err + } + } + + } + + return nil +} + func (m *AddSignalsRequestItem) contextValidateSource(ctx context.Context, formats strfmt.Registry) error { if m.Source != nil { @@ -230,3 +277,43 @@ func (m *AddSignalsRequestItem) UnmarshalBinary(b []byte) error { *m = res return nil } + +// AddSignalsRequestItemContextItems0 add signals request item context items0 +// +// swagger:model AddSignalsRequestItemContextItems0 +type AddSignalsRequestItemContextItems0 struct { + + // key + Key string `json:"key,omitempty"` + + // value + Value string `json:"value,omitempty"` +} + +// Validate validates this add signals request item context items0 +func (m *AddSignalsRequestItemContextItems0) Validate(formats strfmt.Registry) error { + return nil +} + +// ContextValidate validates this add signals request item context items0 based on context it is used +func (m *AddSignalsRequestItemContextItems0) ContextValidate(ctx context.Context, formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *AddSignalsRequestItemContextItems0) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *AddSignalsRequestItemContextItems0) UnmarshalBinary(b []byte) error { + var res AddSignalsRequestItemContextItems0 + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index 670776956..3779b0941 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -4,9 +4,11 @@ import ( "fmt" "os" "path" + "sort" "strings" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/grokky" log "github.com/sirupsen/logrus" @@ -50,6 +52,45 @@ func Init(c map[string]interface{}) (*UnixParserCtx, error) { return &r, nil } +// Return new parsers +// nodes and povfwnodes are already initialized in parser.LoadStages +func NewParsers() *Parsers { + parsers := &Parsers{ + Ctx: &UnixParserCtx{}, + Povfwctx: &UnixParserCtx{}, + StageFiles: make([]Stagefile, 0), + PovfwStageFiles: make([]Stagefile, 0), + } + for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} { + for _, hubParserItem := range cwhub.GetItemMap(itemType) { + if hubParserItem.Installed { + stagefile := Stagefile{ + Filename: hubParserItem.LocalPath, + Stage: hubParserItem.Stage, + } + if itemType == cwhub.PARSERS { + parsers.StageFiles = append(parsers.StageFiles, stagefile) + } + if itemType == cwhub.PARSERS_OVFLW { + parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile) + } + } + } + } + if parsers.StageFiles != nil { + sort.Slice(parsers.StageFiles, func(i, j int) bool { + return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename + }) + } + if parsers.PovfwStageFiles != nil { + sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool { + return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename + }) + } + + return parsers +} + func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { var err error diff --git a/rpm/SPECS/crowdsec.spec b/rpm/SPECS/crowdsec.spec index 41ed04d12..246c7b181 100644 --- a/rpm/SPECS/crowdsec.spec +++ b/rpm/SPECS/crowdsec.spec @@ -45,6 +45,7 @@ sed -i "s#/usr/local/lib/crowdsec/plugins/#%{_libdir}/%{name}/plugins/#g" config rm -rf %{buildroot} mkdir -p %{buildroot}/etc/crowdsec/hub mkdir -p %{buildroot}/etc/crowdsec/patterns +mkdir -p %{buildroot}/etc/crowdsec/console/ mkdir -p %{buildroot}%{_sharedstatedir}/%{name}/data mkdir -p %{buildroot}%{_presetdir} @@ -62,6 +63,7 @@ install -m 600 -D config/config.yaml %{buildroot}%{_sysconfdir}/crowdsec install -m 644 -D config/simulation.yaml %{buildroot}%{_sysconfdir}/crowdsec install -m 644 -D config/profiles.yaml %{buildroot}%{_sysconfdir}/crowdsec install -m 644 -D config/console.yaml %{buildroot}%{_sysconfdir}/crowdsec +install -m 644 -D config/context.yaml %{buildroot}%{_sysconfdir}/crowdsec/console/ install -m 750 -D config/%{name}.cron.daily %{buildroot}%{_sysconfdir}/cron.daily/%{name} install -m 644 -D %{SOURCE1} %{buildroot}%{_presetdir} @@ -115,6 +117,7 @@ rm -rf %{buildroot} %config(noreplace) %{_sysconfdir}/%{name}/simulation.yaml %config(noreplace) %{_sysconfdir}/%{name}/profiles.yaml %config(noreplace) %{_sysconfdir}/%{name}/console.yaml +%config(noreplace) %{_sysconfdir}/%{name}/console/context.yaml %config(noreplace) %{_presetdir}/80-%{name}.preset %config(noreplace) %{_sysconfdir}/%{name}/notifications/http.yaml %config(noreplace) %{_sysconfdir}/%{name}/notifications/slack.yaml diff --git a/tests/bats/81_alert_context.bats b/tests/bats/81_alert_context.bats new file mode 100644 index 000000000..56463131a --- /dev/null +++ b/tests/bats/81_alert_context.bats @@ -0,0 +1,67 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +fake_log() { + for _ in $(seq 1 6); do + echo "$(LC_ALL=C date '+%b %d %H:%M:%S ')"'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.172 port 35424' + done +} + +setup_file() { + load "../lib/setup_file.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + ./instance-data load +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "$FILE 1.1.1.172 has context" { + tmpfile=$(TMPDIR="${BATS_TEST_TMPDIR}" mktemp) + touch "${tmpfile}" + + ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') + + cat <<-EOT >"${ACQUIS_YAML}" + filename: $tmpfile + labels: + type: syslog + EOT + + CONTEXT_YAML=$(config_get '.crowdsec_service.console_context_path') + + cat <<-EOT >"${CONTEXT_YAML}" + target_user: + - evt.Parsed.sshd_invalid_user + source_ip: + - evt.Parsed.sshd_client_ip + source_host: + - evt.Meta.machine + EOT + + ./instance-crowdsec start + sleep 2 + fake_log >>"${tmpfile}" + sleep 2 + rm -f -- "${tmpfile}" + + run -0 cscli alerts list -o json + run -0 jq '.[0].id' <(output) + ALERT_ID="$output" + run -0 cscli alerts inspect "$ALERT_ID" -o json + run -0 jq -c '.meta | sort_by(.key) | map([.key,.value])' <(output) + + assert_json '[["source_host","[\"sd-126005\"]"],["source_ip","[\"1.1.1.172\"]"],["target_user","[\"netflix\"]"]]' +} \ No newline at end of file diff --git a/tests/lib/config/config-local b/tests/lib/config/config-local index 0298fc4d4..8fdd3898e 100755 --- a/tests/lib/config/config-local +++ b/tests/lib/config/config-local @@ -61,6 +61,8 @@ config_generate() { ../config/online_api_credentials.yaml \ "${CONFIG_DIR}/" + cp ../config/context.yaml "${CONFIG_DIR}/console/" + # the default acquis file contains files that are not readable by everyone touch "$LOG_DIR/empty.log" cat <<-EOT >"$CONFIG_DIR/acquis.yaml" @@ -94,6 +96,7 @@ config_generate() { .api.client.credentials_path=strenv(CONFIG_DIR)+"/local_api_credentials.yaml" | .api.server.profiles_path=strenv(CONFIG_DIR)+"/profiles.yaml" | .api.server.console_path=strenv(CONFIG_DIR)+"/console.yaml" | + .crowdsec_service.console_context_path=strenv(CONFIG_DIR) + "/console/context.yaml" | .api.server.online_client.credentials_path=strenv(CONFIG_DIR)+"/online_api_credentials.yaml" ' ../config/config.yaml >"${CONFIG_DIR}/config.yaml" } @@ -107,6 +110,7 @@ make_init_data() { mkdir -p "${CONFIG_DIR}/notifications" mkdir -p "${CONFIG_DIR}/hub" mkdir -p "${CONFIG_DIR}/patterns" + mkdir -p "${CONFIG_DIR}/console" cp -a "../config/patterns" "${CONFIG_DIR}/" config_generate # XXX errors from instance-db should be reported... diff --git a/windows/installer/product.wxs b/windows/installer/product.wxs index 074e892c8..5f37f7348 100644 --- a/windows/installer/product.wxs +++ b/windows/installer/product.wxs @@ -54,6 +54,11 @@ + + + + + @@ -163,6 +168,8 @@ + + diff --git a/wizard.sh b/wizard.sh index 7a314d86a..638c33ef2 100755 --- a/wizard.sh +++ b/wizard.sh @@ -28,6 +28,7 @@ CROWDSEC_CONFIG_PATH="${CROWDSEC_PATH}" CROWDSEC_LOG_FILE="/var/log/crowdsec.log" LAPI_LOG_FILE="/var/log/crowdsec_api.log" CROWDSEC_PLUGIN_DIR="${CROWDSEC_USR_DIR}/plugins" +CROWDSEC_CONSOLE_DIR="${CROWDSEC_PATH}/console" CROWDSEC_BIN="./cmd/crowdsec/crowdsec" CSCLI_BIN="./cmd/crowdsec-cli/cscli" @@ -403,6 +404,7 @@ install_crowdsec() { mkdir -p "${CROWDSEC_CONFIG_PATH}/postoverflows" || exit mkdir -p "${CROWDSEC_CONFIG_PATH}/collections" || exit mkdir -p "${CROWDSEC_CONFIG_PATH}/patterns" || exit + mkdir -p "${CROWDSEC_CONSOLE_DIR}" || exit #tmp mkdir -p /tmp/data @@ -419,6 +421,7 @@ install_crowdsec() { install -v -m 644 -D ./config/profiles.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit install -v -m 644 -D ./config/simulation.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit install -v -m 644 -D ./config/"${CONSOLE_FILE}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit + install -v -m 644 -D ./config/context.yaml "${CROWDSEC_CONSOLE_DIR}" 1> /dev/null || exit DATA=${CROWDSEC_DATA_DIR} CFG=${CROWDSEC_CONFIG_PATH} envsubst '$CFG $DATA' < ./config/user.yaml > ${CROWDSEC_CONFIG_PATH}"/user.yaml" || log_fatal "unable to generate user configuration file" if [[ ${DOCKER_MODE} == "false" ]]; then