Alert context (#1895)

Co-authored-by: bui <thibault@crowdsec.net>

cmd/crowdsec-cli/alerts.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -112,6 +113,29 @@ func DisplayOneAlert(alert *models.Alert, withDetail bool) error {
 
 		alertDecisionsTable(color.Output, alert)
 
+		if len(alert.Meta) > 0 {
+			fmt.Printf("\n - Context  :\n")
+			sort.Slice(alert.Meta, func(i, j int) bool {
+				return alert.Meta[i].Key < alert.Meta[j].Key
+			})
+			table := newTable(color.Output)
+			table.SetRowLines(false)
+			table.SetHeaders("Key", "Value")
+			for _, meta := range alert.Meta {
+				var valSlice []string
+				if err := json.Unmarshal([]byte(meta.Value), &valSlice); err != nil {
+					return fmt.Errorf("unknown context value type '%s' : %s", meta.Value, err)
+				}
+				for _, value := range valSlice {
+					table.AddRow(
+						meta.Key,
+						value,
+					)
+				}
+			}
+			table.Render()
+		}
+
 		if withDetail {
 			fmt.Printf("\n - Events  :\n")
 			for _, event := range alert.Events {

cmd/crowdsec-cli/console.go

@@ -46,7 +46,7 @@ func NewConsoleCmd() *cobra.Command {
 				log.Fatalf("No configuration for Central API (CAPI) in '%s'", *csConfig.FilePath)
 			}
 			if csConfig.API.Server.OnlineClient.Credentials == nil {
-				log.Fatal("You must configure Central API (CAPI) with `cscli capi register` before enrolling your instance")
+				log.Fatal("You must configure Central API (CAPI) with `cscli capi register` before accessing console features.")
 			}
 			return nil
 		},
@@ -129,9 +129,9 @@ After running this command your will need to validate the enrollment in the weba
 	var enableAll, disableAll bool
 
 	cmdEnable := &cobra.Command{
-		Use:     "enable [feature-flag]",
+		Use:     "enable [option]",
-		Short:   "Enable a feature flag",
+		Short:   "Enable a console option",
-		Example: "enable tainted",
+		Example: "sudo cscli console enable tainted",
 		Long: `
 Enable given information push to the central API. Allows to empower the console`,
 		ValidArgs:         csconfig.CONSOLE_CONFIGS,
@@ -153,13 +153,13 @@ Enable given information push to the central API. Allows to empower the console`
 			log.Infof(ReloadMessage())
 		},
 	}
-	cmdEnable.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all feature flags")
+	cmdEnable.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all console options")
 	cmdConsole.AddCommand(cmdEnable)
 
 	cmdDisable := &cobra.Command{
-		Use:     "disable [feature-flag]",
+		Use:     "disable [option]",
-		Short:   "Disable a feature flag",
+		Short:   "Disable a console option",
-		Example: "disable tainted",
+		Example: "sudo cscli console disable tainted",
 		Long: `
 Disable given information push to the central API.`,
 		ValidArgs:         csconfig.CONSOLE_CONFIGS,
@@ -183,13 +183,13 @@ Disable given information push to the central API.`,
 			log.Infof(ReloadMessage())
 		},
 	}
-	cmdDisable.Flags().BoolVarP(&disableAll, "all", "a", false, "Enable all feature flags")
+	cmdDisable.Flags().BoolVarP(&disableAll, "all", "a", false, "Disable all console options")
 	cmdConsole.AddCommand(cmdDisable)
 
 	cmdConsoleStatus := &cobra.Command{
-		Use:               "status [feature-flag]",
+		Use:               "status [option]",
-		Short:             "Shows status of one or all feature flags",
+		Short:             "Shows status of one or all console options",
-		Example:           "status tainted",
+		Example:           `sudo cscli console status`,
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {
 			switch csConfig.Cscli.Output {
@@ -212,6 +212,7 @@ Disable given information push to the central API.`,
 					{"share_manual_decisions", fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareManualDecisions)},
 					{"share_custom", fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios)},
 					{"share_tainted", fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios)},
+					{"share_context", fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareContext)},
 				}
 				for _, row := range rows {
 					err = csvwriter.Write(row)
@@ -223,8 +224,8 @@ Disable given information push to the central API.`,
 			}
 		},
 	}
-
 	cmdConsole.AddCommand(cmdConsoleStatus)
+
 	return cmdConsole
 }
 
@@ -270,6 +271,19 @@ func SetConsoleOpts(args []string, wanted bool) {
 				log.Infof("%s set to %t", csconfig.SEND_MANUAL_SCENARIOS, wanted)
 				csConfig.API.Server.ConsoleConfig.ShareManualDecisions = types.BoolPtr(wanted)
 			}
+		case csconfig.SEND_CONTEXT:
+			/*for each flag check if it's already set before setting it*/
+			if csConfig.API.Server.ConsoleConfig.ShareContext != nil {
+				if *csConfig.API.Server.ConsoleConfig.ShareContext == wanted {
+					log.Infof("%s already set to %t", csconfig.SEND_CONTEXT, wanted)
+				} else {
+					log.Infof("%s set to %t", csconfig.SEND_CONTEXT, wanted)
+					*csConfig.API.Server.ConsoleConfig.ShareContext = wanted
+				}
+			} else {
+				log.Infof("%s set to %t", csconfig.SEND_CONTEXT, wanted)
+				csConfig.API.Server.ConsoleConfig.ShareContext = types.BoolPtr(wanted)
+			}
 		default:
 			log.Fatalf("unknown flag %s", arg)
 		}

cmd/crowdsec-cli/console_table.go

@@ -41,6 +41,12 @@ func cmdConsoleStatusTable(out io.Writer, csConfig csconfig.Config) {
 			}
 
 			t.AddRow(option, activated, "Send alerts from tainted scenarios to the console")
+		case csconfig.SEND_CONTEXT:
+			activated := string(emoji.CrossMark)
+			if *csConfig.API.Server.ConsoleConfig.ShareContext {
+				activated = string(emoji.CheckMarkButton)
+			}
+			t.AddRow(option, activated, "Send context with alerts to the console")
 		}
 	}
 

cmd/crowdsec-cli/lapi.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"sort"
 	"strings"
 
 	"github.com/go-openapi/strfmt"
@@ -13,16 +14,20 @@ import (
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
 
+	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
+	"github.com/crowdsecurity/crowdsec/pkg/parser"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
 var LAPIURLPrefix string = "v1"
 
-func runLapiStatus (cmd *cobra.Command, args []string) error {
+func runLapiStatus(cmd *cobra.Command, args []string) error {
 	var err error
 
 	password := strfmt.Password(csConfig.API.Client.Credentials.Password)
@@ -68,7 +73,6 @@ func runLapiStatus (cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-
 func runLapiRegister(cmd *cobra.Command, args []string) error {
 	var err error
 
@@ -160,7 +164,6 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-
 func NewLapiStatusCmd() *cobra.Command {
 	cmdLapiStatus := &cobra.Command{
 		Use:               "status",
@@ -173,7 +176,6 @@ func NewLapiStatusCmd() *cobra.Command {
 	return cmdLapiStatus
 }
 
-
 func NewLapiRegisterCmd() *cobra.Command {
 	cmdLapiRegister := &cobra.Command{
 		Use:   "register",
@@ -193,7 +195,6 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side
 	return cmdLapiRegister
 }
 
-
 func NewLapiCmd() *cobra.Command {
 	var cmdLapi = &cobra.Command{
 		Use:               "lapi [action]",
@@ -210,6 +211,350 @@ func NewLapiCmd() *cobra.Command {
 
 	cmdLapi.AddCommand(NewLapiRegisterCmd())
 	cmdLapi.AddCommand(NewLapiStatusCmd())
+	cmdLapi.AddCommand(NewLapiContextCmd())
 
 	return cmdLapi
 }
+
+func NewLapiContextCmd() *cobra.Command {
+	cmdContext := &cobra.Command{
+		Use:               "context [command]",
+		Short:             "Manage context to send with alerts",
+		DisableAutoGenTag: true,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadCrowdsec(); err != nil {
+				fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", csConfig.Crowdsec.ConsoleContextPath)
+				if err.Error() != fileNotFoundMessage {
+					log.Fatalf("Unable to load CrowdSec Agent: %s", err)
+				}
+			}
+			if csConfig.DisableAgent {
+				log.Fatalf("Agent is disabled and lapi context can only be used on the agent")
+			}
+
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			printHelp(cmd)
+		},
+	}
+
+	var keyToAdd string
+	var valuesToAdd []string
+	cmdContextAdd := &cobra.Command{
+		Use:   "add",
+		Short: "Add context to send with alerts. You must specify the output key with the expr value you want",
+		Example: `cscli lapi context add --key source_ip --value evt.Meta.source_ip
+cscli lapi context add --key file_source --value evt.Line.Src
+		`,
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := alertcontext.ValidateContextExpr(keyToAdd, valuesToAdd); err != nil {
+				log.Fatalf("invalid context configuration :%s", err)
+			}
+			if _, ok := csConfig.Crowdsec.ContextToSend[keyToAdd]; !ok {
+				csConfig.Crowdsec.ContextToSend[keyToAdd] = make([]string, 0)
+				log.Infof("key '%s' added", keyToAdd)
+			}
+			data := csConfig.Crowdsec.ContextToSend[keyToAdd]
+			for _, val := range valuesToAdd {
+				if !inSlice(val, data) {
+					log.Infof("value '%s' added to key '%s'", val, keyToAdd)
+					data = append(data, val)
+				}
+				csConfig.Crowdsec.ContextToSend[keyToAdd] = data
+			}
+			if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
+				log.Fatalf(err.Error())
+			}
+		},
+	}
+	cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
+	cmdContextAdd.Flags().StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key")
+	cmdContextAdd.MarkFlagRequired("key")
+	cmdContextAdd.MarkFlagRequired("value")
+	cmdContext.AddCommand(cmdContextAdd)
+
+	cmdContextStatus := &cobra.Command{
+		Use:               "status",
+		Short:             "List context to send with alerts",
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			if len(csConfig.Crowdsec.ContextToSend) == 0 {
+				fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
+				return
+			}
+
+			dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend)
+			if err != nil {
+				log.Fatalf("unable to show context status: %s", err)
+			}
+
+			fmt.Println(string(dump))
+
+		},
+	}
+	cmdContext.AddCommand(cmdContextStatus)
+
+	var detectAll bool
+	cmdContextDetect := &cobra.Command{
+		Use:   "detect",
+		Short: "Detect available fields from the installed parsers",
+		Example: `cscli lapi context detect --all
+cscli lapi context detect crowdsecurity/sshd-logs
+		`,
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+
+			if !detectAll && len(args) == 0 {
+				log.Infof("Please provide parsers to detect or --all flag.")
+				printHelp(cmd)
+			}
+
+			// to avoid all the log.Info from the loaders functions
+			log.SetLevel(log.ErrorLevel)
+
+			err = exprhelpers.Init(nil)
+			if err != nil {
+				log.Fatalf("Failed to init expr helpers : %s", err)
+			}
+
+			// Populate cwhub package tools
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Fatalf("Failed to load hub index : %s", err)
+			}
+
+			csParsers := parser.NewParsers()
+			if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
+				log.Fatalf("unable to load parsers: %s", err)
+			}
+
+			fieldByParsers := make(map[string][]string)
+			for _, node := range csParsers.Nodes {
+				if !detectAll && !inSlice(node.Name, args) {
+					continue
+				}
+				if !detectAll {
+					args = removeFromSlice(node.Name, args)
+				}
+				fieldByParsers[node.Name] = make([]string, 0)
+				fieldByParsers[node.Name] = detectNode(node, *csParsers.Ctx)
+
+				subNodeFields := detectSubNode(node, *csParsers.Ctx)
+				for _, field := range subNodeFields {
+					if !inSlice(field, fieldByParsers[node.Name]) {
+						fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field)
+					}
+				}
+
+			}
+
+			fmt.Printf("Acquisition :\n\n")
+			fmt.Printf("  - evt.Line.Module\n")
+			fmt.Printf("  - evt.Line.Raw\n")
+			fmt.Printf("  - evt.Line.Src\n")
+			fmt.Println()
+
+			parsersKey := make([]string, 0)
+			for k := range fieldByParsers {
+				parsersKey = append(parsersKey, k)
+			}
+			sort.Strings(parsersKey)
+
+			for _, k := range parsersKey {
+				if len(fieldByParsers[k]) == 0 {
+					continue
+				}
+				fmt.Printf("%s :\n\n", k)
+				values := fieldByParsers[k]
+				sort.Strings(values)
+				for _, value := range values {
+					fmt.Printf("  - %s\n", value)
+				}
+				fmt.Println()
+			}
+
+			if len(args) > 0 {
+				for _, parserNotFound := range args {
+					log.Errorf("parser '%s' not found, can't detect fields", parserNotFound)
+				}
+			}
+		},
+	}
+	cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
+	cmdContext.AddCommand(cmdContextDetect)
+
+	var keysToDelete []string
+	var valuesToDelete []string
+	cmdContextDelete := &cobra.Command{
+		Use:   "delete",
+		Short: "Delete context to send with alerts",
+		Example: `cscli lapi context delete --key source_ip
+cscli lapi context delete --value evt.Line.Src
+		`,
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			if len(keysToDelete) == 0 && len(valuesToDelete) == 0 {
+				log.Fatalf("please provide at least a key or a value to delete")
+			}
+
+			for _, key := range keysToDelete {
+				if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok {
+					delete(csConfig.Crowdsec.ContextToSend, key)
+					log.Infof("key '%s' has been removed", key)
+				} else {
+					log.Warningf("key '%s' doesn't exist", key)
+				}
+			}
+
+			for _, value := range valuesToDelete {
+				valueFound := false
+				for key, context := range csConfig.Crowdsec.ContextToSend {
+					if inSlice(value, context) {
+						valueFound = true
+						csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, context)
+						log.Infof("value '%s' has been removed from key '%s'", value, key)
+					}
+					if len(csConfig.Crowdsec.ContextToSend[key]) == 0 {
+						delete(csConfig.Crowdsec.ContextToSend, key)
+					}
+				}
+				if !valueFound {
+					log.Warningf("value '%s' not found", value)
+				}
+			}
+
+			if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
+				log.Fatalf(err.Error())
+			}
+
+		},
+	}
+	cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete")
+	cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete")
+	cmdContext.AddCommand(cmdContextDelete)
+
+	return cmdContext
+}
+
+func detectStaticField(GrokStatics []types.ExtraField) []string {
+	ret := make([]string, 0)
+	for _, static := range GrokStatics {
+		if static.Parsed != "" {
+			fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed)
+			if !inSlice(fieldName, ret) {
+				ret = append(ret, fieldName)
+			}
+		}
+		if static.Meta != "" {
+			fieldName := fmt.Sprintf("evt.Meta.%s", static.Meta)
+			if !inSlice(fieldName, ret) {
+				ret = append(ret, fieldName)
+			}
+		}
+		if static.TargetByName != "" {
+			fieldName := static.TargetByName
+			if !strings.HasPrefix(fieldName, "evt.") {
+				fieldName = "evt." + fieldName
+			}
+			if !inSlice(fieldName, ret) {
+				ret = append(ret, fieldName)
+			}
+		}
+	}
+
+	return ret
+}
+
+func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
+	var ret = make([]string, 0)
+	if node.Grok.RunTimeRegexp != nil {
+		for _, capturedField := range node.Grok.RunTimeRegexp.Names() {
+			fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
+			if !inSlice(fieldName, ret) {
+				ret = append(ret, fieldName)
+			}
+		}
+	}
+
+	if node.Grok.RegexpName != "" {
+		grokCompiled, err := parserCTX.Grok.Get(node.Grok.RegexpName)
+		if err != nil {
+			log.Warningf("Can't get subgrok: %s", err)
+		}
+		for _, capturedField := range grokCompiled.Names() {
+			fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
+			if !inSlice(fieldName, ret) {
+				ret = append(ret, fieldName)
+			}
+		}
+	}
+
+	if len(node.Grok.Statics) > 0 {
+		staticsField := detectStaticField(node.Grok.Statics)
+		for _, staticField := range staticsField {
+			if !inSlice(staticField, ret) {
+				ret = append(ret, staticField)
+			}
+		}
+	}
+
+	if len(node.Statics) > 0 {
+		staticsField := detectStaticField(node.Statics)
+		for _, staticField := range staticsField {
+			if !inSlice(staticField, ret) {
+				ret = append(ret, staticField)
+			}
+		}
+	}
+
+	return ret
+}
+
+func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
+	var ret = make([]string, 0)
+
+	for _, subnode := range node.LeavesNodes {
+		if subnode.Grok.RunTimeRegexp != nil {
+			for _, capturedField := range subnode.Grok.RunTimeRegexp.Names() {
+				fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
+				if !inSlice(fieldName, ret) {
+					ret = append(ret, fieldName)
+				}
+			}
+		}
+		if subnode.Grok.RegexpName != "" {
+			grokCompiled, err := parserCTX.Grok.Get(subnode.Grok.RegexpName)
+			if err != nil {
+				log.Warningf("Can't get subgrok: %s", err)
+			}
+			for _, capturedField := range grokCompiled.Names() {
+				fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
+				if !inSlice(fieldName, ret) {
+					ret = append(ret, fieldName)
+				}
+			}
+		}
+
+		if len(subnode.Grok.Statics) > 0 {
+			staticsField := detectStaticField(subnode.Grok.Statics)
+			for _, staticField := range staticsField {
+				if !inSlice(staticField, ret) {
+					ret = append(ret, staticField)
+				}
+			}
+		}
+
+		if len(subnode.Statics) > 0 {
+			staticsField := detectStaticField(subnode.Statics)
+			for _, staticField := range staticsField {
+				if !inSlice(staticField, ret) {
+					ret = append(ret, staticField)
+				}
+			}
+		}
+	}
+
+	return ret
+}

cmd/crowdsec-cli/utils.go

@@ -748,3 +748,28 @@ func getDBClient() (*database.Client, error) {
 	}
 	return ret, nil
 }
+
+
+func removeFromSlice(val string, slice []string) []string {
+	var i int
+	var value string
+
+	valueFound := false
+
+	// get the index
+	for i, value = range slice {
+		if value == val {
+			valueFound = true
+			break
+		}
+	}
+
+	if valueFound {
+		slice[i] = slice[len(slice)-1]
+		slice[len(slice)-1] = ""
+		slice = slice[:len(slice)-1]
+	}
+
+	return slice
+
+}

cmd/crowdsec/crowdsec.go

@@ -28,7 +28,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
 	}
 
 	// Start loading configs
-	csParsers := newParsers()
+	csParsers := parser.NewParsers()
 	if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
 		return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err)
 	}

cmd/crowdsec/main.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
-	"sort"
 	"strings"
 	"time"
 
@@ -72,45 +71,6 @@ type Flags struct {
 
 type labelsMap map[string]string
 
-// Return new parsers
-// nodes and povfwnodes are already initialized in parser.LoadStages
-func newParsers() *parser.Parsers {
-	parsers := &parser.Parsers{
-		Ctx:             &parser.UnixParserCtx{},
-		Povfwctx:        &parser.UnixParserCtx{},
-		StageFiles:      make([]parser.Stagefile, 0),
-		PovfwStageFiles: make([]parser.Stagefile, 0),
-	}
-	for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} {
-		for _, hubParserItem := range cwhub.GetItemMap(itemType) {
-			if hubParserItem.Installed {
-				stagefile := parser.Stagefile{
-					Filename: hubParserItem.LocalPath,
-					Stage:    hubParserItem.Stage,
-				}
-				if itemType == cwhub.PARSERS {
-					parsers.StageFiles = append(parsers.StageFiles, stagefile)
-				}
-				if itemType == cwhub.PARSERS_OVFLW {
-					parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
-				}
-			}
-		}
-	}
-	if parsers.StageFiles != nil {
-		sort.Slice(parsers.StageFiles, func(i, j int) bool {
-			return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename
-		})
-	}
-	if parsers.PovfwStageFiles != nil {
-		sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool {
-			return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename
-		})
-	}
-
-	return parsers
-}
-
 func LoadBuckets(cConfig *csconfig.Config) error {
 	var (
 		err   error

config/config.yaml

@@ -16,6 +16,7 @@ config_paths:
   notification_dir: /etc/crowdsec/notifications/
   plugin_dir: /usr/local/lib/crowdsec/plugins/
 crowdsec_service:
+  #console_context_path: /etc/crowdsec/console/context.yaml
   acquisition_path: /etc/crowdsec/acquis.yaml
   acquisition_dir: /etc/crowdsec/acquis.d
   parser_routines: 1

config/config_win.yaml

@@ -13,6 +13,7 @@ config_paths:
   plugin_dir: C:\ProgramData\CrowdSec\plugins\
   notification_dir:  C:\ProgramData\CrowdSec\config\notifications\
 crowdsec_service:
+  #console_context_path: C:\ProgramData\CrowdSec\console\context.yaml
   acquisition_path:  C:\ProgramData\CrowdSec\config\acquis.yaml
   parser_routines: 1
 cscli:

config/console.yaml

@@ -1,3 +1,4 @@
 share_manual_decisions: false
 share_custom: true
 share_tainted: true
+share_context: false

debian/rules

@@ -28,7 +28,7 @@ override_dh_auto_install:
 	mkdir -p debian/crowdsec/usr/share/crowdsec
 	mkdir -p debian/crowdsec/etc/crowdsec/hub/
 	mkdir -p debian/crowdsec/usr/share/crowdsec/config
-
+	mkdir -p debian/crowdsec/etc/crowdsec/console/
 
 	mkdir -p debian/crowdsec/usr/lib/crowdsec/plugins/
 	mkdir -p debian/crowdsec/etc/crowdsec/notifications/
@@ -44,6 +44,7 @@ override_dh_auto_install:
 	install -m 600 config/config.yaml debian/crowdsec/etc/crowdsec/config.yaml
 	cp config/simulation.yaml debian/crowdsec/etc/crowdsec/simulation.yaml
 	cp config/profiles.yaml debian/crowdsec/etc/crowdsec/profiles.yaml
+	cp config/context.yaml debian/crowdsec/etc/crowdsec/console/context.yaml
 	cp config/console.yaml debian/crowdsec/etc/crowdsec/console.yaml
 	cp -a config/patterns debian/crowdsec/etc/crowdsec
 

pkg/alertcontext/alertcontext.go

@@ -0,0 +1,157 @@
+package alertcontext
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	"github.com/antonmedv/expr"
+	"github.com/antonmedv/expr/vm"
+	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
+	"github.com/crowdsecurity/crowdsec/pkg/models"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	maxContextValueLen = 4000
+)
+
+var (
+	alertContext = Context{}
+)
+
+type Context struct {
+	ContextToSend         map[string][]string
+	ContextValueLen       int
+	ContextToSendCompiled map[string][]*vm.Program
+	Log                   *log.Logger
+}
+
+func ValidateContextExpr(key string, expressions []string) error {
+	for _, expression := range expressions {
+		_, err := expr.Compile(expression, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
+		if err != nil {
+			return fmt.Errorf("compilation of '%s' failed: %v", expression, err)
+		}
+	}
+	return nil
+}
+
+func NewAlertContext(contextToSend map[string][]string, valueLength int) error {
+	var clog = log.New()
+	if err := types.ConfigureLogger(clog); err != nil {
+		return fmt.Errorf("couldn't create logger for alert context: %s", err)
+	}
+
+	if valueLength == 0 {
+		clog.Debugf("No console context value length provided, using default: %d", maxContextValueLen)
+		valueLength = maxContextValueLen
+	}
+	if valueLength > maxContextValueLen {
+		clog.Debugf("Provided console context value length (%d) is higher than the maximum, using default: %d", valueLength, maxContextValueLen)
+		valueLength = maxContextValueLen
+	}
+
+	alertContext = Context{
+		ContextToSend:         contextToSend,
+		ContextValueLen:       valueLength,
+		Log:                   clog,
+		ContextToSendCompiled: make(map[string][]*vm.Program),
+	}
+
+	for key, values := range contextToSend {
+		alertContext.ContextToSendCompiled[key] = make([]*vm.Program, 0)
+		for _, value := range values {
+			valueCompiled, err := expr.Compile(value, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
+			if err != nil {
+				return fmt.Errorf("compilation of '%s' context value failed: %v", value, err)
+			}
+			alertContext.ContextToSendCompiled[key] = append(alertContext.ContextToSendCompiled[key], valueCompiled)
+		}
+	}
+
+	return nil
+}
+
+func truncate(values []string, contextValueLen int) (string, error) {
+	var ret string
+	valueByte, err := json.Marshal(values)
+	if err != nil {
+		return "", fmt.Errorf("unable to dump metas: %s", err)
+	}
+	ret = string(valueByte)
+	for {
+		if len(ret) <= contextValueLen {
+			break
+		}
+		// if there is only 1 value left and that the size is too big, truncate it
+		if len(values) == 1 {
+			valueToTruncate := values[0]
+			half := len(valueToTruncate) / 2
+			lastValueTruncated := valueToTruncate[:half] + "..."
+			values = values[:len(values)-1]
+			values = append(values, lastValueTruncated)
+		} else {
+			// if there is multiple value inside, just remove the last one
+			values = values[:len(values)-1]
+		}
+		valueByte, err = json.Marshal(values)
+		if err != nil {
+			return "", fmt.Errorf("unable to dump metas: %s", err)
+		}
+		ret = string(valueByte)
+	}
+	return ret, nil
+}
+
+func EventToContext(events []types.Event) (models.Meta, []error) {
+	var errors []error
+
+	metas := make([]*models.MetaItems0, 0)
+	tmpContext := make(map[string][]string)
+	for _, evt := range events {
+		for key, values := range alertContext.ContextToSendCompiled {
+			if _, ok := tmpContext[key]; !ok {
+				tmpContext[key] = make([]string, 0)
+			}
+			for _, value := range values {
+				var val string
+				output, err := expr.Run(value, exprhelpers.GetExprEnv(map[string]interface{}{"evt": evt}))
+				if err != nil {
+					errors = append(errors, fmt.Errorf("failed to get value for %s : %v", key, err))
+					continue
+				}
+				switch out := output.(type) {
+				case string:
+					val = out
+				case int:
+					val = strconv.Itoa(out)
+				default:
+					errors = append(errors, fmt.Errorf("unexpected return type for %s : %T", key, output))
+					continue
+				}
+				if val != "" && !types.InSlice(val, tmpContext[key]) {
+					tmpContext[key] = append(tmpContext[key], val)
+				}
+			}
+		}
+	}
+	for key, values := range tmpContext {
+		if len(values) == 0 {
+			continue
+		}
+		valueStr, err := truncate(values, alertContext.ContextValueLen)
+		if err != nil {
+			log.Warningf(err.Error())
+		}
+		meta := models.MetaItems0{
+			Key:   key,
+			Value: valueStr,
+		}
+		metas = append(metas, &meta)
+	}
+
+	ret := models.Meta(metas)
+	return ret, errors
+}

pkg/alertcontext/alertcontext_test.go

@@ -0,0 +1,201 @@
+package alertcontext
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/crowdsecurity/crowdsec/pkg/models"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewAlertContext(t *testing.T) {
+	tests := []struct {
+		name          string
+		contextToSend map[string][]string
+		valueLength   int
+		expectedErr   error
+	}{
+		{
+			name: "basic config test",
+			contextToSend: map[string][]string{
+				"test": []string{"evt.Parsed.source_ip"},
+			},
+			valueLength: 100,
+			expectedErr: nil,
+		},
+	}
+
+	for _, test := range tests {
+		fmt.Printf("Running test '%s'\n", test.name)
+		err := NewAlertContext(test.contextToSend, test.valueLength)
+		assert.ErrorIs(t, err, test.expectedErr)
+
+	}
+}
+
+func TestEventToContext(t *testing.T) {
+	tests := []struct {
+		name           string
+		contextToSend  map[string][]string
+		valueLength    int
+		events         []types.Event
+		expectedResult models.Meta
+	}{
+		{
+			name: "basic test",
+			contextToSend: map[string][]string{
+				"source_ip":         []string{"evt.Parsed.source_ip"},
+				"nonexistent_field": []string{"evt.Parsed.nonexist"},
+			},
+			valueLength: 100,
+			events: []types.Event{
+				{
+					Parsed: map[string]string{
+						"source_ip":      "1.2.3.4",
+						"source_machine": "mymachine",
+					},
+				},
+			},
+			expectedResult: []*models.MetaItems0{
+				{
+					Key:   "source_ip",
+					Value: "[\"1.2.3.4\"]",
+				},
+			},
+		},
+		{
+			name: "test many events",
+			contextToSend: map[string][]string{
+				"source_ip":      []string{"evt.Parsed.source_ip"},
+				"source_machine": []string{"evt.Parsed.source_machine"},
+				"cve":            []string{"evt.Parsed.cve"},
+			},
+			valueLength: 100,
+			events: []types.Event{
+				{
+					Parsed: map[string]string{
+						"source_ip":      "1.2.3.4",
+						"source_machine": "mymachine",
+						"cve":            "CVE-2022-1234",
+					},
+				},
+				{
+					Parsed: map[string]string{
+						"source_ip":      "1.2.3.4",
+						"source_machine": "mymachine",
+						"cve":            "CVE-2022-1235",
+					},
+				},
+				{
+					Parsed: map[string]string{
+						"source_ip":      "1.2.3.4",
+						"source_machine": "mymachine",
+						"cve":            "CVE-2022-125",
+					},
+				},
+			},
+			expectedResult: []*models.MetaItems0{
+				{
+					Key:   "source_ip",
+					Value: "[\"1.2.3.4\"]",
+				},
+				{
+					Key:   "source_machine",
+					Value: "[\"mymachine\"]",
+				},
+				{
+					Key:   "cve",
+					Value: "[\"CVE-2022-1234\",\"CVE-2022-1235\",\"CVE-2022-125\"]",
+				},
+			},
+		},
+		{
+			name: "test many events with result above max length (need truncate, keep only 2 on 3 elements)",
+			contextToSend: map[string][]string{
+				"source_ip":      []string{"evt.Parsed.source_ip"},
+				"source_machine": []string{"evt.Parsed.source_machine"},
+				"uri":            []string{"evt.Parsed.uri"},
+			},
+			valueLength: 100,
+			events: []types.Event{
+				{
+					Parsed: map[string]string{
+						"source_ip":      "1.2.3.4",
+						"source_machine": "mymachine",
+						"uri":            "/test/test/test/../../../../../../../../",
+					},
+				},
+				{
+					Parsed: map[string]string{
+						"source_ip":      "1.2.3.4",
+						"source_machine": "mymachine",
+						"uri":            "/admin/admin/admin/../../../../../../../../",
+					},
+				},
+				{
+					Parsed: map[string]string{
+						"source_ip":      "1.2.3.4",
+						"source_machine": "mymachine",
+						"uri":            "/login/login/login/../../../../../../../../../../../",
+					},
+				},
+			},
+			expectedResult: []*models.MetaItems0{
+				{
+					Key:   "source_ip",
+					Value: "[\"1.2.3.4\"]",
+				},
+				{
+					Key:   "source_machine",
+					Value: "[\"mymachine\"]",
+				},
+				{
+					Key:   "uri",
+					Value: "[\"/test/test/test/../../../../../../../../\",\"/admin/admin/admin/../../../../../../../../\"]",
+				},
+			},
+		},
+		{
+			name: "test one events with result above max length (need truncate on one element)",
+			contextToSend: map[string][]string{
+				"source_ip":      []string{"evt.Parsed.source_ip"},
+				"source_machine": []string{"evt.Parsed.source_machine"},
+				"uri":            []string{"evt.Parsed.uri"},
+			},
+			valueLength: 100,
+			events: []types.Event{
+				{
+					Parsed: map[string]string{
+						"source_ip":      "1.2.3.4",
+						"source_machine": "mymachine",
+						"uri":            "/test/test/test/../../../../.should_truncate_just_after_this/../../../..../../../../../../../../../../../../../../../end",
+					},
+				},
+			},
+			expectedResult: []*models.MetaItems0{
+				{
+					Key:   "source_machine",
+					Value: "[\"mymachine\"]",
+				},
+				{
+					Key:   "uri",
+					Value: "[\"/test/test/test/../../../../.should_truncate_just_after_this...\"]",
+				},
+				{
+					Key:   "source_ip",
+					Value: "[\"1.2.3.4\"]",
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		fmt.Printf("Running test '%s'\n", test.name)
+		err := NewAlertContext(test.contextToSend, test.valueLength)
+		assert.ErrorIs(t, err, nil)
+
+		metas, _ := EventToContext(test.events)
+		assert.ElementsMatch(t, test.expectedResult, metas)
+	}
+}

pkg/apiserver/apic.go

@@ -85,8 +85,8 @@ func (a *apic) FetchScenariosListFromDB() ([]string, error) {
 	return scenarios, nil
 }
 
-func alertToSignal(alert *models.Alert, scenarioTrust string) *models.AddSignalsRequestItem {
+func alertToSignal(alert *models.Alert, scenarioTrust string, shareContext bool) *models.AddSignalsRequestItem {
-	return &models.AddSignalsRequestItem{
+	signal := &models.AddSignalsRequestItem{
 		Message:         alert.Message,
 		Scenario:        alert.Scenario,
 		ScenarioHash:    alert.ScenarioHash,
@@ -96,8 +96,19 @@ func alertToSignal(alert *models.Alert, scenarioTrust string) *models.AddSignals
 		StopAt:          alert.StopAt,
 		CreatedAt:       alert.CreatedAt,
 		MachineID:       alert.MachineID,
-		ScenarioTrust:   &scenarioTrust,
+		ScenarioTrust:   scenarioTrust,
 	}
+	if shareContext {
+		signal.Context = make([]*models.AddSignalsRequestItemContextItems0, 0)
+		for _, meta := range alert.Meta {
+			contextItem := models.AddSignalsRequestItemContextItems0{
+				Key:   meta.Key,
+				Value: meta.Value,
+			}
+			signal.Context = append(signal.Context, &contextItem)
+		}
+	}
+	return signal
 }
 
 func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, consoleConfig *csconfig.ConsoleConfig) (*apic, error) {
@@ -176,7 +187,7 @@ func (a *apic) Push() error {
 			var signals []*models.AddSignalsRequestItem
 			for _, alert := range alerts {
 				if ok := shouldShareAlert(alert, a.consoleConfig); ok {
-					signals = append(signals, alertToSignal(alert, getScenarioTrustOfAlert(alert)))
+					signals = append(signals, alertToSignal(alert, getScenarioTrustOfAlert(alert), *a.consoleConfig.ShareContext))
 				}
 			}
 			a.mu.Lock()

pkg/apiserver/apic_test.go

@@ -58,6 +58,7 @@ func getAPIC(t *testing.T) *apic {
 			ShareManualDecisions:  types.BoolPtr(false),
 			ShareTaintedScenarios: types.BoolPtr(false),
 			ShareCustomScenarios:  types.BoolPtr(false),
+			ShareContext:          types.BoolPtr(false),
 		},
 	}
 }

pkg/csconfig/api_test.go

@@ -213,6 +213,7 @@ func TestLoadAPIServer(t *testing.T) {
 					ShareManualDecisions:  types.BoolPtr(false),
 					ShareTaintedScenarios: types.BoolPtr(true),
 					ShareCustomScenarios:  types.BoolPtr(true),
+					ShareContext:          types.BoolPtr(false),
 				},
 				LogDir:   LogDirFullPath,
 				LogMedia: "stdout",

pkg/csconfig/console.go

@@ -14,9 +14,10 @@ const (
 	SEND_CUSTOM_SCENARIOS  = "custom"
 	SEND_TAINTED_SCENARIOS = "tainted"
 	SEND_MANUAL_SCENARIOS  = "manual"
+	SEND_CONTEXT           = "context"
 )
 
-var CONSOLE_CONFIGS = []string{SEND_CUSTOM_SCENARIOS, SEND_MANUAL_SCENARIOS, SEND_TAINTED_SCENARIOS}
+var CONSOLE_CONFIGS = []string{SEND_CUSTOM_SCENARIOS, SEND_MANUAL_SCENARIOS, SEND_TAINTED_SCENARIOS, SEND_CONTEXT}
 
 var DefaultConsoleConfigFilePath = DefaultConfigPath("console.yaml")
 
@@ -24,6 +25,7 @@ type ConsoleConfig struct {
 	ShareManualDecisions  *bool `yaml:"share_manual_decisions"`
 	ShareTaintedScenarios *bool `yaml:"share_tainted"`
 	ShareCustomScenarios  *bool `yaml:"share_custom"`
+	ShareContext          *bool `yaml:"share_context"`
 }
 
 func (c *LocalApiServerCfg) LoadConsoleConfig() error {
@@ -33,6 +35,7 @@ func (c *LocalApiServerCfg) LoadConsoleConfig() error {
 		c.ConsoleConfig.ShareCustomScenarios = types.BoolPtr(true)
 		c.ConsoleConfig.ShareTaintedScenarios = types.BoolPtr(true)
 		c.ConsoleConfig.ShareManualDecisions = types.BoolPtr(false)
+		c.ConsoleConfig.ShareContext = types.BoolPtr(false)
 		return nil
 	}
 
@@ -57,6 +60,12 @@ func (c *LocalApiServerCfg) LoadConsoleConfig() error {
 		log.Debugf("no share_manual scenarios found, setting to false")
 		c.ConsoleConfig.ShareManualDecisions = types.BoolPtr(false)
 	}
+
+	if c.ConsoleConfig.ShareContext == nil {
+		log.Debugf("no 'context' found, setting to false")
+		c.ConsoleConfig.ShareContext = types.BoolPtr(false)
+	}
+
 	log.Debugf("Console configuration '%s' loaded successfully", c.ConsoleConfigPath)
 
 	return nil

pkg/csconfig/crowdsec_service.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
@@ -16,7 +17,8 @@ type CrowdsecServiceCfg struct {
 	Enable                    *bool             `yaml:"enable"`
 	AcquisitionFilePath       string            `yaml:"acquisition_path,omitempty"`
 	AcquisitionDirPath        string            `yaml:"acquisition_dir,omitempty"`
-
+	ConsoleContextPath        string            `yaml:"console_context_path"`
+	ConsoleContextValueLength int               `yaml:"console_context_value_length"`
 	AcquisitionFiles          []string          `yaml:"-"`
 	ParserRoutinesCount       int               `yaml:"parser_routines"`
 	BucketsRoutinesCount      int               `yaml:"buckets_routines"`
@@ -32,6 +34,7 @@ type CrowdsecServiceCfg struct {
 	ConfigDir          string              `yaml:"-"`
 	HubIndexFile       string              `yaml:"-"`
 	SimulationFilePath string              `yaml:"-"`
+	ContextToSend      map[string][]string `yaml:"-"`
 }
 
 func (c *Config) LoadCrowdsec() error {
@@ -152,5 +155,50 @@ func (c *Config) LoadCrowdsec() error {
 		return errors.Wrap(err, "while loading hub")
 	}
 
+	c.Crowdsec.ContextToSend = make(map[string][]string, 0)
+	fallback := false
+	if c.Crowdsec.ConsoleContextPath == "" {
+		// fallback to default config file
+		c.Crowdsec.ConsoleContextPath = filepath.Join(c.Crowdsec.ConfigDir, "console", "context.yaml")
+		fallback = true
+	}
+
+	f, err := filepath.Abs(c.Crowdsec.ConsoleContextPath)
+	if err != nil {
+		return fmt.Errorf("fail to get absolute path of %s: %s", c.Crowdsec.ConsoleContextPath, err)
+	}
+
+	c.Crowdsec.ConsoleContextPath = f
+	yamlFile, err := os.ReadFile(c.Crowdsec.ConsoleContextPath)
+	if err != nil {
+		if fallback {
+			log.Debugf("Default context config file doesn't exist, will not use it")
+		} else {
+			return fmt.Errorf("failed to open context file: %s", err)
+		}
+	} else {
+		err = yaml.Unmarshal(yamlFile, c.Crowdsec.ContextToSend)
+		if err != nil {
+			return fmt.Errorf("unmarshaling labels console config file '%s': %s", c.Crowdsec.ConsoleContextPath, err)
+		}
+	}
+
+	return nil
+}
+
+func (c *CrowdsecServiceCfg) DumpContextConfigFile() error {
+	var out []byte
+	var err error
+
+	if out, err = yaml.Marshal(c.ContextToSend); err != nil {
+		return errors.Wrapf(err, "while marshaling ConsoleConfig (for %s)", c.ConsoleContextPath)
+	}
+
+	if err := os.WriteFile(c.ConsoleContextPath, out, 0600); err != nil {
+		return errors.Wrapf(err, "while dumping console config to %s", c.ConsoleContextPath)
+	}
+
+	log.Infof("%s file saved", c.ConsoleContextPath)
+
 	return nil
 }

pkg/csconfig/crowdsec_service_test.go

@@ -33,6 +33,9 @@ func TestLoadCrowdsec(t *testing.T) {
 	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
 	require.NoError(t, err)
 
+	contextFileFullPath, err := filepath.Abs("./tests/context.yaml")
+	require.NoError(t, err)
+
 	tests := []struct {
 		name           string
 		input          *Config
@@ -55,11 +58,14 @@ func TestLoadCrowdsec(t *testing.T) {
 				Crowdsec: &CrowdsecServiceCfg{
 					AcquisitionFilePath:       "./tests/acquis.yaml",
 					SimulationFilePath:        "./tests/simulation.yaml",
+					ConsoleContextPath:        "./tests/context.yaml",
+					ConsoleContextValueLength: 2500,
 				},
 			},
 			expectedResult: &CrowdsecServiceCfg{
 				Enable:                    types.BoolPtr(true),
 				AcquisitionDirPath:        "",
+				ConsoleContextPath:        contextFileFullPath,
 				AcquisitionFilePath:       acquisFullPath,
 				ConfigDir:                 configDirFullPath,
 				DataDir:                   dataFullPath,
@@ -68,8 +74,12 @@ func TestLoadCrowdsec(t *testing.T) {
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,
 				OutputRoutinesCount:       1,
+				ConsoleContextValueLength: 2500,
 				AcquisitionFiles:          []string{acquisFullPath},
 				SimulationFilePath:        "./tests/simulation.yaml",
+				ContextToSend: map[string][]string{
+					"source_ip": {"evt.Parsed.source_ip"},
+				},
 				SimulationConfig: &SimulationConfig{
 					Simulation: &falseBoolPtr,
 				},
@@ -92,12 +102,14 @@ func TestLoadCrowdsec(t *testing.T) {
 					AcquisitionFilePath: "./tests/acquis.yaml",
 					AcquisitionDirPath:  "./tests/acquis/",
 					SimulationFilePath:  "./tests/simulation.yaml",
+					ConsoleContextPath:  "./tests/context.yaml",
 				},
 			},
 			expectedResult: &CrowdsecServiceCfg{
 				Enable:                    types.BoolPtr(true),
 				AcquisitionDirPath:        acquisDirFullPath,
 				AcquisitionFilePath:       acquisFullPath,
+				ConsoleContextPath:        contextFileFullPath,
 				ConfigDir:                 configDirFullPath,
 				HubIndexFile:              hubIndexFileFullPath,
 				DataDir:                   dataFullPath,
@@ -105,7 +117,11 @@ func TestLoadCrowdsec(t *testing.T) {
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,
 				OutputRoutinesCount:       1,
+				ConsoleContextValueLength: 0,
 				AcquisitionFiles:          []string{acquisFullPath, acquisInDirFullPath},
+				ContextToSend: map[string][]string{
+					"source_ip": {"evt.Parsed.source_ip"},
+				},
 				SimulationFilePath: "./tests/simulation.yaml",
 				SimulationConfig: &SimulationConfig{
 					Simulation: &falseBoolPtr,
@@ -125,7 +141,10 @@ func TestLoadCrowdsec(t *testing.T) {
 						CredentialsFilePath: "./tests/lapi-secrets.yaml",
 					},
 				},
-				Crowdsec: &CrowdsecServiceCfg{},
+				Crowdsec: &CrowdsecServiceCfg{
+					ConsoleContextPath:        contextFileFullPath,
+					ConsoleContextValueLength: 10,
+				},
 			},
 			expectedResult: &CrowdsecServiceCfg{
 				Enable:                    types.BoolPtr(true),
@@ -135,11 +154,16 @@ func TestLoadCrowdsec(t *testing.T) {
 				HubIndexFile:              hubIndexFileFullPath,
 				DataDir:                   dataFullPath,
 				HubDir:                    hubFullPath,
+				ConsoleContextPath:        contextFileFullPath,
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,
 				OutputRoutinesCount:       1,
+				ConsoleContextValueLength: 10,
 				AcquisitionFiles:          []string{},
 				SimulationFilePath:        "",
+				ContextToSend: map[string][]string{
+					"source_ip": {"evt.Parsed.source_ip"},
+				},
 				SimulationConfig: &SimulationConfig{
 					Simulation: &falseBoolPtr,
 				},
@@ -159,6 +183,7 @@ func TestLoadCrowdsec(t *testing.T) {
 					},
 				},
 				Crowdsec: &CrowdsecServiceCfg{
+					ConsoleContextPath:  "",
 					AcquisitionFilePath: "./tests/acquis_not_exist.yaml",
 				},
 			},

pkg/csconfig/tests/context.yaml

@@ -0,0 +1,2 @@
+source_ip:
+ - evt.Parsed.source_ip

pkg/leakybucket/manager_load.go

@@ -11,6 +11,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
+
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
@@ -225,6 +227,11 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
 			ret = append(ret, bucketFactory)
 		}
 	}
+
+	if err := alertcontext.NewAlertContext(cscfg.ContextToSend, cscfg.ConsoleContextValueLength); err != nil {
+		return nil, nil, fmt.Errorf("unable to load alert context: %s", err)
+	}
+
 	log.Warningf("Loaded %d scenarios", len(ret))
 	return ret, response, nil
 }
@@ -349,6 +356,7 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error {
 		return fmt.Errorf("invalid bucket from %s : %v", bucketFactory.Filename, err)
 	}
 	bucketFactory.tomb = tomb
+
 	return nil
 
 }

pkg/leakybucket/overflows.go

@@ -6,6 +6,7 @@ import (
 	"sort"
 	"strconv"
 
+	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/davecgh/go-spew/spew"
@@ -17,7 +18,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 )
 
-//SourceFromEvent extracts and formats a valid models.Source object from an Event
+// SourceFromEvent extracts and formats a valid models.Source object from an Event
 func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, error) {
 	srcs := make(map[string]models.Source)
 	/*if it's already an overflow, we have properly formatted sources.
@@ -160,7 +161,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e
 	return srcs, nil
 }
 
-//EventsFromQueue iterates the queue to collect & prepare meta-datas from alert
+// EventsFromQueue iterates the queue to collect & prepare meta-datas from alert
 func EventsFromQueue(queue *Queue) []*models.Event {
 
 	events := []*models.Event{}
@@ -207,7 +208,7 @@ func EventsFromQueue(queue *Queue) []*models.Event {
 	return events
 }
 
-//alertFormatSource iterates over the queue to collect sources
+// alertFormatSource iterates over the queue to collect sources
 func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, string, error) {
 	var sources map[string]models.Source = make(map[string]models.Source)
 	var source_type string
@@ -233,7 +234,7 @@ func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, st
 	return sources, source_type, nil
 }
 
-//NewAlert will generate a RuntimeAlert and its APIAlert(s) from a bucket that overflowed
+// NewAlert will generate a RuntimeAlert and its APIAlert(s) from a bucket that overflowed
 func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) {
 	var runtimeAlert types.RuntimeAlert
 
@@ -293,6 +294,11 @@ func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) {
 	*apiAlert.Message = fmt.Sprintf("%s %s performed '%s' (%d events over %s) at %s", source_scope, sourceStr, leaky.Name, leaky.Total_count, leaky.Ovflw_ts.Sub(leaky.First_ts), leaky.Last_ts)
 	//Get the events from Leaky/Queue
 	apiAlert.Events = EventsFromQueue(queue)
+	var warnings []error
+	apiAlert.Meta, warnings = alertcontext.EventToContext(leaky.Queue.GetQueue())
+	for _, w := range warnings {
+		log.Warningf("while extracting context from bucket %s : %s", leaky.Name, w)
+	}
 
 	//Loop over the Sources and generate appropriate number of ApiAlerts
 	for _, srcValue := range sources {

pkg/models/add_signals_request_item.go

@@ -7,6 +7,7 @@ package models
 
 import (
 	"context"
+	"strconv"
 
 	"github.com/go-openapi/errors"
 	"github.com/go-openapi/strfmt"
@@ -19,6 +20,12 @@ import (
 // swagger:model AddSignalsRequestItem
 type AddSignalsRequestItem struct {
 
+	// alert id
+	AlertID int64 `json:"alert_id,omitempty"`
+
+	// context
+	Context []*AddSignalsRequestItemContextItems0 `json:"context"`
+
 	// created at
 	CreatedAt string `json:"created_at,omitempty"`
 
@@ -38,8 +45,7 @@ type AddSignalsRequestItem struct {
 	ScenarioHash *string `json:"scenario_hash"`
 
 	// scenario trust
-	// Required: true
+	ScenarioTrust string `json:"scenario_trust,omitempty"`
-	ScenarioTrust *string `json:"scenario_trust"`
 
 	// scenario version
 	// Required: true
@@ -62,6 +68,10 @@ type AddSignalsRequestItem struct {
 func (m *AddSignalsRequestItem) Validate(formats strfmt.Registry) error {
 	var res []error
 
+	if err := m.validateContext(formats); err != nil {
+		res = append(res, err)
+	}
+
 	if err := m.validateMessage(formats); err != nil {
 		res = append(res, err)
 	}
@@ -74,10 +84,6 @@ func (m *AddSignalsRequestItem) Validate(formats strfmt.Registry) error {
 		res = append(res, err)
 	}
 
-	if err := m.validateScenarioTrust(formats); err != nil {
-		res = append(res, err)
-	}
-
 	if err := m.validateScenarioVersion(formats); err != nil {
 		res = append(res, err)
 	}
@@ -100,6 +106,32 @@ func (m *AddSignalsRequestItem) Validate(formats strfmt.Registry) error {
 	return nil
 }
 
+func (m *AddSignalsRequestItem) validateContext(formats strfmt.Registry) error {
+	if swag.IsZero(m.Context) { // not required
+		return nil
+	}
+
+	for i := 0; i < len(m.Context); i++ {
+		if swag.IsZero(m.Context[i]) { // not required
+			continue
+		}
+
+		if m.Context[i] != nil {
+			if err := m.Context[i].Validate(formats); err != nil {
+				if ve, ok := err.(*errors.Validation); ok {
+					return ve.ValidateName("context" + "." + strconv.Itoa(i))
+				} else if ce, ok := err.(*errors.CompositeError); ok {
+					return ce.ValidateName("context" + "." + strconv.Itoa(i))
+				}
+				return err
+			}
+		}
+
+	}
+
+	return nil
+}
+
 func (m *AddSignalsRequestItem) validateMessage(formats strfmt.Registry) error {
 
 	if err := validate.Required("message", "body", m.Message); err != nil {
@@ -127,15 +159,6 @@ func (m *AddSignalsRequestItem) validateScenarioHash(formats strfmt.Registry) er
 	return nil
 }
 
-func (m *AddSignalsRequestItem) validateScenarioTrust(formats strfmt.Registry) error {
-
-	if err := validate.Required("scenario_trust", "body", m.ScenarioTrust); err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func (m *AddSignalsRequestItem) validateScenarioVersion(formats strfmt.Registry) error {
 
 	if err := validate.Required("scenario_version", "body", m.ScenarioVersion); err != nil {
@@ -187,6 +210,10 @@ func (m *AddSignalsRequestItem) validateStopAt(formats strfmt.Registry) error {
 func (m *AddSignalsRequestItem) ContextValidate(ctx context.Context, formats strfmt.Registry) error {
 	var res []error
 
+	if err := m.contextValidateContext(ctx, formats); err != nil {
+		res = append(res, err)
+	}
+
 	if err := m.contextValidateSource(ctx, formats); err != nil {
 		res = append(res, err)
 	}
@@ -197,6 +224,26 @@ func (m *AddSignalsRequestItem) ContextValidate(ctx context.Context, formats str
 	return nil
 }
 
+func (m *AddSignalsRequestItem) contextValidateContext(ctx context.Context, formats strfmt.Registry) error {
+
+	for i := 0; i < len(m.Context); i++ {
+
+		if m.Context[i] != nil {
+			if err := m.Context[i].ContextValidate(ctx, formats); err != nil {
+				if ve, ok := err.(*errors.Validation); ok {
+					return ve.ValidateName("context" + "." + strconv.Itoa(i))
+				} else if ce, ok := err.(*errors.CompositeError); ok {
+					return ce.ValidateName("context" + "." + strconv.Itoa(i))
+				}
+				return err
+			}
+		}
+
+	}
+
+	return nil
+}
+
 func (m *AddSignalsRequestItem) contextValidateSource(ctx context.Context, formats strfmt.Registry) error {
 
 	if m.Source != nil {
@@ -230,3 +277,43 @@ func (m *AddSignalsRequestItem) UnmarshalBinary(b []byte) error {
 	*m = res
 	return nil
 }
+
+// AddSignalsRequestItemContextItems0 add signals request item context items0
+//
+// swagger:model AddSignalsRequestItemContextItems0
+type AddSignalsRequestItemContextItems0 struct {
+
+	// key
+	Key string `json:"key,omitempty"`
+
+	// value
+	Value string `json:"value,omitempty"`
+}
+
+// Validate validates this add signals request item context items0
+func (m *AddSignalsRequestItemContextItems0) Validate(formats strfmt.Registry) error {
+	return nil
+}
+
+// ContextValidate validates this add signals request item context items0 based on context it is used
+func (m *AddSignalsRequestItemContextItems0) ContextValidate(ctx context.Context, formats strfmt.Registry) error {
+	return nil
+}
+
+// MarshalBinary interface implementation
+func (m *AddSignalsRequestItemContextItems0) MarshalBinary() ([]byte, error) {
+	if m == nil {
+		return nil, nil
+	}
+	return swag.WriteJSON(m)
+}
+
+// UnmarshalBinary interface implementation
+func (m *AddSignalsRequestItemContextItems0) UnmarshalBinary(b []byte) error {
+	var res AddSignalsRequestItemContextItems0
+	if err := swag.ReadJSON(b, &res); err != nil {
+		return err
+	}
+	*m = res
+	return nil
+}

pkg/parser/unix_parser.go

@@ -4,9 +4,11 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"sort"
 	"strings"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 
 	"github.com/crowdsecurity/grokky"
 	log "github.com/sirupsen/logrus"
@@ -50,6 +52,45 @@ func Init(c map[string]interface{}) (*UnixParserCtx, error) {
 	return &r, nil
 }
 
+// Return new parsers
+// nodes and povfwnodes are already initialized in parser.LoadStages
+func NewParsers() *Parsers {
+	parsers := &Parsers{
+		Ctx:             &UnixParserCtx{},
+		Povfwctx:        &UnixParserCtx{},
+		StageFiles:      make([]Stagefile, 0),
+		PovfwStageFiles: make([]Stagefile, 0),
+	}
+	for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} {
+		for _, hubParserItem := range cwhub.GetItemMap(itemType) {
+			if hubParserItem.Installed {
+				stagefile := Stagefile{
+					Filename: hubParserItem.LocalPath,
+					Stage:    hubParserItem.Stage,
+				}
+				if itemType == cwhub.PARSERS {
+					parsers.StageFiles = append(parsers.StageFiles, stagefile)
+				}
+				if itemType == cwhub.PARSERS_OVFLW {
+					parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
+				}
+			}
+		}
+	}
+	if parsers.StageFiles != nil {
+		sort.Slice(parsers.StageFiles, func(i, j int) bool {
+			return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename
+		})
+	}
+	if parsers.PovfwStageFiles != nil {
+		sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool {
+			return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename
+		})
+	}
+
+	return parsers
+}
+
 func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
 	var err error
 

rpm/SPECS/crowdsec.spec

@@ -45,6 +45,7 @@ sed -i "s#/usr/local/lib/crowdsec/plugins/#%{_libdir}/%{name}/plugins/#g" config
 rm -rf %{buildroot}
 mkdir -p %{buildroot}/etc/crowdsec/hub
 mkdir -p %{buildroot}/etc/crowdsec/patterns
+mkdir -p %{buildroot}/etc/crowdsec/console/
 mkdir -p %{buildroot}%{_sharedstatedir}/%{name}/data
 mkdir -p %{buildroot}%{_presetdir}
 
@@ -62,6 +63,7 @@ install -m 600 -D config/config.yaml %{buildroot}%{_sysconfdir}/crowdsec
 install -m 644 -D config/simulation.yaml %{buildroot}%{_sysconfdir}/crowdsec
 install -m 644 -D config/profiles.yaml %{buildroot}%{_sysconfdir}/crowdsec
 install -m 644 -D config/console.yaml %{buildroot}%{_sysconfdir}/crowdsec
+install -m 644 -D config/context.yaml %{buildroot}%{_sysconfdir}/crowdsec/console/
 install -m 750 -D config/%{name}.cron.daily %{buildroot}%{_sysconfdir}/cron.daily/%{name}
 install -m 644 -D %{SOURCE1} %{buildroot}%{_presetdir}
 
@@ -115,6 +117,7 @@ rm -rf %{buildroot}
 %config(noreplace) %{_sysconfdir}/%{name}/simulation.yaml
 %config(noreplace) %{_sysconfdir}/%{name}/profiles.yaml
 %config(noreplace) %{_sysconfdir}/%{name}/console.yaml
+%config(noreplace) %{_sysconfdir}/%{name}/console/context.yaml
 %config(noreplace) %{_presetdir}/80-%{name}.preset
 %config(noreplace) %{_sysconfdir}/%{name}/notifications/http.yaml
 %config(noreplace) %{_sysconfdir}/%{name}/notifications/slack.yaml

tests/bats/81_alert_context.bats

@@ -0,0 +1,67 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+fake_log() {
+    for _ in $(seq 1 6); do
+        echo "$(LC_ALL=C date '+%b %d %H:%M:%S ')"'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.172 port 35424'
+    done
+}
+
+setup_file() {
+    load "../lib/setup_file.sh"
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh"
+}
+
+setup() {
+    load "../lib/setup.sh"
+    ./instance-data load
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+#----------
+
+@test "$FILE 1.1.1.172 has context" {
+    tmpfile=$(TMPDIR="${BATS_TEST_TMPDIR}" mktemp)
+    touch "${tmpfile}"
+
+    ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
+
+    cat <<-EOT >"${ACQUIS_YAML}"
+	filename: $tmpfile
+	labels:
+	  type: syslog
+	EOT
+
+    CONTEXT_YAML=$(config_get '.crowdsec_service.console_context_path')
+
+    cat <<-EOT >"${CONTEXT_YAML}"
+	target_user:
+	- evt.Parsed.sshd_invalid_user
+	source_ip:
+	- evt.Parsed.sshd_client_ip
+	source_host:
+	- evt.Meta.machine
+	EOT
+
+    ./instance-crowdsec start
+    sleep 2
+    fake_log >>"${tmpfile}"
+    sleep 2
+    rm -f -- "${tmpfile}"
+
+    run -0 cscli alerts list -o json
+    run -0 jq '.[0].id' <(output)
+    ALERT_ID="$output"
+    run -0 cscli alerts inspect "$ALERT_ID" -o json
+    run -0 jq -c '.meta | sort_by(.key) | map([.key,.value])' <(output)
+
+    assert_json '[["source_host","[\"sd-126005\"]"],["source_ip","[\"1.1.1.172\"]"],["target_user","[\"netflix\"]"]]'
+}

tests/lib/config/config-local

@@ -61,6 +61,8 @@ config_generate() {
        ../config/online_api_credentials.yaml \
        "${CONFIG_DIR}/"
 
+    cp ../config/context.yaml "${CONFIG_DIR}/console/"
+
     # the default acquis file contains files that are not readable by everyone
     touch "$LOG_DIR/empty.log"
     cat <<-EOT >"$CONFIG_DIR/acquis.yaml"
@@ -94,6 +96,7 @@ config_generate() {
     .api.client.credentials_path=strenv(CONFIG_DIR)+"/local_api_credentials.yaml" |
     .api.server.profiles_path=strenv(CONFIG_DIR)+"/profiles.yaml" |
     .api.server.console_path=strenv(CONFIG_DIR)+"/console.yaml" |
+    .crowdsec_service.console_context_path=strenv(CONFIG_DIR) + "/console/context.yaml" |
     .api.server.online_client.credentials_path=strenv(CONFIG_DIR)+"/online_api_credentials.yaml"
     ' ../config/config.yaml >"${CONFIG_DIR}/config.yaml"
 }
@@ -107,6 +110,7 @@ make_init_data() {
     mkdir -p "${CONFIG_DIR}/notifications"
     mkdir -p "${CONFIG_DIR}/hub"
     mkdir -p "${CONFIG_DIR}/patterns"
+    mkdir -p "${CONFIG_DIR}/console"
     cp -a "../config/patterns" "${CONFIG_DIR}/"
     config_generate
     # XXX errors from instance-db should be reported...

windows/installer/product.wxs

@@ -54,6 +54,11 @@
                         <PermissionEx Sddl="D:PAI(A;;FA;;;SY)(A;;FA;;;BA)"/>
                      </File>
                   </Component>
+                  <Directory Id="ConsoleDir" Name="console">
+                     <Component Id="ConsoleContextFile" Guid="f146e12a-8f02-4129-9029-577807966e92">
+                        <File Id="context.yaml" Source="config\context.yaml" />
+                     </Component>
+                  </Directory>
                   <Component Id="OnlineCreds" Guid="a652a6cb-d464-40b1-8f50-78dce0135d20">
                      <File Id="online_api_credentials.yaml" Source="config\online_api_credentials.yaml">
                         <PermissionEx Sddl="D:PAI(A;;FA;;;SY)(A;;FA;;;BA)"/>
@@ -163,6 +168,8 @@
          <ComponentRef Id="NotifConfig" />
          <ComponentRef Id="CreateCrowdsecPluginsDir"/>
          <ComponentRef Id="CreateCrowdsecDataDir" />
+         <ComponentRef Id="ConsoleDir"/>
+         <ComponentRef Id="ConsoleContextFile"/>
          <ComponentRef Id="Csconfig_lapi" />
          <ComponentRef Id="Csconfig_no_lapi" />
          <ComponentGroupRef Id="CrowdsecPatterns" />

wizard.sh

@@ -28,6 +28,7 @@ CROWDSEC_CONFIG_PATH="${CROWDSEC_PATH}"
 CROWDSEC_LOG_FILE="/var/log/crowdsec.log"
 LAPI_LOG_FILE="/var/log/crowdsec_api.log"
 CROWDSEC_PLUGIN_DIR="${CROWDSEC_USR_DIR}/plugins"
+CROWDSEC_CONSOLE_DIR="${CROWDSEC_PATH}/console"
 
 CROWDSEC_BIN="./cmd/crowdsec/crowdsec"
 CSCLI_BIN="./cmd/crowdsec-cli/cscli"
@@ -403,6 +404,7 @@ install_crowdsec() {
     mkdir -p "${CROWDSEC_CONFIG_PATH}/postoverflows" || exit
     mkdir -p "${CROWDSEC_CONFIG_PATH}/collections" || exit
     mkdir -p "${CROWDSEC_CONFIG_PATH}/patterns" || exit
+    mkdir -p "${CROWDSEC_CONSOLE_DIR}" || exit
 
     #tmp
     mkdir -p /tmp/data
@@ -419,6 +421,7 @@ install_crowdsec() {
     install -v -m 644 -D ./config/profiles.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
     install -v -m 644 -D ./config/simulation.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
     install -v -m 644 -D ./config/"${CONSOLE_FILE}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
+    install -v -m 644 -D ./config/context.yaml "${CROWDSEC_CONSOLE_DIR}" 1> /dev/null || exit
 
     DATA=${CROWDSEC_DATA_DIR} CFG=${CROWDSEC_CONFIG_PATH} envsubst '$CFG $DATA' < ./config/user.yaml > ${CROWDSEC_CONFIG_PATH}"/user.yaml" || log_fatal "unable to generate user configuration file"
     if [[ ${DOCKER_MODE} == "false" ]]; then