cwhub: context type (#2631)

* add hub type "context"
* cscli lapi: log.Fatal -> fmt.Errorf; lint
* tests for context.yaml
* load console context from hub
* original & compiled context
* deprecate "cscli lapi context delete"
$ cscli lapi context delete
Command "delete" is deprecated, please manually edit the context file.
* cscli completion: add appsec-rules, appsec-configs, explain, hubtest

cmd/crowdsec-cli/hubcontext.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewContextCLI() *itemCLI {
+	return &itemCLI{
+		name:      cwhub.CONTEXTS,
+		singular:  "context",
+		oneOrMore: "context(s)",
+		help: cliHelp{
+			example: `cscli contexts list -a
+cscli contexts install crowdsecurity/yyy crowdsecurity/zzz
+cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz
+cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz
+cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz
+`,
+		},
+		installHelp: cliHelp{
+			example: `cscli contexts install crowdsecurity/yyy crowdsecurity/zzz`,
+		},
+		removeHelp: cliHelp{
+			example: `cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz`,
+		},
+		upgradeHelp: cliHelp{
+			example: `cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz`,
+		},
+		inspectHelp: cliHelp{
+			example: `cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz`,
+		},
+		listHelp: cliHelp{
+			example: `cscli contexts list
+cscli contexts list -a
+cscli contexts list crowdsecurity/yyy crowdsecurity/zzz
+
+List only enabled contexts unless "-a" or names are specified.`,
+		},
+	}
+}

cmd/crowdsec-cli/lapi.go

@@ -2,10 +2,10 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/url"
 	"os"
-	"slices"
 	"sort"
 	"strings"
 
@@ -13,6 +13,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
+	"slices"
 
 	"github.com/crowdsecurity/go-cs-lib/version"
 
@@ -26,26 +27,24 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
 )
 
-var LAPIURLPrefix string = "v1"
+const LAPIURLPrefix = "v1"
 
 func runLapiStatus(cmd *cobra.Command, args []string) error {
-	var err error
-
 	password := strfmt.Password(csConfig.API.Client.Credentials.Password)
 	apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
 	login := csConfig.API.Client.Credentials.Login
 	if err != nil {
-		log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
+		return fmt.Errorf("parsing api url: %w", err)
 	}
 
 	hub, err := require.Hub(csConfig, nil)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 
 	scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 	if err != nil {
-		log.Fatalf("failed to get scenarios : %s", err)
+		return fmt.Errorf("failed to get scenarios: %w", err)
 	}
 
 	Client, err = apiclient.NewDefaultClient(apiurl,
@@ -53,28 +52,27 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
 		fmt.Sprintf("crowdsec/%s", version.String()),
 		nil)
 	if err != nil {
-		log.Fatalf("init default client: %s", err)
+		return fmt.Errorf("init default client: %w", err)
 	}
 	t := models.WatcherAuthRequest{
 		MachineID: &login,
 		Password:  &password,
 		Scenarios: scenarios,
 	}
+
 	log.Infof("Loaded credentials from %s", csConfig.API.Client.CredentialsFilePath)
 	log.Infof("Trying to authenticate with username %s on %s", login, apiurl)
+
 	_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
 	if err != nil {
-		log.Fatalf("Failed to authenticate to Local API (LAPI) : %s", err)
+		return fmt.Errorf("failed to authenticate to Local API (LAPI): %w", err)
-	} else {
-		log.Infof("You can successfully interact with Local API (LAPI)")
 	}
 
+	log.Infof("You can successfully interact with Local API (LAPI)")
 	return nil
 }
 
 func runLapiRegister(cmd *cobra.Command, args []string) error {
-	var err error
-
 	flags := cmd.Flags()
 
 	apiURL, err := flags.GetString("url")
@@ -95,16 +93,15 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
 	if lapiUser == "" {
 		lapiUser, err = generateID("")
 		if err != nil {
-			log.Fatalf("unable to generate machine id: %s", err)
+			return fmt.Errorf("unable to generate machine id: %w", err)
 		}
 	}
 	password := strfmt.Password(generatePassword(passwordLength))
 	if apiURL == "" {
-		if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {
+		if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil || csConfig.API.Client.Credentials.URL == "" {
-			apiURL = csConfig.API.Client.Credentials.URL
+			return fmt.Errorf("no Local API URL. Please provide it in your configuration or with the -u parameter")
-		} else {
-			log.Fatalf("No Local API URL. Please provide it in your configuration or with the -u parameter")
 		}
+		apiURL = csConfig.API.Client.Credentials.URL
 	}
 	/*URL needs to end with /, but user doesn't care*/
 	if !strings.HasSuffix(apiURL, "/") {
@@ -116,7 +113,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
 	}
 	apiurl, err := url.Parse(apiURL)
 	if err != nil {
-		log.Fatalf("parsing api url: %s", err)
+		return fmt.Errorf("parsing api url: %w", err)
 	}
 	_, err = apiclient.RegisterClient(&apiclient.Config{
 		MachineID:     lapiUser,
@@ -127,7 +124,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
 	}, nil)
 
 	if err != nil {
-		log.Fatalf("api client register: %s", err)
+		return fmt.Errorf("api client register: %w", err)
 	}
 
 	log.Printf("Successfully registered to Local API (LAPI)")
@@ -147,12 +144,12 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
 	}
 	apiConfigDump, err := yaml.Marshal(apiCfg)
 	if err != nil {
-		log.Fatalf("unable to marshal api credentials: %s", err)
+		return fmt.Errorf("unable to marshal api credentials: %w", err)
 	}
 	if dumpFile != "" {
 		err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
 		if err != nil {
-			log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
+			return fmt.Errorf("write api credentials to '%s' failed: %w", dumpFile, err)
 		}
 		log.Printf("Local API credentials written to '%s'", dumpFile)
 	} else {
@@ -195,7 +192,7 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side
 }
 
 func NewLapiCmd() *cobra.Command {
-	var cmdLapi = &cobra.Command{
+	cmdLapi := &cobra.Command{
 		Use:               "lapi [action]",
 		Short:             "Manage interaction with Local API (LAPI)",
 		Args:              cobra.MinimumNArgs(1),
@@ -221,6 +218,7 @@ func AddContext(key string, values []string) error {
 	}
 	if _, ok := csConfig.Crowdsec.ContextToSend[key]; !ok {
 		csConfig.Crowdsec.ContextToSend[key] = make([]string, 0)
+
 		log.Infof("key '%s' added", key)
 	}
 	data := csConfig.Crowdsec.ContextToSend[key]
@@ -247,11 +245,11 @@ func NewLapiContextCmd() *cobra.Command {
 			if err := csConfig.LoadCrowdsec(); err != nil {
 				fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", csConfig.Crowdsec.ConsoleContextPath)
 				if err.Error() != fileNotFoundMessage {
-					log.Fatalf("Unable to load CrowdSec Agent: %s", err)
+					return fmt.Errorf("unable to start CrowdSec agent: %w", err)
 				}
 			}
 			if csConfig.DisableAgent {
-				log.Fatalf("Agent is disabled and lapi context can only be used on the agent")
+				return errors.New("agent is disabled and lapi context can only be used on the agent")
 			}
 
 			return nil
@@ -271,12 +269,21 @@ cscli lapi context add --key file_source --value evt.Line.Src
 cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user 
 		`,
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
+			hub, err := require.Hub(csConfig, nil)
+			if err != nil {
+				return err
+			}
+
+			if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
+				return fmt.Errorf("while loading context: %w", err)
+			}
+
 			if keyToAdd != "" {
 				if err := AddContext(keyToAdd, valuesToAdd); err != nil {
-					log.Fatalf(err.Error())
+					return err
 				}
-				return
+				return nil
 			}
 
 			for _, v := range valuesToAdd {
@@ -284,9 +291,11 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
 				key := keySlice[len(keySlice)-1]
 				value := []string{v}
 				if err := AddContext(key, value); err != nil {
-					log.Fatalf(err.Error())
+					return err
 				}
 			}
+
+			return nil
 		},
 	}
 	cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
@@ -298,19 +307,29 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
 		Use:               "status",
 		Short:             "List context to send with alerts",
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
+			hub, err := require.Hub(csConfig, nil)
+			if err != nil {
+				return err
+			}
+
+			if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
+				return fmt.Errorf("while loading context: %w", err)
+			}
+
 			if len(csConfig.Crowdsec.ContextToSend) == 0 {
 				fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
-				return
+				return nil
 			}
 
 			dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend)
 			if err != nil {
-				log.Fatalf("unable to show context status: %s", err)
+				return fmt.Errorf("unable to show context status: %w", err)
 			}
 
-			fmt.Println(string(dump))
+			fmt.Print(string(dump))
 
+			return nil
 		},
 	}
 	cmdContext.AddCommand(cmdContextStatus)
@@ -323,9 +342,7 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
 cscli lapi context detect crowdsecurity/sshd-logs
 		`,
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
-			var err error
-
 			if !detectAll && len(args) == 0 {
 				log.Infof("Please provide parsers to detect or --all flag.")
 				printHelp(cmd)
@@ -334,19 +351,18 @@ cscli lapi context detect crowdsecurity/sshd-logs
 			// to avoid all the log.Info from the loaders functions
 			log.SetLevel(log.WarnLevel)
 
-			err = exprhelpers.Init(nil)
+			if err := exprhelpers.Init(nil); err != nil {
-			if err != nil {
+				return fmt.Errorf("failed to init expr helpers: %w", err)
-				log.Fatalf("Failed to init expr helpers : %s", err)
 			}
 
 			hub, err := require.Hub(csConfig, nil)
 			if err != nil {
-				log.Fatal(err)
+				return err
 			}
 
 			csParsers := parser.NewParsers(hub)
 			if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
-				log.Fatalf("unable to load parsers: %s", err)
+				return fmt.Errorf("unable to load parsers: %w", err)
 			}
 
 			fieldByParsers := make(map[string][]string)
@@ -366,7 +382,6 @@ cscli lapi context detect crowdsecurity/sshd-logs
 						fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field)
 					}
 				}
-
 			}
 
 			fmt.Printf("Acquisition :\n\n")
@@ -399,59 +414,17 @@ cscli lapi context detect crowdsecurity/sshd-logs
 					log.Errorf("parser '%s' not found, can't detect fields", parserNotFound)
 				}
 			}
+
+			return nil
 		},
 	}
 	cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
 	cmdContext.AddCommand(cmdContextDetect)
 
-	var keysToDelete []string
-	var valuesToDelete []string
 	cmdContextDelete := &cobra.Command{
 		Use:   "delete",
-		Short: "Delete context to send with alerts",
+		Deprecated: "please manually edit the context file.",
-		Example: `cscli lapi context delete --key source_ip
-cscli lapi context delete --value evt.Line.Src
-		`,
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			if len(keysToDelete) == 0 && len(valuesToDelete) == 0 {
-				log.Fatalf("please provide at least a key or a value to delete")
 	}
-
-			for _, key := range keysToDelete {
-				if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok {
-					delete(csConfig.Crowdsec.ContextToSend, key)
-					log.Infof("key '%s' has been removed", key)
-				} else {
-					log.Warningf("key '%s' doesn't exist", key)
-				}
-			}
-
-			for _, value := range valuesToDelete {
-				valueFound := false
-				for key, context := range csConfig.Crowdsec.ContextToSend {
-					if slices.Contains(context, value) {
-						valueFound = true
-						csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, context)
-						log.Infof("value '%s' has been removed from key '%s'", value, key)
-					}
-					if len(csConfig.Crowdsec.ContextToSend[key]) == 0 {
-						delete(csConfig.Crowdsec.ContextToSend, key)
-					}
-				}
-				if !valueFound {
-					log.Warningf("value '%s' not found", value)
-				}
-			}
-
-			if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
-				log.Fatalf(err.Error())
-			}
-
-		},
-	}
-	cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete")
-	cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete")
 	cmdContext.AddCommand(cmdContextDelete)
 
 	return cmdContext
@@ -459,6 +432,7 @@ cscli lapi context delete --value evt.Line.Src
 
 func detectStaticField(GrokStatics []parser.ExtraField) []string {
 	ret := make([]string, 0)
+
 	for _, static := range GrokStatics {
 		if static.Parsed != "" {
 			fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed)
@@ -487,7 +461,8 @@ func detectStaticField(GrokStatics []parser.ExtraField) []string {
 }
 
 func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
-	var ret = make([]string, 0)
+	ret := make([]string, 0)
+
 	if node.Grok.RunTimeRegexp != nil {
 		for _, capturedField := range node.Grok.RunTimeRegexp.Names() {
 			fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)

cmd/crowdsec-cli/main.go

@@ -93,9 +93,10 @@ func initConfig() {
 }
 
 var validArgs = []string{
-	"scenarios", "parsers", "collections", "capi", "lapi", "postoverflows", "machines",
+	"alerts", "appsec-configs", "appsec-rules", "bouncers", "capi", "collections",
-	"metrics", "bouncers", "alerts", "decisions", "simulation", "hub", "dashboard",
+	"completion", "config", "console", "contexts", "dashboard", "decisions", "explain",
-	"config", "completion", "version", "console", "notifications", "support",
+	"hub", "hubtest", "lapi", "machines", "metrics", "notifications", "parsers",
+	"postoverflows", "scenarios", "simulation", "support", "version",
 }
 
 func prepender(filename string) string {
@@ -246,6 +247,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	rootCmd.AddCommand(NewParserCLI().NewCommand())
 	rootCmd.AddCommand(NewScenarioCLI().NewCommand())
 	rootCmd.AddCommand(NewPostOverflowCLI().NewCommand())
+	rootCmd.AddCommand(NewContextCLI().NewCommand())
 	rootCmd.AddCommand(NewAppsecConfigCLI().NewCommand())
 	rootCmd.AddCommand(NewAppsecRuleCLI().NewCommand())
 

cmd/crowdsec-cli/support.go

@@ -37,6 +37,7 @@ const (
 	SUPPORT_OS_INFO_PATH                 = "osinfo.txt"
 	SUPPORT_PARSERS_PATH                 = "hub/parsers.txt"
 	SUPPORT_SCENARIOS_PATH               = "hub/scenarios.txt"
+	SUPPORT_CONTEXTS_PATH                = "hub/scenarios.txt"
 	SUPPORT_COLLECTIONS_PATH             = "hub/collections.txt"
 	SUPPORT_POSTOVERFLOWS_PATH           = "hub/postoverflows.txt"
 	SUPPORT_BOUNCERS_PATH                = "lapi/bouncers.txt"
@@ -272,6 +273,7 @@ func (cli cliSupport) NewDumpCmd() *cobra.Command {
 - Installed parsers list
 - Installed scenarios list
 - Installed postoverflows list
+- Installed context list
 - Bouncers list
 - Machines list
 - CAPI status
@@ -321,6 +323,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
 				infos[SUPPORT_SCENARIOS_PATH] = []byte(err.Error())
 				infos[SUPPORT_POSTOVERFLOWS_PATH] = []byte(err.Error())
+				infos[SUPPORT_CONTEXTS_PATH] = []byte(err.Error())
 				infos[SUPPORT_COLLECTIONS_PATH] = []byte(err.Error())
 			}
 
@@ -356,6 +359,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
 				infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
 				infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
+				infos[SUPPORT_CONTEXTS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
 				infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
 			}
 

cmd/crowdsec/crowdsec.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
 	"github.com/crowdsecurity/crowdsec/pkg/appsec"
+	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@@ -24,6 +25,10 @@ import (
 func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
 	var err error
 
+	if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil {
+		return nil, fmt.Errorf("while loading context: %w", err)
+	}
+
 	// Start loading configs
 	csParsers := parser.NewParsers(hub)
 	if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
@@ -41,6 +46,7 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, er
 	if err := LoadAcquisition(cConfig); err != nil {
 		return nil, fmt.Errorf("while loading acquisition config: %w", err)
 	}
+
 	return csParsers, nil
 }
 

docker/README.md

@@ -320,10 +320,12 @@ config.yaml) each time the container is run.
 | `PARSERS`               | | Parsers to install, separated by space |
 | `SCENARIOS`             | | Scenarios to install, separated by space |
 | `POSTOVERFLOWS`         | | Postoverflows to install, separated by space |
+| `CONTEXTS`              | | Context files to install, separated by space |
 | `DISABLE_COLLECTIONS`   | | Collections to remove, separated by space: `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` |
 | `DISABLE_PARSERS`       | | Parsers to remove, separated by space |
 | `DISABLE_SCENARIOS`     | | Scenarios to remove, separated by space |
 | `DISABLE_POSTOVERFLOWS` | | Postoverflows to remove, separated by space |
+| `DISABLE_POSTOVERFLOWS` | | Context files to remove, separated by space |
 |                         | | |
 | __Log verbosity__       | | |
 | `LEVEL_INFO`            | false | Force INFO level for the container log |

docker/docker_start.sh

@@ -300,7 +300,7 @@ fi
 
 conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)'
 
-## Install collections, parsers, scenarios & postoverflows
+## Install hub items
 cscli hub update
 
 cscli_if_clean collections upgrade crowdsecurity/linux
@@ -328,6 +328,11 @@ if [ "$POSTOVERFLOWS" != "" ]; then
     cscli_if_clean postoverflows install "$(difference "$POSTOVERFLOWS" "$DISABLE_POSTOVERFLOWS")"
 fi
 
+if [ "$CONTEXTS" != "" ]; then
+    # shellcheck disable=SC2086
+    cscli_if_clean contexts install "$(difference "$CONTEXTS" "$DISABLE_CONTEXTS")"
+fi
+
 ## Remove collections, parsers, scenarios & postoverflows
 if [ "$DISABLE_COLLECTIONS" != "" ]; then
     # shellcheck disable=SC2086
@@ -349,6 +354,11 @@ if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
     cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force
 fi
 
+if [ "$DISABLE_CONTEXTS" != "" ]; then
+    # shellcheck disable=SC2086
+    cscli_if_clean contexts remove "$DISABLE_CONTEXTS" --force
+fi
+
 ## Register bouncers via env
 for BOUNCER in $(compgen -A variable | grep -i BOUNCER_KEY); do
     KEY=$(printf '%s' "${!BOUNCER}")

pkg/alertcontext/alertcontext.go

@@ -63,13 +63,21 @@ func NewAlertContext(contextToSend map[string][]string, valueLength int) error {
 	}
 
 	for key, values := range contextToSend {
+		if _, ok := alertContext.ContextToSend[key]; !ok {
+			alertContext.ContextToSend[key] = make([]string, 0)
+		}
+
+		if _, ok := alertContext.ContextToSendCompiled[key]; !ok {
 			alertContext.ContextToSendCompiled[key] = make([]*vm.Program, 0)
+		}
+
 		for _, value := range values {
 			valueCompiled, err := expr.Compile(value, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
 			if err != nil {
 				return fmt.Errorf("compilation of '%s' context value failed: %v", value, err)
 			}
 			alertContext.ContextToSendCompiled[key] = append(alertContext.ContextToSendCompiled[key], valueCompiled)
+			alertContext.ContextToSend[key] = append(alertContext.ContextToSend[key], value)
 		}
 	}
 

pkg/alertcontext/config.go

@@ -0,0 +1,125 @@
+package alertcontext
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v3"
+
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+// this file is here to avoid circular dependencies between the configuration and the hub
+
+// HubItemWrapper is a wrapper around a hub item to unmarshal only the context part
+// because there are other fields like name etc.
+type HubItemWrapper struct {
+	Context map[string][]string `yaml:"context"`
+}
+
+// mergeContext adds the context from src to dest.
+func mergeContext(dest map[string][]string, src map[string][]string) {
+	for k, v := range src {
+		if _, ok := dest[k]; !ok {
+			dest[k] = make([]string, 0)
+		}
+		for _, s := range v {
+			if !slices.Contains(dest[k], s) {
+				dest[k] = append(dest[k], s)
+			}
+		}
+	}
+}
+
+// addContextFromItem merges the context from an item into the context to send to the console.
+func addContextFromItem(toSend map[string][]string, item *cwhub.Item) error {
+	filePath := item.State.LocalPath
+	log.Tracef("loading console context from %s", filePath)
+	content, err := os.ReadFile(filePath)
+	if err != nil {
+		return err
+	}
+
+	wrapper := &HubItemWrapper{}
+
+	err = yaml.Unmarshal(content, wrapper)
+	if err != nil {
+		return fmt.Errorf("%s: %w", filePath, err)
+	}
+
+	mergeContext(toSend, wrapper.Context)
+
+	return nil
+}
+
+// addContextFromFile merges the context from a file into the context to send to the console.
+func addContextFromFile(toSend map[string][]string, filePath string) error {
+	log.Tracef("loading console context from %s", filePath)
+	content, err := os.ReadFile(filePath)
+	if err != nil {
+		return err
+	}
+
+	newContext := make(map[string][]string, 0)
+
+	err = yaml.Unmarshal(content, newContext)
+	if err != nil {
+		return fmt.Errorf("%s: %w", filePath, err)
+	}
+
+	mergeContext(toSend, newContext)
+
+	return nil
+}
+
+
+// LoadConsoleContext loads the context from the hub (if provided) and the file console_context_path.
+func LoadConsoleContext(c *csconfig.Config, hub *cwhub.Hub) error {
+	c.Crowdsec.ContextToSend = make(map[string][]string, 0)
+
+	if hub != nil {
+		items, err := hub.GetInstalledItems(cwhub.CONTEXTS)
+		if err != nil {
+			return err
+		}
+
+		for _, item := range items {
+			// context in item files goes under the key 'context'
+			if err = addContextFromItem(c.Crowdsec.ContextToSend, item); err != nil {
+				return err
+			}
+		}
+	}
+
+	ignoreMissing := false
+
+	if c.Crowdsec.ConsoleContextPath != "" {
+		// if it's provided, it must exist
+		if _, err := os.Stat(c.Crowdsec.ConsoleContextPath); err != nil {
+			return fmt.Errorf("while checking console_context_path: %w", err)
+		}
+	} else {
+		c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
+		ignoreMissing = true
+	}
+
+	if err := addContextFromFile(c.Crowdsec.ContextToSend, c.Crowdsec.ConsoleContextPath); err != nil {
+		if !ignoreMissing || !os.IsNotExist(err) {
+			return err
+		}
+	}
+
+	feedback, err := json.Marshal(c.Crowdsec.ContextToSend)
+	if err != nil {
+		return fmt.Errorf("marshaling console context: %s", err)
+	}
+
+	log.Debugf("console context to send: %s", feedback)
+
+	return nil
+}

pkg/csconfig/crowdsec_service.go

@@ -108,8 +108,9 @@ func (c *Config) LoadCrowdsec() error {
 		c.Crowdsec.OutputRoutinesCount = 1
 	}
 
-	var crowdsecCleanup = []*string{
+	crowdsecCleanup := []*string{
 		&c.Crowdsec.AcquisitionFilePath,
+		&c.Crowdsec.ConsoleContextPath,
 	}
 
 	for _, k := range crowdsecCleanup {
@@ -131,38 +132,10 @@ func (c *Config) LoadCrowdsec() error {
 		c.Crowdsec.AcquisitionFiles[i] = f
 	}
 
-	if err := c.LoadAPIClient(); err != nil {
+	if err = c.LoadAPIClient(); err != nil {
 		return fmt.Errorf("loading api client: %s", err)
 	}
 
-	c.Crowdsec.ContextToSend = make(map[string][]string, 0)
-	fallback := false
-	if c.Crowdsec.ConsoleContextPath == "" {
-		// fallback to default config file
-		c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
-		fallback = true
-	}
-
-	f, err := filepath.Abs(c.Crowdsec.ConsoleContextPath)
-	if err != nil {
-		return fmt.Errorf("fail to get absolute path of %s: %s", c.Crowdsec.ConsoleContextPath, err)
-	}
-
-	c.Crowdsec.ConsoleContextPath = f
-	yamlFile, err := os.ReadFile(c.Crowdsec.ConsoleContextPath)
-	if err != nil {
-		if fallback {
-			log.Debugf("Default context config file doesn't exist, will not use it")
-		} else {
-			return fmt.Errorf("failed to open context file: %s", err)
-		}
-	} else {
-		err = yaml.Unmarshal(yamlFile, c.Crowdsec.ContextToSend)
-		if err != nil {
-			return fmt.Errorf("unmarshaling labels console config file '%s': %s", c.Crowdsec.ConsoleContextPath, err)
-		}
-	}
-
 	return nil
 }
 
@@ -170,10 +143,16 @@ func (c *CrowdsecServiceCfg) DumpContextConfigFile() error {
 	var out []byte
 	var err error
 
+	// XXX: MakeDirs
+
 	if out, err = yaml.Marshal(c.ContextToSend); err != nil {
 		return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleContextPath, err)
 	}
 
+	if err = os.MkdirAll(filepath.Dir(c.ConsoleContextPath), 0700); err != nil {
+		return fmt.Errorf("while creating directories for %s: %w", c.ConsoleContextPath, err)
+	}
+
 	if err := os.WriteFile(c.ConsoleContextPath, out, 0600); err != nil {
 		return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleContextPath, err)
 	}

pkg/csconfig/crowdsec_service_test.go

@@ -60,9 +60,10 @@ func TestLoadCrowdsec(t *testing.T) {
 				ConsoleContextValueLength: 2500,
 				AcquisitionFiles:          []string{acquisFullPath},
 				SimulationFilePath:        "./testdata/simulation.yaml",
-				ContextToSend: map[string][]string{
+				// context is loaded in pkg/alertcontext
-					"source_ip": {"evt.Parsed.source_ip"},
+//				ContextToSend: map[string][]string{
-				},
+//					"source_ip": {"evt.Parsed.source_ip"},
+//				},
 				SimulationConfig: &SimulationConfig{
 					Simulation: ptr.Of(false),
 				},
@@ -98,9 +99,10 @@ func TestLoadCrowdsec(t *testing.T) {
 				OutputRoutinesCount:       1,
 				ConsoleContextValueLength: 0,
 				AcquisitionFiles:          []string{acquisFullPath, acquisInDirFullPath},
-				ContextToSend: map[string][]string{
+				// context is loaded in pkg/alertcontext
-					"source_ip": {"evt.Parsed.source_ip"},
+//				ContextToSend: map[string][]string{
-				},
+//					"source_ip": {"evt.Parsed.source_ip"},
+//				},
 				SimulationFilePath: "./testdata/simulation.yaml",
 				SimulationConfig: &SimulationConfig{
 					Simulation: ptr.Of(false),
@@ -136,9 +138,10 @@ func TestLoadCrowdsec(t *testing.T) {
 				ConsoleContextValueLength: 10,
 				AcquisitionFiles:          []string{},
 				SimulationFilePath:        "",
-				ContextToSend: map[string][]string{
+				// context is loaded in pkg/alertcontext
-					"source_ip": {"evt.Parsed.source_ip"},
+//				ContextToSend: map[string][]string{
-				},
+//					"source_ip": {"evt.Parsed.source_ip"},
+//				},
 				SimulationConfig: &SimulationConfig{
 					Simulation: ptr.Of(false),
 				},

pkg/cwhub/item.go

@@ -16,6 +16,7 @@ const (
 	PARSERS        = "parsers"
 	POSTOVERFLOWS  = "postoverflows"
 	SCENARIOS      = "scenarios"
+	CONTEXTS       = "contexts"
 	APPSEC_CONFIGS = "appsec-configs"
 	APPSEC_RULES   = "appsec-rules"
 )
@@ -29,7 +30,7 @@ const (
 
 var (
 	// The order is important, as it is used to range over sub-items in collections.
-	ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, APPSEC_CONFIGS, APPSEC_RULES, COLLECTIONS}
+	ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, CONTEXTS, APPSEC_CONFIGS, APPSEC_RULES, COLLECTIONS}
 )
 
 type HubItems map[string]map[string]*Item
@@ -120,6 +121,7 @@ type Item struct {
 	PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
 	Scenarios     []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"`
 	Collections   []string `json:"collections,omitempty" yaml:"collections,omitempty"`
+	Contexts      []string `json:"contexts,omitempty" yaml:"contexts,omitempty"`
 	AppsecConfigs []string `json:"appsec-configs,omitempty"   yaml:"appsec-configs,omitempty"`
 	AppsecRules   []string `json:"appsec-rules,omitempty"   yaml:"appsec-rules,omitempty"`
 }
@@ -231,6 +233,15 @@ func (i *Item) SubItems() []*Item {
 		sub = append(sub, s)
 	}
 
+	for _, name := range i.Contexts {
+		s := i.hub.GetItem(CONTEXTS, name)
+		if s == nil {
+			continue
+		}
+
+		sub = append(sub, s)
+	}
+
 	for _, name := range i.AppsecConfigs {
 		s := i.hub.GetItem(APPSEC_CONFIGS, name)
 		if s == nil {
@@ -284,6 +295,12 @@ func (i *Item) logMissingSubItems() {
 		}
 	}
 
+	for _, subName := range i.Contexts {
+		if i.hub.GetItem(CONTEXTS, subName) == nil {
+			log.Errorf("can't find %s in %s, required by %s", subName, CONTEXTS, i.Name)
+		}
+	}
+
 	for _, subName := range i.AppsecConfigs {
 		if i.hub.GetItem(APPSEC_CONFIGS, subName) == nil {
 			log.Errorf("can't find %s in %s, required by %s", subName, APPSEC_CONFIGS, i.Name)

pkg/cwhub/sync.go

@@ -346,7 +346,7 @@ func (i *Item) checkSubItemVersions() error {
 
 // syncDir scans a directory for items, and updates the Hub state accordingly.
 func (h *Hub) syncDir(dir string) error {
-	// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
+	// For each, scan PARSERS, POSTOVERFLOWS... and COLLECTIONS last
 	for _, scan := range ItemTypes {
 		// cpath: top-level item directory, either downloaded or installed items.
 		// i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ...

test/bats.mk

@@ -83,7 +83,7 @@ bats-build: bats-environment
 
 # Create a reusable package with initial configuration + data
 bats-fixture: bats-check-requirements bats-update-tools
-	@echo "Creating functional test fixture..."
+	@echo "Creating functional test fixture."
 	@$(TEST_DIR)/instance-data make
 
 # Remove the local crowdsec installation and the fixture config + data

test/bats/01_cscli.bats

@@ -252,19 +252,20 @@ teardown() {
 
 @test "cscli - malformed LAPI url" {
     LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path')
-    config_set "${LOCAL_API_CREDENTIALS}" '.url="https://127.0.0.1:-80"'
+    config_set "${LOCAL_API_CREDENTIALS}" '.url="http://127.0.0.1:-80"'
 
-    rune -1 cscli lapi status
+    rune -1 cscli lapi status -o json
-    assert_stderr --partial 'parsing api url'
+    rune -0 jq -r '.msg' <(stderr)
-    assert_stderr --partial 'invalid port \":-80\" after host'
+    assert_output 'parsing api url: parse "http://127.0.0.1:-80/": invalid port ":-80" after host'
+}
 
-    rune -1 cscli alerts list
+@test "cscli - bad LAPI password" {
-    assert_stderr --partial 'parsing api url'
+    LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path')
-    assert_stderr --partial 'invalid port \":-80\" after host'
+    config_set "${LOCAL_API_CREDENTIALS}" '.password="meh"'
 
-    rune -1 cscli decisions list
+    rune -1 cscli lapi status -o json
-    assert_stderr --partial 'parsing api url'
+    rune -0 jq -r '.msg' <(stderr)
-    assert_stderr --partial 'invalid port \":-80\" after host'
+    assert_output 'failed to authenticate to Local API (LAPI): API error: incorrect Username or Password'
 }
 
 @test "cscli metrics" {

test/bats/09_context.bats

@@ -0,0 +1,95 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh"
+    CONFIG_DIR=$(config_get '.config_paths.config_dir')
+    export CONFIG_DIR
+    CONTEXT_YAML="$CONFIG_DIR/console/context.yaml"
+    export CONTEXT_YAML
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh"
+}
+
+setup() {
+    load "../lib/setup.sh"
+    load "../lib/bats-file/load.bash"
+    ./instance-data load
+    config_set '.common.log_media="stdout"'
+    mkdir -p "$CONFIG_DIR/console"
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+#----------
+
+@test "detect available context" {
+    rune -0 cscli lapi context detect -a
+    rune -0 yq -o json <(output)
+    assert_json '{"Acquisition":["evt.Line.Module","evt.Line.Raw","evt.Line.Src"]}'
+
+    rune -0 cscli parsers install crowdsecurity/dateparse-enrich
+    rune -0 cscli lapi context detect crowdsecurity/dateparse-enrich
+    rune -0 yq -o json '.crowdsecurity/dateparse-enrich' <(output)
+    assert_json '["evt.MarshaledTime","evt.Meta.timestamp"]'
+}
+
+@test "attempt to load from default context file, ignore if missing" {
+    rune -0 rm -f "$CONTEXT_YAML"
+    rune -0 "$CROWDSEC" -t --trace
+    assert_stderr --partial "loading console context from $CONTEXT_YAML"
+}
+
+@test "error if context file is explicitly set but does not exist" {
+    config_set ".crowdsec_service.console_context_path=strenv(CONTEXT_YAML)"
+    rune -0 rm -f "$CONTEXT_YAML"
+    rune -1 "$CROWDSEC" -t
+    assert_stderr --partial "while checking console_context_path: stat $CONTEXT_YAML: no such file or directory"
+}
+
+@test "context file is bad" {
+    echo "bad yaml" > "$CONTEXT_YAML"
+    rune -1 "$CROWDSEC" -t
+    assert_stderr --partial "while loading context: $CONTEXT_YAML: yaml: unmarshal errors"
+}
+
+@test "context file is good" {
+    echo '{"source_ip":["evt.Parsed.source_ip"]}' > "$CONTEXT_YAML"
+    rune -0 "$CROWDSEC" -t --debug
+    assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}'
+}
+
+@test "context file is from hub (local item)" {
+    mkdir -p "$CONFIG_DIR/contexts"
+    config_set "del(.crowdsec_service.console_context_path)"
+    echo '{"context":{"source_ip":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/foobar.yaml"
+    rune -0 "$CROWDSEC" -t --trace
+    assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/foobar.yaml"
+    assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}'
+}
+
+@test "merge multiple contexts" {
+    mkdir -p "$CONFIG_DIR/contexts"
+    echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml"
+    echo '{"context":{"two":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/two.yaml"
+    rune -0 "$CROWDSEC" -t --trace
+    assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml"
+    assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/two.yaml"
+    assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip"],"two":["evt.Parsed.source_ip"]}'
+}
+
+@test "merge contexts from hub and context.yaml file" {
+    mkdir -p "$CONFIG_DIR/contexts"
+    echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml"
+    echo '{"one":["evt.Parsed.source_ip_2"]}' > "$CONFIG_DIR/console/context.yaml"
+    rune -0 "$CROWDSEC" -t --trace
+    assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml"
+    assert_stderr --partial "loading console context from $CONFIG_DIR/console/context.yaml"
+    assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip","evt.Parsed.source_ip_2"]}'
+}

test/bats/20_hub.bats

@@ -36,7 +36,7 @@ teardown() {
     rune -0 cscli hub list
     assert_output "No items to display"
     rune -0 cscli hub list -o json
-    assert_json '{"appsec-configs":[],"appsec-rules":[],parsers:[],scenarios:[],collections:[],postoverflows:[]}'
+    assert_json '{"appsec-configs":[],"appsec-rules":[],parsers:[],scenarios:[],collections:[],contexts:[],postoverflows:[]}'
     rune -0 cscli hub list -o raw
     assert_output 'name,status,version,description,type'
 
@@ -47,6 +47,7 @@ teardown() {
     assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*SCENARIOS.*crowdsecurity/telnet-bf.*"
     refute_output --partial 'POSTOVERFLOWS'
     refute_output --partial 'COLLECTIONS'
+
     rune -0 cscli hub list -o json
     rune -0 jq -e '(.parsers | length == 1) and (.scenarios | length == 1)' <(output)
     rune -0 cscli hub list -o raw
@@ -55,8 +56,11 @@ teardown() {
     refute_output --partial 'crowdsecurity/iptables'
 
     # all items
+    mkdir -p "$CONFIG_DIR/contexts"
+    # there are no contexts yet, so we create a local one
+    touch "$CONFIG_DIR/contexts/mycontext.yaml"
     rune -0 cscli hub list -a
-    assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*crowdsecurity/iptables.*"
+    assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*CONTEXTS.*mycontext.yaml.*COLLECTIONS.*crowdsecurity/iptables.*"
     rune -0 cscli hub list -a -o json
     rune -0 jq -e '(.parsers | length > 1) and (.scenarios | length > 1)' <(output)
     rune -0 cscli hub list -a -o raw
@@ -107,6 +111,8 @@ teardown() {
     assert_stderr --partial "Upgraded 0 postoverflows"
     assert_stderr --partial "Upgrading scenarios"
     assert_stderr --partial "Upgraded 0 scenarios"
+    assert_stderr --partial "Upgrading contexts"
+    assert_stderr --partial "Upgraded 0 contexts"
     assert_stderr --partial "Upgrading collections"
     assert_stderr --partial "Upgraded 0 collections"
 
@@ -134,10 +140,11 @@ teardown() {
     assert_line "parsers"
     assert_line "postoverflows"
     assert_line "scenarios"
+    assert_line "contexts"
     assert_line "collections"
     rune -0 cscli hub types -o human
     rune -0 yq -o json <(output)
-    assert_json '["parsers","postoverflows","scenarios","appsec-configs","appsec-rules","collections"]'
+    assert_json '["parsers","postoverflows","scenarios","contexts","appsec-configs","appsec-rules","collections"]'
     rune -0 cscli hub types -o json
-    assert_json '["parsers","postoverflows","scenarios","appsec-configs","appsec-rules","collections"]'
+    assert_json '["parsers","postoverflows","scenarios","contexts","appsec-configs","appsec-rules","collections"]'
 }

test/lib/setup_file.sh

@@ -247,12 +247,14 @@ hub_purge_all() {
         "$CONFIG_DIR"/collections/* \
         "$CONFIG_DIR"/parsers/*/* \
         "$CONFIG_DIR"/scenarios/* \
-        "$CONFIG_DIR"/postoverflows/*
+        "$CONFIG_DIR"/postoverflows/* \
+        "$CONFIG_DIR"/contexts/*
     rm -rf \
         "$CONFIG_DIR"/hub/collections/* \
         "$CONFIG_DIR"/hub/parsers/*/* \
         "$CONFIG_DIR"/hub/scenarios/* \
-        "$CONFIG_DIR"/hub/postoverflows/*
+        "$CONFIG_DIR"/hub/postoverflows/* \
+        "$CONFIG_DIR"/hub/contexts/*
     local DATA_DIR
     DATA_DIR=$(config_get .config_paths.data_dir)
     # should remove everything except the db (find $DATA_DIR -not -name "crowdsec.db*" -delete),