deprecate "cscli lapi context delete"

$ cscli lapi context delete
Command "delete" is deprecated, please manually edit the context file.

cmd/crowdsec-cli/hubappsec.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
+	"gopkg.in/yaml.v3"
+
+	"github.com/crowdsecurity/crowdsec/pkg/appsec"
+	"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewAppsecConfigCLI() *itemCLI {
+	return &itemCLI{
+		name:      cwhub.APPSEC_CONFIGS,
+		singular:  "appsec-config",
+		oneOrMore: "appsec-config(s)",
+		help: cliHelp{
+			example: `cscli appsec-configs list -a
+cscli appsec-configs install crowdsecurity/vpatch
+cscli appsec-configs inspect crowdsecurity/vpatch
+cscli appsec-configs upgrade crowdsecurity/vpatch
+cscli appsec-configs remove crowdsecurity/vpatch
+`,
+		},
+		installHelp: cliHelp{
+			example: `cscli appsec-configs install crowdsecurity/vpatch`,
+		},
+		removeHelp: cliHelp{
+			example: `cscli appsec-configs remove crowdsecurity/vpatch`,
+		},
+		upgradeHelp: cliHelp{
+			example: `cscli appsec-configs upgrade crowdsecurity/vpatch`,
+		},
+		inspectHelp: cliHelp{
+			example: `cscli appsec-configs inspect crowdsecurity/vpatch`,
+		},
+		listHelp: cliHelp{
+			example: `cscli appsec-configs list
+cscli appsec-configs list -a
+cscli appsec-configs list crowdsecurity/vpatch`,
+		},
+	}
+}
+
+func NewAppsecRuleCLI() *itemCLI {
+	inspectDetail := func(item *cwhub.Item) error {
+		appsecRule := appsec.AppsecCollectionConfig{}
+		yamlContent, err := os.ReadFile(item.State.LocalPath)
+		if err != nil {
+			return fmt.Errorf("unable to read file %s : %s", item.State.LocalPath, err)
+		}
+		if err := yaml.Unmarshal(yamlContent, &appsecRule); err != nil {
+			return fmt.Errorf("unable to unmarshal yaml file %s : %s", item.State.LocalPath, err)
+		}
+
+		for _, ruleType := range appsec_rule.SupportedTypes() {
+			fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType))
+			for _, rule := range appsecRule.Rules {
+				convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name)
+				if err != nil {
+					return fmt.Errorf("unable to convert rule %s : %s", rule.Name, err)
+				}
+				fmt.Println(convertedRule)
+			}
+		}
+
+		return nil
+	}
+
+	return &itemCLI{
+		name:      "appsec-rules",
+		singular:  "appsec-rule",
+		oneOrMore: "appsec-rule(s)",
+		help: cliHelp{
+			example: `cscli appsec-rules list -a
+cscli appsec-rules install crowdsecurity/crs
+cscli appsec-rules inspect crowdsecurity/crs
+cscli appsec-rules upgrade crowdsecurity/crs
+cscli appsec-rules remove crowdsecurity/crs
+`,
+		},
+		installHelp: cliHelp{
+			example: `cscli appsec-rules install crowdsecurity/crs`,
+		},
+		removeHelp: cliHelp{
+			example: `cscli appsec-rules remove crowdsecurity/crs`,
+		},
+		upgradeHelp: cliHelp{
+			example: `cscli appsec-rules upgrade crowdsecurity/crs`,
+		},
+		inspectHelp: cliHelp{
+			example: `cscli appsec-rules inspect crowdsecurity/crs`,
+		},
+		inspectDetail: inspectDetail,
+		listHelp: cliHelp{
+			example: `cscli appsec-rules list
+cscli appsec-rules list -a
+cscli appsec-rules list crowdsecurity/crs`,
+		},
+	}
+}

cmd/crowdsec-cli/hubcollection.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewCollectionCLI() *itemCLI {
+	return &itemCLI{
+		name:      cwhub.COLLECTIONS,
+		singular:  "collection",
+		oneOrMore: "collection(s)",
+		help: cliHelp{
+			example: `cscli collections list -a
+cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
+`,
+		},
+		installHelp: cliHelp{
+			example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		removeHelp: cliHelp{
+			example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		upgradeHelp: cliHelp{
+			example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		inspectHelp: cliHelp{
+			example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		listHelp: cliHelp{
+			example: `cscli collections list
+cscli collections list -a
+cscli collections list crowdsecurity/http-cve crowdsecurity/iptables
+
+List only enabled collections unless "-a" or names are specified.`,
+		},
+	}
+}

cmd/crowdsec-cli/hubcontext.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewContextCLI() *itemCLI {
+	return &itemCLI{
+		name:      cwhub.CONTEXTS,
+		singular:  "context",
+		oneOrMore: "context(s)",
+		help: cliHelp{
+			example: `cscli contexts list -a
+cscli contexts install crowdsecurity/yyy crowdsecurity/zzz
+cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz
+cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz
+cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz
+`,
+		},
+		installHelp: cliHelp{
+			example: `cscli contexts install crowdsecurity/yyy crowdsecurity/zzz`,
+		},
+		removeHelp: cliHelp{
+			example: `cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz`,
+		},
+		upgradeHelp: cliHelp{
+			example: `cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz`,
+		},
+		inspectHelp: cliHelp{
+			example: `cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz`,
+		},
+		listHelp: cliHelp{
+			example: `cscli contexts list
+cscli contexts list -a
+cscli contexts list crowdsecurity/yyy crowdsecurity/zzz
+
+List only enabled contexts unless "-a" or names are specified.`,
+		},
+	}
+}

cmd/crowdsec-cli/hubparser.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewParserCLI() *itemCLI {
+	return &itemCLI{
+		name:      cwhub.PARSERS,
+		singular:  "parser",
+		oneOrMore: "parser(s)",
+		help: cliHelp{
+			example: `cscli parsers list -a
+cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+`,
+		},
+		installHelp: cliHelp{
+			example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+		removeHelp: cliHelp{
+			example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+		upgradeHelp: cliHelp{
+			example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+		inspectHelp: cliHelp{
+			example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
+		},
+		listHelp: cliHelp{
+			example: `cscli parsers list
+cscli parsers list -a
+cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+
+List only enabled parsers unless "-a" or names are specified.`,
+		},
+	}
+}

cmd/crowdsec-cli/hubpostoverflow.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewPostOverflowCLI() *itemCLI {
+	return &itemCLI{
+		name:      cwhub.POSTOVERFLOWS,
+		singular:  "postoverflow",
+		oneOrMore: "postoverflow(s)",
+		help: cliHelp{
+			example: `cscli postoverflows list -a
+cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
+`,
+		},
+		installHelp: cliHelp{
+			example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		removeHelp: cliHelp{
+			example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		upgradeHelp: cliHelp{
+			example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		inspectHelp: cliHelp{
+			example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		listHelp: cliHelp{
+			example: `cscli postoverflows list
+cscli postoverflows list -a
+cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns
+
+List only enabled postoverflows unless "-a" or names are specified.`,
+		},
+	}
+}

cmd/crowdsec-cli/hubscenario.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewScenarioCLI() *itemCLI {
+	return &itemCLI{
+		name:      cwhub.SCENARIOS,
+		singular:  "scenario",
+		oneOrMore: "scenario(s)",
+		help: cliHelp{
+			example: `cscli scenarios list -a
+cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
+`,
+		},
+		installHelp: cliHelp{
+			example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		removeHelp: cliHelp{
+			example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		upgradeHelp: cliHelp{
+			example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		inspectHelp: cliHelp{
+			example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		listHelp: cliHelp{
+			example: `cscli scenarios list
+cscli scenarios list -a
+cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing
+
+List only enabled scenarios unless "-a" or names are specified.`,
+		},
+	}
+}

cmd/crowdsec-cli/hubtest.go

@@ -19,6 +19,9 @@ import (
 )
 
 var HubTest hubtest.HubTest
+var HubAppsecTests hubtest.HubTest
+var hubPtr *hubtest.HubTest
+var isAppsecTest bool
 
 func NewHubTestCmd() *cobra.Command {
 	var hubPath string
@@ -33,11 +36,20 @@ func NewHubTestCmd() *cobra.Command {
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			var err error
-			HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath)
+			HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, false)
 			if err != nil {
 				return fmt.Errorf("unable to load hubtest: %+v", err)
 			}
 
+			HubAppsecTests, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, true)
+			if err != nil {
+				return fmt.Errorf("unable to load appsec specific hubtest: %+v", err)
+			}
+			/*commands will use the hubPtr, will point to the default hubTest object, or the one dedicated to appsec tests*/
+			hubPtr = &HubTest
+			if isAppsecTest {
+				hubPtr = &HubAppsecTests
+			}
 			return nil
 		},
 	}
@@ -45,6 +57,7 @@ func NewHubTestCmd() *cobra.Command {
 	cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
 	cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
 	cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
+	cmdHubTest.PersistentFlags().BoolVar(&isAppsecTest, "appsec", false, "Command relates to appsec tests")
 
 	cmdHubTest.AddCommand(NewHubTestCreateCmd())
 	cmdHubTest.AddCommand(NewHubTestRunCmd())
@@ -76,7 +89,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			testName := args[0]
-			testPath := filepath.Join(HubTest.HubTestPath, testName)
+			testPath := filepath.Join(hubPtr.HubTestPath, testName)
 			if _, err := os.Stat(testPath); os.IsExist(err) {
 				return fmt.Errorf("test '%s' already exists in '%s', exiting", testName, testPath)
 			}
@@ -89,53 +102,76 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 				return fmt.Errorf("unable to create folder '%s': %+v", testPath, err)
 			}
 
-			// create empty log file
-			logFileName := fmt.Sprintf("%s.log", testName)
-			logFilePath := filepath.Join(testPath, logFileName)
-			logFile, err := os.Create(logFilePath)
-			if err != nil {
-				return err
-			}
-			logFile.Close()
-
-			// create empty parser assertion file
-			parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName)
-			parserAssertFile, err := os.Create(parserAssertFilePath)
-			if err != nil {
-				return err
-			}
-			parserAssertFile.Close()
-
-			// create empty scenario assertion file
-			scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName)
-			scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
-			if err != nil {
-				return err
-			}
-			scenarioAssertFile.Close()
-
-			parsers = append(parsers, "crowdsecurity/syslog-logs")
-			parsers = append(parsers, "crowdsecurity/dateparse-enrich")
-
-			if len(scenarios) == 0 {
-				scenarios = append(scenarios, "")
-			}
-
-			if len(postoverflows) == 0 {
-				postoverflows = append(postoverflows, "")
-			}
-
-			configFileData := &hubtest.HubTestItemConfig{
-				Parsers:       parsers,
-				Scenarios:     scenarios,
-				PostOVerflows: postoverflows,
-				LogFile:       logFileName,
-				LogType:       logType,
-				IgnoreParsers: ignoreParsers,
-				Labels:        labels,
-			}
-
 			configFilePath := filepath.Join(testPath, "config.yaml")
+
+			configFileData := &hubtest.HubTestItemConfig{}
+			if logType == "appsec" {
+				//create empty nuclei template file
+				nucleiFileName := fmt.Sprintf("%s.yaml", testName)
+				nucleiFilePath := filepath.Join(testPath, nucleiFileName)
+				nucleiFile, err := os.Create(nucleiFilePath)
+				if err != nil {
+					return err
+				}
+				nucleiFile.Close()
+				configFileData.AppsecRules = []string{"your_rule_here.yaml"}
+				configFileData.NucleiTemplate = nucleiFileName
+				fmt.Println()
+				fmt.Printf("  Test name                   :  %s\n", testName)
+				fmt.Printf("  Test path                   :  %s\n", testPath)
+				fmt.Printf("  Nuclei Template             :  %s\n", nucleiFileName)
+			} else {
+				// create empty log file
+				logFileName := fmt.Sprintf("%s.log", testName)
+				logFilePath := filepath.Join(testPath, logFileName)
+				logFile, err := os.Create(logFilePath)
+				if err != nil {
+					return err
+				}
+				logFile.Close()
+
+				// create empty parser assertion file
+				parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName)
+				parserAssertFile, err := os.Create(parserAssertFilePath)
+				if err != nil {
+					return err
+				}
+				parserAssertFile.Close()
+				// create empty scenario assertion file
+				scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName)
+				scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
+				if err != nil {
+					return err
+				}
+				scenarioAssertFile.Close()
+
+				parsers = append(parsers, "crowdsecurity/syslog-logs")
+				parsers = append(parsers, "crowdsecurity/dateparse-enrich")
+
+				if len(scenarios) == 0 {
+					scenarios = append(scenarios, "")
+				}
+
+				if len(postoverflows) == 0 {
+					postoverflows = append(postoverflows, "")
+				}
+				configFileData.Parsers = parsers
+				configFileData.Scenarios = scenarios
+				configFileData.PostOverflows = postoverflows
+				configFileData.LogFile = logFileName
+				configFileData.LogType = logType
+				configFileData.IgnoreParsers = ignoreParsers
+				configFileData.Labels = labels
+				fmt.Println()
+				fmt.Printf("  Test name                   :  %s\n", testName)
+				fmt.Printf("  Test path                   :  %s\n", testPath)
+				fmt.Printf("  Log file                    :  %s (please fill it with logs)\n", logFilePath)
+				fmt.Printf("  Parser assertion file       :  %s (please fill it with assertion)\n", parserAssertFilePath)
+				fmt.Printf("  Scenario assertion file     :  %s (please fill it with assertion)\n", scenarioAssertFilePath)
+				fmt.Printf("  Configuration File          :  %s (please fill it with parsers, scenarios...)\n", configFilePath)
+
+			}
+
 			fd, err := os.Create(configFilePath)
 			if err != nil {
 				return fmt.Errorf("open: %s", err)
@@ -151,14 +187,6 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 			if err := fd.Close(); err != nil {
 				return fmt.Errorf("close: %s", err)
 			}
-			fmt.Println()
-			fmt.Printf("  Test name                   :  %s\n", testName)
-			fmt.Printf("  Test path                   :  %s\n", testPath)
-			fmt.Printf("  Log file                    :  %s (please fill it with logs)\n", logFilePath)
-			fmt.Printf("  Parser assertion file       :  %s (please fill it with assertion)\n", parserAssertFilePath)
-			fmt.Printf("  Scenario assertion file     :  %s (please fill it with assertion)\n", scenarioAssertFilePath)
-			fmt.Printf("  Configuration File          :  %s (please fill it with parsers, scenarios...)\n", configFilePath)
-
 			return nil
 		},
 	}
@@ -188,12 +216,12 @@ func NewHubTestRunCmd() *cobra.Command {
 			}
 
 			if runAll {
-				if err := HubTest.LoadAllTests(); err != nil {
+				if err := hubPtr.LoadAllTests(); err != nil {
 					return fmt.Errorf("unable to load all tests: %+v", err)
 				}
 			} else {
 				for _, testName := range args {
-					_, err := HubTest.LoadTestItem(testName)
+					_, err := hubPtr.LoadTestItem(testName)
 					if err != nil {
 						return fmt.Errorf("unable to load test '%s': %s", testName, err)
 					}
@@ -202,8 +230,7 @@ func NewHubTestRunCmd() *cobra.Command {
 
 			// set timezone to avoid DST issues
 			os.Setenv("TZ", "UTC")
-
-			for _, test := range HubTest.Tests {
+			for _, test := range hubPtr.Tests {
 				if csConfig.Cscli.Output == "human" {
 					log.Infof("Running test '%s'", test.Name)
 				}
@@ -218,8 +245,8 @@ func NewHubTestRunCmd() *cobra.Command {
 		PersistentPostRunE: func(cmd *cobra.Command, args []string) error {
 			success := true
 			testResult := make(map[string]bool)
-			for _, test := range HubTest.Tests {
-				if test.AutoGen {
+			for _, test := range hubPtr.Tests {
+				if test.AutoGen && !isAppsecTest {
 					if test.ParserAssert.AutoGenAssert {
 						log.Warningf("Assert file '%s' is empty, generating assertion:", test.ParserAssert.File)
 						fmt.Println()
@@ -341,7 +368,7 @@ func NewHubTestCleanCmd() *cobra.Command {
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			for _, testName := range args {
-				test, err := HubTest.LoadTestItem(testName)
+				test, err := hubPtr.LoadTestItem(testName)
 				if err != nil {
 					return fmt.Errorf("unable to load test '%s': %s", testName, err)
 				}
@@ -364,17 +391,23 @@ func NewHubTestInfoCmd() *cobra.Command {
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
+
 			for _, testName := range args {
-				test, err := HubTest.LoadTestItem(testName)
+				test, err := hubPtr.LoadTestItem(testName)
 				if err != nil {
 					return fmt.Errorf("unable to load test '%s': %s", testName, err)
 				}
 				fmt.Println()
 				fmt.Printf("  Test name                   :  %s\n", test.Name)
 				fmt.Printf("  Test path                   :  %s\n", test.Path)
-				fmt.Printf("  Log file                    :  %s\n", filepath.Join(test.Path, test.Config.LogFile))
-				fmt.Printf("  Parser assertion file       :  %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
-				fmt.Printf("  Scenario assertion file     :  %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
+				if isAppsecTest {
+					fmt.Printf("  Nuclei Template             :  %s\n", test.Config.NucleiTemplate)
+					fmt.Printf("  Appsec Rules                  :  %s\n", strings.Join(test.Config.AppsecRules, ", "))
+				} else {
+					fmt.Printf("  Log file                    :  %s\n", filepath.Join(test.Path, test.Config.LogFile))
+					fmt.Printf("  Parser assertion file       :  %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
+					fmt.Printf("  Scenario assertion file     :  %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
+				}
 				fmt.Printf("  Configuration File          :  %s\n", filepath.Join(test.Path, "config.yaml"))
 			}
 
@@ -391,15 +424,15 @@ func NewHubTestListCmd() *cobra.Command {
 		Short:             "list",
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := HubTest.LoadAllTests(); err != nil {
+			if err := hubPtr.LoadAllTests(); err != nil {
 				return fmt.Errorf("unable to load all tests: %s", err)
 			}
 
 			switch csConfig.Cscli.Output {
 			case "human":
-				hubTestListTable(color.Output, HubTest.Tests)
+				hubTestListTable(color.Output, hubPtr.Tests)
 			case "json":
-				j, err := json.MarshalIndent(HubTest.Tests, " ", "  ")
+				j, err := json.MarshalIndent(hubPtr.Tests, " ", "  ")
 				if err != nil {
 					return err
 				}
@@ -419,23 +452,27 @@ func NewHubTestCoverageCmd() *cobra.Command {
 	var showParserCov bool
 	var showScenarioCov bool
 	var showOnlyPercent bool
+	var showAppsecCov bool
 
 	var cmdHubTestCoverage = &cobra.Command{
 		Use:               "coverage",
 		Short:             "coverage",
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
+			//for this one we explicitly don't do for appsec
 			if err := HubTest.LoadAllTests(); err != nil {
 				return fmt.Errorf("unable to load all tests: %+v", err)
 			}
 			var err error
 			scenarioCoverage := []hubtest.Coverage{}
 			parserCoverage := []hubtest.Coverage{}
+			appsecRuleCoverage := []hubtest.Coverage{}
 			scenarioCoveragePercent := 0
 			parserCoveragePercent := 0
+			appsecRuleCoveragePercent := 0
 
 			// if both are false (flag by default), show both
-			showAll := !showScenarioCov && !showParserCov
+			showAll := !showScenarioCov && !showParserCov && !showAppsecCov
 
 			if showParserCov || showAll {
 				parserCoverage, err = HubTest.GetParsersCoverage()
@@ -467,13 +504,30 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100)))
 			}
 
+			if showAppsecCov || showAll {
+				appsecRuleCoverage, err = HubTest.GetAppsecCoverage()
+				if err != nil {
+					return fmt.Errorf("while getting scenario coverage: %s", err)
+				}
+
+				appsecRuleTested := 0
+				for _, test := range appsecRuleCoverage {
+					if test.TestsCount > 0 {
+						appsecRuleTested++
+					}
+				}
+				appsecRuleCoveragePercent = int(math.Round((float64(appsecRuleTested) / float64(len(appsecRuleCoverage)) * 100)))
+			}
+
 			if showOnlyPercent {
 				if showAll {
-					fmt.Printf("parsers=%d%%\nscenarios=%d%%", parserCoveragePercent, scenarioCoveragePercent)
+					fmt.Printf("parsers=%d%%\nscenarios=%d%%\nappsec_rules=%d%%", parserCoveragePercent, scenarioCoveragePercent, appsecRuleCoveragePercent)
 				} else if showParserCov {
 					fmt.Printf("parsers=%d%%", parserCoveragePercent)
 				} else if showScenarioCov {
 					fmt.Printf("scenarios=%d%%", scenarioCoveragePercent)
+				} else if showAppsecCov {
+					fmt.Printf("appsec_rules=%d%%", appsecRuleCoveragePercent)
 				}
 				os.Exit(0)
 			}
@@ -487,6 +541,11 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				if showScenarioCov || showAll {
 					hubTestScenarioCoverageTable(color.Output, scenarioCoverage)
 				}
+
+				if showAppsecCov || showAll {
+					hubTestAppsecRuleCoverageTable(color.Output, appsecRuleCoverage)
+				}
+
 				fmt.Println()
 				if showParserCov || showAll {
 					fmt.Printf("PARSERS    : %d%% of coverage\n", parserCoveragePercent)
@@ -494,6 +553,9 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				if showScenarioCov || showAll {
 					fmt.Printf("SCENARIOS  : %d%% of coverage\n", scenarioCoveragePercent)
 				}
+				if showAppsecCov || showAll {
+					fmt.Printf("APPSEC RULES  : %d%% of coverage\n", appsecRuleCoveragePercent)
+				}
 			case "json":
 				dump, err := json.MarshalIndent(parserCoverage, "", " ")
 				if err != nil {
@@ -505,6 +567,11 @@ func NewHubTestCoverageCmd() *cobra.Command {
 					return err
 				}
 				fmt.Printf("%s", dump)
+				dump, err = json.MarshalIndent(appsecRuleCoverage, "", " ")
+				if err != nil {
+					return err
+				}
+				fmt.Printf("%s", dump)
 			default:
 				return fmt.Errorf("only human/json output modes are supported")
 			}
@@ -516,6 +583,7 @@ func NewHubTestCoverageCmd() *cobra.Command {
 	cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
 	cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
 	cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
+	cmdHubTestCoverage.PersistentFlags().BoolVar(&showAppsecCov, "appsec", false, "Show only appsec coverage")
 
 	return cmdHubTestCoverage
 }
@@ -529,7 +597,7 @@ func NewHubTestEvalCmd() *cobra.Command {
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			for _, testName := range args {
-				test, err := HubTest.LoadTestItem(testName)
+				test, err := hubPtr.LoadTestItem(testName)
 				if err != nil {
 					return fmt.Errorf("can't load test: %+v", err)
 				}

cmd/crowdsec-cli/hubtest_table.go

@@ -61,6 +61,26 @@ func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
 	t.Render()
 }
 
+func hubTestAppsecRuleCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
+	t := newLightTable(out)
+	t.SetHeaders("Appsec Rule", "Status", "Number of tests")
+	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
+	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
+
+	parserTested := 0
+
+	for _, test := range coverage {
+		status := emoji.RedCircle.String()
+		if test.TestsCount > 0 {
+			status = emoji.GreenCircle.String()
+			parserTested++
+		}
+		t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
+	}
+
+	t.Render()
+}
+
 func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
 	t := newLightTable(out)
 	t.SetHeaders("Scenario", "Status", "Number of tests")

cmd/crowdsec-cli/item_metrics.go

@@ -32,6 +32,8 @@ func ShowMetrics(hubItem *cwhub.Item) error {
 				return err
 			}
 		}
+	case cwhub.APPSEC_RULES:
+		log.Error("FIXME: not implemented yet")
 	default:
 		// no metrics for this item type
 	}

cmd/crowdsec-cli/itemcli.go

@@ -0,0 +1,454 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/fatih/color"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/crowdsecurity/go-cs-lib/coalesce"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+type cliHelp struct {
+	// Example is required, the others have a default value
+	// generated from the item type
+	use     string
+	short   string
+	long    string
+	example string
+}
+
+type itemCLI struct {
+	name          string // plural, as used in the hub index
+	singular      string
+	oneOrMore     string // parenthetical pluralizaion: "parser(s)"
+	help          cliHelp
+	installHelp   cliHelp
+	removeHelp    cliHelp
+	upgradeHelp   cliHelp
+	inspectHelp   cliHelp
+	inspectDetail func(item *cwhub.Item) error
+	listHelp      cliHelp
+}
+
+func (it itemCLI) NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.help.use, fmt.Sprintf("%s <action> [item]...", it.name)),
+		Short:             coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)),
+		Long:              it.help.long,
+		Example:           it.help.example,
+		Args:              cobra.MinimumNArgs(1),
+		Aliases:           []string{it.singular},
+		DisableAutoGenTag: true,
+	}
+
+	cmd.AddCommand(it.NewInstallCmd())
+	cmd.AddCommand(it.NewRemoveCmd())
+	cmd.AddCommand(it.NewUpgradeCmd())
+	cmd.AddCommand(it.NewInspectCmd())
+	cmd.AddCommand(it.NewListCmd())
+
+	return cmd
+}
+
+func (it itemCLI) Install(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	downloadOnly, err := flags.GetBool("download-only")
+	if err != nil {
+		return err
+	}
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	ignoreError, err := flags.GetBool("ignore")
+	if err != nil {
+		return err
+	}
+
+	hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
+	if err != nil {
+		return err
+	}
+
+	for _, name := range args {
+		item := hub.GetItem(it.name, name)
+		if item == nil {
+			msg := suggestNearestMessage(hub, it.name, name)
+			if !ignoreError {
+				return fmt.Errorf(msg)
+			}
+
+			log.Errorf(msg)
+
+			continue
+		}
+
+		if err := item.Install(force, downloadOnly); err != nil {
+			if !ignoreError {
+				return fmt.Errorf("error while installing '%s': %w", item.Name, err)
+			}
+			log.Errorf("Error while installing '%s': %s", item.Name, err)
+		}
+	}
+
+	log.Infof(ReloadMessage())
+	return nil
+}
+
+func (it itemCLI) NewInstallCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.installHelp.use, "install [item]..."),
+		Short:             coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)),
+		Example:           it.installHelp.example,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compAllItems(it.name, args, toComplete)
+		},
+		RunE: it.Install,
+	}
+
+	flags := cmd.Flags()
+	flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
+	flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
+	flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name))
+
+	return cmd
+}
+
+// return the names of the installed parents of an item, used to check if we can remove it
+func istalledParentNames(item *cwhub.Item) []string {
+	ret := make([]string, 0)
+
+	for _, parent := range item.Ancestors() {
+		if parent.State.Installed {
+			ret = append(ret, parent.Name)
+		}
+	}
+
+	return ret
+}
+
+func (it itemCLI) Remove(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	purge, err := flags.GetBool("purge")
+	if err != nil {
+		return err
+	}
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	hub, err := require.Hub(csConfig, nil)
+	if err != nil {
+		return err
+	}
+
+	if all {
+		getter := hub.GetInstalledItems
+		if purge {
+			getter = hub.GetAllItems
+		}
+
+		items, err := getter(it.name)
+		if err != nil {
+			return err
+		}
+
+		removed := 0
+
+		for _, item := range items {
+			didRemove, err := item.Remove(purge, force)
+			if err != nil {
+				return err
+			}
+			if didRemove {
+				log.Infof("Removed %s", item.Name)
+				removed++
+			}
+		}
+
+		log.Infof("Removed %d %s", removed, it.name)
+		if removed > 0 {
+			log.Infof(ReloadMessage())
+		}
+
+		return nil
+	}
+
+	if len(args) == 0 {
+		return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular)
+	}
+
+	removed := 0
+
+	for _, itemName := range args {
+		item := hub.GetItem(it.name, itemName)
+		if item == nil {
+			return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
+		}
+
+		parents := istalledParentNames(item)
+
+		if !force && len(parents) > 0 {
+			log.Warningf("%s belongs to collections: %s", item.Name, parents)
+			log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular)
+			continue
+		}
+
+		didRemove, err := item.Remove(purge, force)
+		if err != nil {
+			return err
+		}
+
+		if didRemove {
+			log.Infof("Removed %s", item.Name)
+			removed++
+		}
+	}
+
+	log.Infof("Removed %d %s", removed, it.name)
+	if removed > 0 {
+		log.Infof(ReloadMessage())
+	}
+
+	return nil
+}
+
+func (it itemCLI) NewRemoveCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.removeHelp.use, "remove [item]..."),
+		Short:             coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)),
+		Example:           it.removeHelp.example,
+		Aliases:           []string{"delete"},
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(it.name, args, toComplete)
+		},
+		RunE: it.Remove,
+	}
+
+	flags := cmd.Flags()
+	flags.Bool("purge", false, "Delete source file too")
+	flags.Bool("force", false, "Force remove: remove tainted and outdated files")
+	flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name))
+
+	return cmd
+}
+
+func (it itemCLI) Upgrade(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
+	if err != nil {
+		return err
+	}
+
+	if all {
+		items, err := hub.GetInstalledItems(it.name)
+		if err != nil {
+			return err
+		}
+
+		updated := 0
+
+		for _, item := range items {
+			didUpdate, err := item.Upgrade(force)
+			if err != nil {
+				return err
+			}
+			if didUpdate {
+				updated++
+			}
+		}
+
+		log.Infof("Updated %d %s", updated, it.name)
+
+		if updated > 0 {
+			log.Infof(ReloadMessage())
+		}
+
+		return nil
+	}
+
+	if len(args) == 0 {
+		return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular)
+	}
+
+	updated := 0
+
+	for _, itemName := range args {
+		item := hub.GetItem(it.name, itemName)
+		if item == nil {
+			return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
+		}
+
+		didUpdate, err := item.Upgrade(force)
+		if err != nil {
+			return err
+		}
+
+		if didUpdate {
+			log.Infof("Updated %s", item.Name)
+			updated++
+		}
+	}
+	if updated > 0 {
+		log.Infof(ReloadMessage())
+	}
+
+	return nil
+}
+
+func (it itemCLI) NewUpgradeCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.upgradeHelp.use, "upgrade [item]..."),
+		Short:             coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)),
+		Example:           it.upgradeHelp.example,
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(it.name, args, toComplete)
+		},
+		RunE: it.Upgrade,
+	}
+
+	flags := cmd.Flags()
+	flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name))
+	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
+
+	return cmd
+}
+
+func (it itemCLI) Inspect(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	url, err := flags.GetString("url")
+	if err != nil {
+		return err
+	}
+
+	if url != "" {
+		csConfig.Cscli.PrometheusUrl = url
+	}
+
+	noMetrics, err := flags.GetBool("no-metrics")
+	if err != nil {
+		return err
+	}
+
+	hub, err := require.Hub(csConfig, nil)
+	if err != nil {
+		return err
+	}
+
+	for _, name := range args {
+		item := hub.GetItem(it.name, name)
+		if item == nil {
+			return fmt.Errorf("can't find '%s' in %s", name, it.name)
+		}
+		if err = InspectItem(item, !noMetrics); err != nil {
+			return err
+		}
+
+		if it.inspectDetail != nil {
+			if err = it.inspectDetail(item); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (it itemCLI) NewInspectCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.inspectHelp.use, "inspect [item]..."),
+		Short:             coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)),
+		Example:           it.inspectHelp.example,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(it.name, args, toComplete)
+		},
+		RunE: it.Inspect,
+	}
+
+	flags := cmd.Flags()
+	flags.StringP("url", "u", "", "Prometheus url")
+	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
+
+	return cmd
+}
+
+func (it itemCLI) List(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	hub, err := require.Hub(csConfig, nil)
+	if err != nil {
+		return err
+	}
+
+	items := make(map[string][]*cwhub.Item)
+
+	items[it.name], err = selectItems(hub, it.name, args, !all)
+	if err != nil {
+		return err
+	}
+
+	if err = listItems(color.Output, []string{it.name}, items, false); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (it itemCLI) NewListCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.listHelp.use, "list [item... | -a]"),
+		Short:             coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)),
+		Long:              coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)),
+		Example:           it.listHelp.example,
+		DisableAutoGenTag: true,
+		RunE:              it.List,
+	}
+
+	flags := cmd.Flags()
+	flags.BoolP("all", "a", false, "List disabled items as well")
+
+	return cmd
+}

cmd/crowdsec-cli/itemcommands.go

@@ -1,609 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/go-cs-lib/coalesce"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-type cmdHelp struct {
-	// Example is required, the others have a default value
-	// generated from the item type
-	use     string
-	short   string
-	long    string
-	example string
-}
-
-type hubItemType struct {
-	name        string // plural, as used in the hub index
-	singular    string
-	oneOrMore   string // parenthetical pluralizaion: "parser(s)"
-	help        cmdHelp
-	installHelp cmdHelp
-	removeHelp  cmdHelp
-	upgradeHelp cmdHelp
-	inspectHelp cmdHelp
-	listHelp    cmdHelp
-}
-
-var hubItemTypes = map[string]hubItemType{
-	"parsers": {
-		name:      cwhub.PARSERS,
-		singular:  "parser",
-		oneOrMore: "parser(s)",
-		help: cmdHelp{
-			example: `cscli parsers list -a
-cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
-cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
-cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
-cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
-`,
-		},
-		installHelp: cmdHelp{
-			example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
-		},
-		removeHelp: cmdHelp{
-			example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
-		},
-		upgradeHelp: cmdHelp{
-			example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
-		},
-		inspectHelp: cmdHelp{
-			example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
-		},
-		listHelp: cmdHelp{
-			example: `cscli parsers list
-cscli parsers list -a
-cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs
-
-List only enabled parsers unless "-a" or names are specified.`,
-		},
-	},
-	"postoverflows": {
-		name:      cwhub.POSTOVERFLOWS,
-		singular:  "postoverflow",
-		oneOrMore: "postoverflow(s)",
-		help: cmdHelp{
-			example: `cscli postoverflows list -a
-cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
-cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
-cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
-cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
-`,
-		},
-		installHelp: cmdHelp{
-			example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
-		},
-		removeHelp: cmdHelp{
-			example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
-		},
-		upgradeHelp: cmdHelp{
-			example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
-		},
-		inspectHelp: cmdHelp{
-			example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
-		},
-		listHelp: cmdHelp{
-			example: `cscli postoverflows list
-cscli postoverflows list -a
-cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns
-
-List only enabled postoverflows unless "-a" or names are specified.`,
-		},
-	},
-	"scenarios": {
-		name:      cwhub.SCENARIOS,
-		singular:  "scenario",
-		oneOrMore: "scenario(s)",
-		help: cmdHelp{
-			example: `cscli scenarios list -a
-cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
-cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
-cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
-cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
-`,
-		},
-		installHelp: cmdHelp{
-			example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
-		},
-		removeHelp: cmdHelp{
-			example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
-		},
-		upgradeHelp: cmdHelp{
-			example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
-		},
-		inspectHelp: cmdHelp{
-			example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
-		},
-		listHelp: cmdHelp{
-			example: `cscli scenarios list
-cscli scenarios list -a
-cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing
-
-List only enabled scenarios unless "-a" or names are specified.`,
-		},
-	},
-	"collections": {
-		name:      cwhub.COLLECTIONS,
-		singular:  "collection",
-		oneOrMore: "collection(s)",
-		help: cmdHelp{
-			example: `cscli collections list -a
-cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
-cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
-cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
-cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
-`,
-		},
-		installHelp: cmdHelp{
-			example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
-		},
-		removeHelp: cmdHelp{
-			example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
-		},
-		upgradeHelp: cmdHelp{
-			example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
-		},
-		inspectHelp: cmdHelp{
-			example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
-		},
-		listHelp: cmdHelp{
-			example: `cscli collections list
-cscli collections list -a
-cscli collections list crowdsecurity/http-cve crowdsecurity/iptables
-
-List only enabled collections unless "-a" or names are specified.`,
-		},
-	},
-}
-
-func NewItemsCmd(typeName string) *cobra.Command {
-	it := hubItemTypes[typeName]
-
-	cmd := &cobra.Command{
-		Use:               coalesce.String(it.help.use, fmt.Sprintf("%s <action> [item]...", it.name)),
-		Short:             coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)),
-		Long:              it.help.long,
-		Example:           it.help.example,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{it.singular},
-		DisableAutoGenTag: true,
-	}
-
-	cmd.AddCommand(NewItemsInstallCmd(typeName))
-	cmd.AddCommand(NewItemsRemoveCmd(typeName))
-	cmd.AddCommand(NewItemsUpgradeCmd(typeName))
-	cmd.AddCommand(NewItemsInspectCmd(typeName))
-	cmd.AddCommand(NewItemsListCmd(typeName))
-
-	return cmd
-}
-
-func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
-	run := func(cmd *cobra.Command, args []string) error {
-		flags := cmd.Flags()
-
-		downloadOnly, err := flags.GetBool("download-only")
-		if err != nil {
-			return err
-		}
-
-		force, err := flags.GetBool("force")
-		if err != nil {
-			return err
-		}
-
-		ignoreError, err := flags.GetBool("ignore")
-		if err != nil {
-			return err
-		}
-
-		hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
-		if err != nil {
-			return err
-		}
-
-		for _, name := range args {
-			item := hub.GetItem(it.name, name)
-			if item == nil {
-				msg := suggestNearestMessage(hub, it.name, name)
-				if !ignoreError {
-					return fmt.Errorf(msg)
-				}
-
-				log.Errorf(msg)
-
-				continue
-			}
-
-			if err := item.Install(force, downloadOnly); err != nil {
-				if !ignoreError {
-					return fmt.Errorf("error while installing '%s': %w", item.Name, err)
-				}
-				log.Errorf("Error while installing '%s': %s", item.Name, err)
-			}
-		}
-
-		log.Infof(ReloadMessage())
-		return nil
-	}
-
-	return run
-}
-
-func NewItemsInstallCmd(typeName string) *cobra.Command {
-	it := hubItemTypes[typeName]
-
-	cmd := &cobra.Command{
-		Use:               coalesce.String(it.installHelp.use, "install [item]..."),
-		Short:             coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)),
-		Long:              coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)),
-		Example:           it.installHelp.example,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(typeName, args, toComplete)
-		},
-		RunE: itemsInstallRunner(it),
-	}
-
-	flags := cmd.Flags()
-	flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
-	flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
-	flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name))
-
-	return cmd
-}
-
-// return the names of the installed parents of an item, used to check if we can remove it
-func istalledParentNames(item *cwhub.Item) []string {
-	ret := make([]string, 0)
-
-	for _, parent := range item.Ancestors() {
-		if parent.State.Installed {
-			ret = append(ret, parent.Name)
-		}
-	}
-
-	return ret
-}
-
-func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
-	run := func(cmd *cobra.Command, args []string) error {
-		flags := cmd.Flags()
-
-		purge, err := flags.GetBool("purge")
-		if err != nil {
-			return err
-		}
-
-		force, err := flags.GetBool("force")
-		if err != nil {
-			return err
-		}
-
-		all, err := flags.GetBool("all")
-		if err != nil {
-			return err
-		}
-
-		hub, err := require.Hub(csConfig, nil)
-		if err != nil {
-			return err
-		}
-
-		if all {
-			getter := hub.GetInstalledItems
-			if purge {
-				getter = hub.GetAllItems
-			}
-
-			items, err := getter(it.name)
-			if err != nil {
-				return err
-			}
-
-			removed := 0
-
-			for _, item := range items {
-				didRemove, err := item.Remove(purge, force)
-				if err != nil {
-					return err
-				}
-				if didRemove {
-					log.Infof("Removed %s", item.Name)
-					removed++
-				}
-			}
-
-			log.Infof("Removed %d %s", removed, it.name)
-			if removed > 0 {
-				log.Infof(ReloadMessage())
-			}
-
-			return nil
-		}
-
-		if len(args) == 0 {
-			return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular)
-		}
-
-		removed := 0
-
-		for _, itemName := range args {
-			item := hub.GetItem(it.name, itemName)
-			if item == nil {
-				return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
-			}
-
-			parents := istalledParentNames(item)
-
-			if !force && len(parents) > 0 {
-				log.Warningf("%s belongs to collections: %s", item.Name, parents)
-				log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular)
-				continue
-			}
-
-			didRemove, err := item.Remove(purge, force)
-			if err != nil {
-				return err
-			}
-
-			if didRemove {
-				log.Infof("Removed %s", item.Name)
-				removed++
-			}
-		}
-
-		log.Infof("Removed %d %s", removed, it.name)
-		if removed > 0 {
-			log.Infof(ReloadMessage())
-		}
-
-		return nil
-	}
-	return run
-}
-
-func NewItemsRemoveCmd(typeName string) *cobra.Command {
-	it := hubItemTypes[typeName]
-
-	cmd := &cobra.Command{
-		Use:               coalesce.String(it.removeHelp.use, "remove [item]..."),
-		Short:             coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)),
-		Long:              coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)),
-		Example:           it.removeHelp.example,
-		Aliases:           []string{"delete"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(it.name, args, toComplete)
-		},
-		RunE: itemsRemoveRunner(it),
-	}
-
-	flags := cmd.Flags()
-	flags.Bool("purge", false, "Delete source file too")
-	flags.Bool("force", false, "Force remove: remove tainted and outdated files")
-	flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name))
-
-	return cmd
-}
-
-func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
-	run := func(cmd *cobra.Command, args []string) error {
-		flags := cmd.Flags()
-
-		force, err := flags.GetBool("force")
-		if err != nil {
-			return err
-		}
-
-		all, err := flags.GetBool("all")
-		if err != nil {
-			return err
-		}
-
-		hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
-		if err != nil {
-			return err
-		}
-
-		if all {
-			items, err := hub.GetInstalledItems(it.name)
-			if err != nil {
-				return err
-			}
-
-			updated := 0
-
-			for _, item := range items {
-				didUpdate, err := item.Upgrade(force)
-				if err != nil {
-					return err
-				}
-				if didUpdate {
-					updated++
-				}
-			}
-
-			log.Infof("Updated %d %s", updated, it.name)
-
-			if updated > 0 {
-				log.Infof(ReloadMessage())
-			}
-
-			return nil
-		}
-
-		if len(args) == 0 {
-			return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular)
-		}
-
-		updated := 0
-
-		for _, itemName := range args {
-			item := hub.GetItem(it.name, itemName)
-			if item == nil {
-				return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
-			}
-
-			didUpdate, err := item.Upgrade(force)
-			if err != nil {
-				return err
-			}
-
-			if didUpdate {
-				log.Infof("Updated %s", item.Name)
-				updated++
-			}
-		}
-		if updated > 0 {
-			log.Infof(ReloadMessage())
-		}
-
-		return nil
-	}
-
-	return run
-}
-
-func NewItemsUpgradeCmd(typeName string) *cobra.Command {
-	it := hubItemTypes[typeName]
-
-	cmd := &cobra.Command{
-		Use:               coalesce.String(it.upgradeHelp.use, "upgrade [item]..."),
-		Short:             coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)),
-		Long:              coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)),
-		Example:           it.upgradeHelp.example,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(it.name, args, toComplete)
-		},
-		RunE: itemsUpgradeRunner(it),
-	}
-
-	flags := cmd.Flags()
-	flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name))
-	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
-
-	return cmd
-}
-
-func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
-	run := func(cmd *cobra.Command, args []string) error {
-		flags := cmd.Flags()
-
-		url, err := flags.GetString("url")
-		if err != nil {
-			return err
-		}
-
-		if url != "" {
-			csConfig.Cscli.PrometheusUrl = url
-		}
-
-		noMetrics, err := flags.GetBool("no-metrics")
-		if err != nil {
-			return err
-		}
-
-		hub, err := require.Hub(csConfig, nil)
-		if err != nil {
-			return err
-		}
-
-		for _, name := range args {
-			item := hub.GetItem(it.name, name)
-			if item == nil {
-				return fmt.Errorf("can't find '%s' in %s", name, it.name)
-			}
-			if err = InspectItem(item, !noMetrics); err != nil {
-				return err
-			}
-		}
-
-		return nil
-	}
-
-	return run
-}
-
-func NewItemsInspectCmd(typeName string) *cobra.Command {
-	it := hubItemTypes[typeName]
-
-	cmd := &cobra.Command{
-		Use:               coalesce.String(it.inspectHelp.use, "inspect [item]..."),
-		Short:             coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)),
-		Long:              coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)),
-		Example:           it.inspectHelp.example,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(it.name, args, toComplete)
-		},
-		RunE: itemsInspectRunner(it),
-	}
-
-	flags := cmd.Flags()
-	flags.StringP("url", "u", "", "Prometheus url")
-	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
-
-	return cmd
-}
-
-func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
-	run := func(cmd *cobra.Command, args []string) error {
-		flags := cmd.Flags()
-
-		all, err := flags.GetBool("all")
-		if err != nil {
-			return err
-		}
-
-		hub, err := require.Hub(csConfig, nil)
-		if err != nil {
-			return err
-		}
-
-		items := make(map[string][]*cwhub.Item)
-
-		items[it.name], err = selectItems(hub, it.name, args, !all)
-		if err != nil {
-			return err
-		}
-
-		if err = listItems(color.Output, []string{it.name}, items, false); err != nil {
-			return err
-		}
-
-		return nil
-	}
-
-	return run
-}
-
-func NewItemsListCmd(typeName string) *cobra.Command {
-	it := hubItemTypes[typeName]
-
-	cmd := &cobra.Command{
-		Use:               coalesce.String(it.listHelp.use, "list [item... | -a]"),
-		Short:             coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)),
-		Long:              coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)),
-		Example:           it.listHelp.example,
-		DisableAutoGenTag: true,
-		RunE:              itemsListRunner(it),
-	}
-
-	flags := cmd.Flags()
-	flags.BoolP("all", "a", false, "List disabled items as well")
-
-	return cmd
-}

cmd/crowdsec-cli/lapi.go

@@ -2,10 +2,10 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/url"
 	"os"
-	"slices"
 	"sort"
 	"strings"
 
@@ -13,6 +13,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
+	"slices"
 
 	"github.com/crowdsecurity/go-cs-lib/version"
 
@@ -26,26 +27,24 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
 )
 
-var LAPIURLPrefix string = "v1"
+const LAPIURLPrefix = "v1"
 
 func runLapiStatus(cmd *cobra.Command, args []string) error {
-	var err error
-
 	password := strfmt.Password(csConfig.API.Client.Credentials.Password)
 	apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
 	login := csConfig.API.Client.Credentials.Login
 	if err != nil {
-		log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
+		return fmt.Errorf("parsing api url: %w", err)
 	}
 
 	hub, err := require.Hub(csConfig, nil)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 
 	scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 	if err != nil {
-		log.Fatalf("failed to get scenarios : %s", err)
+		return fmt.Errorf("failed to get scenarios: %w", err)
 	}
 
 	Client, err = apiclient.NewDefaultClient(apiurl,
@@ -53,28 +52,27 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
 		fmt.Sprintf("crowdsec/%s", version.String()),
 		nil)
 	if err != nil {
-		log.Fatalf("init default client: %s", err)
+		return fmt.Errorf("init default client: %w", err)
 	}
 	t := models.WatcherAuthRequest{
 		MachineID: &login,
 		Password:  &password,
 		Scenarios: scenarios,
 	}
+
 	log.Infof("Loaded credentials from %s", csConfig.API.Client.CredentialsFilePath)
 	log.Infof("Trying to authenticate with username %s on %s", login, apiurl)
+
 	_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
 	if err != nil {
-		log.Fatalf("Failed to authenticate to Local API (LAPI) : %s", err)
-	} else {
-		log.Infof("You can successfully interact with Local API (LAPI)")
+		return fmt.Errorf("failed to authenticate to Local API (LAPI): %w", err)
 	}
 
+	log.Infof("You can successfully interact with Local API (LAPI)")
 	return nil
 }
 
 func runLapiRegister(cmd *cobra.Command, args []string) error {
-	var err error
-
 	flags := cmd.Flags()
 
 	apiURL, err := flags.GetString("url")
@@ -95,16 +93,15 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
 	if lapiUser == "" {
 		lapiUser, err = generateID("")
 		if err != nil {
-			log.Fatalf("unable to generate machine id: %s", err)
+			return fmt.Errorf("unable to generate machine id: %w", err)
 		}
 	}
 	password := strfmt.Password(generatePassword(passwordLength))
 	if apiURL == "" {
-		if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {
-			apiURL = csConfig.API.Client.Credentials.URL
-		} else {
-			log.Fatalf("No Local API URL. Please provide it in your configuration or with the -u parameter")
+		if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil || csConfig.API.Client.Credentials.URL == "" {
+			return fmt.Errorf("no Local API URL. Please provide it in your configuration or with the -u parameter")
 		}
+		apiURL = csConfig.API.Client.Credentials.URL
 	}
 	/*URL needs to end with /, but user doesn't care*/
 	if !strings.HasSuffix(apiURL, "/") {
@@ -116,7 +113,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
 	}
 	apiurl, err := url.Parse(apiURL)
 	if err != nil {
-		log.Fatalf("parsing api url: %s", err)
+		return fmt.Errorf("parsing api url: %w", err)
 	}
 	_, err = apiclient.RegisterClient(&apiclient.Config{
 		MachineID:     lapiUser,
@@ -127,7 +124,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
 	}, nil)
 
 	if err != nil {
-		log.Fatalf("api client register: %s", err)
+		return fmt.Errorf("api client register: %w", err)
 	}
 
 	log.Printf("Successfully registered to Local API (LAPI)")
@@ -147,12 +144,12 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
 	}
 	apiConfigDump, err := yaml.Marshal(apiCfg)
 	if err != nil {
-		log.Fatalf("unable to marshal api credentials: %s", err)
+		return fmt.Errorf("unable to marshal api credentials: %w", err)
 	}
 	if dumpFile != "" {
 		err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
 		if err != nil {
-			log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
+			return fmt.Errorf("write api credentials to '%s' failed: %w", dumpFile, err)
 		}
 		log.Printf("Local API credentials written to '%s'", dumpFile)
 	} else {
@@ -195,7 +192,7 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side
 }
 
 func NewLapiCmd() *cobra.Command {
-	var cmdLapi = &cobra.Command{
+	cmdLapi := &cobra.Command{
 		Use:               "lapi [action]",
 		Short:             "Manage interaction with Local API (LAPI)",
 		Args:              cobra.MinimumNArgs(1),
@@ -221,6 +218,7 @@ func AddContext(key string, values []string) error {
 	}
 	if _, ok := csConfig.Crowdsec.ContextToSend[key]; !ok {
 		csConfig.Crowdsec.ContextToSend[key] = make([]string, 0)
+
 		log.Infof("key '%s' added", key)
 	}
 	data := csConfig.Crowdsec.ContextToSend[key]
@@ -247,11 +245,11 @@ func NewLapiContextCmd() *cobra.Command {
 			if err := csConfig.LoadCrowdsec(); err != nil {
 				fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", csConfig.Crowdsec.ConsoleContextPath)
 				if err.Error() != fileNotFoundMessage {
-					log.Fatalf("Unable to load CrowdSec Agent: %s", err)
+					return fmt.Errorf("unable to start CrowdSec agent: %w", err)
 				}
 			}
 			if csConfig.DisableAgent {
-				log.Fatalf("Agent is disabled and lapi context can only be used on the agent")
+				return errors.New("agent is disabled and lapi context can only be used on the agent")
 			}
 
 			return nil
@@ -271,12 +269,21 @@ cscli lapi context add --key file_source --value evt.Line.Src
 cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user 
 		`,
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
+			hub, err := require.Hub(csConfig, nil)
+			if err != nil {
+				return err
+			}
+
+			if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
+				return fmt.Errorf("while loading context: %w", err)
+			}
+
 			if keyToAdd != "" {
 				if err := AddContext(keyToAdd, valuesToAdd); err != nil {
-					log.Fatalf(err.Error())
+					return err
 				}
-				return
+				return nil
 			}
 
 			for _, v := range valuesToAdd {
@@ -284,9 +291,11 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
 				key := keySlice[len(keySlice)-1]
 				value := []string{v}
 				if err := AddContext(key, value); err != nil {
-					log.Fatalf(err.Error())
+					return err
 				}
 			}
+
+			return nil
 		},
 	}
 	cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
@@ -298,19 +307,29 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
 		Use:               "status",
 		Short:             "List context to send with alerts",
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
+			hub, err := require.Hub(csConfig, nil)
+			if err != nil {
+				return err
+			}
+
+			if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
+				return fmt.Errorf("while loading context: %w", err)
+			}
+
 			if len(csConfig.Crowdsec.ContextToSend) == 0 {
 				fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
-				return
+				return nil
 			}
 
 			dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend)
 			if err != nil {
-				log.Fatalf("unable to show context status: %s", err)
+				return fmt.Errorf("unable to show context status: %w", err)
 			}
 
-			fmt.Println(string(dump))
+			fmt.Print(string(dump))
 
+			return nil
 		},
 	}
 	cmdContext.AddCommand(cmdContextStatus)
@@ -323,9 +342,7 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
 cscli lapi context detect crowdsecurity/sshd-logs
 		`,
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			var err error
-
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if !detectAll && len(args) == 0 {
 				log.Infof("Please provide parsers to detect or --all flag.")
 				printHelp(cmd)
@@ -334,19 +351,18 @@ cscli lapi context detect crowdsecurity/sshd-logs
 			// to avoid all the log.Info from the loaders functions
 			log.SetLevel(log.WarnLevel)
 
-			err = exprhelpers.Init(nil)
-			if err != nil {
-				log.Fatalf("Failed to init expr helpers : %s", err)
+			if err := exprhelpers.Init(nil); err != nil {
+				return fmt.Errorf("failed to init expr helpers: %w", err)
 			}
 
 			hub, err := require.Hub(csConfig, nil)
 			if err != nil {
-				log.Fatal(err)
+				return err
 			}
 
 			csParsers := parser.NewParsers(hub)
 			if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
-				log.Fatalf("unable to load parsers: %s", err)
+				return fmt.Errorf("unable to load parsers: %w", err)
 			}
 
 			fieldByParsers := make(map[string][]string)
@@ -366,7 +382,6 @@ cscli lapi context detect crowdsecurity/sshd-logs
 						fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field)
 					}
 				}
-
 			}
 
 			fmt.Printf("Acquisition :\n\n")
@@ -399,59 +414,17 @@ cscli lapi context detect crowdsecurity/sshd-logs
 					log.Errorf("parser '%s' not found, can't detect fields", parserNotFound)
 				}
 			}
+
+			return nil
 		},
 	}
 	cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
 	cmdContext.AddCommand(cmdContextDetect)
 
-	var keysToDelete []string
-	var valuesToDelete []string
 	cmdContextDelete := &cobra.Command{
 		Use:   "delete",
-		Short: "Delete context to send with alerts",
-		Example: `cscli lapi context delete --key source_ip
-cscli lapi context delete --value evt.Line.Src
-		`,
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			if len(keysToDelete) == 0 && len(valuesToDelete) == 0 {
-				log.Fatalf("please provide at least a key or a value to delete")
-			}
-
-			for _, key := range keysToDelete {
-				if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok {
-					delete(csConfig.Crowdsec.ContextToSend, key)
-					log.Infof("key '%s' has been removed", key)
-				} else {
-					log.Warningf("key '%s' doesn't exist", key)
-				}
-			}
-
-			for _, value := range valuesToDelete {
-				valueFound := false
-				for key, context := range csConfig.Crowdsec.ContextToSend {
-					if slices.Contains(context, value) {
-						valueFound = true
-						csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, context)
-						log.Infof("value '%s' has been removed from key '%s'", value, key)
-					}
-					if len(csConfig.Crowdsec.ContextToSend[key]) == 0 {
-						delete(csConfig.Crowdsec.ContextToSend, key)
-					}
-				}
-				if !valueFound {
-					log.Warningf("value '%s' not found", value)
-				}
-			}
-
-			if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
-				log.Fatalf(err.Error())
-			}
-
-		},
+		Deprecated: "please manually edit the context file.",
 	}
-	cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete")
-	cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete")
 	cmdContext.AddCommand(cmdContextDelete)
 
 	return cmdContext
@@ -459,6 +432,7 @@ cscli lapi context delete --value evt.Line.Src
 
 func detectStaticField(GrokStatics []parser.ExtraField) []string {
 	ret := make([]string, 0)
+
 	for _, static := range GrokStatics {
 		if static.Parsed != "" {
 			fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed)
@@ -487,7 +461,8 @@ func detectStaticField(GrokStatics []parser.ExtraField) []string {
 }
 
 func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
-	var ret = make([]string, 0)
+	ret := make([]string, 0)
+
 	if node.Grok.RunTimeRegexp != nil {
 		for _, capturedField := range node.Grok.RunTimeRegexp.Names() {
 			fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)

cmd/crowdsec-cli/main.go

@@ -6,12 +6,13 @@ import (
 	"path/filepath"
 	"strings"
 
+	"slices"
+
 	"github.com/fatih/color"
 	cc "github.com/ivanpirog/coloredcobra"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/cobra/doc"
-	"slices"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
@@ -92,7 +93,7 @@ func initConfig() {
 }
 
 var validArgs = []string{
-	"scenarios", "parsers", "collections", "capi", "lapi", "postoverflows", "machines",
+	"scenarios", "parsers", "collections", "capi", "contexts", "lapi", "postoverflows", "machines",
 	"metrics", "bouncers", "alerts", "decisions", "simulation", "hub", "dashboard",
 	"config", "completion", "version", "console", "notifications", "support",
 }
@@ -240,10 +241,14 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	rootCmd.AddCommand(NewHubTestCmd())
 	rootCmd.AddCommand(NewNotificationsCmd())
 	rootCmd.AddCommand(NewSupportCmd())
-	rootCmd.AddCommand(NewItemsCmd("collections"))
-	rootCmd.AddCommand(NewItemsCmd("parsers"))
-	rootCmd.AddCommand(NewItemsCmd("scenarios"))
-	rootCmd.AddCommand(NewItemsCmd("postoverflows"))
+
+	rootCmd.AddCommand(NewCollectionCLI().NewCommand())
+	rootCmd.AddCommand(NewParserCLI().NewCommand())
+	rootCmd.AddCommand(NewScenarioCLI().NewCommand())
+	rootCmd.AddCommand(NewPostOverflowCLI().NewCommand())
+	rootCmd.AddCommand(NewContextCLI().NewCommand())
+	rootCmd.AddCommand(NewAppsecConfigCLI().NewCommand())
+	rootCmd.AddCommand(NewAppsecRuleCLI().NewCommand())
 
 	if fflag.CscliSetup.IsEnabled() {
 		rootCmd.AddCommand(NewSetupCmd())

cmd/crowdsec-cli/metrics.go

@@ -63,6 +63,8 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
 	lapi_machine_stats := map[string]map[string]map[string]int{}
 	lapi_bouncer_stats := map[string]map[string]map[string]int{}
 	decisions_stats := map[string]map[string]map[string]int{}
+	appsec_engine_stats := map[string]map[string]int{}
+	appsec_rule_stats := map[string]map[string]map[string]int{}
 	alerts_stats := map[string]int{}
 	stash_stats := map[string]struct {
 		Type  string
@@ -226,10 +228,30 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
 					Type  string
 					Count int
 				}{Type: mtype, Count: ival}
+			case "cs_appsec_reqs_total":
+				if _, ok := appsec_engine_stats[metric.Labels["appsec_engine"]]; !ok {
+					appsec_engine_stats[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
+				}
+				appsec_engine_stats[metric.Labels["appsec_engine"]]["processed"] = ival
+			case "cs_appsec_block_total":
+				if _, ok := appsec_engine_stats[metric.Labels["appsec_engine"]]; !ok {
+					appsec_engine_stats[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
+				}
+				appsec_engine_stats[metric.Labels["appsec_engine"]]["blocked"] = ival
+			case "cs_appsec_rule_hits":
+				appsecEngine := metric.Labels["appsec_engine"]
+				ruleID := metric.Labels["rule_name"]
+				if _, ok := appsec_rule_stats[appsecEngine]; !ok {
+					appsec_rule_stats[appsecEngine] = make(map[string]map[string]int, 0)
+				}
+				if _, ok := appsec_rule_stats[appsecEngine][ruleID]; !ok {
+					appsec_rule_stats[appsecEngine][ruleID] = make(map[string]int, 0)
+				}
+				appsec_rule_stats[appsecEngine][ruleID]["triggered"] = ival
 			default:
+				log.Debugf("unknown: %+v", fam.Name)
 				continue
 			}
-
 		}
 	}
 
@@ -244,6 +266,8 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
 		decisionStatsTable(out, decisions_stats)
 		alertStatsTable(out, alerts_stats)
 		stashStatsTable(out, stash_stats)
+		appsecMetricsToTable(out, appsec_engine_stats)
+		appsecRulesToTable(out, appsec_rule_stats)
 		return nil
 	}
 
@@ -282,7 +306,6 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
 
 var noUnit bool
 
-
 func runMetrics(cmd *cobra.Command, args []string) error {
 	flags := cmd.Flags()
 
@@ -314,7 +337,6 @@ func runMetrics(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-
 func NewMetricsCmd() *cobra.Command {
 	cmdMetrics := &cobra.Command{
 		Use:               "metrics",
@@ -322,7 +344,7 @@ func NewMetricsCmd() *cobra.Command {
 		Long:              `Fetch metrics from the prometheus server and display them in a human-friendly way`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: runMetrics,
+		RunE:              runMetrics,
 	}
 
 	flags := cmdMetrics.PersistentFlags()

cmd/crowdsec-cli/metrics_table.go

@@ -90,7 +90,7 @@ func bucketStatsTable(out io.Writer, stats map[string]map[string]int) {
 	keys := []string{"curr_count", "overflow", "instantiation", "pour", "underflow"}
 
 	if numRows, err := metricsToTable(t, stats, keys); err != nil {
-		log.Warningf("while collecting acquis stats: %s", err)
+		log.Warningf("while collecting bucket stats: %s", err)
 	} else if numRows > 0 {
 		renderTableTitle(out, "\nBucket Metrics:")
 		t.Render()
@@ -113,6 +113,37 @@ func acquisStatsTable(out io.Writer, stats map[string]map[string]int) {
 	}
 }
 
+func appsecMetricsToTable(out io.Writer, metrics map[string]map[string]int) {
+	t := newTable(out)
+	t.SetRowLines(false)
+	t.SetHeaders("Appsec Engine", "Processed", "Blocked")
+	t.SetAlignment(table.AlignLeft, table.AlignLeft)
+	keys := []string{"processed", "blocked"}
+	if numRows, err := metricsToTable(t, metrics, keys); err != nil {
+		log.Warningf("while collecting appsec stats: %s", err)
+	} else if numRows > 0 {
+		renderTableTitle(out, "\nAppsec Metrics:")
+		t.Render()
+	}
+}
+
+func appsecRulesToTable(out io.Writer, metrics map[string]map[string]map[string]int) {
+	for appsecEngine, appsecEngineRulesStats := range metrics {
+		t := newTable(out)
+		t.SetRowLines(false)
+		t.SetHeaders("Rule ID", "Triggered")
+		t.SetAlignment(table.AlignLeft, table.AlignLeft)
+		keys := []string{"triggered"}
+		if numRows, err := metricsToTable(t, appsecEngineRulesStats, keys); err != nil {
+			log.Warningf("while collecting appsec rules stats: %s", err)
+		} else if numRows > 0 {
+			renderTableTitle(out, fmt.Sprintf("\nAppsec '%s' Rules Metrics:", appsecEngine))
+			t.Render()
+		}
+	}
+
+}
+
 func parserStatsTable(out io.Writer, stats map[string]map[string]int) {
 	t := newTable(out)
 	t.SetRowLines(false)
@@ -122,7 +153,7 @@ func parserStatsTable(out io.Writer, stats map[string]map[string]int) {
 	keys := []string{"hits", "parsed", "unparsed"}
 
 	if numRows, err := metricsToTable(t, stats, keys); err != nil {
-		log.Warningf("while collecting acquis stats: %s", err)
+		log.Warningf("while collecting parsers stats: %s", err)
 	} else if numRows > 0 {
 		renderTableTitle(out, "\nParser Metrics:")
 		t.Render()

cmd/crowdsec-cli/support.go

@@ -37,6 +37,7 @@ const (
 	SUPPORT_OS_INFO_PATH                 = "osinfo.txt"
 	SUPPORT_PARSERS_PATH                 = "hub/parsers.txt"
 	SUPPORT_SCENARIOS_PATH               = "hub/scenarios.txt"
+	SUPPORT_CONTEXTS_PATH                = "hub/scenarios.txt"
 	SUPPORT_COLLECTIONS_PATH             = "hub/collections.txt"
 	SUPPORT_POSTOVERFLOWS_PATH           = "hub/postoverflows.txt"
 	SUPPORT_BOUNCERS_PATH                = "lapi/bouncers.txt"
@@ -260,6 +261,7 @@ func NewSupportCmd() *cobra.Command {
 - Installed parsers list
 - Installed scenarios list
 - Installed postoverflows list
+- Installed context list
 - Bouncers list
 - Machines list
 - CAPI status
@@ -309,6 +311,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
 				infos[SUPPORT_SCENARIOS_PATH] = []byte(err.Error())
 				infos[SUPPORT_POSTOVERFLOWS_PATH] = []byte(err.Error())
+				infos[SUPPORT_CONTEXTS_PATH] = []byte(err.Error())
 				infos[SUPPORT_COLLECTIONS_PATH] = []byte(err.Error())
 			}
 
@@ -344,6 +347,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
 				infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
 				infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
+				infos[SUPPORT_CONTEXTS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
 				infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
 			}
 

cmd/crowdsec/crowdsec.go

@@ -13,6 +13,8 @@ import (
 	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
+	"github.com/crowdsecurity/crowdsec/pkg/appsec"
+	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@@ -23,6 +25,10 @@ import (
 func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
 	var err error
 
+	if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil {
+		return nil, fmt.Errorf("while loading context: %w", err)
+	}
+
 	// Start loading configs
 	csParsers := parser.NewParsers(hub)
 	if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
@@ -33,9 +39,14 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, er
 		return nil, fmt.Errorf("while loading scenarios: %w", err)
 	}
 
+	if err := appsec.LoadAppsecRules(hub); err != nil {
+		return nil, fmt.Errorf("while loading appsec rules: %w", err)
+	}
+
 	if err := LoadAcquisition(cConfig); err != nil {
 		return nil, fmt.Errorf("while loading acquisition config: %w", err)
 	}
+
 	return csParsers, nil
 }
 

cmd/crowdsec/metrics.go

@@ -161,7 +161,8 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
 			leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow,
 			v1.LapiRouteHits,
 			leaky.BucketsCurrentCount,
-			cache.CacheMetrics, exprhelpers.RegexpCacheMetrics)
+			cache.CacheMetrics, exprhelpers.RegexpCacheMetrics,
+		)
 	} else {
 		log.Infof("Loading prometheus collectors")
 		prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
@@ -170,7 +171,8 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
 			v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions, v1.LapiResponseTime,
 			leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount,
 			globalActiveDecisions, globalAlerts,
-			cache.CacheMetrics, exprhelpers.RegexpCacheMetrics)
+			cache.CacheMetrics, exprhelpers.RegexpCacheMetrics,
+		)
 
 	}
 }

cmd/crowdsec/parse.go

@@ -22,6 +22,13 @@ LOOP:
 			if !event.Process {
 				continue
 			}
+			/*Application security engine is going to generate 2 events:
+			- one that is treated as a log and can go to scenarios
+			- another one that will go directly to LAPI*/
+			if event.Type == types.APPSEC {
+				outputEventChan <- event
+				continue
+			}
 			if event.Line.Module == "" {
 				log.Errorf("empty event.Line.Module field, the acquisition module must set it ! : %+v", event.Line)
 				continue

docker/README.md

@@ -320,10 +320,12 @@ config.yaml) each time the container is run.
 | `PARSERS`               | | Parsers to install, separated by space |
 | `SCENARIOS`             | | Scenarios to install, separated by space |
 | `POSTOVERFLOWS`         | | Postoverflows to install, separated by space |
+| `CONTEXTS`              | | Context files to install, separated by space |
 | `DISABLE_COLLECTIONS`   | | Collections to remove, separated by space: `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` |
 | `DISABLE_PARSERS`       | | Parsers to remove, separated by space |
 | `DISABLE_SCENARIOS`     | | Scenarios to remove, separated by space |
 | `DISABLE_POSTOVERFLOWS` | | Postoverflows to remove, separated by space |
+| `DISABLE_POSTOVERFLOWS` | | Context files to remove, separated by space |
 |                         | | |
 | __Log verbosity__       | | |
 | `LEVEL_INFO`            | false | Force INFO level for the container log |

docker/docker_start.sh

@@ -300,7 +300,7 @@ fi
 
 conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)'
 
-## Install collections, parsers, scenarios & postoverflows
+## Install hub items
 cscli hub update
 
 cscli_if_clean collections upgrade crowdsecurity/linux
@@ -328,6 +328,11 @@ if [ "$POSTOVERFLOWS" != "" ]; then
     cscli_if_clean postoverflows install "$(difference "$POSTOVERFLOWS" "$DISABLE_POSTOVERFLOWS")"
 fi
 
+if [ "$CONTEXTS" != "" ]; then
+    # shellcheck disable=SC2086
+    cscli_if_clean contexts install "$(difference "$CONTEXTS" "$DISABLE_CONTEXTS")"
+fi
+
 ## Remove collections, parsers, scenarios & postoverflows
 if [ "$DISABLE_COLLECTIONS" != "" ]; then
     # shellcheck disable=SC2086
@@ -349,6 +354,11 @@ if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
     cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force
 fi
 
+if [ "$DISABLE_CONTEXTS" != "" ]; then
+    # shellcheck disable=SC2086
+    cscli_if_clean contexts remove "$DISABLE_CONTEXTS" --force
+fi
+
 ## Register bouncers via env
 for BOUNCER in $(compgen -A variable | grep -i BOUNCER_KEY); do
     KEY=$(printf '%s' "${!BOUNCER}")

go.mod

@@ -80,14 +80,19 @@ require (
 	github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
 	github.com/wasilibs/go-re2 v1.3.0
 	github.com/xhit/go-simple-mail/v2 v2.16.0
-	golang.org/x/crypto v0.15.0
+	golang.org/x/crypto v0.16.0
 	golang.org/x/mod v0.11.0
-	golang.org/x/sys v0.14.0
+	golang.org/x/sys v0.15.0
 	google.golang.org/grpc v1.56.3
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
 	gopkg.in/yaml.v2 v2.4.0
+)
+
+require (
+	github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135508-23eef9bf7f39
+	golang.org/x/text v0.14.0
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/v3 v3.5.0
 	k8s.io/apiserver v0.28.4
@@ -103,6 +108,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bytedance/sonic v1.9.1 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
+	github.com/corazawaf/libinjection-go v0.1.2 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/creack/pty v1.1.18 // indirect
@@ -149,7 +155,7 @@ require (
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
-	github.com/magefile/mage v1.14.0 // indirect
+	github.com/magefile/mage v1.15.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
@@ -168,6 +174,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/petar-dambovaliev/aho-corasick v0.0.0-20230725210150-fb29fc3c913e // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
@@ -181,7 +188,9 @@ require (
 	github.com/spf13/cast v1.3.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tetratelabs/wazero v1.2.1 // indirect
-	github.com/tidwall/gjson v1.13.0 // indirect
+	github.com/tidwall/gjson v1.17.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
 	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 // indirect
@@ -192,10 +201,9 @@ require (
 	github.com/zclconf/go-cty v1.8.0 // indirect
 	go.mongodb.org/mongo-driver v1.9.4 // indirect
 	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/net v0.18.0 // indirect
-	golang.org/x/sync v0.2.0 // indirect
-	golang.org/x/term v0.14.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/sync v0.5.0 // indirect
+	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
@@ -207,6 +215,7 @@ require (
 	k8s.io/apimachinery v0.28.4 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
+	rsc.io/binaryregexp v0.2.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )

go.sum

@@ -84,6 +84,8 @@ github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhD
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
 github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
+github.com/corazawaf/libinjection-go v0.1.2 h1:oeiV9pc5rvJ+2oqOqXEAMJousPpGiup6f7Y3nZj5GoM=
+github.com/corazawaf/libinjection-go v0.1.2/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nNZEmJNuWbNPOItn7aw=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
@@ -96,6 +98,14 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
+github.com/crowdsecurity/coraza/v3 v3.0.0-20231114091225-b0f8bc435a75 h1:Kp1sY2PE1H5nbr7xgAQeEWDqDW/o3HNL1rHvcVqzWT4=
+github.com/crowdsecurity/coraza/v3 v3.0.0-20231114091225-b0f8bc435a75/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI=
+github.com/crowdsecurity/coraza/v3 v3.0.0-20231204125126-35deffad7734 h1:THMSMkBW/DLG5NvMAr/Mdg/eQOrEnMJ9Y+UdFG4yV8k=
+github.com/crowdsecurity/coraza/v3 v3.0.0-20231204125126-35deffad7734/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI=
+github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135226-6c45fc2dedf9 h1:vFJiYtKOW5DwGQ9gxQi8+XDNc+YvuXXsJyWXXuiOn+M=
+github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135226-6c45fc2dedf9/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI=
+github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135508-23eef9bf7f39 h1:vY0KZvoS4Xl9IfGucBA4l1CV1auRPPJtjZSTz/Rl6iQ=
+github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135508-23eef9bf7f39/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
 github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8=
@@ -125,6 +135,8 @@ github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkK
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
 github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI=
+github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
@@ -459,8 +471,8 @@ github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffkt
 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
-github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo=
-github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
+github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
+github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -496,6 +508,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
+github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
+github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
@@ -548,6 +562,8 @@ github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUr
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
 github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
+github.com/petar-dambovaliev/aho-corasick v0.0.0-20230725210150-fb29fc3c913e h1:POJco99aNgosh92lGqmx7L1ei+kCymivB/419SD15PQ=
+github.com/petar-dambovaliev/aho-corasick v0.0.0-20230725210150-fb29fc3c913e/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
 github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
@@ -652,13 +668,14 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
 github.com/tetratelabs/wazero v1.2.1 h1:J4X2hrGzJvt+wqltuvcSjHQ7ujQxA9gb6PeMs4qlUWs=
 github.com/tetratelabs/wazero v1.2.1/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ=
 github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M=
-github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
+github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
-github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
 github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
 github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
@@ -748,6 +765,8 @@ golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
 golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
+golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
+golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
@@ -783,6 +802,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
 golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -792,8 +813,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
-golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
+golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -835,6 +856,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -844,6 +867,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
 golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
+golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -943,6 +968,8 @@ k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
 k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk=
 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=

pkg/acquisition/acquisition.go

@@ -18,6 +18,7 @@ import (
 	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
+	appsecacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/appsec"
 	cloudwatchacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/cloudwatch"
 	dockeracquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/docker"
 	fileacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file"
@@ -76,6 +77,7 @@ var AcquisitionSources = map[string]func() DataSource{
 	"k8s-audit":   func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{} },
 	"loki":        func() DataSource { return &lokiacquisition.LokiSource{} },
 	"s3":          func() DataSource { return &s3acquisition.S3Source{} },
+	"appsec":      func() DataSource { return &appsecacquisition.AppsecSource{} },
 }
 
 var transformRuntimes = map[string]*vm.Program{}

pkg/acquisition/modules/appsec/appsec.go

@@ -0,0 +1,371 @@
+package appsecacquisition
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+
+	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
+	"github.com/crowdsecurity/crowdsec/pkg/appsec"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/crowdsecurity/go-cs-lib/trace"
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/tomb.v2"
+	"gopkg.in/yaml.v2"
+)
+
+const (
+	InBand    = "inband"
+	OutOfBand = "outofband"
+)
+
+var (
+	DefaultAuthCacheDuration = (1 * time.Minute)
+)
+
+// configuration structure of the acquis for the application security engine
+type AppsecSourceConfig struct {
+	ListenAddr                        string         `yaml:"listen_addr"`
+	CertFilePath                      string         `yaml:"cert_file"`
+	KeyFilePath                       string         `yaml:"key_file"`
+	Path                              string         `yaml:"path"`
+	Routines                          int            `yaml:"routines"`
+	AppsecConfig                      string         `yaml:"appsec_config"`
+	AppsecConfigPath                  string         `yaml:"appsec_config_path"`
+	AuthCacheDuration                 *time.Duration `yaml:"auth_cache_duration"`
+	configuration.DataSourceCommonCfg `yaml:",inline"`
+}
+
+// runtime structure of AppsecSourceConfig
+type AppsecSource struct {
+	config        AppsecSourceConfig
+	logger        *log.Entry
+	mux           *http.ServeMux
+	server        *http.Server
+	outChan       chan types.Event
+	InChan        chan appsec.ParsedRequest
+	AppsecRuntime *appsec.AppsecRuntimeConfig
+	AppsecConfigs map[string]appsec.AppsecConfig
+	lapiURL       string
+	AuthCache     AuthCache
+	AppsecRunners []AppsecRunner //one for each go-routine
+}
+
+// Struct to handle cache of authentication
+type AuthCache struct {
+	APIKeys map[string]time.Time
+	mu      sync.RWMutex
+}
+
+func NewAuthCache() AuthCache {
+	return AuthCache{
+		APIKeys: make(map[string]time.Time, 0),
+		mu:      sync.RWMutex{},
+	}
+}
+
+func (ac *AuthCache) Set(apiKey string, expiration time.Time) {
+	ac.mu.Lock()
+	ac.APIKeys[apiKey] = expiration
+	ac.mu.Unlock()
+}
+
+func (ac *AuthCache) Get(apiKey string) (time.Time, bool) {
+	ac.mu.RLock()
+	expiration, exists := ac.APIKeys[apiKey]
+	ac.mu.RUnlock()
+	return expiration, exists
+}
+
+// @tko + @sbl : we might want to get rid of that or improve it
+type BodyResponse struct {
+	Action string `json:"action"`
+}
+
+func (w *AppsecSource) UnmarshalConfig(yamlConfig []byte) error {
+
+	err := yaml.UnmarshalStrict(yamlConfig, &w.config)
+	if err != nil {
+		return errors.Wrap(err, "Cannot parse appsec configuration")
+	}
+
+	if w.config.ListenAddr == "" {
+		w.config.ListenAddr = "127.0.0.1:7422"
+	}
+
+	if w.config.Path == "" {
+		w.config.Path = "/"
+	}
+
+	if w.config.Path[0] != '/' {
+		w.config.Path = "/" + w.config.Path
+	}
+
+	if w.config.Mode == "" {
+		w.config.Mode = configuration.TAIL_MODE
+	}
+
+	// always have at least one appsec routine
+	if w.config.Routines == 0 {
+		w.config.Routines = 1
+	}
+
+	if w.config.AppsecConfig == "" && w.config.AppsecConfigPath == "" {
+		return fmt.Errorf("appsec_config or appsec_config_path must be set")
+	}
+
+	if w.config.Name == "" {
+		w.config.Name = fmt.Sprintf("%s%s", w.config.ListenAddr, w.config.Path)
+	}
+
+	csConfig := csconfig.GetConfig()
+	w.lapiURL = fmt.Sprintf("%sv1/decisions/stream", csConfig.API.Client.Credentials.URL)
+	w.AuthCache = NewAuthCache()
+
+	return nil
+}
+
+func (w *AppsecSource) GetMetrics() []prometheus.Collector {
+	return []prometheus.Collector{AppsecReqCounter, AppsecBlockCounter, AppsecRuleHits, AppsecOutbandParsingHistogram, AppsecInbandParsingHistogram, AppsecGlobalParsingHistogram}
+}
+
+func (w *AppsecSource) GetAggregMetrics() []prometheus.Collector {
+	return []prometheus.Collector{AppsecReqCounter, AppsecBlockCounter, AppsecRuleHits, AppsecOutbandParsingHistogram, AppsecInbandParsingHistogram, AppsecGlobalParsingHistogram}
+}
+
+func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry) error {
+	err := w.UnmarshalConfig(yamlConfig)
+	if err != nil {
+		return errors.Wrap(err, "unable to parse appsec configuration")
+	}
+	w.logger = logger
+
+	w.logger.Tracef("Appsec configuration: %+v", w.config)
+
+	if w.config.AuthCacheDuration == nil {
+		w.config.AuthCacheDuration = &DefaultAuthCacheDuration
+		w.logger.Infof("Cache duration for auth not set, using default: %v", *w.config.AuthCacheDuration)
+	}
+
+	w.mux = http.NewServeMux()
+
+	w.server = &http.Server{
+		Addr:    w.config.ListenAddr,
+		Handler: w.mux,
+	}
+
+	w.InChan = make(chan appsec.ParsedRequest)
+	appsecCfg := appsec.AppsecConfig{Logger: w.logger.WithField("component", "appsec_config")}
+
+	//let's load the associated appsec_config:
+	if w.config.AppsecConfigPath != "" {
+		err := appsecCfg.LoadByPath(w.config.AppsecConfigPath)
+		if err != nil {
+			return fmt.Errorf("unable to load appsec_config : %s", err)
+		}
+	} else if w.config.AppsecConfig != "" {
+		err := appsecCfg.Load(w.config.AppsecConfig)
+		if err != nil {
+			return fmt.Errorf("unable to load appsec_config : %s", err)
+		}
+	} else {
+		return fmt.Errorf("no appsec_config provided")
+	}
+
+	w.AppsecRuntime, err = appsecCfg.Build()
+	if err != nil {
+		return fmt.Errorf("unable to build appsec_config : %s", err)
+	}
+
+	err = w.AppsecRuntime.ProcessOnLoadRules()
+
+	if err != nil {
+		return fmt.Errorf("unable to process on load rules : %s", err)
+	}
+
+	w.AppsecRunners = make([]AppsecRunner, w.config.Routines)
+
+	for nbRoutine := 0; nbRoutine < w.config.Routines; nbRoutine++ {
+		appsecRunnerUUID := uuid.New().String()
+		//we copy AppsecRutime for each runner
+		wrt := *w.AppsecRuntime
+		wrt.Logger = w.logger.Dup().WithField("runner_uuid", appsecRunnerUUID)
+		runner := AppsecRunner{
+			inChan: w.InChan,
+			UUID:   appsecRunnerUUID,
+			logger: w.logger.WithFields(log.Fields{
+				"runner_uuid": appsecRunnerUUID,
+			}),
+			AppsecRuntime: &wrt,
+			Labels:        w.config.Labels,
+		}
+		err := runner.Init(appsecCfg.GetDataDir())
+		if err != nil {
+			return fmt.Errorf("unable to initialize runner : %s", err)
+		}
+		w.AppsecRunners[nbRoutine] = runner
+	}
+
+	w.logger.Infof("Created %d appsec runners", len(w.AppsecRunners))
+
+	//We don´t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec
+	w.mux.HandleFunc(w.config.Path, w.appsecHandler)
+	return nil
+}
+
+func (w *AppsecSource) ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uuid string) error {
+	return fmt.Errorf("AppSec datasource does not support command line acquisition")
+}
+
+func (w *AppsecSource) GetMode() string {
+	return w.config.Mode
+}
+
+func (w *AppsecSource) GetName() string {
+	return "appsec"
+}
+
+func (w *AppsecSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error {
+	return fmt.Errorf("AppSec datasource does not support command line acquisition")
+}
+
+func (w *AppsecSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
+	w.outChan = out
+	t.Go(func() error {
+		defer trace.CatchPanic("crowdsec/acquis/appsec/live")
+
+		w.logger.Infof("%d appsec runner to start", len(w.AppsecRunners))
+		for _, runner := range w.AppsecRunners {
+			runner := runner
+			runner.outChan = out
+			t.Go(func() error {
+				defer trace.CatchPanic("crowdsec/acquis/appsec/live/runner")
+				return runner.Run(t)
+			})
+		}
+
+		w.logger.Infof("Starting Appsec server on %s%s", w.config.ListenAddr, w.config.Path)
+		t.Go(func() error {
+			var err error
+			if w.config.CertFilePath != "" && w.config.KeyFilePath != "" {
+				err = w.server.ListenAndServeTLS(w.config.CertFilePath, w.config.KeyFilePath)
+			} else {
+				err = w.server.ListenAndServe()
+			}
+
+			if err != nil && err != http.ErrServerClosed {
+				return errors.Wrap(err, "Appsec server failed")
+			}
+			return nil
+		})
+		<-t.Dying()
+		w.logger.Infof("Stopping Appsec server on %s%s", w.config.ListenAddr, w.config.Path)
+		w.server.Shutdown(context.TODO())
+		return nil
+	})
+	return nil
+}
+
+func (w *AppsecSource) CanRun() error {
+	return nil
+}
+
+func (w *AppsecSource) GetUuid() string {
+	return w.config.UniqueId
+}
+
+func (w *AppsecSource) Dump() interface{} {
+	return w
+}
+
+func (w *AppsecSource) IsAuth(apiKey string) bool {
+	client := &http.Client{
+		Timeout: 200 * time.Millisecond,
+	}
+
+	req, err := http.NewRequest(http.MethodHead, w.lapiURL, nil)
+	if err != nil {
+		log.Errorf("Error creating request: %s", err)
+		return false
+	}
+
+	req.Header.Add("X-Api-Key", apiKey)
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Errorf("Error performing request: %s", err)
+		return false
+	}
+	defer resp.Body.Close()
+
+	return resp.StatusCode == http.StatusOK
+
+}
+
+// should this be in the runner ?
+func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) {
+	w.logger.Debugf("Received request from '%s' on %s", r.RemoteAddr, r.URL.Path)
+
+	apiKey := r.Header.Get(appsec.APIKeyHeaderName)
+	clientIP := r.Header.Get(appsec.IPHeaderName)
+	remoteIP := r.RemoteAddr
+	if apiKey == "" {
+		w.logger.Errorf("Unauthorized request from '%s' (real IP = %s)", remoteIP, clientIP)
+		rw.WriteHeader(http.StatusUnauthorized)
+		return
+	}
+	expiration, exists := w.AuthCache.Get(apiKey)
+	// if the apiKey is not in cache or has expired, just recheck the auth
+	if !exists || time.Now().After(expiration) {
+		if !w.IsAuth(apiKey) {
+			rw.WriteHeader(http.StatusUnauthorized)
+			w.logger.Errorf("Unauthorized request from '%s' (real IP = %s)", remoteIP, clientIP)
+			return
+		}
+
+		// apiKey is valid, store it in cache
+		w.AuthCache.Set(apiKey, time.Now().Add(*w.config.AuthCacheDuration))
+	}
+
+	// parse the request only once
+	parsedRequest, err := appsec.NewParsedRequestFromRequest(r)
+	if err != nil {
+		log.Errorf("%s", err)
+		rw.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	parsedRequest.AppsecEngine = w.config.Name
+
+	logger := w.logger.WithFields(log.Fields{
+		"request_uuid": parsedRequest.UUID,
+		"client_ip":    parsedRequest.ClientIP,
+	})
+
+	AppsecReqCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.AppsecEngine}).Inc()
+
+	w.InChan <- parsedRequest
+
+	response := <-parsedRequest.ResponseChannel
+	if response.InBandInterrupt {
+		AppsecBlockCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.AppsecEngine}).Inc()
+	}
+
+	appsecResponse := w.AppsecRuntime.GenerateResponse(response, logger)
+
+	rw.WriteHeader(appsecResponse.HTTPStatus)
+	body, err := json.Marshal(BodyResponse{Action: appsecResponse.Action})
+	if err != nil {
+		logger.Errorf("unable to marshal response: %s", err)
+		rw.WriteHeader(http.StatusInternalServerError)
+	} else {
+		rw.Write(body)
+	}
+
+}

pkg/acquisition/modules/appsec/appsec_runner.go

@@ -0,0 +1,350 @@
+package appsecacquisition
+
+import (
+	"fmt"
+	"os"
+	"slices"
+	"time"
+
+	"github.com/crowdsecurity/coraza/v3"
+	corazatypes "github.com/crowdsecurity/coraza/v3/types"
+	"github.com/crowdsecurity/crowdsec/pkg/appsec"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/tomb.v2"
+)
+
+// that's the runtime structure of the Application security engine as seen from the acquis
+type AppsecRunner struct {
+	outChan             chan types.Event
+	inChan              chan appsec.ParsedRequest
+	UUID                string
+	AppsecRuntime       *appsec.AppsecRuntimeConfig //this holds the actual appsec runtime config, rules, remediations, hooks etc.
+	AppsecInbandEngine  coraza.WAF
+	AppsecOutbandEngine coraza.WAF
+	Labels              map[string]string
+	logger              *log.Entry
+}
+
+func (r *AppsecRunner) Init(datadir string) error {
+	var err error
+	fs := os.DirFS(datadir)
+
+	inBandRules := ""
+	outOfBandRules := ""
+
+	for _, collection := range r.AppsecRuntime.InBandRules {
+		inBandRules += collection.String()
+	}
+
+	for _, collection := range r.AppsecRuntime.OutOfBandRules {
+		outOfBandRules += collection.String()
+	}
+	inBandLogger := r.logger.Dup().WithField("band", "inband")
+	outBandLogger := r.logger.Dup().WithField("band", "outband")
+
+	//setting up inband engine
+	inbandCfg := coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs).WithDebugLogger(appsec.NewCrzLogger(inBandLogger))
+	if !r.AppsecRuntime.Config.InbandOptions.DisableBodyInspection {
+		inbandCfg = inbandCfg.WithRequestBodyAccess()
+	} else {
+		log.Warningf("Disabling body inspection, Inband rules will not be able to match on body's content.")
+	}
+	if r.AppsecRuntime.Config.InbandOptions.RequestBodyInMemoryLimit != nil {
+		inbandCfg = inbandCfg.WithRequestBodyInMemoryLimit(*r.AppsecRuntime.Config.InbandOptions.RequestBodyInMemoryLimit)
+	}
+	r.AppsecInbandEngine, err = coraza.NewWAF(inbandCfg)
+	if err != nil {
+		return fmt.Errorf("unable to initialize inband engine : %w", err)
+	}
+
+	//setting up outband engine
+	outbandCfg := coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs).WithDebugLogger(appsec.NewCrzLogger(outBandLogger))
+	if !r.AppsecRuntime.Config.OutOfBandOptions.DisableBodyInspection {
+		outbandCfg = outbandCfg.WithRequestBodyAccess()
+	} else {
+		log.Warningf("Disabling body inspection, Out of band rules will not be able to match on body's content.")
+	}
+	if r.AppsecRuntime.Config.OutOfBandOptions.RequestBodyInMemoryLimit != nil {
+		outbandCfg = outbandCfg.WithRequestBodyInMemoryLimit(*r.AppsecRuntime.Config.OutOfBandOptions.RequestBodyInMemoryLimit)
+	}
+	r.AppsecOutbandEngine, err = coraza.NewWAF(outbandCfg)
+
+	if r.AppsecRuntime.DisabledInBandRulesTags != nil {
+		for _, tag := range r.AppsecRuntime.DisabledInBandRulesTags {
+			r.AppsecInbandEngine.GetRuleGroup().DeleteByTag(tag)
+		}
+	}
+
+	if r.AppsecRuntime.DisabledOutOfBandRulesTags != nil {
+		for _, tag := range r.AppsecRuntime.DisabledOutOfBandRulesTags {
+			r.AppsecOutbandEngine.GetRuleGroup().DeleteByTag(tag)
+		}
+	}
+
+	if r.AppsecRuntime.DisabledInBandRuleIds != nil {
+		for _, id := range r.AppsecRuntime.DisabledInBandRuleIds {
+			r.AppsecInbandEngine.GetRuleGroup().DeleteByID(id)
+		}
+	}
+
+	if r.AppsecRuntime.DisabledOutOfBandRuleIds != nil {
+		for _, id := range r.AppsecRuntime.DisabledOutOfBandRuleIds {
+			r.AppsecOutbandEngine.GetRuleGroup().DeleteByID(id)
+		}
+	}
+
+	r.logger.Tracef("Loaded inband rules: %+v", r.AppsecInbandEngine.GetRuleGroup().GetRules())
+	r.logger.Tracef("Loaded outband rules: %+v", r.AppsecOutbandEngine.GetRuleGroup().GetRules())
+
+	if err != nil {
+		return fmt.Errorf("unable to initialize outband engine : %w", err)
+	}
+
+	return nil
+}
+
+func (r *AppsecRunner) processRequest(tx appsec.ExtendedTransaction, request *appsec.ParsedRequest) error {
+	var in *corazatypes.Interruption
+	var err error
+	request.Tx = tx
+
+	if request.Tx.IsRuleEngineOff() {
+		r.logger.Debugf("rule engine is off, skipping")
+		return nil
+	}
+
+	defer func() {
+		request.Tx.ProcessLogging()
+		//We don't close the transaction here, as it will reset coraza internal state and break variable tracking
+	}()
+
+	//pre eval (expr) rules
+	err = r.AppsecRuntime.ProcessPreEvalRules(request)
+	if err != nil {
+		r.logger.Errorf("unable to process PreEval rules: %s", err)
+		//FIXME: should we abort here ?
+	}
+
+	request.Tx.Tx.ProcessConnection(request.RemoteAddr, 0, "", 0)
+
+	for k, v := range request.Args {
+		for _, vv := range v {
+			request.Tx.AddGetRequestArgument(k, vv)
+		}
+	}
+
+	request.Tx.ProcessURI(request.URI, request.Method, request.Proto)
+
+	for k, vr := range request.Headers {
+		for _, v := range vr {
+			request.Tx.AddRequestHeader(k, v)
+		}
+	}
+
+	if request.ClientHost != "" {
+		request.Tx.AddRequestHeader("Host", request.ClientHost)
+		request.Tx.SetServerName(request.ClientHost)
+	}
+
+	if request.TransferEncoding != nil {
+		request.Tx.AddRequestHeader("Transfer-Encoding", request.TransferEncoding[0])
+	}
+
+	in = request.Tx.ProcessRequestHeaders()
+
+	if in != nil {
+		r.logger.Infof("inband rules matched for headers : %s", in.Action)
+		return nil
+	}
+
+	if request.Body != nil && len(request.Body) > 0 {
+		in, _, err = request.Tx.WriteRequestBody(request.Body)
+		if err != nil {
+			r.logger.Errorf("unable to write request body : %s", err)
+			return err
+		}
+		if in != nil {
+			return nil
+		}
+	}
+
+	in, err = request.Tx.ProcessRequestBody()
+
+	if err != nil {
+		r.logger.Errorf("unable to process request body : %s", err)
+		return err
+	}
+
+	if in != nil {
+		r.logger.Debugf("rules matched for body : %d", in.RuleID)
+	}
+
+	err = r.AppsecRuntime.ProcessPostEvalRules(request)
+	if err != nil {
+		r.logger.Errorf("unable to process PostEval rules: %s", err)
+	}
+
+	return nil
+}
+
+func (r *AppsecRunner) ProcessInBandRules(request *appsec.ParsedRequest) error {
+	tx := appsec.NewExtendedTransaction(r.AppsecInbandEngine, request.UUID)
+	r.AppsecRuntime.InBandTx = tx
+	err := r.processRequest(tx, request)
+	return err
+}
+
+func (r *AppsecRunner) ProcessOutOfBandRules(request *appsec.ParsedRequest) error {
+	r.logger.Debugf("Processing out of band rules")
+	tx := appsec.NewExtendedTransaction(r.AppsecOutbandEngine, request.UUID)
+	r.AppsecRuntime.OutOfBandTx = tx
+	err := r.processRequest(tx, request)
+	return err
+}
+
+func (r *AppsecRunner) handleInBandInterrupt(request *appsec.ParsedRequest) {
+	//create the associated event for crowdsec itself
+	evt, err := EventFromRequest(request, r.Labels)
+	if err != nil {
+		//let's not interrupt the pipeline for this
+		r.logger.Errorf("unable to create event from request : %s", err)
+	}
+	err = r.AccumulateTxToEvent(&evt, request)
+	if err != nil {
+		r.logger.Errorf("unable to accumulate tx to event : %s", err)
+	}
+	if in := request.Tx.Interruption(); in != nil {
+		r.logger.Debugf("inband rules matched : %d", in.RuleID)
+		r.AppsecRuntime.Response.InBandInterrupt = true
+		r.AppsecRuntime.Response.HTTPResponseCode = r.AppsecRuntime.Config.BlockedHTTPCode
+		r.AppsecRuntime.Response.Action = r.AppsecRuntime.DefaultRemediation
+
+		if _, ok := r.AppsecRuntime.RemediationById[in.RuleID]; ok {
+			r.AppsecRuntime.Response.Action = r.AppsecRuntime.RemediationById[in.RuleID]
+		}
+
+		for tag, remediation := range r.AppsecRuntime.RemediationByTag {
+			if slices.Contains[[]string, string](in.Tags, tag) {
+				r.AppsecRuntime.Response.Action = remediation
+			}
+		}
+
+		err = r.AppsecRuntime.ProcessOnMatchRules(request, evt)
+		if err != nil {
+			r.logger.Errorf("unable to process OnMatch rules: %s", err)
+			return
+		}
+		// Should the in band match trigger an event ?
+		if r.AppsecRuntime.Response.SendEvent {
+			r.outChan <- evt
+		}
+
+		// Should the in band match trigger an overflow ?
+		if r.AppsecRuntime.Response.SendAlert {
+			appsecOvlfw, err := AppsecEventGeneration(evt)
+			if err != nil {
+				r.logger.Errorf("unable to generate appsec event : %s", err)
+				return
+			}
+			r.outChan <- *appsecOvlfw
+		}
+	}
+}
+
+func (r *AppsecRunner) handleOutBandInterrupt(request *appsec.ParsedRequest) {
+	evt, err := EventFromRequest(request, r.Labels)
+	if err != nil {
+		//let's not interrupt the pipeline for this
+		r.logger.Errorf("unable to create event from request : %s", err)
+	}
+	err = r.AccumulateTxToEvent(&evt, request)
+	if err != nil {
+		r.logger.Errorf("unable to accumulate tx to event : %s", err)
+	}
+	if in := request.Tx.Interruption(); in != nil {
+		r.logger.Debugf("inband rules matched : %d", in.RuleID)
+		r.AppsecRuntime.Response.OutOfBandInterrupt = true
+
+		err = r.AppsecRuntime.ProcessOnMatchRules(request, evt)
+		if err != nil {
+			r.logger.Errorf("unable to process OnMatch rules: %s", err)
+			return
+		}
+		// Should the match trigger an event ?
+		if r.AppsecRuntime.Response.SendEvent {
+			r.outChan <- evt
+		}
+
+		// Should the match trigger an overflow ?
+		if r.AppsecRuntime.Response.SendAlert {
+			appsecOvlfw, err := AppsecEventGeneration(evt)
+			if err != nil {
+				r.logger.Errorf("unable to generate appsec event : %s", err)
+				return
+			}
+			r.outChan <- *appsecOvlfw
+		}
+	}
+}
+
+func (r *AppsecRunner) handleRequest(request *appsec.ParsedRequest) {
+	r.AppsecRuntime.Logger = r.AppsecRuntime.Logger.WithField("request_uuid", request.UUID)
+	logger := r.logger.WithField("request_uuid", request.UUID)
+	logger.Debug("Request received in runner")
+	r.AppsecRuntime.ClearResponse()
+
+	request.IsInBand = true
+	request.IsOutBand = false
+
+	//to measure the time spent in the Application Security Engine
+	startParsing := time.Now()
+
+	//inband appsec rules
+	err := r.ProcessInBandRules(request)
+	if err != nil {
+		logger.Errorf("unable to process InBand rules: %s", err)
+		return
+	}
+
+	if request.Tx.IsInterrupted() {
+		r.handleInBandInterrupt(request)
+	}
+
+	elapsed := time.Since(startParsing)
+	AppsecInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds())
+
+	// send back the result to the HTTP handler for the InBand part
+	request.ResponseChannel <- r.AppsecRuntime.Response
+
+	//Now let's process the out of band rules
+
+	request.IsInBand = false
+	request.IsOutBand = true
+	r.AppsecRuntime.Response.SendAlert = false
+	r.AppsecRuntime.Response.SendEvent = true
+
+	err = r.ProcessOutOfBandRules(request)
+	if err != nil {
+		logger.Errorf("unable to process OutOfBand rules: %s", err)
+		return
+	}
+
+	if request.Tx.IsInterrupted() {
+		r.handleOutBandInterrupt(request)
+	}
+}
+
+func (r *AppsecRunner) Run(t *tomb.Tomb) error {
+	r.logger.Infof("Appsec Runner ready to process event")
+	for {
+		select {
+		case <-t.Dying():
+			r.logger.Infof("Appsec Runner is dying")
+			return nil
+		case request := <-r.inChan:
+			r.handleRequest(&request)
+		}
+	}
+}

pkg/acquisition/modules/appsec/metrics.go

@@ -0,0 +1,54 @@
+package appsecacquisition
+
+import "github.com/prometheus/client_golang/prometheus"
+
+var AppsecGlobalParsingHistogram = prometheus.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Help:    "Time spent processing a request by the Application Security Engine.",
+		Name:    "cs_appsec_parsing_time_seconds",
+		Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01},
+	},
+	[]string{"source"},
+)
+
+var AppsecInbandParsingHistogram = prometheus.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Help:    "Time spent processing a request by the inband Application Security Engine.",
+		Name:    "cs_appsec_inband_parsing_time_seconds",
+		Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01},
+	},
+	[]string{"source"},
+)
+
+var AppsecOutbandParsingHistogram = prometheus.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Help:    "Time spent processing a request by the Application Security Engine.",
+		Name:    "cs_appsec_outband_parsing_time_seconds",
+		Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01},
+	},
+	[]string{"source"},
+)
+
+var AppsecReqCounter = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "cs_appsec_reqs_total",
+		Help: "Total events processed by the Application Security Engine.",
+	},
+	[]string{"source", "appsec_engine"},
+)
+
+var AppsecBlockCounter = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "cs_appsec_block_total",
+		Help: "Total events blocked by the Application Security Engine.",
+	},
+	[]string{"source", "appsec_engine"},
+)
+
+var AppsecRuleHits = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "cs_appsec_rule_hits",
+		Help: "Count of triggered rule, by rule_name, type (inband/outofband), appsec_engine and source",
+	},
+	[]string{"rule_name", "type", "appsec_engine", "source"},
+)

pkg/acquisition/modules/appsec/rx_operator.go

@@ -0,0 +1,94 @@
+package appsecacquisition
+
+import (
+	"fmt"
+	"strconv"
+	"unicode/utf8"
+
+	"github.com/crowdsecurity/coraza/v3/experimental/plugins"
+	"github.com/crowdsecurity/coraza/v3/experimental/plugins/plugintypes"
+	"github.com/wasilibs/go-re2"
+	"github.com/wasilibs/go-re2/experimental"
+)
+
+type rx struct {
+	re *re2.Regexp
+}
+
+var _ plugintypes.Operator = (*rx)(nil)
+
+func newRX(options plugintypes.OperatorOptions) (plugintypes.Operator, error) {
+	// (?sm) enables multiline mode which makes 942522-7 work, see
+	// - https://stackoverflow.com/a/27680233
+	// - https://groups.google.com/g/golang-nuts/c/jiVdamGFU9E
+	data := fmt.Sprintf("(?sm)%s", options.Arguments)
+
+	var re *re2.Regexp
+	var err error
+
+	if matchesArbitraryBytes(data) {
+		re, err = experimental.CompileLatin1(data)
+	} else {
+		re, err = re2.Compile(data)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return &rx{re: re}, nil
+}
+
+func (o *rx) Evaluate(tx plugintypes.TransactionState, value string) bool {
+	if tx.Capturing() {
+		match := o.re.FindStringSubmatch(value)
+		if len(match) == 0 {
+			return false
+		}
+		for i, c := range match {
+			if i == 9 {
+				return true
+			}
+			tx.CaptureField(i, c)
+		}
+		return true
+	} else {
+		return o.re.MatchString(value)
+	}
+}
+
+// RegisterRX registers the rx operator using a WASI implementation instead of Go.
+func RegisterRX() {
+	plugins.RegisterOperator("rx", newRX)
+}
+
+// matchesArbitraryBytes checks for control sequences for byte matches in the expression.
+// If the sequences are not valid utf8, it returns true.
+func matchesArbitraryBytes(expr string) bool {
+	decoded := make([]byte, 0, len(expr))
+	for i := 0; i < len(expr); i++ {
+		c := expr[i]
+		if c != '\\' {
+			decoded = append(decoded, c)
+			continue
+		}
+		if i+3 >= len(expr) {
+			decoded = append(decoded, expr[i:]...)
+			break
+		}
+		if expr[i+1] != 'x' {
+			decoded = append(decoded, expr[i])
+			continue
+		}
+
+		v, mb, _, err := strconv.UnquoteChar(expr[i:], 0)
+		if err != nil || mb {
+			// Wasn't a byte escape sequence, shouldn't happen in practice.
+			decoded = append(decoded, expr[i])
+			continue
+		}
+
+		decoded = append(decoded, byte(v))
+		i += 3
+	}
+
+	return !utf8.Valid(decoded)
+}

pkg/acquisition/modules/appsec/utils.go

@@ -0,0 +1,278 @@
+package appsecacquisition
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/crowdsecurity/coraza/v3/collection"
+	"github.com/crowdsecurity/coraza/v3/types/variables"
+	"github.com/crowdsecurity/crowdsec/pkg/appsec"
+	"github.com/crowdsecurity/crowdsec/pkg/models"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/crowdsecurity/go-cs-lib/ptr"
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+)
+
+func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) {
+	//if the request didnd't trigger inband rules, we don't want to generate an event to LAPI/CAPI
+	if !inEvt.Appsec.HasInBandMatches {
+		return nil, nil
+	}
+	evt := types.Event{}
+	evt.Type = types.APPSEC
+	evt.Process = true
+	source := models.Source{
+		Value: ptr.Of(inEvt.Parsed["source_ip"]),
+		IP:    inEvt.Parsed["source_ip"],
+		Scope: ptr.Of(types.Ip),
+	}
+
+	evt.Overflow.Sources = make(map[string]models.Source)
+	evt.Overflow.Sources["ip"] = source
+
+	alert := models.Alert{}
+	alert.Capacity = ptr.Of(int32(1))
+	alert.Events = make([]*models.Event, 0)
+	alert.Meta = make(models.Meta, 0)
+	for _, key := range []string{"target_uri", "method"} {
+
+		valueByte, err := json.Marshal([]string{inEvt.Parsed[key]})
+		if err != nil {
+			log.Debugf("unable to serialize key %s", key)
+			continue
+		}
+
+		meta := models.MetaItems0{
+			Key:   key,
+			Value: string(valueByte),
+		}
+		alert.Meta = append(alert.Meta, &meta)
+	}
+	matchedZones := inEvt.Appsec.GetMatchedZones()
+	if matchedZones != nil {
+		valueByte, err := json.Marshal(matchedZones)
+		if err != nil {
+			log.Debugf("unable to serialize key matched_zones")
+		} else {
+			meta := models.MetaItems0{
+				Key:   "matched_zones",
+				Value: string(valueByte),
+			}
+			alert.Meta = append(alert.Meta, &meta)
+		}
+	}
+	for _, key := range evt.Appsec.MatchedRules.GetMatchedZones() {
+		valueByte, err := json.Marshal([]string{key})
+		if err != nil {
+			log.Debugf("unable to serialize key %s", key)
+			continue
+		}
+		meta := models.MetaItems0{
+			Key:   "matched_zones",
+			Value: string(valueByte),
+		}
+		alert.Meta = append(alert.Meta, &meta)
+	}
+	alert.EventsCount = ptr.Of(int32(1))
+	alert.Leakspeed = ptr.Of("")
+	alert.Scenario = ptr.Of(inEvt.Appsec.MatchedRules.GetName())
+	alert.ScenarioHash = ptr.Of(inEvt.Appsec.MatchedRules.GetHash())
+	alert.ScenarioVersion = ptr.Of(inEvt.Appsec.MatchedRules.GetVersion())
+	alert.Simulated = ptr.Of(false)
+	alert.Source = &source
+	msg := fmt.Sprintf("AppSec block: %s from %s (%s)", inEvt.Appsec.MatchedRules.GetName(),
+		alert.Source.IP, inEvt.Parsed["remediation_cmpt_ip"])
+	alert.Message = &msg
+	alert.StartAt = ptr.Of(time.Now().UTC().Format(time.RFC3339))
+	alert.StopAt = ptr.Of(time.Now().UTC().Format(time.RFC3339))
+	evt.Overflow.APIAlerts = []models.Alert{alert}
+	evt.Overflow.Alert = &alert
+	return &evt, nil
+}
+
+func EventFromRequest(r *appsec.ParsedRequest, labels map[string]string) (types.Event, error) {
+	evt := types.Event{}
+	//we might want to change this based on in-band vs out-of-band ?
+	evt.Type = types.LOG
+	evt.ExpectMode = types.LIVE
+	//def needs fixing
+	evt.Stage = "s00-raw"
+	evt.Parsed = map[string]string{
+		"source_ip":           r.ClientIP,
+		"target_host":         r.Host,
+		"target_uri":          r.URI,
+		"method":              r.Method,
+		"req_uuid":            r.Tx.ID(),
+		"source":              "crowdsec-appsec",
+		"remediation_cmpt_ip": r.RemoteAddrNormalized,
+		//TBD:
+		//http_status
+		//user_agent
+
+	}
+	evt.Line = types.Line{
+		Time: time.Now(),
+		//should we add some info like listen addr/port/path ?
+		Labels:  labels,
+		Process: true,
+		Module:  "appsec",
+		Src:     "appsec",
+		Raw:     "dummy-appsec-data", //we discard empty Line.Raw items :)
+	}
+	evt.Appsec = types.AppsecEvent{}
+
+	return evt, nil
+}
+
+func LogAppsecEvent(evt *types.Event, logger *log.Entry) {
+	req := evt.Parsed["target_uri"]
+	if len(req) > 12 {
+		req = req[:10] + ".."
+	}
+
+	if evt.Meta["appsec_interrupted"] == "true" {
+		logger.WithFields(log.Fields{
+			"module":     "appsec",
+			"source":     evt.Parsed["source_ip"],
+			"target_uri": req,
+		}).Infof("%s blocked on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Appsec.MatchedRules), evt.Appsec.GetRuleIDs())
+	} else if evt.Parsed["outofband_interrupted"] == "true" {
+		logger.WithFields(log.Fields{
+			"module":     "appsec",
+			"source":     evt.Parsed["source_ip"],
+			"target_uri": req,
+		}).Infof("%s out-of-band blocking rules on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Appsec.MatchedRules), evt.Appsec.GetRuleIDs())
+	} else {
+		logger.WithFields(log.Fields{
+			"module":     "appsec",
+			"source":     evt.Parsed["source_ip"],
+			"target_uri": req,
+		}).Debugf("%s triggered non-blocking rules on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Appsec.MatchedRules), evt.Appsec.GetRuleIDs())
+	}
+
+}
+
+func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedRequest) error {
+
+	if evt == nil {
+		//an error was already emitted, let's not spam the logs
+		return nil
+	}
+
+	if !req.Tx.IsInterrupted() {
+		//if the phase didn't generate an interruption, we don't have anything to add to the event
+		return nil
+	}
+	//if one interruption was generated, event is good for processing :)
+	evt.Process = true
+
+	if evt.Meta == nil {
+		evt.Meta = map[string]string{}
+	}
+	if evt.Parsed == nil {
+		evt.Parsed = map[string]string{}
+	}
+	if req.IsInBand {
+		evt.Meta["appsec_interrupted"] = "true"
+		evt.Meta["appsec_action"] = req.Tx.Interruption().Action
+		evt.Parsed["inband_interrupted"] = "true"
+		evt.Parsed["inband_action"] = req.Tx.Interruption().Action
+	} else {
+		evt.Parsed["outofband_interrupted"] = "true"
+		evt.Parsed["outofband_action"] = req.Tx.Interruption().Action
+	}
+
+	if evt.Appsec.Vars == nil {
+		evt.Appsec.Vars = map[string]string{}
+	}
+
+	req.Tx.Variables().All(func(v variables.RuleVariable, col collection.Collection) bool {
+		for _, variable := range col.FindAll() {
+			key := ""
+			if variable.Key() == "" {
+				key = variable.Variable().Name()
+			} else {
+				key = variable.Variable().Name() + "." + variable.Key()
+			}
+			if variable.Value() == "" {
+				continue
+			}
+			for _, collectionToKeep := range r.AppsecRuntime.CompiledVariablesTracking {
+				match := collectionToKeep.MatchString(key)
+				if match {
+					evt.Appsec.Vars[key] = variable.Value()
+					r.logger.Debugf("%s.%s = %s", variable.Variable().Name(), variable.Key(), variable.Value())
+				} else {
+					r.logger.Debugf("%s.%s != %s (%s) (not kept)", variable.Variable().Name(), variable.Key(), collectionToKeep, variable.Value())
+				}
+			}
+		}
+		return true
+	})
+
+	for _, rule := range req.Tx.MatchedRules() {
+		if rule.Message() == "" {
+			r.logger.Tracef("discarding rule %d", rule.Rule().ID())
+			continue
+		}
+		kind := "outofband"
+		if req.IsInBand {
+			kind = "inband"
+			evt.Appsec.HasInBandMatches = true
+		} else {
+			evt.Appsec.HasOutBandMatches = true
+		}
+
+		name := "NOT_SET"
+		version := "NOT_SET"
+		hash := "NOT_SET"
+		ruleNameProm := fmt.Sprintf("%d", rule.Rule().ID())
+
+		if details, ok := appsec.AppsecRulesDetails[rule.Rule().ID()]; ok {
+			//Only set them for custom rules, not for rules written in seclang
+			name = details.Name
+			version = details.Version
+			hash = details.Hash
+			ruleNameProm = details.Name
+			r.logger.Debugf("custom rule for event, setting name: %s, version: %s, hash: %s", name, version, hash)
+		}
+
+		AppsecRuleHits.With(prometheus.Labels{"rule_name": ruleNameProm, "type": kind, "source": req.RemoteAddrNormalized, "appsec_engine": req.AppsecEngine}).Inc()
+
+		matchedZones := make([]string, 0)
+		for _, matchData := range rule.MatchedDatas() {
+			zone := matchData.Variable().Name()
+			varName := matchData.Key()
+			if varName != "" {
+				zone += "." + varName
+			}
+			matchedZones = append(matchedZones, zone)
+		}
+
+		corazaRule := map[string]interface{}{
+			"id":            rule.Rule().ID(),
+			"uri":           evt.Parsed["uri"],
+			"rule_type":     kind,
+			"method":        evt.Parsed["method"],
+			"disruptive":    rule.Disruptive(),
+			"tags":          rule.Rule().Tags(),
+			"file":          rule.Rule().File(),
+			"file_line":     rule.Rule().Line(),
+			"revision":      rule.Rule().Revision(),
+			"secmark":       rule.Rule().SecMark(),
+			"accuracy":      rule.Rule().Accuracy(),
+			"msg":           rule.Message(),
+			"severity":      rule.Rule().Severity().String(),
+			"name":          name,
+			"hash":          hash,
+			"version":       version,
+			"matched_zones": matchedZones,
+		}
+		evt.Appsec.MatchedRules = append(evt.Appsec.MatchedRules, corazaRule)
+	}
+
+	return nil
+
+}

pkg/alertcontext/alertcontext.go

@@ -63,13 +63,21 @@ func NewAlertContext(contextToSend map[string][]string, valueLength int) error {
 	}
 
 	for key, values := range contextToSend {
-		alertContext.ContextToSendCompiled[key] = make([]*vm.Program, 0)
+		if _, ok := alertContext.ContextToSend[key]; !ok {
+			alertContext.ContextToSend[key] = make([]string, 0)
+		}
+
+		if _, ok := alertContext.ContextToSendCompiled[key]; !ok {
+			alertContext.ContextToSendCompiled[key] = make([]*vm.Program, 0)
+		}
+
 		for _, value := range values {
 			valueCompiled, err := expr.Compile(value, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
 			if err != nil {
 				return fmt.Errorf("compilation of '%s' context value failed: %v", value, err)
 			}
 			alertContext.ContextToSendCompiled[key] = append(alertContext.ContextToSendCompiled[key], valueCompiled)
+			alertContext.ContextToSend[key] = append(alertContext.ContextToSend[key], value)
 		}
 	}
 

pkg/alertcontext/config.go

@@ -0,0 +1,125 @@
+package alertcontext
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v3"
+
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+// this file is here to avoid circular dependencies between the configuration and the hub
+
+// HubItemWrapper is a wrapper around a hub item to unmarshal only the context part
+// because there are other fields like name etc.
+type HubItemWrapper struct {
+	Context map[string][]string `yaml:"context"`
+}
+
+// mergeContext adds the context from src to dest.
+func mergeContext(dest map[string][]string, src map[string][]string) {
+	for k, v := range src {
+		if _, ok := dest[k]; !ok {
+			dest[k] = make([]string, 0)
+		}
+		for _, s := range v {
+			if !slices.Contains(dest[k], s) {
+				dest[k] = append(dest[k], s)
+			}
+		}
+	}
+}
+
+// addContextFromItem merges the context from an item into the context to send to the console.
+func addContextFromItem(toSend map[string][]string, item *cwhub.Item) error {
+	filePath := item.State.LocalPath
+	log.Tracef("loading console context from %s", filePath)
+	content, err := os.ReadFile(filePath)
+	if err != nil {
+		return err
+	}
+
+	wrapper := &HubItemWrapper{}
+
+	err = yaml.Unmarshal(content, wrapper)
+	if err != nil {
+		return fmt.Errorf("%s: %w", filePath, err)
+	}
+
+	mergeContext(toSend, wrapper.Context)
+
+	return nil
+}
+
+// addContextFromFile merges the context from a file into the context to send to the console.
+func addContextFromFile(toSend map[string][]string, filePath string) error {
+	log.Tracef("loading console context from %s", filePath)
+	content, err := os.ReadFile(filePath)
+	if err != nil {
+		return err
+	}
+
+	newContext := make(map[string][]string, 0)
+
+	err = yaml.Unmarshal(content, newContext)
+	if err != nil {
+		return fmt.Errorf("%s: %w", filePath, err)
+	}
+
+	mergeContext(toSend, newContext)
+
+	return nil
+}
+
+
+// LoadConsoleContext loads the context from the hub (if provided) and the file console_context_path.
+func LoadConsoleContext(c *csconfig.Config, hub *cwhub.Hub) error {
+	c.Crowdsec.ContextToSend = make(map[string][]string, 0)
+
+	if hub != nil {
+		items, err := hub.GetInstalledItems(cwhub.CONTEXTS)
+		if err != nil {
+			return err
+		}
+
+		for _, item := range items {
+			// context in item files goes under the key 'context'
+			if err = addContextFromItem(c.Crowdsec.ContextToSend, item); err != nil {
+				return err
+			}
+		}
+	}
+
+	ignoreMissing := false
+
+	if c.Crowdsec.ConsoleContextPath != "" {
+		// if it's provided, it must exist
+		if _, err := os.Stat(c.Crowdsec.ConsoleContextPath); err != nil {
+			return fmt.Errorf("while checking console_context_path: %w", err)
+		}
+	} else {
+		c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
+		ignoreMissing = true
+	}
+
+	if err := addContextFromFile(c.Crowdsec.ContextToSend, c.Crowdsec.ConsoleContextPath); err != nil {
+		if !ignoreMissing || !os.IsNotExist(err) {
+			return err
+		}
+	}
+
+	feedback, err := json.Marshal(c.Crowdsec.ContextToSend)
+	if err != nil {
+		return fmt.Errorf("marshaling console context: %s", err)
+	}
+
+	log.Debugf("console context to send: %s", feedback)
+
+	return nil
+}

pkg/appsec/appsec.go

@@ -0,0 +1,579 @@
+package appsec
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+
+	"github.com/antonmedv/expr"
+	"github.com/antonmedv/expr/vm"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+)
+
+type Hook struct {
+	Filter     string      `yaml:"filter"`
+	FilterExpr *vm.Program `yaml:"-"`
+
+	OnSuccess string        `yaml:"on_success"`
+	Apply     []string      `yaml:"apply"`
+	ApplyExpr []*vm.Program `yaml:"-"`
+}
+
+const (
+	hookOnLoad = iota
+	hookPreEval
+	hookPostEval
+	hookOnMatch
+)
+
+// @tko : todo - debug mode
+func (h *Hook) Build(hookStage int) error {
+
+	ctx := map[string]interface{}{}
+	switch hookStage {
+	case hookOnLoad:
+		ctx = GetOnLoadEnv(&AppsecRuntimeConfig{})
+	case hookPreEval:
+		ctx = GetPreEvalEnv(&AppsecRuntimeConfig{}, &ParsedRequest{})
+	case hookPostEval:
+		ctx = GetPostEvalEnv(&AppsecRuntimeConfig{}, &ParsedRequest{})
+	case hookOnMatch:
+		ctx = GetOnMatchEnv(&AppsecRuntimeConfig{}, &ParsedRequest{}, types.Event{})
+	}
+	opts := exprhelpers.GetExprOptions(ctx)
+	if h.Filter != "" {
+		program, err := expr.Compile(h.Filter, opts...) //FIXME: opts
+		if err != nil {
+			return fmt.Errorf("unable to compile filter %s : %w", h.Filter, err)
+		}
+		h.FilterExpr = program
+	}
+	for _, apply := range h.Apply {
+		program, err := expr.Compile(apply, opts...)
+		if err != nil {
+			return fmt.Errorf("unable to compile apply %s : %w", apply, err)
+		}
+		h.ApplyExpr = append(h.ApplyExpr, program)
+	}
+	return nil
+}
+
+type AppsecTempResponse struct {
+	InBandInterrupt    bool
+	OutOfBandInterrupt bool
+	Action             string //allow, deny, captcha, log
+	HTTPResponseCode   int
+	SendEvent          bool //do we send an internal event on rule match
+	SendAlert          bool //do we send an alert on rule match
+}
+
+type AppsecSubEngineOpts struct {
+	DisableBodyInspection    bool `yaml:"disable_body_inspection"`
+	RequestBodyInMemoryLimit *int `yaml:"request_body_in_memory_limit"`
+}
+
+// runtime version of AppsecConfig
+type AppsecRuntimeConfig struct {
+	Name           string
+	OutOfBandRules []AppsecCollection
+
+	InBandRules []AppsecCollection
+
+	DefaultRemediation        string
+	RemediationByTag          map[string]string //Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME
+	RemediationById           map[int]string
+	CompiledOnLoad            []Hook
+	CompiledPreEval           []Hook
+	CompiledPostEval          []Hook
+	CompiledOnMatch           []Hook
+	CompiledVariablesTracking []*regexp.Regexp
+	Config                    *AppsecConfig
+	//CorazaLogger              debuglog.Logger
+
+	//those are ephemeral, created/destroyed with every req
+	OutOfBandTx ExtendedTransaction //is it a good idea ?
+	InBandTx    ExtendedTransaction //is it a good idea ?
+	Response    AppsecTempResponse
+	//should we store matched rules here ?
+
+	Logger *log.Entry
+
+	//Set by on_load to ignore some rules on loading
+	DisabledInBandRuleIds   []int
+	DisabledInBandRulesTags []string //Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME
+
+	DisabledOutOfBandRuleIds   []int
+	DisabledOutOfBandRulesTags []string //Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME
+}
+
+type AppsecConfig struct {
+	Name               string              `yaml:"name"`
+	OutOfBandRules     []string            `yaml:"outofband_rules"`
+	InBandRules        []string            `yaml:"inband_rules"`
+	DefaultRemediation string              `yaml:"default_remediation"`
+	DefaultPassAction  string              `yaml:"default_pass_action"`
+	BlockedHTTPCode    int                 `yaml:"blocked_http_code"`
+	PassedHTTPCode     int                 `yaml:"passed_http_code"`
+	OnLoad             []Hook              `yaml:"on_load"`
+	PreEval            []Hook              `yaml:"pre_eval"`
+	PostEval           []Hook              `yaml:"post_eval"`
+	OnMatch            []Hook              `yaml:"on_match"`
+	VariablesTracking  []string            `yaml:"variables_tracking"`
+	InbandOptions      AppsecSubEngineOpts `yaml:"inband_options"`
+	OutOfBandOptions   AppsecSubEngineOpts `yaml:"outofband_options"`
+
+	LogLevel *log.Level `yaml:"log_level"`
+	Logger   *log.Entry `yaml:"-"`
+}
+
+func (w *AppsecRuntimeConfig) ClearResponse() {
+	log.Debugf("#-> %p", w)
+	w.Response = AppsecTempResponse{}
+	log.Debugf("-> %p", w.Config)
+	w.Response.Action = w.Config.DefaultPassAction
+	w.Response.HTTPResponseCode = w.Config.PassedHTTPCode
+	w.Response.SendEvent = true
+	w.Response.SendAlert = true
+}
+
+func (wc *AppsecConfig) LoadByPath(file string) error {
+
+	wc.Logger.Debugf("loading config %s", file)
+
+	yamlFile, err := os.ReadFile(file)
+	if err != nil {
+		return fmt.Errorf("unable to read file %s : %s", file, err)
+	}
+	err = yaml.UnmarshalStrict(yamlFile, wc)
+	if err != nil {
+		return fmt.Errorf("unable to parse yaml file %s : %s", file, err)
+	}
+
+	if wc.Name == "" {
+		return fmt.Errorf("name cannot be empty")
+	}
+	if wc.LogLevel == nil {
+		lvl := wc.Logger.Logger.GetLevel()
+		wc.LogLevel = &lvl
+	}
+	wc.Logger = wc.Logger.Dup().WithField("name", wc.Name)
+	wc.Logger.Logger.SetLevel(*wc.LogLevel)
+	if wc.DefaultRemediation == "" {
+		return fmt.Errorf("default_remediation cannot be empty")
+	}
+	switch wc.DefaultRemediation {
+	case "ban", "captcha", "log":
+		//those are the officially supported remediation(s)
+	default:
+		wc.Logger.Warningf("default '%s' remediation of %s is none of [ban,captcha,log] ensure bouncer compatbility!", wc.DefaultRemediation, file)
+	}
+	if wc.BlockedHTTPCode == 0 {
+		wc.BlockedHTTPCode = 403
+	}
+	if wc.PassedHTTPCode == 0 {
+		wc.PassedHTTPCode = 200
+	}
+	if wc.DefaultPassAction == "" {
+		wc.DefaultPassAction = "allow"
+	}
+	return nil
+}
+
+func (wc *AppsecConfig) Load(configName string) error {
+	appsecConfigs := hub.GetItemMap(cwhub.APPSEC_CONFIGS)
+
+	for _, hubAppsecConfigItem := range appsecConfigs {
+		if !hubAppsecConfigItem.State.Installed {
+			continue
+		}
+		if hubAppsecConfigItem.Name != configName {
+			continue
+		}
+		wc.Logger.Infof("loading %s", hubAppsecConfigItem.State.LocalPath)
+		err := wc.LoadByPath(hubAppsecConfigItem.State.LocalPath)
+		if err != nil {
+			return fmt.Errorf("unable to load appsec-config %s : %s", hubAppsecConfigItem.State.LocalPath, err)
+		}
+		return nil
+	}
+
+	return fmt.Errorf("no appsec-config found for %s", configName)
+}
+
+func (wc *AppsecConfig) GetDataDir() string {
+	return hub.GetDataDir()
+}
+
+func (wc *AppsecConfig) Build() (*AppsecRuntimeConfig, error) {
+	ret := &AppsecRuntimeConfig{Logger: wc.Logger.WithField("component", "appsec_runtime_config")}
+	ret.Name = wc.Name
+	ret.Config = wc
+	ret.DefaultRemediation = wc.DefaultRemediation
+
+	wc.Logger.Tracef("Loading config %+v", wc)
+	//load rules
+	for _, rule := range wc.OutOfBandRules {
+		wc.Logger.Infof("loading outofband rule %s", rule)
+		collections, err := LoadCollection(rule, wc.Logger.WithField("component", "appsec_collection_loader"))
+		if err != nil {
+			return nil, fmt.Errorf("unable to load outofband rule %s : %s", rule, err)
+		}
+		ret.OutOfBandRules = append(ret.OutOfBandRules, collections...)
+	}
+
+	wc.Logger.Infof("Loaded %d outofband rules", len(ret.OutOfBandRules))
+	for _, rule := range wc.InBandRules {
+		wc.Logger.Infof("loading inband rule %s", rule)
+		collections, err := LoadCollection(rule, wc.Logger.WithField("component", "appsec_collection_loader"))
+		if err != nil {
+			return nil, fmt.Errorf("unable to load inband rule %s : %s", rule, err)
+		}
+		ret.InBandRules = append(ret.InBandRules, collections...)
+	}
+
+	wc.Logger.Infof("Loaded %d inband rules", len(ret.InBandRules))
+
+	//load hooks
+	for _, hook := range wc.OnLoad {
+		err := hook.Build(hookOnLoad)
+		if err != nil {
+			return nil, fmt.Errorf("unable to build on_load hook : %s", err)
+		}
+		ret.CompiledOnLoad = append(ret.CompiledOnLoad, hook)
+	}
+
+	for _, hook := range wc.PreEval {
+		err := hook.Build(hookPreEval)
+		if err != nil {
+			return nil, fmt.Errorf("unable to build pre_eval hook : %s", err)
+		}
+		ret.CompiledPreEval = append(ret.CompiledPreEval, hook)
+	}
+
+	for _, hook := range wc.PostEval {
+		err := hook.Build(hookPostEval)
+		if err != nil {
+			return nil, fmt.Errorf("unable to build post_eval hook : %s", err)
+		}
+		ret.CompiledPostEval = append(ret.CompiledPostEval, hook)
+	}
+
+	for _, hook := range wc.OnMatch {
+		err := hook.Build(hookOnMatch)
+		if err != nil {
+			return nil, fmt.Errorf("unable to build on_match hook : %s", err)
+		}
+		ret.CompiledOnMatch = append(ret.CompiledOnMatch, hook)
+	}
+
+	//variable tracking
+	for _, variable := range wc.VariablesTracking {
+		compiledVariableRule, err := regexp.Compile(variable)
+		if err != nil {
+			return nil, fmt.Errorf("cannot compile variable regexp %s: %w", variable, err)
+		}
+		ret.CompiledVariablesTracking = append(ret.CompiledVariablesTracking, compiledVariableRule)
+	}
+	return ret, nil
+}
+
+func (w *AppsecRuntimeConfig) ProcessOnLoadRules() error {
+	for _, rule := range w.CompiledOnLoad {
+		if rule.FilterExpr != nil {
+			output, err := exprhelpers.Run(rule.FilterExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel)
+			if err != nil {
+				return fmt.Errorf("unable to run appsec on_load filter %s : %w", rule.Filter, err)
+			}
+			switch t := output.(type) {
+			case bool:
+				if !t {
+					log.Debugf("filter didnt match")
+					continue
+				}
+			default:
+				log.Errorf("Filter must return a boolean, can't filter")
+				continue
+			}
+		}
+		for _, applyExpr := range rule.ApplyExpr {
+			_, err := exprhelpers.Run(applyExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel)
+			if err != nil {
+				log.Errorf("unable to apply appsec on_load expr: %s", err)
+				continue
+			}
+		}
+	}
+	return nil
+}
+
+func (w *AppsecRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt types.Event) error {
+
+	for _, rule := range w.CompiledOnMatch {
+		if rule.FilterExpr != nil {
+			output, err := exprhelpers.Run(rule.FilterExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel)
+			if err != nil {
+				return fmt.Errorf("unable to run appsec on_match filter %s : %w", rule.Filter, err)
+			}
+			switch t := output.(type) {
+			case bool:
+				if !t {
+					log.Debugf("filter didnt match")
+					continue
+				}
+			default:
+				log.Errorf("Filter must return a boolean, can't filter")
+				continue
+			}
+		}
+		for _, applyExpr := range rule.ApplyExpr {
+			_, err := exprhelpers.Run(applyExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel)
+			if err != nil {
+				log.Errorf("unable to apply appsec on_match expr: %s", err)
+				continue
+			}
+		}
+	}
+	return nil
+}
+
+func (w *AppsecRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error {
+	for _, rule := range w.CompiledPreEval {
+		if rule.FilterExpr != nil {
+			output, err := exprhelpers.Run(rule.FilterExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel)
+			if err != nil {
+				return fmt.Errorf("unable to run appsec pre_eval filter %s : %w", rule.Filter, err)
+			}
+			switch t := output.(type) {
+			case bool:
+				if !t {
+					log.Debugf("filter didnt match")
+					continue
+				}
+			default:
+				log.Errorf("Filter must return a boolean, can't filter")
+				continue
+			}
+		}
+		// here means there is no filter or the filter matched
+		for _, applyExpr := range rule.ApplyExpr {
+			_, err := exprhelpers.Run(applyExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel)
+			if err != nil {
+				log.Errorf("unable to apply appsec pre_eval expr: %s", err)
+				continue
+			}
+		}
+	}
+
+	return nil
+}
+
+func (w *AppsecRuntimeConfig) ProcessPostEvalRules(request *ParsedRequest) error {
+	for _, rule := range w.CompiledPostEval {
+		if rule.FilterExpr != nil {
+			output, err := exprhelpers.Run(rule.FilterExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel)
+			if err != nil {
+				return fmt.Errorf("unable to run appsec post_eval filter %s : %w", rule.Filter, err)
+			}
+			switch t := output.(type) {
+			case bool:
+				if !t {
+					log.Debugf("filter didnt match")
+					continue
+				}
+			default:
+				log.Errorf("Filter must return a boolean, can't filter")
+				continue
+			}
+		}
+		// here means there is no filter or the filter matched
+		for _, applyExpr := range rule.ApplyExpr {
+			_, err := exprhelpers.Run(applyExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel)
+			if err != nil {
+				log.Errorf("unable to apply appsec post_eval expr: %s", err)
+				continue
+			}
+		}
+	}
+
+	return nil
+}
+
+func (w *AppsecRuntimeConfig) RemoveInbandRuleByID(id int) error {
+	w.Logger.Debugf("removing inband rule %d", id)
+	return w.InBandTx.RemoveRuleByIDWithError(id)
+}
+
+func (w *AppsecRuntimeConfig) RemoveOutbandRuleByID(id int) error {
+	w.Logger.Debugf("removing outband rule %d", id)
+	return w.OutOfBandTx.RemoveRuleByIDWithError(id)
+}
+
+func (w *AppsecRuntimeConfig) RemoveInbandRuleByTag(tag string) error {
+	w.Logger.Debugf("removing inband rule with tag %s", tag)
+	return w.InBandTx.RemoveRuleByTagWithError(tag)
+}
+
+func (w *AppsecRuntimeConfig) RemoveOutbandRuleByTag(tag string) error {
+	w.Logger.Debugf("removing outband rule with tag %s", tag)
+	return w.OutOfBandTx.RemoveRuleByTagWithError(tag)
+}
+
+func (w *AppsecRuntimeConfig) RemoveInbandRuleByName(name string) error {
+	tag := fmt.Sprintf("crowdsec-%s", name)
+	w.Logger.Debugf("removing inband rule %s", tag)
+	return w.InBandTx.RemoveRuleByTagWithError(tag)
+}
+
+func (w *AppsecRuntimeConfig) RemoveOutbandRuleByName(name string) error {
+	tag := fmt.Sprintf("crowdsec-%s", name)
+	w.Logger.Debugf("removing outband rule %s", tag)
+	return w.OutOfBandTx.RemoveRuleByTagWithError(tag)
+}
+
+func (w *AppsecRuntimeConfig) CancelEvent() error {
+	w.Logger.Debugf("canceling event")
+	w.Response.SendEvent = false
+	return nil
+}
+
+// Disable a rule at load time, meaning it will not run for any request
+func (w *AppsecRuntimeConfig) DisableInBandRuleByID(id int) error {
+	w.DisabledInBandRuleIds = append(w.DisabledInBandRuleIds, id)
+	return nil
+}
+
+// Disable a rule at load time, meaning it will not run for any request
+func (w *AppsecRuntimeConfig) DisableInBandRuleByName(name string) error {
+	tagValue := fmt.Sprintf("crowdsec-%s", name)
+	w.DisabledInBandRulesTags = append(w.DisabledInBandRulesTags, tagValue)
+	return nil
+}
+
+// Disable a rule at load time, meaning it will not run for any request
+func (w *AppsecRuntimeConfig) DisableInBandRuleByTag(tag string) error {
+	w.DisabledInBandRulesTags = append(w.DisabledInBandRulesTags, tag)
+	return nil
+}
+
+// Disable a rule at load time, meaning it will not run for any request
+func (w *AppsecRuntimeConfig) DisableOutBandRuleByID(id int) error {
+	w.DisabledOutOfBandRuleIds = append(w.DisabledOutOfBandRuleIds, id)
+	return nil
+}
+
+// Disable a rule at load time, meaning it will not run for any request
+func (w *AppsecRuntimeConfig) DisableOutBandRuleByName(name string) error {
+	tagValue := fmt.Sprintf("crowdsec-%s", name)
+	w.DisabledOutOfBandRulesTags = append(w.DisabledOutOfBandRulesTags, tagValue)
+	return nil
+}
+
+// Disable a rule at load time, meaning it will not run for any request
+func (w *AppsecRuntimeConfig) DisableOutBandRuleByTag(tag string) error {
+	w.DisabledOutOfBandRulesTags = append(w.DisabledOutOfBandRulesTags, tag)
+	return nil
+}
+
+func (w *AppsecRuntimeConfig) SendEvent() error {
+	w.Logger.Debugf("sending event")
+	w.Response.SendEvent = true
+	return nil
+}
+
+func (w *AppsecRuntimeConfig) SendAlert() error {
+	w.Logger.Debugf("sending alert")
+	w.Response.SendAlert = true
+	return nil
+}
+
+func (w *AppsecRuntimeConfig) CancelAlert() error {
+	w.Logger.Debugf("canceling alert")
+	w.Response.SendAlert = false
+	return nil
+}
+
+func (w *AppsecRuntimeConfig) SetActionByTag(tag string, action string) error {
+	if w.RemediationByTag == nil {
+		w.RemediationByTag = make(map[string]string)
+	}
+	w.Logger.Debugf("setting action of %s to %s", tag, action)
+	w.RemediationByTag[tag] = action
+	return nil
+}
+
+func (w *AppsecRuntimeConfig) SetActionByID(id int, action string) error {
+	if w.RemediationById == nil {
+		w.RemediationById = make(map[int]string)
+	}
+	w.Logger.Debugf("setting action of %d to %s", id, action)
+	w.RemediationById[id] = action
+	return nil
+}
+
+func (w *AppsecRuntimeConfig) SetActionByName(name string, action string) error {
+	if w.RemediationByTag == nil {
+		w.RemediationByTag = make(map[string]string)
+	}
+	tag := fmt.Sprintf("crowdsec-%s", name)
+	w.Logger.Debugf("setting action of %s to %s", tag, action)
+	w.RemediationByTag[tag] = action
+	return nil
+}
+
+func (w *AppsecRuntimeConfig) SetAction(action string) error {
+	//log.Infof("setting to %s", action)
+	w.Logger.Debugf("setting action to %s", action)
+	switch action {
+	case "allow":
+		w.Response.Action = action
+		w.Response.HTTPResponseCode = w.Config.PassedHTTPCode
+		//@tko how should we handle this ? it seems bouncer only understand bans, but it might be misleading ?
+	case "deny", "ban", "block":
+		w.Response.Action = "ban"
+	case "log":
+		w.Response.Action = action
+		w.Response.HTTPResponseCode = w.Config.PassedHTTPCode
+	case "captcha":
+		w.Response.Action = action
+	default:
+		return fmt.Errorf("unknown action %s", action)
+	}
+	return nil
+}
+
+func (w *AppsecRuntimeConfig) SetHTTPCode(code int) error {
+	w.Logger.Debugf("setting http code to %d", code)
+	w.Response.HTTPResponseCode = code
+	return nil
+}
+
+type BodyResponse struct {
+	Action     string `json:"action"`
+	HTTPStatus int    `json:"http_status"`
+}
+
+func (w *AppsecRuntimeConfig) GenerateResponse(response AppsecTempResponse, logger *log.Entry) BodyResponse {
+	resp := BodyResponse{}
+	//if there is no interrupt, we should allow with default code
+	if !response.InBandInterrupt {
+		resp.Action = w.Config.DefaultPassAction
+		resp.HTTPStatus = w.Config.PassedHTTPCode
+		return resp
+	}
+	resp.Action = response.Action
+	if resp.Action == "" {
+		resp.Action = w.Config.DefaultRemediation
+	}
+	logger.Debugf("action is %s", resp.Action)
+
+	resp.HTTPStatus = response.HTTPResponseCode
+	if resp.HTTPStatus == 0 {
+		resp.HTTPStatus = w.Config.BlockedHTTPCode
+	}
+	logger.Debugf("http status is %d", resp.HTTPStatus)
+	return resp
+}

pkg/appsec/appsec_rule/appsec_rule.go

@@ -0,0 +1,67 @@
+package appsec_rule
+
+import (
+	"fmt"
+)
+
+/*
+rules:
+ - name: "test"
+   and:
+   	- zones:
+   		- BODY_ARGS
+   	  variables:
+		- foo
+		- bar
+   	  transform:
+   		- lowercase|uppercase|b64decode|...
+	  match:
+	    type: regex
+	   	value: "[^a-zA-Z]"
+	- zones:
+	   - ARGS
+	  variables:
+	   - bla
+
+*/
+
+type match struct {
+	Type  string `yaml:"type"`
+	Value string `yaml:"value"`
+}
+
+type CustomRule struct {
+	Name string `yaml:"name"`
+
+	Zones     []string `yaml:"zones"`
+	Variables []string `yaml:"variables"`
+
+	Match     match        `yaml:"match"`
+	Transform []string     `yaml:"transform"` //t:lowercase, t:uppercase, etc
+	And       []CustomRule `yaml:"and,omitempty"`
+	Or        []CustomRule `yaml:"or,omitempty"`
+	BodyType  string       `yaml:"body_type,omitempty"`
+}
+
+func (v *CustomRule) Convert(ruleType string, appsecRuleName string) (string, []uint32, error) {
+
+	if v.Zones == nil && v.And == nil && v.Or == nil {
+		return "", nil, fmt.Errorf("no zones defined")
+	}
+
+	if v.Match.Type == "" && v.And == nil && v.Or == nil {
+		return "", nil, fmt.Errorf("no match type defined")
+	}
+
+	if v.Match.Value == "" && v.And == nil && v.Or == nil {
+		return "", nil, fmt.Errorf("no match value defined")
+	}
+
+	switch ruleType {
+	case ModsecurityRuleType:
+		r := ModsecurityRule{}
+		return r.Build(v, appsecRuleName)
+	default:
+		return "", nil, fmt.Errorf("unknown rule format '%s'", ruleType)
+	}
+}

pkg/appsec/appsec_rule/modsec_rule_test.go

@@ -0,0 +1,118 @@
+package appsec_rule
+
+import "testing"
+
+func TestVPatchRuleString(t *testing.T) {
+	tests := []struct {
+		name     string
+		rule     CustomRule
+		expected string
+	}{
+		{
+			name: "Base Rule",
+			rule: CustomRule{
+				Zones:     []string{"ARGS"},
+				Variables: []string{"foo"},
+				Match:     match{Type: "regex", Value: "[^a-zA-Z]"},
+				Transform: []string{"lowercase"},
+			},
+			expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:2203944045,phase:2,deny,log,msg:'Base Rule',tag:'crowdsec-Base Rule',t:lowercase"`,
+		},
+		{
+			name: "Multiple Zones",
+			rule: CustomRule{
+				Zones:     []string{"ARGS", "BODY_ARGS"},
+				Variables: []string{"foo"},
+				Match:     match{Type: "regex", Value: "[^a-zA-Z]"},
+				Transform: []string{"lowercase"},
+			},
+			expected: `SecRule ARGS_GET:foo|ARGS_POST:foo "@rx [^a-zA-Z]" "id:3387135861,phase:2,deny,log,msg:'Multiple Zones',tag:'crowdsec-Multiple Zones',t:lowercase"`,
+		},
+		{
+			name: "Basic AND",
+			rule: CustomRule{
+				And: []CustomRule{
+					{
+
+						Zones:     []string{"ARGS"},
+						Variables: []string{"foo"},
+						Match:     match{Type: "regex", Value: "[^a-zA-Z]"},
+						Transform: []string{"lowercase"},
+					},
+					{
+						Zones:     []string{"ARGS"},
+						Variables: []string{"bar"},
+						Match:     match{Type: "regex", Value: "[^a-zA-Z]"},
+						Transform: []string{"lowercase"},
+					},
+				},
+			},
+			expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:4145519614,phase:2,deny,log,msg:'Basic AND',tag:'crowdsec-Basic AND',t:lowercase,chain"
+SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:1865217529,phase:2,deny,log,msg:'Basic AND',tag:'crowdsec-Basic AND',t:lowercase"`,
+		},
+		{
+			name: "Basic OR",
+			rule: CustomRule{
+				Or: []CustomRule{
+					{
+						Zones:     []string{"ARGS"},
+						Variables: []string{"foo"},
+						Match:     match{Type: "regex", Value: "[^a-zA-Z]"},
+						Transform: []string{"lowercase"},
+					},
+					{
+						Zones:     []string{"ARGS"},
+						Variables: []string{"bar"},
+						Match:     match{Type: "regex", Value: "[^a-zA-Z]"},
+						Transform: []string{"lowercase"},
+					},
+				},
+			},
+			expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:651140804,phase:2,deny,log,msg:'Basic OR',tag:'crowdsec-Basic OR',t:lowercase,skip:1"
+SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:271441587,phase:2,deny,log,msg:'Basic OR',tag:'crowdsec-Basic OR',t:lowercase"`,
+		},
+		{
+			name: "OR AND mix",
+			rule: CustomRule{
+				And: []CustomRule{
+					{
+						Zones:     []string{"ARGS"},
+						Variables: []string{"foo"},
+						Match:     match{Type: "regex", Value: "[^a-zA-Z]"},
+						Transform: []string{"lowercase"},
+						Or: []CustomRule{
+							{
+								Zones:     []string{"ARGS"},
+								Variables: []string{"foo"},
+								Match:     match{Type: "regex", Value: "[^a-zA-Z]"},
+								Transform: []string{"lowercase"},
+							},
+							{
+								Zones:     []string{"ARGS"},
+								Variables: []string{"bar"},
+								Match:     match{Type: "regex", Value: "[^a-zA-Z]"},
+								Transform: []string{"lowercase"},
+							},
+						},
+					},
+				},
+			},
+			expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:1714963250,phase:2,deny,log,msg:'OR AND mix',tag:'crowdsec-OR AND mix',t:lowercase,skip:1"
+SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:1519945803,phase:2,deny,log,msg:'OR AND mix',tag:'crowdsec-OR AND mix',t:lowercase"
+SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:1519945803,phase:2,deny,log,msg:'OR AND mix',tag:'crowdsec-OR AND mix',t:lowercase"`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual, _, err := tt.rule.Convert(ModsecurityRuleType, tt.name)
+
+			if err != nil {
+				t.Errorf("Error converting rule: %s", err)
+			}
+			if actual != tt.expected {
+				t.Errorf("Expected:\n%s\nGot:\n%s", tt.expected, actual)
+			}
+		})
+	}
+}

pkg/appsec/appsec_rule/modsecurity.go

@@ -0,0 +1,181 @@
+package appsec_rule
+
+import (
+	"fmt"
+	"hash/fnv"
+	"strings"
+)
+
+type ModsecurityRule struct {
+	ids []uint32
+}
+
+var zonesMap map[string]string = map[string]string{
+	"ARGS":            "ARGS_GET",
+	"ARGS_NAMES":      "ARGS_GET_NAMES",
+	"BODY_ARGS":       "ARGS_POST",
+	"BODY_ARGS_NAMES": "ARGS_POST_NAMES",
+	"HEADERS":         "REQUEST_HEADERS",
+	"METHOD":          "REQUEST_METHOD",
+	"PROTOCOL":        "REQUEST_PROTOCOL",
+	"URI":             "REQUEST_URI",
+}
+
+var transformMap map[string]string = map[string]string{
+	"lowercase": "t:lowercase",
+	"uppercase": "t:uppercase",
+	"b64decode": "t:base64Decode",
+	"hexdecode": "t:hexDecode",
+	"length":    "t:length",
+}
+
+var matchMap map[string]string = map[string]string{
+	"regex":           "@rx",
+	"equal":           "@streq",
+	"startsWith":      "@beginsWith",
+	"endsWith":        "@endsWith",
+	"contains":        "@contains",
+	"libinjectionSQL": "@detectSQLi",
+	"libinjectionXSS": "@detectXSS",
+	"gt":              "@gt",
+	"lt":              "@lt",
+	"ge":              "@ge",
+	"le":              "@le",
+}
+
+var bodyTypeMatch map[string]string = map[string]string{
+	"json":       "JSON",
+	"xml":        "XML",
+	"multipart":  "MULTIPART",
+	"urlencoded": "URLENCODED",
+}
+
+func (m *ModsecurityRule) Build(rule *CustomRule, appsecRuleName string) (string, []uint32, error) {
+
+	rules, err := m.buildRules(rule, appsecRuleName, false, 0, 0)
+
+	if err != nil {
+		return "", nil, err
+	}
+
+	//We return the id of the first generated rule, as it's the interesting one in case of chain or skip
+	return strings.Join(rules, "\n"), m.ids, nil
+}
+
+func (m *ModsecurityRule) generateRuleID(rule *CustomRule, appsecRuleName string, depth int) uint32 {
+	h := fnv.New32a()
+	h.Write([]byte(appsecRuleName))
+	h.Write([]byte(rule.Match.Type))
+	h.Write([]byte(rule.Match.Value))
+	h.Write([]byte(fmt.Sprintf("%d", depth)))
+	for _, zone := range rule.Zones {
+		h.Write([]byte(zone))
+	}
+	for _, transform := range rule.Transform {
+		h.Write([]byte(transform))
+	}
+	id := h.Sum32()
+	m.ids = append(m.ids, id)
+	return id
+}
+
+func (m *ModsecurityRule) buildRules(rule *CustomRule, appsecRuleName string, and bool, toSkip int, depth int) ([]string, error) {
+	ret := make([]string, 0)
+
+	if len(rule.And) != 0 && len(rule.Or) != 0 {
+		return nil, fmt.Errorf("cannot have both 'and' and 'or' in the same rule")
+	}
+
+	if rule.And != nil {
+		for c, andRule := range rule.And {
+			depth++
+			lastRule := c == len(rule.And)-1 // || len(rule.Or) == 0
+			rules, err := m.buildRules(&andRule, appsecRuleName, !lastRule, 0, depth)
+			if err != nil {
+				return nil, err
+			}
+			ret = append(ret, rules...)
+		}
+	}
+
+	if rule.Or != nil {
+		for c, orRule := range rule.Or {
+			depth++
+			skip := len(rule.Or) - c - 1
+			rules, err := m.buildRules(&orRule, appsecRuleName, false, skip, depth)
+			if err != nil {
+				return nil, err
+			}
+			ret = append(ret, rules...)
+		}
+	}
+
+	r := strings.Builder{}
+
+	r.WriteString("SecRule ")
+
+	if rule.Zones == nil {
+		return ret, nil
+	}
+
+	for idx, zone := range rule.Zones {
+		mappedZone, ok := zonesMap[zone]
+		if !ok {
+			return nil, fmt.Errorf("unknown zone '%s'", zone)
+		}
+		if len(rule.Variables) == 0 {
+			r.WriteString(mappedZone)
+		} else {
+			for j, variable := range rule.Variables {
+				if idx > 0 || j > 0 {
+					r.WriteByte('|')
+				}
+				r.WriteString(fmt.Sprintf("%s:%s", mappedZone, variable))
+			}
+		}
+	}
+	r.WriteByte(' ')
+
+	if rule.Match.Type != "" {
+		if match, ok := matchMap[rule.Match.Type]; ok {
+			r.WriteString(fmt.Sprintf(`"%s %s"`, match, rule.Match.Value))
+		} else {
+			return nil, fmt.Errorf("unknown match type '%s'", rule.Match.Type)
+		}
+	}
+
+	//Should phase:2 be configurable?
+	r.WriteString(fmt.Sprintf(` "id:%d,phase:2,deny,log,msg:'%s',tag:'crowdsec-%s'`, m.generateRuleID(rule, appsecRuleName, depth), appsecRuleName, appsecRuleName))
+
+	if rule.Transform != nil {
+		for _, transform := range rule.Transform {
+			r.WriteByte(',')
+			if mappedTransform, ok := transformMap[transform]; ok {
+				r.WriteString(mappedTransform)
+			} else {
+				return nil, fmt.Errorf("unknown transform '%s'", transform)
+			}
+		}
+	}
+
+	if rule.BodyType != "" {
+		if mappedBodyType, ok := bodyTypeMatch[rule.BodyType]; ok {
+			r.WriteString(fmt.Sprintf(",ctl:requestBodyProcessor=%s", mappedBodyType))
+		} else {
+			return nil, fmt.Errorf("unknown body type '%s'", rule.BodyType)
+		}
+	}
+
+	if and {
+		r.WriteString(",chain")
+	}
+
+	if toSkip > 0 {
+		r.WriteString(fmt.Sprintf(",skip:%d", toSkip))
+	}
+
+	r.WriteByte('"')
+
+	ret = append(ret, r.String())
+	return ret, nil
+}

pkg/appsec/appsec_rule/types.go

@@ -0,0 +1,9 @@
+package appsec_rule
+
+const (
+	ModsecurityRuleType = "modsecurity"
+)
+
+func SupportedTypes() []string {
+	return []string{ModsecurityRuleType}
+}

pkg/appsec/appsec_rules_collection.go

@@ -0,0 +1,144 @@
+package appsec
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
+	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type AppsecCollection struct {
+	collectionName string
+	Rules          []string
+}
+
+var APPSEC_RULE = "appsec-rule"
+
+// to be filled w/ seb update
+type AppsecCollectionConfig struct {
+	Type              string                   `yaml:"type"`
+	Name              string                   `yaml:"name"`
+	Debug             bool                     `yaml:"debug"`
+	Description       string                   `yaml:"description"`
+	SecLangFilesRules []string                 `yaml:"seclang_files_rules"`
+	SecLangRules      []string                 `yaml:"seclang_rules"`
+	Rules             []appsec_rule.CustomRule `yaml:"rules"`
+
+	Labels map[string]interface{} `yaml:"labels"` //Labels is K:V list aiming at providing context the overflow
+
+	Data    interface{} `yaml:"data"` //Ignore it
+	hash    string      `yaml:"-"`
+	version string      `yaml:"-"`
+}
+
+type RulesDetails struct {
+	LogLevel log.Level
+	Hash     string
+	Version  string
+	Name     string
+}
+
+// Should it be a global ?
+// Is using the id is a good idea ? might be too specific to coraza and not easily reusable
+var AppsecRulesDetails = make(map[int]RulesDetails)
+
+func LoadCollection(pattern string, logger *log.Entry) ([]AppsecCollection, error) {
+	ret := make([]AppsecCollection, 0)
+
+	for _, appsecRule := range appsecRules {
+
+		tmpMatch, err := exprhelpers.Match(pattern, appsecRule.Name)
+
+		if err != nil {
+			logger.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err)
+			continue
+		}
+
+		matched, ok := tmpMatch.(bool)
+
+		if !ok {
+			logger.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err)
+			continue
+		}
+
+		if !matched {
+			continue
+		}
+
+		appsecCol := AppsecCollection{
+			collectionName: appsecRule.Name,
+		}
+
+		if appsecRule.SecLangFilesRules != nil {
+			for _, rulesFile := range appsecRule.SecLangFilesRules {
+				logger.Debugf("Adding rules from %s", rulesFile)
+				fullPath := filepath.Join(hub.GetDataDir(), rulesFile)
+				c, err := os.ReadFile(fullPath)
+				if err != nil {
+					logger.Errorf("unable to read file %s : %s", rulesFile, err)
+					continue
+				}
+				for _, line := range strings.Split(string(c), "\n") {
+					if strings.HasPrefix(line, "#") {
+						continue
+					}
+					if strings.TrimSpace(line) == "" {
+						continue
+					}
+					appsecCol.Rules = append(appsecCol.Rules, line)
+				}
+			}
+		}
+
+		if appsecRule.SecLangRules != nil {
+			logger.Tracef("Adding inline rules %+v", appsecRule.SecLangRules)
+			appsecCol.Rules = append(appsecCol.Rules, appsecRule.SecLangRules...)
+		}
+
+		if appsecRule.Rules != nil {
+			for _, rule := range appsecRule.Rules {
+				strRule, rulesId, err := rule.Convert(appsec_rule.ModsecurityRuleType, appsecRule.Name)
+				if err != nil {
+					logger.Errorf("unable to convert rule %s : %s", rule.Name, err)
+					return nil, err
+				}
+				logger.Debugf("Adding rule %s", strRule)
+				appsecCol.Rules = append(appsecCol.Rules, strRule)
+
+				//We only take the first id, as it's the one of the "main" rule
+				if _, ok := AppsecRulesDetails[int(rulesId[0])]; !ok {
+					AppsecRulesDetails[int(rulesId[0])] = RulesDetails{
+						LogLevel: log.InfoLevel,
+						Hash:     appsecRule.hash,
+						Version:  appsecRule.version,
+						Name:     appsecRule.Name,
+					}
+				} else {
+					logger.Warnf("conflicting id %d for rule %s !", rulesId[0], rule.Name)
+				}
+
+				for _, id := range rulesId {
+					SetRuleDebug(int(id), appsecRule.Debug)
+				}
+			}
+		}
+		ret = append(ret, appsecCol)
+	}
+	if len(ret) == 0 {
+		return nil, fmt.Errorf("no appsec-rules found for pattern %s", pattern)
+	}
+	return ret, nil
+}
+
+func (w AppsecCollection) String() string {
+	ret := ""
+	for _, rule := range w.Rules {
+		ret += rule + "\n"
+	}
+	return ret
+}

pkg/appsec/coraza_logger.go

@@ -0,0 +1,194 @@
+package appsec
+
+import (
+	"fmt"
+	"io"
+
+	dbg "github.com/crowdsecurity/coraza/v3/debuglog"
+	log "github.com/sirupsen/logrus"
+)
+
+var DebugRules map[int]bool = map[int]bool{}
+
+func SetRuleDebug(id int, debug bool) {
+	DebugRules[id] = debug
+}
+
+func GetRuleDebug(id int) bool {
+	if val, ok := DebugRules[id]; ok {
+		return val
+	}
+	return false
+}
+
+// type ContextField func(Event) Event
+
+type crzLogEvent struct {
+	fields log.Fields
+	logger *log.Entry
+	muted  bool
+	level  log.Level
+}
+
+func (e *crzLogEvent) Msg(msg string) {
+	if e.muted {
+		return
+	}
+
+	/*this is a hack. As we want to have per-level rule debug but it's not allowed by coraza/modsec, if a rule ID is flagged to be in debug mode, the
+	.Int("rule_id", <ID>) call will set the log_level of the event to debug. However, given the logger is global to the appsec-runner,
+	we are switching forth and back the log level of the logger*/
+	oldLvl := e.logger.Logger.GetLevel()
+
+	if e.level != oldLvl {
+		e.logger.Logger.SetLevel(e.level)
+	}
+
+	if len(e.fields) == 0 {
+		e.logger.Log(e.level, msg)
+	} else {
+		e.logger.WithFields(e.fields).Log(e.level, msg)
+	}
+
+	if e.level != oldLvl {
+		e.logger.Logger.SetLevel(oldLvl)
+		e.level = oldLvl
+	}
+}
+
+func (e *crzLogEvent) Str(key, val string) dbg.Event {
+	if e.muted {
+		return e
+	}
+	e.fields[key] = val
+	return e
+}
+
+func (e *crzLogEvent) Err(err error) dbg.Event {
+	if e.muted {
+		return e
+	}
+	e.fields["error"] = err
+	return e
+}
+
+func (e *crzLogEvent) Bool(key string, b bool) dbg.Event {
+	if e.muted {
+		return e
+	}
+	e.fields[key] = b
+	return e
+}
+
+func (e *crzLogEvent) Int(key string, i int) dbg.Event {
+	if e.muted {
+		//this allows us to have per-rule debug logging
+		if key == "rule_id" && GetRuleDebug(i) {
+			e.muted = false
+			e.fields = map[string]interface{}{}
+			e.level = log.DebugLevel
+		} else {
+			return e
+		}
+	}
+	e.fields[key] = i
+	return e
+}
+
+func (e *crzLogEvent) Uint(key string, i uint) dbg.Event {
+	if e.muted {
+		return e
+	}
+	e.fields[key] = i
+	return e
+}
+
+func (e *crzLogEvent) Stringer(key string, val fmt.Stringer) dbg.Event {
+	if e.muted {
+		return e
+	}
+	e.fields[key] = val
+	return e
+}
+
+func (e crzLogEvent) IsEnabled() bool {
+	return !e.muted
+}
+
+type crzLogger struct {
+	logger        *log.Entry
+	defaultFields log.Fields
+	logLevel      log.Level
+}
+
+func NewCrzLogger(logger *log.Entry) crzLogger {
+	return crzLogger{logger: logger, logLevel: logger.Logger.GetLevel()}
+}
+
+func (c crzLogger) NewMutedEvt(lvl log.Level) dbg.Event {
+	return &crzLogEvent{muted: true, logger: c.logger, level: lvl}
+}
+func (c crzLogger) NewEvt(lvl log.Level) dbg.Event {
+	evt := &crzLogEvent{fields: map[string]interface{}{}, logger: c.logger, level: lvl}
+	if c.defaultFields != nil {
+		for k, v := range c.defaultFields {
+			evt.fields[k] = v
+		}
+	}
+	return evt
+}
+
+func (c crzLogger) WithOutput(w io.Writer) dbg.Logger {
+	return c
+}
+
+func (c crzLogger) WithLevel(lvl dbg.Level) dbg.Logger {
+	c.logLevel = log.Level(lvl)
+	c.logger.Logger.SetLevel(c.logLevel)
+	return c
+}
+
+func (c crzLogger) With(fs ...dbg.ContextField) dbg.Logger {
+	var e dbg.Event = c.NewEvt(c.logLevel)
+	for _, f := range fs {
+		e = f(e)
+	}
+	c.defaultFields = e.(*crzLogEvent).fields
+	return c
+}
+
+func (c crzLogger) Trace() dbg.Event {
+	if c.logLevel < log.TraceLevel {
+		return c.NewMutedEvt(log.TraceLevel)
+	}
+	return c.NewEvt(log.TraceLevel)
+}
+
+func (c crzLogger) Debug() dbg.Event {
+	if c.logLevel < log.DebugLevel {
+		return c.NewMutedEvt(log.DebugLevel)
+
+	}
+	return c.NewEvt(log.DebugLevel)
+}
+
+func (c crzLogger) Info() dbg.Event {
+	if c.logLevel < log.InfoLevel {
+		return c.NewMutedEvt(log.InfoLevel)
+	}
+	return c.NewEvt(log.InfoLevel)
+}
+
+func (c crzLogger) Warn() dbg.Event {
+	if c.logLevel < log.WarnLevel {
+		return c.NewMutedEvt(log.WarnLevel)
+	}
+	return c.NewEvt(log.WarnLevel)
+}
+
+func (c crzLogger) Error() dbg.Event {
+	if c.logLevel < log.ErrorLevel {
+		return c.NewMutedEvt(log.ErrorLevel)
+	}
+	return c.NewEvt(log.ErrorLevel)
+}

pkg/appsec/loader.go

@@ -0,0 +1,52 @@
+package appsec
+
+import (
+	"os"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+)
+
+var appsecRules map[string]AppsecCollectionConfig = make(map[string]AppsecCollectionConfig) //FIXME: would probably be better to have a struct for this
+
+var hub *cwhub.Hub //FIXME: this is a temporary hack to make the hub available in the package
+
+func LoadAppsecRules(hubInstance *cwhub.Hub) error {
+
+	hub = hubInstance
+
+	for _, hubAppsecRuleItem := range hub.GetItemMap(cwhub.APPSEC_RULES) {
+		if !hubAppsecRuleItem.State.Installed {
+			continue
+		}
+
+		content, err := os.ReadFile(hubAppsecRuleItem.State.LocalPath)
+
+		if err != nil {
+			log.Warnf("unable to read file %s : %s", hubAppsecRuleItem.State.LocalPath, err)
+			continue
+		}
+
+		var rule AppsecCollectionConfig
+
+		err = yaml.UnmarshalStrict(content, &rule)
+
+		if err != nil {
+			log.Warnf("unable to unmarshal file %s : %s", hubAppsecRuleItem.State.LocalPath, err)
+			continue
+		}
+
+		rule.hash = hubAppsecRuleItem.State.LocalHash
+		rule.version = hubAppsecRuleItem.Version
+
+		log.Infof("Adding %s to appsec rules", rule.Name)
+
+		appsecRules[rule.Name] = rule
+	}
+
+	if len(appsecRules) == 0 {
+		log.Debugf("No appsec rules found")
+	}
+	return nil
+}

pkg/appsec/request.go

@@ -0,0 +1,345 @@
+package appsec
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"regexp"
+
+	"github.com/google/uuid"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	URIHeaderName    = "X-Crowdsec-Appsec-Uri"
+	VerbHeaderName   = "X-Crowdsec-Appsec-Verb"
+	HostHeaderName   = "X-Crowdsec-Appsec-Host"
+	IPHeaderName     = "X-Crowdsec-Appsec-Ip"
+	APIKeyHeaderName = "X-Crowdsec-Appsec-Api-Key"
+)
+
+type ParsedRequest struct {
+	RemoteAddr           string                  `json:"remote_addr,omitempty"`
+	Host                 string                  `json:"host,omitempty"`
+	ClientIP             string                  `json:"client_ip,omitempty"`
+	URI                  string                  `json:"uri,omitempty"`
+	Args                 url.Values              `json:"args,omitempty"`
+	ClientHost           string                  `json:"client_host,omitempty"`
+	Headers              http.Header             `json:"headers,omitempty"`
+	URL                  *url.URL                `json:"url,omitempty"`
+	Method               string                  `json:"method,omitempty"`
+	Proto                string                  `json:"proto,omitempty"`
+	Body                 []byte                  `json:"body,omitempty"`
+	TransferEncoding     []string                `json:"transfer_encoding,omitempty"`
+	UUID                 string                  `json:"uuid,omitempty"`
+	Tx                   ExtendedTransaction     `json:"transaction,omitempty"`
+	ResponseChannel      chan AppsecTempResponse `json:"-"`
+	IsInBand             bool                    `json:"-"`
+	IsOutBand            bool                    `json:"-"`
+	AppsecEngine         string                  `json:"appsec_engine,omitempty"`
+	RemoteAddrNormalized string                  `json:"normalized_remote_addr,omitempty"`
+}
+
+type ReqDumpFilter struct {
+	req                   *ParsedRequest
+	HeadersContentFilters []string
+	HeadersNameFilters    []string
+	HeadersDrop           bool
+
+	BodyDrop bool
+	//BodyContentFilters []string TBD
+
+	ArgsContentFilters []string
+	ArgsNameFilters    []string
+	ArgsDrop           bool
+}
+
+func (r *ParsedRequest) DumpRequest(params ...any) *ReqDumpFilter {
+	filter := ReqDumpFilter{}
+	filter.BodyDrop = true
+	filter.HeadersNameFilters = []string{"cookie", "authorization"}
+	filter.req = r
+	return &filter
+}
+
+// clear filters
+func (r *ReqDumpFilter) NoFilters() *ReqDumpFilter {
+	r2 := ReqDumpFilter{}
+	r2.req = r.req
+	return &r2
+}
+
+func (r *ReqDumpFilter) WithEmptyHeadersFilters() *ReqDumpFilter {
+	r.HeadersContentFilters = []string{}
+	return r
+}
+
+func (r *ReqDumpFilter) WithHeadersContentFilters(filter string) *ReqDumpFilter {
+	r.HeadersContentFilters = append(r.HeadersContentFilters, filter)
+	return r
+}
+
+func (r *ReqDumpFilter) WithHeadersNameFilter(filter string) *ReqDumpFilter {
+	r.HeadersNameFilters = append(r.HeadersNameFilters, filter)
+	return r
+}
+
+func (r *ReqDumpFilter) WithNoHeaders() *ReqDumpFilter {
+	r.HeadersDrop = true
+	return r
+}
+
+func (r *ReqDumpFilter) WithHeaders() *ReqDumpFilter {
+	r.HeadersDrop = false
+	r.HeadersNameFilters = []string{}
+	return r
+}
+
+func (r *ReqDumpFilter) WithBody() *ReqDumpFilter {
+	r.BodyDrop = false
+	return r
+}
+
+func (r *ReqDumpFilter) WithNoBody() *ReqDumpFilter {
+	r.BodyDrop = true
+	return r
+}
+
+func (r *ReqDumpFilter) WithEmptyArgsFilters() *ReqDumpFilter {
+	r.ArgsContentFilters = []string{}
+	return r
+}
+
+func (r *ReqDumpFilter) WithArgsContentFilters(filter string) *ReqDumpFilter {
+	r.ArgsContentFilters = append(r.ArgsContentFilters, filter)
+	return r
+}
+
+func (r *ReqDumpFilter) WithArgsNameFilter(filter string) *ReqDumpFilter {
+	r.ArgsNameFilters = append(r.ArgsNameFilters, filter)
+	return r
+}
+
+func (r *ReqDumpFilter) FilterBody(out *ParsedRequest) error {
+	if r.BodyDrop {
+		return nil
+	}
+	out.Body = r.req.Body
+	return nil
+}
+
+func (r *ReqDumpFilter) FilterArgs(out *ParsedRequest) error {
+	if r.ArgsDrop {
+		return nil
+	}
+	if len(r.ArgsContentFilters) == 0 && len(r.ArgsNameFilters) == 0 {
+		out.Args = r.req.Args
+		return nil
+	}
+	out.Args = make(url.Values)
+	for k, vals := range r.req.Args {
+		reject := false
+		//exclude by match on name
+		for _, filter := range r.ArgsNameFilters {
+			ok, err := regexp.MatchString("(?i)"+filter, k)
+			if err != nil {
+				log.Debugf("error while matching string '%s' with '%s': %s", filter, k, err)
+				continue
+			}
+			if ok {
+				reject = true
+				break
+			}
+		}
+
+		for _, v := range vals {
+			//exclude by content
+			for _, filter := range r.ArgsContentFilters {
+				ok, err := regexp.MatchString("(?i)"+filter, v)
+				if err != nil {
+					log.Debugf("error while matching string '%s' with '%s': %s", filter, v, err)
+					continue
+				}
+				if ok {
+					reject = true
+					break
+				}
+
+			}
+		}
+		//if it was not rejected, let's add it
+		if !reject {
+			out.Args[k] = vals
+		}
+	}
+	return nil
+}
+
+func (r *ReqDumpFilter) FilterHeaders(out *ParsedRequest) error {
+	if r.HeadersDrop {
+		return nil
+	}
+
+	if len(r.HeadersContentFilters) == 0 && len(r.HeadersNameFilters) == 0 {
+		out.Headers = r.req.Headers
+		return nil
+	}
+
+	out.Headers = make(http.Header)
+	for k, vals := range r.req.Headers {
+		reject := false
+		//exclude by match on name
+		for _, filter := range r.HeadersNameFilters {
+			ok, err := regexp.MatchString("(?i)"+filter, k)
+			if err != nil {
+				log.Debugf("error while matching string '%s' with '%s': %s", filter, k, err)
+				continue
+			}
+			if ok {
+				reject = true
+				break
+			}
+		}
+
+		for _, v := range vals {
+			//exclude by content
+			for _, filter := range r.HeadersContentFilters {
+				ok, err := regexp.MatchString("(?i)"+filter, v)
+				if err != nil {
+					log.Debugf("error while matching string '%s' with '%s': %s", filter, v, err)
+					continue
+				}
+				if ok {
+					reject = true
+					break
+				}
+
+			}
+		}
+		//if it was not rejected, let's add it
+		if !reject {
+			out.Headers[k] = vals
+		}
+	}
+	return nil
+}
+
+func (r *ReqDumpFilter) GetFilteredRequest() *ParsedRequest {
+	//if there are no filters, we return the original request
+	if len(r.HeadersContentFilters) == 0 &&
+		len(r.HeadersNameFilters) == 0 &&
+		len(r.ArgsContentFilters) == 0 &&
+		len(r.ArgsNameFilters) == 0 &&
+		!r.BodyDrop && !r.HeadersDrop && !r.ArgsDrop {
+		log.Warningf("no filters, returning original request")
+		return r.req
+	}
+
+	r2 := ParsedRequest{}
+	r.FilterHeaders(&r2)
+	r.FilterBody(&r2)
+	r.FilterArgs(&r2)
+	return &r2
+}
+
+func (r *ReqDumpFilter) ToJSON() error {
+	fd, err := os.CreateTemp("/tmp/", "crowdsec_req_dump_*.json")
+	if err != nil {
+		return fmt.Errorf("while creating temp file: %w", err)
+	}
+	defer fd.Close()
+	enc := json.NewEncoder(fd)
+	enc.SetIndent("", "  ")
+
+	req := r.GetFilteredRequest()
+
+	log.Warningf("dumping : %+v", req)
+
+	if err := enc.Encode(req); err != nil {
+		return fmt.Errorf("while encoding request: %w", err)
+	}
+	log.Warningf("request dumped to %s", fd.Name())
+	return nil
+}
+
+// Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the App security Engine
+func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) {
+	var err error
+	body := make([]byte, 0)
+
+	if r.Body != nil {
+		body, err = io.ReadAll(r.Body)
+		if err != nil {
+			return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err)
+		}
+	}
+
+	// the real source of the request is set in 'x-client-ip'
+	clientIP := r.Header.Get(IPHeaderName)
+	if clientIP == "" {
+		return ParsedRequest{}, fmt.Errorf("missing '%s' header", IPHeaderName)
+	}
+	// the real target Host of the request is set in 'x-client-host'
+	clientHost := r.Header.Get(HostHeaderName)
+	if clientHost == "" {
+		return ParsedRequest{}, fmt.Errorf("missing '%s' header", HostHeaderName)
+	}
+	// the real URI of the request is set in 'x-client-uri'
+	clientURI := r.Header.Get(URIHeaderName)
+	if clientURI == "" {
+		return ParsedRequest{}, fmt.Errorf("missing '%s' header", URIHeaderName)
+	}
+	// the real VERB of the request is set in 'x-client-uri'
+	clientMethod := r.Header.Get(VerbHeaderName)
+	if clientMethod == "" {
+		return ParsedRequest{}, fmt.Errorf("missing '%s' header", VerbHeaderName)
+	}
+
+	// delete those headers before coraza process the request
+	delete(r.Header, IPHeaderName)
+	delete(r.Header, HostHeaderName)
+	delete(r.Header, URIHeaderName)
+	delete(r.Header, VerbHeaderName)
+
+	parsedURL, err := url.Parse(clientURI)
+	if err != nil {
+		return ParsedRequest{}, fmt.Errorf("unable to parse url '%s': %s", clientURI, err)
+	}
+
+	remoteAddrNormalized := ""
+	host, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		log.Errorf("Invalid appsec remote IP source %v: %s", r.RemoteAddr, err.Error())
+		remoteAddrNormalized = r.RemoteAddr
+	} else {
+		ip := net.ParseIP(host)
+		if ip == nil {
+			log.Errorf("Invalid appsec remote IP address source %v: %s", r.RemoteAddr, err.Error())
+			remoteAddrNormalized = r.RemoteAddr
+		} else {
+			remoteAddrNormalized = ip.String()
+		}
+	}
+
+	return ParsedRequest{
+		RemoteAddr:           r.RemoteAddr,
+		UUID:                 uuid.New().String(),
+		ClientHost:           clientHost,
+		ClientIP:             clientIP,
+		URI:                  parsedURL.Path,
+		Method:               clientMethod,
+		Host:                 r.Host,
+		Headers:              r.Header,
+		URL:                  r.URL,
+		Proto:                r.Proto,
+		Body:                 body,
+		Args:                 parsedURL.Query(), //TODO: Check if there's not potential bypass as it excludes malformed args
+		TransferEncoding:     r.TransferEncoding,
+		ResponseChannel:      make(chan AppsecTempResponse),
+		RemoteAddrNormalized: remoteAddrNormalized,
+	}, nil
+}

pkg/appsec/request_test.go

@@ -0,0 +1,181 @@
+package appsec
+
+import "testing"
+
+func TestBodyDumper(t *testing.T) {
+
+	tests := []struct {
+		name   string
+		req    *ParsedRequest
+		expect *ParsedRequest
+		filter func(r *ReqDumpFilter) *ReqDumpFilter
+	}{
+		{
+			name: "default filter (cookie+authorization stripped + no body)",
+			req: &ParsedRequest{
+				Body:    []byte("yo some body"),
+				Headers: map[string][]string{"cookie": {"toto"}, "authorization": {"tata"}, "foo": {"bar", "baz"}},
+			},
+			expect: &ParsedRequest{
+				Body:    []byte{},
+				Headers: map[string][]string{"foo": {"bar", "baz"}},
+			},
+			filter: func(r *ReqDumpFilter) *ReqDumpFilter {
+				return r
+			},
+		},
+		{
+			name: "explicit empty filter",
+			req: &ParsedRequest{
+				Body:    []byte("yo some body"),
+				Headers: map[string][]string{"cookie": {"toto"}, "authorization": {"tata"}, "foo": {"bar", "baz"}},
+			},
+			expect: &ParsedRequest{
+				Body:    []byte("yo some body"),
+				Headers: map[string][]string{"cookie": {"toto"}, "authorization": {"tata"}, "foo": {"bar", "baz"}},
+			},
+			filter: func(r *ReqDumpFilter) *ReqDumpFilter {
+				return r.NoFilters()
+			},
+		},
+		{
+			name: "filter header",
+			req: &ParsedRequest{
+				Body:    []byte{},
+				Headers: map[string][]string{"test1": {"toto"}, "test2": {"tata"}},
+			},
+			expect: &ParsedRequest{
+				Body:    []byte{},
+				Headers: map[string][]string{"test1": {"toto"}},
+			},
+			filter: func(r *ReqDumpFilter) *ReqDumpFilter {
+				return r.WithNoBody().WithHeadersNameFilter("test2")
+			},
+		},
+		{
+			name: "filter header content",
+			req: &ParsedRequest{
+				Body:    []byte{},
+				Headers: map[string][]string{"test1": {"toto"}, "test2": {"tata"}},
+			},
+			expect: &ParsedRequest{
+				Body:    []byte{},
+				Headers: map[string][]string{"test1": {"toto"}},
+			},
+			filter: func(r *ReqDumpFilter) *ReqDumpFilter {
+				return r.WithHeadersContentFilters("tata")
+			},
+		},
+		{
+			name: "with headers",
+			req: &ParsedRequest{
+				Body:    []byte{},
+				Headers: map[string][]string{"cookie1": {"lol"}},
+			},
+			expect: &ParsedRequest{
+				Body:    []byte{},
+				Headers: map[string][]string{"cookie1": {"lol"}},
+			},
+			filter: func(r *ReqDumpFilter) *ReqDumpFilter {
+				return r.WithHeaders()
+			},
+		},
+		{
+			name: "drop headers",
+			req: &ParsedRequest{
+				Body:    []byte{},
+				Headers: map[string][]string{"toto": {"lol"}},
+			},
+			expect: &ParsedRequest{
+				Body:    []byte{},
+				Headers: map[string][]string{},
+			},
+			filter: func(r *ReqDumpFilter) *ReqDumpFilter {
+				return r.WithNoHeaders()
+			},
+		},
+		{
+			name: "with body",
+			req: &ParsedRequest{
+				Body:    []byte("toto"),
+				Headers: map[string][]string{"toto": {"lol"}},
+			},
+			expect: &ParsedRequest{
+				Body:    []byte("toto"),
+				Headers: map[string][]string{"toto": {"lol"}},
+			},
+			filter: func(r *ReqDumpFilter) *ReqDumpFilter {
+				return r.WithBody()
+			},
+		},
+		{
+			name: "with empty args filter",
+			req: &ParsedRequest{
+				Args: map[string][]string{"toto": {"lol"}},
+			},
+			expect: &ParsedRequest{
+				Args: map[string][]string{"toto": {"lol"}},
+			},
+			filter: func(r *ReqDumpFilter) *ReqDumpFilter {
+				return r.WithEmptyArgsFilters()
+			},
+		},
+		{
+			name: "with args name filter",
+			req: &ParsedRequest{
+				Args: map[string][]string{"toto": {"lol"}, "totolol": {"lol"}},
+			},
+			expect: &ParsedRequest{
+				Args: map[string][]string{"totolol": {"lol"}},
+			},
+			filter: func(r *ReqDumpFilter) *ReqDumpFilter {
+				return r.WithArgsNameFilter("toto")
+			},
+		},
+		{
+			name: "WithEmptyHeadersFilters",
+			req: &ParsedRequest{
+				Args: map[string][]string{"cookie": {"lol"}, "totolol": {"lol"}},
+			},
+			expect: &ParsedRequest{
+				Args: map[string][]string{"cookie": {"lol"}, "totolol": {"lol"}},
+			},
+			filter: func(r *ReqDumpFilter) *ReqDumpFilter {
+				return r.WithEmptyHeadersFilters()
+			},
+		},
+		{
+			name: "WithArgsContentFilters",
+			req: &ParsedRequest{
+				Args: map[string][]string{"test": {"lol"}, "test2": {"toto"}},
+			},
+			expect: &ParsedRequest{
+				Args: map[string][]string{"test": {"lol"}},
+			},
+			filter: func(r *ReqDumpFilter) *ReqDumpFilter {
+				return r.WithArgsContentFilters("toto")
+			},
+		},
+	}
+
+	for idx, test := range tests {
+
+		t.Run(test.name, func(t *testing.T) {
+			orig_dr := test.req.DumpRequest()
+			result := test.filter(orig_dr).GetFilteredRequest()
+
+			if len(result.Body) != len(test.expect.Body) {
+				t.Fatalf("test %d (%s) failed, got %d, expected %d", idx, test.name, len(test.req.Body), len(test.expect.Body))
+			}
+			if len(result.Headers) != len(test.expect.Headers) {
+				t.Fatalf("test %d (%s) failed, got %d, expected %d", idx, test.name, len(test.req.Headers), len(test.expect.Headers))
+			}
+			for k, v := range result.Headers {
+				if len(v) != len(test.expect.Headers[k]) {
+					t.Fatalf("test %d (%s) failed, got %d, expected %d", idx, test.name, len(v), len(test.expect.Headers[k]))
+				}
+			}
+		})
+	}
+
+}

pkg/appsec/tx.go

@@ -0,0 +1,93 @@
+package appsec
+
+import (
+	"github.com/crowdsecurity/coraza/v3"
+	"github.com/crowdsecurity/coraza/v3/experimental"
+	"github.com/crowdsecurity/coraza/v3/experimental/plugins/plugintypes"
+	"github.com/crowdsecurity/coraza/v3/types"
+)
+
+type ExtendedTransaction struct {
+	Tx experimental.FullTransaction
+}
+
+func NewExtendedTransaction(engine coraza.WAF, uuid string) ExtendedTransaction {
+	inBoundTx := engine.NewTransactionWithID(uuid)
+	expTx := inBoundTx.(experimental.FullTransaction)
+	tx := NewTransaction(expTx)
+	return tx
+}
+
+func NewTransaction(tx experimental.FullTransaction) ExtendedTransaction {
+	return ExtendedTransaction{Tx: tx}
+}
+
+func (t *ExtendedTransaction) RemoveRuleByIDWithError(id int) error {
+	t.Tx.RemoveRuleByID(id)
+	return nil
+}
+
+func (t *ExtendedTransaction) RemoveRuleByTagWithError(tag string) error {
+	t.Tx.RemoveRuleByTag(tag)
+	return nil
+}
+
+func (t *ExtendedTransaction) IsRuleEngineOff() bool {
+	return t.Tx.IsRuleEngineOff()
+}
+
+func (t *ExtendedTransaction) ProcessLogging() {
+	t.Tx.ProcessLogging()
+}
+
+func (t *ExtendedTransaction) ProcessConnection(client string, cPort int, server string, sPort int) {
+	t.Tx.ProcessConnection(client, cPort, server, sPort)
+}
+
+func (t *ExtendedTransaction) AddGetRequestArgument(name string, value string) {
+	t.Tx.AddGetRequestArgument(name, value)
+}
+
+func (t *ExtendedTransaction) ProcessURI(uri string, method string, httpVersion string) {
+	t.Tx.ProcessURI(uri, method, httpVersion)
+}
+
+func (t *ExtendedTransaction) AddRequestHeader(name string, value string) {
+	t.Tx.AddRequestHeader(name, value)
+}
+
+func (t *ExtendedTransaction) SetServerName(name string) {
+	t.Tx.SetServerName(name)
+}
+
+func (t *ExtendedTransaction) ProcessRequestHeaders() *types.Interruption {
+	return t.Tx.ProcessRequestHeaders()
+}
+
+func (t *ExtendedTransaction) ProcessRequestBody() (*types.Interruption, error) {
+	return t.Tx.ProcessRequestBody()
+}
+
+func (t *ExtendedTransaction) WriteRequestBody(body []byte) (*types.Interruption, int, error) {
+	return t.Tx.WriteRequestBody(body)
+}
+
+func (t *ExtendedTransaction) Interruption() *types.Interruption {
+	return t.Tx.Interruption()
+}
+
+func (t *ExtendedTransaction) IsInterrupted() bool {
+	return t.Tx.IsInterrupted()
+}
+
+func (t *ExtendedTransaction) Variables() plugintypes.TransactionVariables {
+	return t.Tx.Variables()
+}
+
+func (t *ExtendedTransaction) MatchedRules() []types.MatchedRule {
+	return t.Tx.MatchedRules()
+}
+
+func (t *ExtendedTransaction) ID() string {
+	return t.Tx.ID()
+}

pkg/appsec/waf_helpers.go

@@ -0,0 +1,59 @@
+package appsec
+
+import (
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+)
+
+func GetOnLoadEnv(w *AppsecRuntimeConfig) map[string]interface{} {
+	return map[string]interface{}{
+		"RemoveInBandRuleByID":    w.DisableInBandRuleByID,
+		"RemoveInBandRuleByTag":   w.DisableInBandRuleByTag,
+		"RemoveInBandRuleByName":  w.DisableInBandRuleByName,
+		"RemoveOutBandRuleByID":   w.DisableOutBandRuleByID,
+		"RemoveOutBandRuleByTag":  w.DisableOutBandRuleByTag,
+		"RemoveOutBandRuleByName": w.DisableOutBandRuleByName,
+		"SetRemediationByTag":     w.SetActionByTag,
+		"SetRemediationByID":      w.SetActionByID,
+		"SetRemediationByName":    w.SetActionByName,
+	}
+}
+
+func GetPreEvalEnv(w *AppsecRuntimeConfig, request *ParsedRequest) map[string]interface{} {
+	return map[string]interface{}{
+		"IsInBand":                request.IsInBand,
+		"IsOutBand":               request.IsOutBand,
+		"RemoveInBandRuleByID":    w.RemoveInbandRuleByID,
+		"RemoveInBandRuleByName":  w.RemoveInbandRuleByName,
+		"RemoveInBandRuleByTag":   w.RemoveInbandRuleByTag,
+		"RemoveOutBandRuleByID":   w.RemoveOutbandRuleByID,
+		"RemoveOutBandRuleByTag":  w.RemoveOutbandRuleByTag,
+		"RemoveOutBandRuleByName": w.RemoveOutbandRuleByName,
+		"SetRemediationByTag":     w.SetActionByTag,
+		"SetRemediationByID":      w.SetActionByID,
+		"SetRemediationByName":    w.SetActionByName,
+	}
+}
+
+func GetPostEvalEnv(w *AppsecRuntimeConfig, request *ParsedRequest) map[string]interface{} {
+	return map[string]interface{}{
+		"IsInBand":    request.IsInBand,
+		"IsOutBand":   request.IsOutBand,
+		"DumpRequest": request.DumpRequest,
+	}
+}
+
+func GetOnMatchEnv(w *AppsecRuntimeConfig, request *ParsedRequest, evt types.Event) map[string]interface{} {
+	return map[string]interface{}{
+		"evt":            evt,
+		"req":            request,
+		"IsInBand":       request.IsInBand,
+		"IsOutBand":      request.IsOutBand,
+		"SetRemediation": w.SetAction,
+		"SetReturnCode":  w.SetHTTPCode,
+		"CancelEvent":    w.CancelEvent,
+		"SendEvent":      w.SendEvent,
+		"CancelAlert":    w.CancelAlert,
+		"SendAlert":      w.SendAlert,
+		"DumpRequest":    request.DumpRequest,
+	}
+}

pkg/csconfig/config.go

@@ -21,6 +21,8 @@ var defaultConfigDir = "/etc/crowdsec"
 // defaultDataDir is the base path to all data files, to be overridden in the Makefile */
 var defaultDataDir = "/var/lib/crowdsec/data/"
 
+var globalConfig = Config{}
+
 // Config contains top-level defaults -> overridden by configuration file -> overridden by CLI flags
 type Config struct {
 	//just a path to ourselves :p
@@ -89,9 +91,15 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool
 		return nil, "", err
 	}
 
+	globalConfig = cfg
+
 	return &cfg, configData, nil
 }
 
+func GetConfig() Config {
+	return globalConfig
+}
+
 func NewDefaultConfig() *Config {
 	logLevel := log.InfoLevel
 	commonCfg := CommonCfg{

pkg/csconfig/crowdsec_service.go

@@ -108,8 +108,9 @@ func (c *Config) LoadCrowdsec() error {
 		c.Crowdsec.OutputRoutinesCount = 1
 	}
 
-	var crowdsecCleanup = []*string{
+	crowdsecCleanup := []*string{
 		&c.Crowdsec.AcquisitionFilePath,
+		&c.Crowdsec.ConsoleContextPath,
 	}
 
 	for _, k := range crowdsecCleanup {
@@ -131,38 +132,10 @@ func (c *Config) LoadCrowdsec() error {
 		c.Crowdsec.AcquisitionFiles[i] = f
 	}
 
-	if err := c.LoadAPIClient(); err != nil {
+	if err = c.LoadAPIClient(); err != nil {
 		return fmt.Errorf("loading api client: %s", err)
 	}
 
-	c.Crowdsec.ContextToSend = make(map[string][]string, 0)
-	fallback := false
-	if c.Crowdsec.ConsoleContextPath == "" {
-		// fallback to default config file
-		c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
-		fallback = true
-	}
-
-	f, err := filepath.Abs(c.Crowdsec.ConsoleContextPath)
-	if err != nil {
-		return fmt.Errorf("fail to get absolute path of %s: %s", c.Crowdsec.ConsoleContextPath, err)
-	}
-
-	c.Crowdsec.ConsoleContextPath = f
-	yamlFile, err := os.ReadFile(c.Crowdsec.ConsoleContextPath)
-	if err != nil {
-		if fallback {
-			log.Debugf("Default context config file doesn't exist, will not use it")
-		} else {
-			return fmt.Errorf("failed to open context file: %s", err)
-		}
-	} else {
-		err = yaml.Unmarshal(yamlFile, c.Crowdsec.ContextToSend)
-		if err != nil {
-			return fmt.Errorf("unmarshaling labels console config file '%s': %s", c.Crowdsec.ConsoleContextPath, err)
-		}
-	}
-
 	return nil
 }
 
@@ -170,10 +143,16 @@ func (c *CrowdsecServiceCfg) DumpContextConfigFile() error {
 	var out []byte
 	var err error
 
+	// XXX: MakeDirs
+
 	if out, err = yaml.Marshal(c.ContextToSend); err != nil {
 		return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleContextPath, err)
 	}
 
+	if err = os.MkdirAll(filepath.Dir(c.ConsoleContextPath), 0700); err != nil {
+		return fmt.Errorf("while creating directories for %s: %w", c.ConsoleContextPath, err)
+	}
+
 	if err := os.WriteFile(c.ConsoleContextPath, out, 0600); err != nil {
 		return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleContextPath, err)
 	}

pkg/csconfig/crowdsec_service_test.go

@@ -60,9 +60,10 @@ func TestLoadCrowdsec(t *testing.T) {
 				ConsoleContextValueLength: 2500,
 				AcquisitionFiles:          []string{acquisFullPath},
 				SimulationFilePath:        "./testdata/simulation.yaml",
-				ContextToSend: map[string][]string{
-					"source_ip": {"evt.Parsed.source_ip"},
-				},
+				// context is loaded in pkg/alertcontext
+//				ContextToSend: map[string][]string{
+//					"source_ip": {"evt.Parsed.source_ip"},
+//				},
 				SimulationConfig: &SimulationConfig{
 					Simulation: ptr.Of(false),
 				},
@@ -98,9 +99,10 @@ func TestLoadCrowdsec(t *testing.T) {
 				OutputRoutinesCount:       1,
 				ConsoleContextValueLength: 0,
 				AcquisitionFiles:          []string{acquisFullPath, acquisInDirFullPath},
-				ContextToSend: map[string][]string{
-					"source_ip": {"evt.Parsed.source_ip"},
-				},
+				// context is loaded in pkg/alertcontext
+//				ContextToSend: map[string][]string{
+//					"source_ip": {"evt.Parsed.source_ip"},
+//				},
 				SimulationFilePath: "./testdata/simulation.yaml",
 				SimulationConfig: &SimulationConfig{
 					Simulation: ptr.Of(false),
@@ -136,9 +138,10 @@ func TestLoadCrowdsec(t *testing.T) {
 				ConsoleContextValueLength: 10,
 				AcquisitionFiles:          []string{},
 				SimulationFilePath:        "",
-				ContextToSend: map[string][]string{
-					"source_ip": {"evt.Parsed.source_ip"},
-				},
+				// context is loaded in pkg/alertcontext
+//				ContextToSend: map[string][]string{
+//					"source_ip": {"evt.Parsed.source_ip"},
+//				},
 				SimulationConfig: &SimulationConfig{
 					Simulation: ptr.Of(false),
 				},

pkg/cwhub/item.go

@@ -12,10 +12,13 @@ import (
 
 const (
 	// managed item types.
-	COLLECTIONS   = "collections"
-	PARSERS       = "parsers"
-	POSTOVERFLOWS = "postoverflows"
-	SCENARIOS     = "scenarios"
+	COLLECTIONS    = "collections"
+	PARSERS        = "parsers"
+	POSTOVERFLOWS  = "postoverflows"
+	SCENARIOS      = "scenarios"
+	CONTEXTS       = "contexts"
+	APPSEC_CONFIGS = "appsec-configs"
+	APPSEC_RULES   = "appsec-rules"
 )
 
 const (
@@ -27,7 +30,7 @@ const (
 
 var (
 	// The order is important, as it is used to range over sub-items in collections.
-	ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
+	ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, CONTEXTS, APPSEC_CONFIGS, APPSEC_RULES, COLLECTIONS}
 )
 
 type HubItems map[string]map[string]*Item
@@ -118,6 +121,9 @@ type Item struct {
 	PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
 	Scenarios     []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"`
 	Collections   []string `json:"collections,omitempty" yaml:"collections,omitempty"`
+	Contexts      []string `json:"contexts,omitempty" yaml:"contexts,omitempty"`
+	AppsecConfigs []string `json:"appsec-configs,omitempty"   yaml:"appsec-configs,omitempty"`
+	AppsecRules   []string `json:"appsec-rules,omitempty"   yaml:"appsec-rules,omitempty"`
 }
 
 // installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local
@@ -227,6 +233,33 @@ func (i *Item) SubItems() []*Item {
 		sub = append(sub, s)
 	}
 
+	for _, name := range i.Contexts {
+		s := i.hub.GetItem(CONTEXTS, name)
+		if s == nil {
+			continue
+		}
+
+		sub = append(sub, s)
+	}
+
+	for _, name := range i.AppsecConfigs {
+		s := i.hub.GetItem(APPSEC_CONFIGS, name)
+		if s == nil {
+			continue
+		}
+
+		sub = append(sub, s)
+	}
+
+	for _, name := range i.AppsecRules {
+		s := i.hub.GetItem(APPSEC_RULES, name)
+		if s == nil {
+			continue
+		}
+
+		sub = append(sub, s)
+	}
+
 	for _, name := range i.Collections {
 		s := i.hub.GetItem(COLLECTIONS, name)
 		if s == nil {
@@ -262,6 +295,24 @@ func (i *Item) logMissingSubItems() {
 		}
 	}
 
+	for _, subName := range i.Contexts {
+		if i.hub.GetItem(CONTEXTS, subName) == nil {
+			log.Errorf("can't find %s in %s, required by %s", subName, CONTEXTS, i.Name)
+		}
+	}
+
+	for _, subName := range i.AppsecConfigs {
+		if i.hub.GetItem(APPSEC_CONFIGS, subName) == nil {
+			log.Errorf("can't find %s in %s, required by %s", subName, APPSEC_CONFIGS, i.Name)
+		}
+	}
+
+	for _, subName := range i.AppsecRules {
+		if i.hub.GetItem(APPSEC_RULES, subName) == nil {
+			log.Errorf("can't find %s in %s, required by %s", subName, APPSEC_RULES, i.Name)
+		}
+	}
+
 	for _, subName := range i.Collections {
 		if i.hub.GetItem(COLLECTIONS, subName) == nil {
 			log.Errorf("can't find %s in %s, required by %s", subName, COLLECTIONS, i.Name)

pkg/cwhub/sync.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strings"
 
@@ -112,15 +113,13 @@ func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) {
 
 	log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
 
-	if ret.stage == SCENARIOS {
-		ret.ftype = SCENARIOS
+	if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS {
+		if !slices.Contains(ItemTypes, ret.stage) {
+			return nil, fmt.Errorf("unknown configuration type for file '%s'", path)
+		}
+
+		ret.ftype = ret.stage
 		ret.stage = ""
-	} else if ret.stage == COLLECTIONS {
-		ret.ftype = COLLECTIONS
-		ret.stage = ""
-	} else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS {
-		// it's a PARSER / POSTOVERFLOW with a stage
-		return nil, fmt.Errorf("unknown configuration type for file '%s'", path)
 	}
 
 	log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype)
@@ -347,7 +346,7 @@ func (i *Item) checkSubItemVersions() error {
 
 // syncDir scans a directory for items, and updates the Hub state accordingly.
 func (h *Hub) syncDir(dir string) error {
-	// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
+	// For each, scan PARSERS, POSTOVERFLOWS... and COLLECTIONS last
 	for _, scan := range ItemTypes {
 		// cpath: top-level item directory, either downloaded or installed items.
 		// i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ...

pkg/exprhelpers/expr_lib.go

@@ -20,6 +20,21 @@ var exprFuncs = []exprCustomFunc{
 			new(func(string) (*cticlient.SmokeItem, error)),
 		},
 	},
+	{
+		name:      "Flatten",
+		function:  Flatten,
+		signature: []interface{}{},
+	},
+	{
+		name:      "Distinct",
+		function:  Distinct,
+		signature: []interface{}{},
+	},
+	{
+		name:      "FlattenDistinct",
+		function:  FlattenDistinct,
+		signature: []interface{}{},
+	},
 	{
 		name:     "Distance",
 		function: Distance,

pkg/exprhelpers/helpers.go

@@ -9,6 +9,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"reflect"
 	"regexp"
 	"strconv"
 	"strings"
@@ -176,6 +177,54 @@ func FileInit(fileFolder string, filename string, fileType string) error {
 	return nil
 }
 
+// Expr helpers
+
+func Distinct(params ...any) (any, error) {
+
+	if rt := reflect.TypeOf(params[0]).Kind(); rt != reflect.Slice && rt != reflect.Array {
+		return nil, nil
+	}
+	array := params[0].([]interface{})
+	if array == nil {
+		return []interface{}{}, nil
+	}
+
+	var exists map[any]bool = make(map[any]bool)
+	var ret []interface{} = make([]interface{}, 0)
+
+	for _, val := range array {
+		if _, ok := exists[val]; !ok {
+			exists[val] = true
+			ret = append(ret, val)
+		}
+	}
+	return ret, nil
+
+}
+
+func FlattenDistinct(params ...any) (any, error) {
+	return Distinct(flatten(nil, reflect.ValueOf(params))) //nolint:asasalint
+}
+
+func Flatten(params ...any) (any, error) {
+	return flatten(nil, reflect.ValueOf(params)), nil
+}
+
+func flatten(args []interface{}, v reflect.Value) []interface{} {
+	if v.Kind() == reflect.Interface {
+		v = v.Elem()
+	}
+
+	if v.Kind() == reflect.Array || v.Kind() == reflect.Slice {
+		for i := 0; i < v.Len(); i++ {
+			args = flatten(args, v.Index(i))
+		}
+	} else {
+		args = append(args, v.Interface())
+	}
+
+	return args
+}
 func existsInFileMaps(filename string, ftype string) (bool, error) {
 	ok := false
 	var err error

pkg/hubtest/coverage.go

@@ -7,9 +7,10 @@ import (
 	"path/filepath"
 	"strings"
 
-	log "github.com/sirupsen/logrus"
-
+	"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
 )
 
 type Coverage struct {
@@ -18,6 +19,65 @@ type Coverage struct {
 	PresentIn  map[string]bool //poorman's set
 }
 
+func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) {
+	if len(h.HubIndex.GetItemMap(cwhub.APPSEC_RULES)) == 0 {
+		return nil, fmt.Errorf("no appsec rules in hub index")
+	}
+
+	// populate from hub, iterate in alphabetical order
+	pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.APPSEC_RULES))
+	coverage := make([]Coverage, len(pkeys))
+
+	for i, name := range pkeys {
+		coverage[i] = Coverage{
+			Name:       name,
+			TestsCount: 0,
+			PresentIn:  make(map[string]bool),
+		}
+	}
+
+	// parser the expressions a-la-oneagain
+	appsecTestConfigs, err := filepath.Glob(".appsec-tests/*/config.yaml")
+	if err != nil {
+		return nil, fmt.Errorf("while find appsec-tests config: %s", err)
+	}
+
+	for _, appsecTestConfigPath := range appsecTestConfigs {
+		configFileData := &HubTestItemConfig{}
+		yamlFile, err := os.ReadFile(appsecTestConfigPath)
+		if err != nil {
+			log.Printf("unable to open appsec test config file '%s': %s", appsecTestConfigPath, err)
+			continue
+		}
+		err = yaml.Unmarshal(yamlFile, configFileData)
+		if err != nil {
+			return nil, fmt.Errorf("unmarshal: %v", err)
+		}
+
+		for _, appsecRulesFile := range configFileData.AppsecRules {
+			appsecRuleData := &appsec_rule.CustomRule{}
+			yamlFile, err := os.ReadFile(appsecRulesFile)
+			if err != nil {
+				log.Printf("unable to open appsec rule '%s': %s", appsecRulesFile, err)
+			}
+			err = yaml.Unmarshal(yamlFile, appsecRuleData)
+			if err != nil {
+				return nil, fmt.Errorf("unmarshal: %v", err)
+			}
+			appsecRuleName := appsecRuleData.Name
+
+			for idx, cov := range coverage {
+				if cov.Name == appsecRuleName {
+					coverage[idx].TestsCount++
+					coverage[idx].PresentIn[appsecTestConfigPath] = true
+				}
+			}
+		}
+	}
+
+	return coverage, nil
+}
+
 func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
 	if len(h.HubIndex.GetItemMap(cwhub.PARSERS)) == 0 {
 		return nil, fmt.Errorf("no parsers in hub index")
@@ -105,7 +165,7 @@ func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
 }
 
 func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) {
-	if len(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) == 0  {
+	if len(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) == 0 {
 		return nil, fmt.Errorf("no scenarios in hub index")
 	}
 
@@ -127,7 +187,6 @@ func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) {
 		return nil, fmt.Errorf("while find scenario asserts : %s", err)
 	}
 
-
 	for _, assert := range passerts {
 		file, err := os.Open(assert)
 		if err != nil {

pkg/hubtest/hubtest.go

@@ -11,25 +11,30 @@ import (
 )
 
 type HubTest struct {
-	CrowdSecPath           string
-	CscliPath              string
-	HubPath                string
-	HubTestPath            string
-	HubIndexFile           string
-	TemplateConfigPath     string
-	TemplateProfilePath    string
-	TemplateSimulationPath string
-	HubIndex               *cwhub.Hub
-	Tests                  []*HubTestItem
+	CrowdSecPath              string
+	CscliPath                 string
+	HubPath                   string
+	HubTestPath               string //generic parser/scenario tests .tests
+	HubAppsecTestPath         string //dir specific to appsec tests .appsec-tests
+	HubIndexFile              string
+	TemplateConfigPath        string
+	TemplateProfilePath       string
+	TemplateSimulationPath    string
+	TemplateAcquisPath        string
+	TemplateAppsecProfilePath string
+	HubIndex                  *cwhub.Hub
+	Tests                     []*HubTestItem
 }
 
 const (
-	templateConfigFile     = "template_config.yaml"
-	templateSimulationFile = "template_simulation.yaml"
-	templateProfileFile    = "template_profiles.yaml"
+	templateConfigFile        = "template_config.yaml"
+	templateSimulationFile    = "template_simulation.yaml"
+	templateProfileFile       = "template_profiles.yaml"
+	templateAcquisFile        = "template_acquis.yaml"
+	templateAppsecProfilePath = "template_appsec-profile.yaml"
 )
 
-func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, error) {
+func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecTest bool) (HubTest, error) {
 	hubPath, err := filepath.Abs(hubPath)
 	if err != nil {
 		return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err)
@@ -39,9 +44,6 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest,
 	if _, err = os.Stat(hubPath); os.IsNotExist(err) {
 		return HubTest{}, fmt.Errorf("path to hub '%s' doesn't exist, can't run", hubPath)
 	}
-
-	HubTestPath := filepath.Join(hubPath, "./.tests/")
-
 	// we can't use hubtest without crowdsec binary
 	if _, err = exec.LookPath(crowdsecPath); err != nil {
 		if _, err = os.Stat(crowdsecPath); os.IsNotExist(err) {
@@ -56,6 +58,39 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest,
 		}
 	}
 
+	if isAppsecTest {
+		HubTestPath := filepath.Join(hubPath, "./.appsec-tests/")
+		hubIndexFile := filepath.Join(hubPath, ".index.json")
+
+		local := &csconfig.LocalHubCfg{
+			HubDir:         hubPath,
+			HubIndexFile:   hubIndexFile,
+			InstallDir:     HubTestPath,
+			InstallDataDir: HubTestPath,
+		}
+
+		hub, err := cwhub.NewHub(local, nil, false)
+		if err != nil {
+			return HubTest{}, fmt.Errorf("unable to load hub: %s", err)
+		}
+
+		return HubTest{
+			CrowdSecPath:              crowdsecPath,
+			CscliPath:                 cscliPath,
+			HubPath:                   hubPath,
+			HubTestPath:               HubTestPath,
+			HubIndexFile:              hubIndexFile,
+			TemplateConfigPath:        filepath.Join(HubTestPath, templateConfigFile),
+			TemplateProfilePath:       filepath.Join(HubTestPath, templateProfileFile),
+			TemplateSimulationPath:    filepath.Join(HubTestPath, templateSimulationFile),
+			TemplateAppsecProfilePath: filepath.Join(HubTestPath, templateAppsecProfilePath),
+			TemplateAcquisPath:        filepath.Join(HubTestPath, templateAcquisFile),
+			HubIndex:                  hub,
+		}, nil
+	}
+
+	HubTestPath := filepath.Join(hubPath, "./.tests/")
+
 	hubIndexFile := filepath.Join(hubPath, ".index.json")
 
 	local := &csconfig.LocalHubCfg{
@@ -70,19 +105,15 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest,
 		return HubTest{}, fmt.Errorf("unable to load hub: %s", err)
 	}
 
-	templateConfigFilePath := filepath.Join(HubTestPath, templateConfigFile)
-	templateProfilePath := filepath.Join(HubTestPath, templateProfileFile)
-	templateSimulationPath := filepath.Join(HubTestPath, templateSimulationFile)
-
 	return HubTest{
 		CrowdSecPath:           crowdsecPath,
 		CscliPath:              cscliPath,
 		HubPath:                hubPath,
 		HubTestPath:            HubTestPath,
 		HubIndexFile:           hubIndexFile,
-		TemplateConfigPath:     templateConfigFilePath,
-		TemplateProfilePath:    templateProfilePath,
-		TemplateSimulationPath: templateSimulationPath,
+		TemplateConfigPath:     filepath.Join(HubTestPath, templateConfigFile),
+		TemplateProfilePath:    filepath.Join(HubTestPath, templateProfileFile),
+		TemplateSimulationPath: filepath.Join(HubTestPath, templateSimulationFile),
 		HubIndex:               hub,
 	}, nil
 }

pkg/hubtest/hubtest_item.go

@@ -1,7 +1,9 @@
 package hubtest
 
 import (
+	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -16,14 +18,17 @@ import (
 )
 
 type HubTestItemConfig struct {
-	Parsers         []string            `yaml:"parsers"`
-	Scenarios       []string            `yaml:"scenarios"`
-	PostOVerflows   []string            `yaml:"postoverflows"`
-	LogFile         string              `yaml:"log_file"`
-	LogType         string              `yaml:"log_type"`
-	Labels          map[string]string   `yaml:"labels"`
-	IgnoreParsers   bool                `yaml:"ignore_parsers"`   // if we test a scenario, we don't want to assert on Parser
-	OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00
+	Parsers               []string            `yaml:"parsers,omitempty"`
+	Scenarios             []string            `yaml:"scenarios,omitempty"`
+	PostOverflows         []string            `yaml:"postoverflows,omitempty"`
+	AppsecRules           []string            `yaml:"appsec-rules,omitempty"`
+	NucleiTemplate        string              `yaml:"nuclei_template,omitempty"`
+	ExpectedNucleiFailure bool                `yaml:"expect_failure,omitempty"`
+	LogFile               string              `yaml:"log_file,omitempty"`
+	LogType               string              `yaml:"log_type,omitempty"`
+	Labels                map[string]string   `yaml:"labels,omitempty"`
+	IgnoreParsers         bool                `yaml:"ignore_parsers,omitempty"`   // if we test a scenario, we don't want to assert on Parser
+	OverrideStatics       []parser.ExtraField `yaml:"override_statics,omitempty"` //Allow to override statics. Executed before s00
 }
 
 type HubTestItem struct {
@@ -40,6 +45,7 @@ type HubTestItem struct {
 	RuntimeConfigFilePath     string
 	RuntimeProfileFilePath    string
 	RuntimeSimulationFilePath string
+	RuntimeAcquisFilePath     string
 	RuntimeHubConfig          *csconfig.LocalHubCfg
 
 	ResultsPath          string
@@ -47,13 +53,15 @@ type HubTestItem struct {
 	ScenarioResultFile   string
 	BucketPourResultFile string
 
-	HubPath                string
-	HubTestPath            string
-	HubIndexFile           string
-	TemplateConfigPath     string
-	TemplateProfilePath    string
-	TemplateSimulationPath string
-	HubIndex               *cwhub.Hub
+	HubPath                   string
+	HubTestPath               string
+	HubIndexFile              string
+	TemplateConfigPath        string
+	TemplateProfilePath       string
+	TemplateSimulationPath    string
+	TemplateAcquisPath        string
+	TemplateAppsecProfilePath string
+	HubIndex                  *cwhub.Hub
 
 	Config *HubTestItemConfig
 
@@ -75,6 +83,11 @@ const (
 	ScenarioResultFileName = "bucket-dump.yaml"
 
 	BucketPourResultFileName = "bucketpour-dump.yaml"
+
+	TestBouncerApiKey = "this_is_a_bad_password"
+
+	DefaultNucleiTarget = "http://127.0.0.1:80/"
+	DefaultAppsecHost   = "127.0.0.1:4241"
 )
 
 func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
@@ -115,6 +128,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
 		RuntimeConfigFilePath:     filepath.Join(runtimeFolder, "config.yaml"),
 		RuntimeProfileFilePath:    filepath.Join(runtimeFolder, "profiles.yaml"),
 		RuntimeSimulationFilePath: filepath.Join(runtimeFolder, "simulation.yaml"),
+		RuntimeAcquisFilePath:     filepath.Join(runtimeFolder, "acquis.yaml"),
 		ResultsPath:               resultPath,
 		ParserResultFile:          filepath.Join(resultPath, ParserResultFileName),
 		ScenarioResultFile:        filepath.Join(resultPath, ScenarioResultFileName),
@@ -125,17 +139,19 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
 			InstallDir:     runtimeFolder,
 			InstallDataDir: filepath.Join(runtimeFolder, "data"),
 		},
-		Config:                 configFileData,
-		HubPath:                hubTest.HubPath,
-		HubTestPath:            hubTest.HubTestPath,
-		HubIndexFile:           hubTest.HubIndexFile,
-		TemplateConfigPath:     hubTest.TemplateConfigPath,
-		TemplateProfilePath:    hubTest.TemplateProfilePath,
-		TemplateSimulationPath: hubTest.TemplateSimulationPath,
-		HubIndex:               hubTest.HubIndex,
-		ScenarioAssert:         ScenarioAssert,
-		ParserAssert:           ParserAssert,
-		CustomItemsLocation:    []string{hubTest.HubPath, testPath},
+		Config:                    configFileData,
+		HubPath:                   hubTest.HubPath,
+		HubTestPath:               hubTest.HubTestPath,
+		HubIndexFile:              hubTest.HubIndexFile,
+		TemplateConfigPath:        hubTest.TemplateConfigPath,
+		TemplateProfilePath:       hubTest.TemplateProfilePath,
+		TemplateSimulationPath:    hubTest.TemplateSimulationPath,
+		TemplateAcquisPath:        hubTest.TemplateAcquisPath,
+		TemplateAppsecProfilePath: hubTest.TemplateAppsecProfilePath,
+		HubIndex:                  hubTest.HubIndex,
+		ScenarioAssert:            ScenarioAssert,
+		ParserAssert:              ParserAssert,
+		CustomItemsLocation:       []string{hubTest.HubPath, testPath},
 	}, nil
 }
 
@@ -297,8 +313,81 @@ func (t *HubTestItem) InstallHub() error {
 		}
 	}
 
+	// install appsec-rules in runtime environment
+	for _, appsecrule := range t.Config.AppsecRules {
+		log.Infof("adding rule '%s'", appsecrule)
+		if appsecrule == "" {
+			continue
+		}
+
+		if hubAppsecRule, ok := t.HubIndex.GetItemMap(cwhub.APPSEC_RULES)[appsecrule]; ok {
+			appsecRuleSource, err := filepath.Abs(filepath.Join(t.HubPath, hubAppsecRule.RemotePath))
+			if err != nil {
+				return fmt.Errorf("can't get absolute path of '%s': %s", appsecRuleSource, err)
+			}
+
+			appsecRuleFilename := filepath.Base(appsecRuleSource)
+
+			// runtime/hub/appsec-rules/author/appsec-rule
+			hubDirAppsecRuleDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubAppsecRule.RemotePath))
+
+			// runtime/appsec-rules/
+			appsecRuleDirDest := fmt.Sprintf("%s/appsec-rules/", t.RuntimePath)
+
+			if err := os.MkdirAll(hubDirAppsecRuleDest, os.ModePerm); err != nil {
+				return fmt.Errorf("unable to create folder '%s': %s", hubDirAppsecRuleDest, err)
+			}
+
+			if err := os.MkdirAll(appsecRuleDirDest, os.ModePerm); err != nil {
+				return fmt.Errorf("unable to create folder '%s': %s", appsecRuleDirDest, err)
+			}
+
+			// runtime/hub/appsec-rules/crowdsecurity/rule.yaml
+			hubDirAppsecRulePath := filepath.Join(appsecRuleDirDest, appsecRuleFilename)
+			if err := Copy(appsecRuleSource, hubDirAppsecRulePath); err != nil {
+				return fmt.Errorf("unable to copy '%s' to '%s': %s", appsecRuleSource, hubDirAppsecRulePath, err)
+			}
+
+			// runtime/appsec-rules/rule.yaml
+			appsecRulePath := filepath.Join(appsecRuleDirDest, appsecRuleFilename)
+			if err := os.Symlink(hubDirAppsecRulePath, appsecRulePath); err != nil {
+				if !os.IsExist(err) {
+					return fmt.Errorf("unable to symlink appsec-rule '%s' to '%s': %s", hubDirAppsecRulePath, appsecRulePath, err)
+				}
+			}
+		} else {
+			customAppsecRuleExist := false
+			for _, customPath := range t.CustomItemsLocation {
+				// we check if its a custom appsec-rule
+				customAppsecRulePath := filepath.Join(customPath, appsecrule)
+				if _, err := os.Stat(customAppsecRulePath); os.IsNotExist(err) {
+					continue
+				}
+				customAppsecRulePathSplit := strings.Split(customAppsecRulePath, "/")
+				customAppsecRuleName := customAppsecRulePathSplit[len(customAppsecRulePathSplit)-1]
+
+				appsecRuleDirDest := fmt.Sprintf("%s/appsec-rules/", t.RuntimePath)
+				if err := os.MkdirAll(appsecRuleDirDest, os.ModePerm); err != nil {
+					return fmt.Errorf("unable to create folder '%s': %s", appsecRuleDirDest, err)
+				}
+
+				// runtime/appsec-rules/
+				customAppsecRuleDest := fmt.Sprintf("%s/appsec-rules/%s", t.RuntimePath, customAppsecRuleName)
+				// if path to postoverflow exist, copy it
+				if err := Copy(customAppsecRulePath, customAppsecRuleDest); err != nil {
+					continue
+				}
+				customAppsecRuleExist = true
+				break
+			}
+			if !customAppsecRuleExist {
+				return fmt.Errorf("couldn't find custom appsec-rule '%s' in the following location: %+v", appsecrule, t.CustomItemsLocation)
+			}
+		}
+	}
+
 	// install postoverflows in runtime environment
-	for _, postoverflow := range t.Config.PostOVerflows {
+	for _, postoverflow := range t.Config.PostOverflows {
 		if postoverflow == "" {
 			continue
 		}
@@ -449,16 +538,114 @@ func (t *HubTestItem) Clean() error {
 	return os.RemoveAll(t.RuntimePath)
 }
 
-func (t *HubTestItem) Run() error {
-	t.Success = false
-	t.ErrorsList = make([]string, 0)
+func (t *HubTestItem) RunWithNucleiTemplate() error {
 
 	testPath := filepath.Join(t.HubTestPath, t.Name)
 	if _, err := os.Stat(testPath); os.IsNotExist(err) {
 		return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath)
 	}
 
-	currentDir, err := os.Getwd()
+	if err := os.Chdir(testPath); err != nil {
+		return fmt.Errorf("can't 'cd' to '%s': %s", testPath, err)
+	}
+
+	//machine add
+	cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"}
+	cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...)
+
+	output, err := cscliRegisterCmd.CombinedOutput()
+	if err != nil {
+		if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") {
+			fmt.Println(string(output))
+			return fmt.Errorf("fail to run '%s' for test '%s': %v", cscliRegisterCmd.String(), t.Name, err)
+		}
+	}
+
+	//hardcode bouncer key
+	cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "bouncers", "add", "appsectests", "-k", TestBouncerApiKey}
+	cscliBouncerCmd := exec.Command(t.CscliPath, cmdArgs...)
+
+	output, err = cscliBouncerCmd.CombinedOutput()
+	if err != nil {
+		if !strings.Contains(string(output), "unable to create bouncer: bouncer appsectests already exists") {
+			fmt.Println(string(output))
+			return fmt.Errorf("fail to run '%s' for test '%s': %v", cscliRegisterCmd.String(), t.Name, err)
+		}
+	}
+
+	//start crowdsec service
+	cmdArgs = []string{"-c", t.RuntimeConfigFilePath}
+	crowdsecDaemon := exec.Command(t.CrowdSecPath, cmdArgs...)
+
+	crowdsecDaemon.Start()
+
+	//wait for the appsec port to be available
+	if _, err := IsAlive(DefaultAppsecHost); err != nil {
+		return fmt.Errorf("appsec is down: %s", err)
+	}
+
+	// check if the target is available
+	nucleiTargetParsedURL, err := url.Parse(DefaultNucleiTarget)
+	if err != nil {
+		return fmt.Errorf("unable to parse target '%s': %s", DefaultNucleiTarget, err)
+	}
+	nucleiTargetHost := nucleiTargetParsedURL.Host
+	if _, err := IsAlive(nucleiTargetHost); err != nil {
+		return fmt.Errorf("target is down: %s", err)
+	}
+
+	nucleiConfig := NucleiConfig{
+		Path:      "nuclei",
+		OutputDir: t.RuntimePath,
+		CmdLineOptions: []string{"-ev", //allow variables from environment
+			"-nc",    //no colors in output
+			"-dresp", //dump response
+			"-j",     //json output
+		},
+	}
+
+	err = nucleiConfig.RunNucleiTemplate(t.Name, t.Config.NucleiTemplate, DefaultNucleiTarget)
+	crowdsecLogFile := fmt.Sprintf("%s/log/crowdsec.log", nucleiConfig.OutputDir)
+	if t.Config.ExpectedNucleiFailure {
+		if err != nil && errors.Is(err, NucleiTemplateFail) {
+			log.Infof("Appsec test %s failed as expected", t.Name)
+			t.Success = true
+		} else {
+			log.Errorf("Appsec test %s failed:  %s", t.Name, err)
+			crowdsecLog, err := os.ReadFile(crowdsecLogFile)
+			if err != nil {
+				log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err)
+			} else {
+				log.Errorf("crowdsec log file '%s'", crowdsecLogFile)
+				log.Errorf("%s\n", string(crowdsecLog))
+			}
+		}
+	} else {
+		if err == nil {
+			log.Infof("Appsec test %s succeeded", t.Name)
+			t.Success = true
+		} else {
+			log.Errorf("Appsec test %s failed:  %s", t.Name, err)
+			crowdsecLog, err := os.ReadFile(crowdsecLogFile)
+			if err != nil {
+				log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err)
+			} else {
+				log.Errorf("crowdsec log file '%s'", crowdsecLogFile)
+				log.Errorf("%s\n", string(crowdsecLog))
+			}
+		}
+	}
+	crowdsecDaemon.Process.Kill()
+	return nil
+}
+
+func (t *HubTestItem) RunWithLogFile() error {
+	testPath := filepath.Join(t.HubTestPath, t.Name)
+	if _, err := os.Stat(testPath); os.IsNotExist(err) {
+		return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath)
+	}
+
+	currentDir, err := os.Getwd() //xx
 	if err != nil {
 		return fmt.Errorf("can't get current directory: %+v", err)
 	}
@@ -650,3 +837,92 @@ func (t *HubTestItem) Run() error {
 
 	return nil
 }
+
+func (t *HubTestItem) Run() error {
+	var err error
+	t.Success = false
+	t.ErrorsList = make([]string, 0)
+
+	// create runtime folder
+	if err = os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil {
+		return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err)
+	}
+
+	// create runtime data folder
+	if err = os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil {
+		return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeDataPath, err)
+	}
+
+	// create runtime hub folder
+	if err = os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil {
+		return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeHubPath, err)
+	}
+
+	if err = Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil {
+		return fmt.Errorf("unable to copy .index.json file in '%s': %s", filepath.Join(t.RuntimeHubPath, ".index.json"), err)
+	}
+
+	// create results folder
+	if err = os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil {
+		return fmt.Errorf("unable to create folder '%s': %+v", t.ResultsPath, err)
+	}
+
+	// copy template config file to runtime folder
+	if err = Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil {
+		return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err)
+	}
+
+	// copy template profile file to runtime folder
+	if err = Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil {
+		return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err)
+	}
+
+	// copy template simulation file to runtime folder
+	if err = Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil {
+		return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err)
+	}
+
+	crowdsecPatternsFolder := csconfig.DefaultConfigPath("patterns")
+
+	// copy template patterns folder to runtime folder
+	if err = CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil {
+		return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err)
+	}
+
+	// create the appsec-configs dir
+	if err = os.MkdirAll(filepath.Join(t.RuntimePath, "appsec-configs"), os.ModePerm); err != nil {
+		return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err)
+	}
+
+	//if it's an appsec rule test, we need acquis and appsec profile
+	if len(t.Config.AppsecRules) > 0 {
+		// copy template acquis file to runtime folder
+		log.Infof("copying %s to %s", t.TemplateAcquisPath, t.RuntimeAcquisFilePath)
+		if err = Copy(t.TemplateAcquisPath, t.RuntimeAcquisFilePath); err != nil {
+			return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateAcquisPath, t.RuntimeAcquisFilePath, err)
+		}
+
+		log.Infof("copying %s to %s", t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml"))
+		// copy template appsec-config file to runtime folder
+		if err = Copy(t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml")); err != nil {
+			return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml"), err)
+		}
+	} else { //otherwise we drop a blank acquis file
+		if err = os.WriteFile(t.RuntimeAcquisFilePath, []byte(""), os.ModePerm); err != nil {
+			return fmt.Errorf("unable to write blank acquis file '%s': %s", t.RuntimeAcquisFilePath, err)
+		}
+	}
+
+	// install the hub in the runtime folder
+	if err = t.InstallHub(); err != nil {
+		return fmt.Errorf("unable to install hub in '%s': %s", t.RuntimeHubPath, err)
+	}
+
+	if t.Config.LogFile != "" {
+		return t.RunWithLogFile()
+	} else if t.Config.NucleiTemplate != "" {
+		return t.RunWithNucleiTemplate()
+	} else {
+		return fmt.Errorf("log file or nuclei template must be set in '%s'", t.Name)
+	}
+}

pkg/hubtest/nucleirunner.go

@@ -0,0 +1,66 @@
+package hubtest
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type NucleiConfig struct {
+	Path           string   `yaml:"nuclei_path"`
+	OutputDir      string   `yaml:"output_dir"`
+	CmdLineOptions []string `yaml:"cmdline_options"`
+}
+
+var NucleiTemplateFail = errors.New("Nuclei template failed")
+
+func (nc *NucleiConfig) RunNucleiTemplate(testName string, templatePath string, target string) error {
+	tstamp := time.Now().Unix()
+	//templatePath is the full path to the template, we just want the name ie. "sqli-random-test"
+	tmp := strings.Split(templatePath, "/")
+	template := strings.Split(tmp[len(tmp)-1], ".")[0]
+
+	outputPrefix := fmt.Sprintf("%s/%s_%s-%d", nc.OutputDir, testName, template, tstamp)
+
+	args := []string{
+		"-u", target,
+		"-t", templatePath,
+		"-o", outputPrefix + ".json",
+	}
+	args = append(args, nc.CmdLineOptions...)
+	cmd := exec.Command(nc.Path, args...)
+
+	var out bytes.Buffer
+	var outErr bytes.Buffer
+
+	cmd.Stdout = &out
+	cmd.Stderr = &outErr
+
+	err := cmd.Run()
+
+	if err := os.WriteFile(outputPrefix+"_stdout.txt", out.Bytes(), 0644); err != nil {
+		log.Warningf("Error writing stdout: %s", err)
+	}
+
+	if err := os.WriteFile(outputPrefix+"_stderr.txt", outErr.Bytes(), 0644); err != nil {
+		log.Warningf("Error writing stderr: %s", err)
+	}
+
+	if err != nil {
+		log.Warningf("Error running nuclei: %s", err)
+		log.Warningf("Stdout saved to %s", outputPrefix+"_stdout.txt")
+		log.Warningf("Stderr saved to %s", outputPrefix+"_stderr.txt")
+		log.Warningf("Nuclei generated output saved to %s", outputPrefix+".json")
+		return err
+	} else if len(out.String()) == 0 {
+		//No stdout means no finding, it means our test failed
+		return NucleiTemplateFail
+	}
+	return nil
+}

pkg/hubtest/utils.go

@@ -2,9 +2,13 @@ package hubtest
 
 import (
 	"fmt"
+	"net"
 	"os"
 	"path/filepath"
 	"sort"
+	"time"
+
+	log "github.com/sirupsen/logrus"
 )
 
 func sortedMapKeys[V any](m map[string]V) []string {
@@ -106,3 +110,19 @@ func CopyDir(src string, dest string) error {
 
 	return nil
 }
+
+func IsAlive(target string) (bool, error) {
+	start := time.Now()
+	for {
+		conn, err := net.Dial("tcp", target)
+		if err == nil {
+			log.Debugf("appsec is up after %s", time.Since(start))
+			conn.Close()
+			return true, nil
+		}
+		time.Sleep(500 * time.Millisecond)
+		if time.Since(start) > 10*time.Second {
+			return false, fmt.Errorf("took more than 10s for %s to be available", target)
+		}
+	}
+}

pkg/parser/node.go

@@ -214,6 +214,10 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
 			switch out := output.(type) {
 			case string:
 				gstr = out
+			case int:
+				gstr = fmt.Sprintf("%d", out)
+			case float64, float32:
+				gstr = fmt.Sprintf("%f", out)
 			default:
 				clog.Errorf("unexpected return type for RunTimeValue : %T", output)
 			}

pkg/parser/runtime.go

@@ -127,6 +127,8 @@ func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error {
 				value = out
 			case int:
 				value = strconv.Itoa(out)
+			case float64, float32:
+				value = fmt.Sprintf("%f", out)
 			case map[string]interface{}:
 				clog.Warnf("Expression '%s' returned a map, please use ToJsonString() to convert it to string if you want to keep it as is, or refine your expression to extract a string", static.ExpValue)
 			case []interface{}:
@@ -134,7 +136,7 @@ func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error {
 			case nil:
 				clog.Debugf("Expression '%s' returned nil, skipping", static.ExpValue)
 			default:
-				clog.Errorf("unexpected return type for RunTimeValue : %T", output)
+				clog.Errorf("unexpected return type for '%s' : %T", static.ExpValue, output)
 				return errors.New("unexpected return type for RunTimeValue")
 			}
 		}

pkg/types/appsec_event.go

@@ -0,0 +1,240 @@
+package types
+
+import (
+	"regexp"
+
+	log "github.com/sirupsen/logrus"
+)
+
+/*
+ 1. If user triggered a rule that is for a CVE, that has high confidence and that is blocking, ban
+ 2. If user triggered 3 distinct rules with medium confidence across 3 different requests, ban
+
+
+any(evt.Waf.ByTag("CVE"), {.confidence == "high" && .action == "block"})
+
+len(evt.Waf.ByTagRx("*CVE*").ByConfidence("high").ByAction("block")) > 1
+
+*/
+
+type MatchedRules []map[string]interface{}
+
+type AppsecEvent struct {
+	HasInBandMatches, HasOutBandMatches bool
+	MatchedRules
+	Vars map[string]string
+}
+type Field string
+
+func (f Field) String() string {
+	return string(f)
+}
+
+const (
+	ID         Field = "id"
+	RuleType   Field = "rule_type"
+	Tags       Field = "tags"
+	File       Field = "file"
+	Confidence Field = "confidence"
+	Revision   Field = "revision"
+	SecMark    Field = "secmark"
+	Accuracy   Field = "accuracy"
+	Msg        Field = "msg"
+	Severity   Field = "severity"
+	Kind       Field = "kind"
+)
+
+func (w AppsecEvent) GetVar(varName string) string {
+	if w.Vars == nil {
+		return ""
+	}
+	if val, ok := w.Vars[varName]; ok {
+		return val
+	}
+	log.Infof("var %s not found. Available variables: %+v", varName, w.Vars)
+	return ""
+
+}
+
+// getters
+func (w MatchedRules) GetField(field Field) []interface{} {
+	ret := make([]interface{}, 0)
+	for _, rule := range w {
+		ret = append(ret, rule[field.String()])
+	}
+	return ret
+}
+
+func (w MatchedRules) GetURI() string {
+	for _, rule := range w {
+		return rule["uri"].(string)
+	}
+	return ""
+}
+
+func (w MatchedRules) GetHash() string {
+	for _, rule := range w {
+		//@sbl : let's fix this
+		return rule["hash"].(string)
+	}
+	return ""
+}
+
+func (w MatchedRules) GetVersion() string {
+	for _, rule := range w {
+		//@sbl : let's fix this
+		return rule["version"].(string)
+	}
+	return ""
+}
+
+func (w MatchedRules) GetName() string {
+	for _, rule := range w {
+		//@sbl : let's fix this
+		return rule["name"].(string)
+	}
+	return ""
+}
+
+func (w MatchedRules) GetMethod() string {
+	for _, rule := range w {
+		return rule["method"].(string)
+	}
+	return ""
+}
+
+func (w MatchedRules) GetRuleIDs() []int {
+	ret := make([]int, 0)
+	for _, rule := range w {
+		ret = append(ret, rule["id"].(int))
+	}
+	return ret
+}
+
+func (w MatchedRules) Kinds() []string {
+	ret := make([]string, 0)
+	for _, rule := range w {
+		exists := false
+		for _, val := range ret {
+			if val == rule["kind"] {
+				exists = true
+				break
+			}
+		}
+		if !exists {
+			ret = append(ret, rule["kind"].(string))
+		}
+	}
+	return ret
+}
+
+func (w MatchedRules) GetMatchedZones() []string {
+	ret := make([]string, 0)
+
+	for _, rule := range w {
+		ret = append(ret, rule["matched_zones"].([]string)...)
+	}
+	return ret
+}
+
+// filters
+func (w MatchedRules) ByID(id int) MatchedRules {
+	ret := MatchedRules{}
+
+	for _, rule := range w {
+		if rule["id"] == id {
+			ret = append(ret, rule)
+		}
+	}
+	return ret
+}
+
+func (w MatchedRules) ByKind(kind string) MatchedRules {
+	ret := MatchedRules{}
+	for _, rule := range w {
+		if rule["kind"] == kind {
+			ret = append(ret, rule)
+		}
+	}
+	return ret
+}
+
+func (w MatchedRules) ByTags(match []string) MatchedRules {
+	ret := MatchedRules{}
+	for _, rule := range w {
+		for _, tag := range rule["tags"].([]string) {
+			for _, match_tag := range match {
+				if tag == match_tag {
+					ret = append(ret, rule)
+					break
+				}
+			}
+		}
+	}
+	return ret
+}
+
+func (w MatchedRules) ByTag(match string) MatchedRules {
+	ret := MatchedRules{}
+	for _, rule := range w {
+		for _, tag := range rule["tags"].([]string) {
+			if tag == match {
+				ret = append(ret, rule)
+				break
+			}
+		}
+	}
+	return ret
+}
+
+func (w MatchedRules) ByTagRx(rx string) MatchedRules {
+	ret := MatchedRules{}
+	re := regexp.MustCompile(rx)
+	if re == nil {
+		return ret
+	}
+	for _, rule := range w {
+		for _, tag := range rule["tags"].([]string) {
+			log.Debugf("ByTagRx: %s = %s -> %t", rx, tag, re.MatchString(tag))
+			if re.MatchString(tag) {
+				ret = append(ret, rule)
+				break
+			}
+		}
+	}
+	return ret
+}
+
+func (w MatchedRules) ByDisruptiveness(is bool) MatchedRules {
+	ret := MatchedRules{}
+	for _, rule := range w {
+		if rule["disruptive"] == is {
+			ret = append(ret, rule)
+		}
+	}
+	log.Debugf("ByDisruptiveness(%t) -> %d", is, len(ret))
+
+	return ret
+}
+
+func (w MatchedRules) BySeverity(severity string) MatchedRules {
+	ret := MatchedRules{}
+	for _, rule := range w {
+		if rule["severity"] == severity {
+			ret = append(ret, rule)
+		}
+	}
+	log.Debugf("BySeverity(%s) -> %d", severity, len(ret))
+	return ret
+}
+
+func (w MatchedRules) ByAccuracy(accuracy string) MatchedRules {
+	ret := MatchedRules{}
+	for _, rule := range w {
+		if rule["accuracy"] == accuracy {
+			ret = append(ret, rule)
+		}
+	}
+	log.Debugf("ByAccuracy(%s) -> %d", accuracy, len(ret))
+	return ret
+}

pkg/types/event.go

@@ -13,6 +13,7 @@ import (
 const (
 	LOG = iota
 	OVFLW
+	APPSEC
 )
 
 // Event is the structure representing a runtime event (log or overflow)
@@ -40,6 +41,7 @@ type Event struct {
 	StrTimeFormat string       `yaml:"StrTimeFormat,omitempty" json:"StrTimeFormat,omitempty"`
 	MarshaledTime string       `yaml:"MarshaledTime,omitempty" json:"MarshaledTime,omitempty"`
 	Process       bool         `yaml:"Process,omitempty" json:"Process,omitempty"` //can be set to false to avoid processing line
+	Appsec        AppsecEvent  `yaml:"Appsec,omitempty" json:"Appsec,omitempty"`
 	/* Meta is the only part that will make it to the API - it should be normalized */
 	Meta map[string]string `yaml:"Meta,omitempty" json:"Meta,omitempty"`
 }

test/bats.mk

@@ -83,7 +83,7 @@ bats-build: bats-environment
 
 # Create a reusable package with initial configuration + data
 bats-fixture: bats-check-requirements bats-update-tools
-	@echo "Creating functional test fixture..."
+	@echo "Creating functional test fixture."
 	@$(TEST_DIR)/instance-data make
 
 # Remove the local crowdsec installation and the fixture config + data

test/bats/01_cscli.bats

@@ -252,19 +252,20 @@ teardown() {
 
 @test "cscli - malformed LAPI url" {
     LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path')
-    config_set "${LOCAL_API_CREDENTIALS}" '.url="https://127.0.0.1:-80"'
+    config_set "${LOCAL_API_CREDENTIALS}" '.url="http://127.0.0.1:-80"'
 
-    rune -1 cscli lapi status
-    assert_stderr --partial 'parsing api url'
-    assert_stderr --partial 'invalid port \":-80\" after host'
+    rune -1 cscli lapi status -o json
+    rune -0 jq -r '.msg' <(stderr)
+    assert_output 'parsing api url: parse "http://127.0.0.1:-80/": invalid port ":-80" after host'
+}
 
-    rune -1 cscli alerts list
-    assert_stderr --partial 'parsing api url'
-    assert_stderr --partial 'invalid port \":-80\" after host'
+@test "cscli - bad LAPI password" {
+    LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path')
+    config_set "${LOCAL_API_CREDENTIALS}" '.password="meh"'
 
-    rune -1 cscli decisions list
-    assert_stderr --partial 'parsing api url'
-    assert_stderr --partial 'invalid port \":-80\" after host'
+    rune -1 cscli lapi status -o json
+    rune -0 jq -r '.msg' <(stderr)
+    assert_output 'failed to authenticate to Local API (LAPI): API error: incorrect Username or Password'
 }
 
 @test "cscli metrics" {

test/bats/09_context.bats

@@ -0,0 +1,95 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh"
+    CONFIG_DIR=$(config_get '.config_paths.config_dir')
+    export CONFIG_DIR
+    CONTEXT_YAML="$CONFIG_DIR/console/context.yaml"
+    export CONTEXT_YAML
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh"
+}
+
+setup() {
+    load "../lib/setup.sh"
+    load "../lib/bats-file/load.bash"
+    ./instance-data load
+    config_set '.common.log_media="stdout"'
+    mkdir -p "$CONFIG_DIR/console"
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+#----------
+
+@test "detect available context" {
+    rune -0 cscli lapi context detect -a
+    rune -0 yq -o json <(output)
+    assert_json '{"Acquisition":["evt.Line.Module","evt.Line.Raw","evt.Line.Src"]}'
+
+    rune -0 cscli parsers install crowdsecurity/dateparse-enrich
+    rune -0 cscli lapi context detect crowdsecurity/dateparse-enrich
+    rune -0 yq -o json '.crowdsecurity/dateparse-enrich' <(output)
+    assert_json '["evt.MarshaledTime","evt.Meta.timestamp"]'
+}
+
+@test "attempt to load from default context file, ignore if missing" {
+    rune -0 rm -f "$CONTEXT_YAML"
+    rune -0 "$CROWDSEC" -t --trace
+    assert_stderr --partial "loading console context from $CONTEXT_YAML"
+}
+
+@test "error if context file is explicitly set but does not exist" {
+    config_set ".crowdsec_service.console_context_path=strenv(CONTEXT_YAML)"
+    rune -0 rm -f "$CONTEXT_YAML"
+    rune -1 "$CROWDSEC" -t
+    assert_stderr --partial "while checking console_context_path: stat $CONTEXT_YAML: no such file or directory"
+}
+
+@test "context file is bad" {
+    echo "bad yaml" > "$CONTEXT_YAML"
+    rune -1 "$CROWDSEC" -t
+    assert_stderr --partial "while loading context: $CONTEXT_YAML: yaml: unmarshal errors"
+}
+
+@test "context file is good" {
+    echo '{"source_ip":["evt.Parsed.source_ip"]}' > "$CONTEXT_YAML"
+    rune -0 "$CROWDSEC" -t --debug
+    assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}'
+}
+
+@test "context file is from hub (local item)" {
+    mkdir -p "$CONFIG_DIR/contexts"
+    config_set "del(.crowdsec_service.console_context_path)"
+    echo '{"context":{"source_ip":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/foobar.yaml"
+    rune -0 "$CROWDSEC" -t --trace
+    assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/foobar.yaml"
+    assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}'
+}
+
+@test "merge multiple contexts" {
+    mkdir -p "$CONFIG_DIR/contexts"
+    echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml"
+    echo '{"context":{"two":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/two.yaml"
+    rune -0 "$CROWDSEC" -t --trace
+    assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml"
+    assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/two.yaml"
+    assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip"],"two":["evt.Parsed.source_ip"]}'
+}
+
+@test "merge contexts from hub and context.yaml file" {
+    mkdir -p "$CONFIG_DIR/contexts"
+    echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml"
+    echo '{"one":["evt.Parsed.source_ip_2"]}' > "$CONFIG_DIR/console/context.yaml"
+    rune -0 "$CROWDSEC" -t --trace
+    assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml"
+    assert_stderr --partial "loading console context from $CONFIG_DIR/console/context.yaml"
+    assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip","evt.Parsed.source_ip_2"]}'
+}

test/bats/20_hub.bats

@@ -36,7 +36,7 @@ teardown() {
     rune -0 cscli hub list
     assert_output "No items to display"
     rune -0 cscli hub list -o json
-    assert_json '{parsers:[],scenarios:[],collections:[],postoverflows:[]}'
+    assert_json '{"appsec-configs":[],"appsec-rules":[],parsers:[],scenarios:[],collections:[],contexts:[],postoverflows:[]}'
     rune -0 cscli hub list -o raw
     assert_output 'name,status,version,description,type'
 
@@ -47,6 +47,7 @@ teardown() {
     assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*SCENARIOS.*crowdsecurity/telnet-bf.*"
     refute_output --partial 'POSTOVERFLOWS'
     refute_output --partial 'COLLECTIONS'
+
     rune -0 cscli hub list -o json
     rune -0 jq -e '(.parsers | length == 1) and (.scenarios | length == 1)' <(output)
     rune -0 cscli hub list -o raw
@@ -55,8 +56,11 @@ teardown() {
     refute_output --partial 'crowdsecurity/iptables'
 
     # all items
+    mkdir -p "$CONFIG_DIR/contexts"
+    # there are no contexts yet, so we create a local one
+    touch "$CONFIG_DIR/contexts/mycontext.yaml"
     rune -0 cscli hub list -a
-    assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*crowdsecurity/iptables.*"
+    assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*CONTEXTS.*mycontext.yaml.*COLLECTIONS.*crowdsecurity/iptables.*"
     rune -0 cscli hub list -a -o json
     rune -0 jq -e '(.parsers | length > 1) and (.scenarios | length > 1)' <(output)
     rune -0 cscli hub list -a -o raw
@@ -107,6 +111,8 @@ teardown() {
     assert_stderr --partial "Upgraded 0 postoverflows"
     assert_stderr --partial "Upgrading scenarios"
     assert_stderr --partial "Upgraded 0 scenarios"
+    assert_stderr --partial "Upgrading contexts"
+    assert_stderr --partial "Upgraded 0 contexts"
     assert_stderr --partial "Upgrading collections"
     assert_stderr --partial "Upgraded 0 collections"
 
@@ -134,10 +140,11 @@ teardown() {
     assert_line "parsers"
     assert_line "postoverflows"
     assert_line "scenarios"
+    assert_line "contexts"
     assert_line "collections"
     rune -0 cscli hub types -o human
     rune -0 yq -o json <(output)
-    assert_json '["parsers","postoverflows","scenarios","collections"]'
+    assert_json '["parsers","postoverflows","scenarios","contexts","appsec-configs","appsec-rules","collections"]'
     rune -0 cscli hub types -o json
-    assert_json '["parsers","postoverflows","scenarios","collections"]'
+    assert_json '["parsers","postoverflows","scenarios","contexts","appsec-configs","appsec-rules","collections"]'
 }

test/lib/bats-assert

@@ -1 +1 @@
-Subproject commit 78fa631d1370562d2cd4a1390989e706158e7bf0
+Subproject commit 44913ffe6020d1561c4c4d1e26cda8e07a1f374f

test/lib/setup_file.sh

@@ -247,12 +247,14 @@ hub_purge_all() {
         "$CONFIG_DIR"/collections/* \
         "$CONFIG_DIR"/parsers/*/* \
         "$CONFIG_DIR"/scenarios/* \
-        "$CONFIG_DIR"/postoverflows/*
+        "$CONFIG_DIR"/postoverflows/* \
+        "$CONFIG_DIR"/contexts/*
     rm -rf \
         "$CONFIG_DIR"/hub/collections/* \
         "$CONFIG_DIR"/hub/parsers/*/* \
         "$CONFIG_DIR"/hub/scenarios/* \
-        "$CONFIG_DIR"/hub/postoverflows/*
+        "$CONFIG_DIR"/hub/postoverflows/* \
+        "$CONFIG_DIR"/hub/contexts/*
     local DATA_DIR
     DATA_DIR=$(config_get .config_paths.data_dir)
     # should remove everything except the db (find $DATA_DIR -not -name "crowdsec.db*" -delete),