From 8cca4346a5ab0ed618b726e52fe93c14a5a972e5 Mon Sep 17 00:00:00 2001 From: "Thibault \"bui\" Koechlin" Date: Thu, 7 Dec 2023 12:21:04 +0100 Subject: [PATCH] Application Security Engine Support (#2273) Add a new datasource that: - Receives HTTP requests from remediation components - Apply rules on them to determine whether they are malicious or not - Rules can be evaluated in-band (the remediation component will block the request directly) or out-band (the RC will let the request through, but crowdsec can still process the rule matches with scenarios) The PR also adds support for 2 new hub items: - appsec-configs: Configure the Application Security Engine (which rules to load, in which phase) - appsec-rules: a rule that is added in the Application Security Engine (can use either our own format, or seclang) --------- Co-authored-by: alteredCoder Co-authored-by: Sebastien Blot Co-authored-by: mmetc <92726601+mmetc@users.noreply.github.com> Co-authored-by: Marco Mariani --- cmd/crowdsec-cli/hubappsec.go | 105 +++ cmd/crowdsec-cli/hubcollection.go | 40 ++ cmd/crowdsec-cli/hubparser.go | 40 ++ cmd/crowdsec-cli/hubpostoverflow.go | 40 ++ cmd/crowdsec-cli/hubscenario.go | 40 ++ cmd/crowdsec-cli/hubtest.go | 214 +++--- cmd/crowdsec-cli/hubtest_table.go | 20 + cmd/crowdsec-cli/item_metrics.go | 2 + cmd/crowdsec-cli/itemcli.go | 454 +++++++++++++ cmd/crowdsec-cli/itemcommands.go | 609 ------------------ cmd/crowdsec-cli/main.go | 14 +- cmd/crowdsec-cli/metrics.go | 30 +- cmd/crowdsec-cli/metrics_table.go | 35 +- cmd/crowdsec/crowdsec.go | 5 + cmd/crowdsec/metrics.go | 6 +- cmd/crowdsec/parse.go | 7 + go.mod | 25 +- go.sum | 43 +- pkg/acquisition/acquisition.go | 2 + pkg/acquisition/modules/appsec/appsec.go | 371 +++++++++++ .../modules/appsec/appsec_runner.go | 350 ++++++++++ pkg/acquisition/modules/appsec/metrics.go | 54 ++ pkg/acquisition/modules/appsec/rx_operator.go | 94 +++ pkg/acquisition/modules/appsec/utils.go | 280 ++++++++ pkg/appsec/appsec.go | 579 +++++++++++++++++ pkg/appsec/appsec_rule/appsec_rule.go | 67 ++ pkg/appsec/appsec_rule/modsec_rule_test.go | 118 ++++ pkg/appsec/appsec_rule/modsecurity.go | 181 ++++++ pkg/appsec/appsec_rule/types.go | 9 + pkg/appsec/appsec_rules_collection.go | 144 +++++ pkg/appsec/coraza_logger.go | 194 ++++++ pkg/appsec/loader.go | 52 ++ pkg/appsec/request.go | 345 ++++++++++ pkg/appsec/request_test.go | 181 ++++++ pkg/appsec/tx.go | 93 +++ pkg/appsec/waf_helpers.go | 59 ++ pkg/csconfig/config.go | 8 + pkg/cwhub/item.go | 44 +- pkg/cwhub/sync.go | 15 +- pkg/exprhelpers/expr_lib.go | 15 + pkg/exprhelpers/helpers.go | 49 ++ pkg/hubtest/coverage.go | 67 +- pkg/hubtest/hubtest.go | 79 ++- pkg/hubtest/hubtest_item.go | 338 +++++++++- pkg/hubtest/nucleirunner.go | 66 ++ pkg/hubtest/utils.go | 20 + pkg/parser/node.go | 4 + pkg/parser/runtime.go | 4 +- pkg/types/appsec_event.go | 240 +++++++ pkg/types/event.go | 2 + test/bats/20_hub.bats | 6 +- test/lib/bats-assert | 2 +- 52 files changed, 5074 insertions(+), 787 deletions(-) create mode 100644 cmd/crowdsec-cli/hubappsec.go create mode 100644 cmd/crowdsec-cli/hubcollection.go create mode 100644 cmd/crowdsec-cli/hubparser.go create mode 100644 cmd/crowdsec-cli/hubpostoverflow.go create mode 100644 cmd/crowdsec-cli/hubscenario.go create mode 100644 cmd/crowdsec-cli/itemcli.go delete mode 100644 cmd/crowdsec-cli/itemcommands.go create mode 100644 pkg/acquisition/modules/appsec/appsec.go create mode 100644 pkg/acquisition/modules/appsec/appsec_runner.go create mode 100644 pkg/acquisition/modules/appsec/metrics.go create mode 100644 pkg/acquisition/modules/appsec/rx_operator.go create mode 100644 pkg/acquisition/modules/appsec/utils.go create mode 100644 pkg/appsec/appsec.go create mode 100644 pkg/appsec/appsec_rule/appsec_rule.go create mode 100644 pkg/appsec/appsec_rule/modsec_rule_test.go create mode 100644 pkg/appsec/appsec_rule/modsecurity.go create mode 100644 pkg/appsec/appsec_rule/types.go create mode 100644 pkg/appsec/appsec_rules_collection.go create mode 100644 pkg/appsec/coraza_logger.go create mode 100644 pkg/appsec/loader.go create mode 100644 pkg/appsec/request.go create mode 100644 pkg/appsec/request_test.go create mode 100644 pkg/appsec/tx.go create mode 100644 pkg/appsec/waf_helpers.go create mode 100644 pkg/hubtest/nucleirunner.go create mode 100644 pkg/types/appsec_event.go diff --git a/cmd/crowdsec-cli/hubappsec.go b/cmd/crowdsec-cli/hubappsec.go new file mode 100644 index 000000000..1fc8b6c00 --- /dev/null +++ b/cmd/crowdsec-cli/hubappsec.go @@ -0,0 +1,105 @@ +package main + +import ( + "fmt" + "os" + + "golang.org/x/text/cases" + "golang.org/x/text/language" + "gopkg.in/yaml.v3" + + "github.com/crowdsecurity/crowdsec/pkg/appsec" + "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewAppsecConfigCLI() *itemCLI { + return &itemCLI{ + name: cwhub.APPSEC_CONFIGS, + singular: "appsec-config", + oneOrMore: "appsec-config(s)", + help: cliHelp{ + example: `cscli appsec-configs list -a +cscli appsec-configs install crowdsecurity/vpatch +cscli appsec-configs inspect crowdsecurity/vpatch +cscli appsec-configs upgrade crowdsecurity/vpatch +cscli appsec-configs remove crowdsecurity/vpatch +`, + }, + installHelp: cliHelp{ + example: `cscli appsec-configs install crowdsecurity/vpatch`, + }, + removeHelp: cliHelp{ + example: `cscli appsec-configs remove crowdsecurity/vpatch`, + }, + upgradeHelp: cliHelp{ + example: `cscli appsec-configs upgrade crowdsecurity/vpatch`, + }, + inspectHelp: cliHelp{ + example: `cscli appsec-configs inspect crowdsecurity/vpatch`, + }, + listHelp: cliHelp{ + example: `cscli appsec-configs list +cscli appsec-configs list -a +cscli appsec-configs list crowdsecurity/vpatch`, + }, + } +} + +func NewAppsecRuleCLI() *itemCLI { + inspectDetail := func(item *cwhub.Item) error { + appsecRule := appsec.AppsecCollectionConfig{} + yamlContent, err := os.ReadFile(item.State.LocalPath) + if err != nil { + return fmt.Errorf("unable to read file %s : %s", item.State.LocalPath, err) + } + if err := yaml.Unmarshal(yamlContent, &appsecRule); err != nil { + return fmt.Errorf("unable to unmarshal yaml file %s : %s", item.State.LocalPath, err) + } + + for _, ruleType := range appsec_rule.SupportedTypes() { + fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType)) + for _, rule := range appsecRule.Rules { + convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name) + if err != nil { + return fmt.Errorf("unable to convert rule %s : %s", rule.Name, err) + } + fmt.Println(convertedRule) + } + } + + return nil + } + + return &itemCLI{ + name: "appsec-rules", + singular: "appsec-rule", + oneOrMore: "appsec-rule(s)", + help: cliHelp{ + example: `cscli appsec-rules list -a +cscli appsec-rules install crowdsecurity/crs +cscli appsec-rules inspect crowdsecurity/crs +cscli appsec-rules upgrade crowdsecurity/crs +cscli appsec-rules remove crowdsecurity/crs +`, + }, + installHelp: cliHelp{ + example: `cscli appsec-rules install crowdsecurity/crs`, + }, + removeHelp: cliHelp{ + example: `cscli appsec-rules remove crowdsecurity/crs`, + }, + upgradeHelp: cliHelp{ + example: `cscli appsec-rules upgrade crowdsecurity/crs`, + }, + inspectHelp: cliHelp{ + example: `cscli appsec-rules inspect crowdsecurity/crs`, + }, + inspectDetail: inspectDetail, + listHelp: cliHelp{ + example: `cscli appsec-rules list +cscli appsec-rules list -a +cscli appsec-rules list crowdsecurity/crs`, + }, + } +} diff --git a/cmd/crowdsec-cli/hubcollection.go b/cmd/crowdsec-cli/hubcollection.go new file mode 100644 index 000000000..a86947975 --- /dev/null +++ b/cmd/crowdsec-cli/hubcollection.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewCollectionCLI() *itemCLI { + return &itemCLI{ + name: cwhub.COLLECTIONS, + singular: "collection", + oneOrMore: "collection(s)", + help: cliHelp{ + example: `cscli collections list -a +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables +cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables +`, + }, + installHelp: cliHelp{ + example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`, + }, + removeHelp: cliHelp{ + example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`, + }, + upgradeHelp: cliHelp{ + example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`, + }, + inspectHelp: cliHelp{ + example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`, + }, + listHelp: cliHelp{ + example: `cscli collections list +cscli collections list -a +cscli collections list crowdsecurity/http-cve crowdsecurity/iptables + +List only enabled collections unless "-a" or names are specified.`, + }, + } +} diff --git a/cmd/crowdsec-cli/hubparser.go b/cmd/crowdsec-cli/hubparser.go new file mode 100644 index 000000000..d2af0b6fb --- /dev/null +++ b/cmd/crowdsec-cli/hubparser.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewParserCLI() *itemCLI { + return &itemCLI{ + name: cwhub.PARSERS, + singular: "parser", + oneOrMore: "parser(s)", + help: cliHelp{ + example: `cscli parsers list -a +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs +`, + }, + installHelp: cliHelp{ + example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + removeHelp: cliHelp{ + example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + upgradeHelp: cliHelp{ + example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + }, + inspectHelp: cliHelp{ + example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`, + }, + listHelp: cliHelp{ + example: `cscli parsers list +cscli parsers list -a +cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs + +List only enabled parsers unless "-a" or names are specified.`, + }, + } +} diff --git a/cmd/crowdsec-cli/hubpostoverflow.go b/cmd/crowdsec-cli/hubpostoverflow.go new file mode 100644 index 000000000..326c3fec5 --- /dev/null +++ b/cmd/crowdsec-cli/hubpostoverflow.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewPostOverflowCLI() *itemCLI { + return &itemCLI{ + name: cwhub.POSTOVERFLOWS, + singular: "postoverflow", + oneOrMore: "postoverflow(s)", + help: cliHelp{ + example: `cscli postoverflows list -a +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns +`, + }, + installHelp: cliHelp{ + example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + removeHelp: cliHelp{ + example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + upgradeHelp: cliHelp{ + example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + inspectHelp: cliHelp{ + example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + }, + listHelp: cliHelp{ + example: `cscli postoverflows list +cscli postoverflows list -a +cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns + +List only enabled postoverflows unless "-a" or names are specified.`, + }, + } +} diff --git a/cmd/crowdsec-cli/hubscenario.go b/cmd/crowdsec-cli/hubscenario.go new file mode 100644 index 000000000..14b246f0f --- /dev/null +++ b/cmd/crowdsec-cli/hubscenario.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func NewScenarioCLI() *itemCLI { + return &itemCLI{ + name: cwhub.SCENARIOS, + singular: "scenario", + oneOrMore: "scenario(s)", + help: cliHelp{ + example: `cscli scenarios list -a +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing +`, + }, + installHelp: cliHelp{ + example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + removeHelp: cliHelp{ + example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + upgradeHelp: cliHelp{ + example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + inspectHelp: cliHelp{ + example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`, + }, + listHelp: cliHelp{ + example: `cscli scenarios list +cscli scenarios list -a +cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing + +List only enabled scenarios unless "-a" or names are specified.`, + }, + } +} diff --git a/cmd/crowdsec-cli/hubtest.go b/cmd/crowdsec-cli/hubtest.go index 8b574c3ee..295a8f89d 100644 --- a/cmd/crowdsec-cli/hubtest.go +++ b/cmd/crowdsec-cli/hubtest.go @@ -19,6 +19,9 @@ import ( ) var HubTest hubtest.HubTest +var HubAppsecTests hubtest.HubTest +var hubPtr *hubtest.HubTest +var isAppsecTest bool func NewHubTestCmd() *cobra.Command { var hubPath string @@ -33,11 +36,20 @@ func NewHubTestCmd() *cobra.Command { DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { var err error - HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath) + HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, false) if err != nil { return fmt.Errorf("unable to load hubtest: %+v", err) } + HubAppsecTests, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, true) + if err != nil { + return fmt.Errorf("unable to load appsec specific hubtest: %+v", err) + } + /*commands will use the hubPtr, will point to the default hubTest object, or the one dedicated to appsec tests*/ + hubPtr = &HubTest + if isAppsecTest { + hubPtr = &HubAppsecTests + } return nil }, } @@ -45,6 +57,7 @@ func NewHubTestCmd() *cobra.Command { cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder") cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec") cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli") + cmdHubTest.PersistentFlags().BoolVar(&isAppsecTest, "appsec", false, "Command relates to appsec tests") cmdHubTest.AddCommand(NewHubTestCreateCmd()) cmdHubTest.AddCommand(NewHubTestRunCmd()) @@ -76,7 +89,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { testName := args[0] - testPath := filepath.Join(HubTest.HubTestPath, testName) + testPath := filepath.Join(hubPtr.HubTestPath, testName) if _, err := os.Stat(testPath); os.IsExist(err) { return fmt.Errorf("test '%s' already exists in '%s', exiting", testName, testPath) } @@ -89,53 +102,76 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios return fmt.Errorf("unable to create folder '%s': %+v", testPath, err) } - // create empty log file - logFileName := fmt.Sprintf("%s.log", testName) - logFilePath := filepath.Join(testPath, logFileName) - logFile, err := os.Create(logFilePath) - if err != nil { - return err - } - logFile.Close() - - // create empty parser assertion file - parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName) - parserAssertFile, err := os.Create(parserAssertFilePath) - if err != nil { - return err - } - parserAssertFile.Close() - - // create empty scenario assertion file - scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName) - scenarioAssertFile, err := os.Create(scenarioAssertFilePath) - if err != nil { - return err - } - scenarioAssertFile.Close() - - parsers = append(parsers, "crowdsecurity/syslog-logs") - parsers = append(parsers, "crowdsecurity/dateparse-enrich") - - if len(scenarios) == 0 { - scenarios = append(scenarios, "") - } - - if len(postoverflows) == 0 { - postoverflows = append(postoverflows, "") - } - - configFileData := &hubtest.HubTestItemConfig{ - Parsers: parsers, - Scenarios: scenarios, - PostOVerflows: postoverflows, - LogFile: logFileName, - LogType: logType, - IgnoreParsers: ignoreParsers, - Labels: labels, - } - configFilePath := filepath.Join(testPath, "config.yaml") + + configFileData := &hubtest.HubTestItemConfig{} + if logType == "appsec" { + //create empty nuclei template file + nucleiFileName := fmt.Sprintf("%s.yaml", testName) + nucleiFilePath := filepath.Join(testPath, nucleiFileName) + nucleiFile, err := os.Create(nucleiFilePath) + if err != nil { + return err + } + nucleiFile.Close() + configFileData.AppsecRules = []string{"your_rule_here.yaml"} + configFileData.NucleiTemplate = nucleiFileName + fmt.Println() + fmt.Printf(" Test name : %s\n", testName) + fmt.Printf(" Test path : %s\n", testPath) + fmt.Printf(" Nuclei Template : %s\n", nucleiFileName) + } else { + // create empty log file + logFileName := fmt.Sprintf("%s.log", testName) + logFilePath := filepath.Join(testPath, logFileName) + logFile, err := os.Create(logFilePath) + if err != nil { + return err + } + logFile.Close() + + // create empty parser assertion file + parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName) + parserAssertFile, err := os.Create(parserAssertFilePath) + if err != nil { + return err + } + parserAssertFile.Close() + // create empty scenario assertion file + scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName) + scenarioAssertFile, err := os.Create(scenarioAssertFilePath) + if err != nil { + return err + } + scenarioAssertFile.Close() + + parsers = append(parsers, "crowdsecurity/syslog-logs") + parsers = append(parsers, "crowdsecurity/dateparse-enrich") + + if len(scenarios) == 0 { + scenarios = append(scenarios, "") + } + + if len(postoverflows) == 0 { + postoverflows = append(postoverflows, "") + } + configFileData.Parsers = parsers + configFileData.Scenarios = scenarios + configFileData.PostOverflows = postoverflows + configFileData.LogFile = logFileName + configFileData.LogType = logType + configFileData.IgnoreParsers = ignoreParsers + configFileData.Labels = labels + fmt.Println() + fmt.Printf(" Test name : %s\n", testName) + fmt.Printf(" Test path : %s\n", testPath) + fmt.Printf(" Log file : %s (please fill it with logs)\n", logFilePath) + fmt.Printf(" Parser assertion file : %s (please fill it with assertion)\n", parserAssertFilePath) + fmt.Printf(" Scenario assertion file : %s (please fill it with assertion)\n", scenarioAssertFilePath) + fmt.Printf(" Configuration File : %s (please fill it with parsers, scenarios...)\n", configFilePath) + + } + fd, err := os.Create(configFilePath) if err != nil { return fmt.Errorf("open: %s", err) @@ -151,14 +187,6 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios if err := fd.Close(); err != nil { return fmt.Errorf("close: %s", err) } - fmt.Println() - fmt.Printf(" Test name : %s\n", testName) - fmt.Printf(" Test path : %s\n", testPath) - fmt.Printf(" Log file : %s (please fill it with logs)\n", logFilePath) - fmt.Printf(" Parser assertion file : %s (please fill it with assertion)\n", parserAssertFilePath) - fmt.Printf(" Scenario assertion file : %s (please fill it with assertion)\n", scenarioAssertFilePath) - fmt.Printf(" Configuration File : %s (please fill it with parsers, scenarios...)\n", configFilePath) - return nil }, } @@ -188,12 +216,12 @@ func NewHubTestRunCmd() *cobra.Command { } if runAll { - if err := HubTest.LoadAllTests(); err != nil { + if err := hubPtr.LoadAllTests(); err != nil { return fmt.Errorf("unable to load all tests: %+v", err) } } else { for _, testName := range args { - _, err := HubTest.LoadTestItem(testName) + _, err := hubPtr.LoadTestItem(testName) if err != nil { return fmt.Errorf("unable to load test '%s': %s", testName, err) } @@ -202,8 +230,7 @@ func NewHubTestRunCmd() *cobra.Command { // set timezone to avoid DST issues os.Setenv("TZ", "UTC") - - for _, test := range HubTest.Tests { + for _, test := range hubPtr.Tests { if csConfig.Cscli.Output == "human" { log.Infof("Running test '%s'", test.Name) } @@ -218,8 +245,8 @@ func NewHubTestRunCmd() *cobra.Command { PersistentPostRunE: func(cmd *cobra.Command, args []string) error { success := true testResult := make(map[string]bool) - for _, test := range HubTest.Tests { - if test.AutoGen { + for _, test := range hubPtr.Tests { + if test.AutoGen && !isAppsecTest { if test.ParserAssert.AutoGenAssert { log.Warningf("Assert file '%s' is empty, generating assertion:", test.ParserAssert.File) fmt.Println() @@ -341,7 +368,7 @@ func NewHubTestCleanCmd() *cobra.Command { DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { for _, testName := range args { - test, err := HubTest.LoadTestItem(testName) + test, err := hubPtr.LoadTestItem(testName) if err != nil { return fmt.Errorf("unable to load test '%s': %s", testName, err) } @@ -364,17 +391,23 @@ func NewHubTestInfoCmd() *cobra.Command { Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { + for _, testName := range args { - test, err := HubTest.LoadTestItem(testName) + test, err := hubPtr.LoadTestItem(testName) if err != nil { return fmt.Errorf("unable to load test '%s': %s", testName, err) } fmt.Println() fmt.Printf(" Test name : %s\n", test.Name) fmt.Printf(" Test path : %s\n", test.Path) - fmt.Printf(" Log file : %s\n", filepath.Join(test.Path, test.Config.LogFile)) - fmt.Printf(" Parser assertion file : %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName)) - fmt.Printf(" Scenario assertion file : %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName)) + if isAppsecTest { + fmt.Printf(" Nuclei Template : %s\n", test.Config.NucleiTemplate) + fmt.Printf(" Appsec Rules : %s\n", strings.Join(test.Config.AppsecRules, ", ")) + } else { + fmt.Printf(" Log file : %s\n", filepath.Join(test.Path, test.Config.LogFile)) + fmt.Printf(" Parser assertion file : %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName)) + fmt.Printf(" Scenario assertion file : %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName)) + } fmt.Printf(" Configuration File : %s\n", filepath.Join(test.Path, "config.yaml")) } @@ -391,15 +424,15 @@ func NewHubTestListCmd() *cobra.Command { Short: "list", DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - if err := HubTest.LoadAllTests(); err != nil { + if err := hubPtr.LoadAllTests(); err != nil { return fmt.Errorf("unable to load all tests: %s", err) } switch csConfig.Cscli.Output { case "human": - hubTestListTable(color.Output, HubTest.Tests) + hubTestListTable(color.Output, hubPtr.Tests) case "json": - j, err := json.MarshalIndent(HubTest.Tests, " ", " ") + j, err := json.MarshalIndent(hubPtr.Tests, " ", " ") if err != nil { return err } @@ -419,23 +452,27 @@ func NewHubTestCoverageCmd() *cobra.Command { var showParserCov bool var showScenarioCov bool var showOnlyPercent bool + var showAppsecCov bool var cmdHubTestCoverage = &cobra.Command{ Use: "coverage", Short: "coverage", DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { + //for this one we explicitly don't do for appsec if err := HubTest.LoadAllTests(); err != nil { return fmt.Errorf("unable to load all tests: %+v", err) } var err error scenarioCoverage := []hubtest.Coverage{} parserCoverage := []hubtest.Coverage{} + appsecRuleCoverage := []hubtest.Coverage{} scenarioCoveragePercent := 0 parserCoveragePercent := 0 + appsecRuleCoveragePercent := 0 // if both are false (flag by default), show both - showAll := !showScenarioCov && !showParserCov + showAll := !showScenarioCov && !showParserCov && !showAppsecCov if showParserCov || showAll { parserCoverage, err = HubTest.GetParsersCoverage() @@ -467,13 +504,30 @@ func NewHubTestCoverageCmd() *cobra.Command { scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100))) } + if showAppsecCov || showAll { + appsecRuleCoverage, err = HubTest.GetAppsecCoverage() + if err != nil { + return fmt.Errorf("while getting scenario coverage: %s", err) + } + + appsecRuleTested := 0 + for _, test := range appsecRuleCoverage { + if test.TestsCount > 0 { + appsecRuleTested++ + } + } + appsecRuleCoveragePercent = int(math.Round((float64(appsecRuleTested) / float64(len(appsecRuleCoverage)) * 100))) + } + if showOnlyPercent { if showAll { - fmt.Printf("parsers=%d%%\nscenarios=%d%%", parserCoveragePercent, scenarioCoveragePercent) + fmt.Printf("parsers=%d%%\nscenarios=%d%%\nappsec_rules=%d%%", parserCoveragePercent, scenarioCoveragePercent, appsecRuleCoveragePercent) } else if showParserCov { fmt.Printf("parsers=%d%%", parserCoveragePercent) } else if showScenarioCov { fmt.Printf("scenarios=%d%%", scenarioCoveragePercent) + } else if showAppsecCov { + fmt.Printf("appsec_rules=%d%%", appsecRuleCoveragePercent) } os.Exit(0) } @@ -487,6 +541,11 @@ func NewHubTestCoverageCmd() *cobra.Command { if showScenarioCov || showAll { hubTestScenarioCoverageTable(color.Output, scenarioCoverage) } + + if showAppsecCov || showAll { + hubTestAppsecRuleCoverageTable(color.Output, appsecRuleCoverage) + } + fmt.Println() if showParserCov || showAll { fmt.Printf("PARSERS : %d%% of coverage\n", parserCoveragePercent) @@ -494,6 +553,9 @@ func NewHubTestCoverageCmd() *cobra.Command { if showScenarioCov || showAll { fmt.Printf("SCENARIOS : %d%% of coverage\n", scenarioCoveragePercent) } + if showAppsecCov || showAll { + fmt.Printf("APPSEC RULES : %d%% of coverage\n", appsecRuleCoveragePercent) + } case "json": dump, err := json.MarshalIndent(parserCoverage, "", " ") if err != nil { @@ -505,6 +567,11 @@ func NewHubTestCoverageCmd() *cobra.Command { return err } fmt.Printf("%s", dump) + dump, err = json.MarshalIndent(appsecRuleCoverage, "", " ") + if err != nil { + return err + } + fmt.Printf("%s", dump) default: return fmt.Errorf("only human/json output modes are supported") } @@ -516,6 +583,7 @@ func NewHubTestCoverageCmd() *cobra.Command { cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage") cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage") cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage") + cmdHubTestCoverage.PersistentFlags().BoolVar(&showAppsecCov, "appsec", false, "Show only appsec coverage") return cmdHubTestCoverage } @@ -529,7 +597,7 @@ func NewHubTestEvalCmd() *cobra.Command { DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { for _, testName := range args { - test, err := HubTest.LoadTestItem(testName) + test, err := hubPtr.LoadTestItem(testName) if err != nil { return fmt.Errorf("can't load test: %+v", err) } diff --git a/cmd/crowdsec-cli/hubtest_table.go b/cmd/crowdsec-cli/hubtest_table.go index 9b31a79a2..4034da7e5 100644 --- a/cmd/crowdsec-cli/hubtest_table.go +++ b/cmd/crowdsec-cli/hubtest_table.go @@ -61,6 +61,26 @@ func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) { t.Render() } +func hubTestAppsecRuleCoverageTable(out io.Writer, coverage []hubtest.Coverage) { + t := newLightTable(out) + t.SetHeaders("Appsec Rule", "Status", "Number of tests") + t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) + t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) + + parserTested := 0 + + for _, test := range coverage { + status := emoji.RedCircle.String() + if test.TestsCount > 0 { + status = emoji.GreenCircle.String() + parserTested++ + } + t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) + } + + t.Render() +} + func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) { t := newLightTable(out) t.SetHeaders("Scenario", "Status", "Number of tests") diff --git a/cmd/crowdsec-cli/item_metrics.go b/cmd/crowdsec-cli/item_metrics.go index 51b652abc..34484f63d 100644 --- a/cmd/crowdsec-cli/item_metrics.go +++ b/cmd/crowdsec-cli/item_metrics.go @@ -32,6 +32,8 @@ func ShowMetrics(hubItem *cwhub.Item) error { return err } } + case cwhub.APPSEC_RULES: + log.Error("FIXME: not implemented yet") default: // no metrics for this item type } diff --git a/cmd/crowdsec-cli/itemcli.go b/cmd/crowdsec-cli/itemcli.go new file mode 100644 index 000000000..0870fdeb4 --- /dev/null +++ b/cmd/crowdsec-cli/itemcli.go @@ -0,0 +1,454 @@ +package main + +import ( + "fmt" + + "github.com/fatih/color" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/crowdsecurity/go-cs-lib/coalesce" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +type cliHelp struct { + // Example is required, the others have a default value + // generated from the item type + use string + short string + long string + example string +} + +type itemCLI struct { + name string // plural, as used in the hub index + singular string + oneOrMore string // parenthetical pluralizaion: "parser(s)" + help cliHelp + installHelp cliHelp + removeHelp cliHelp + upgradeHelp cliHelp + inspectHelp cliHelp + inspectDetail func(item *cwhub.Item) error + listHelp cliHelp +} + +func (it itemCLI) NewCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: coalesce.String(it.help.use, fmt.Sprintf("%s [item]...", it.name)), + Short: coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)), + Long: it.help.long, + Example: it.help.example, + Args: cobra.MinimumNArgs(1), + Aliases: []string{it.singular}, + DisableAutoGenTag: true, + } + + cmd.AddCommand(it.NewInstallCmd()) + cmd.AddCommand(it.NewRemoveCmd()) + cmd.AddCommand(it.NewUpgradeCmd()) + cmd.AddCommand(it.NewInspectCmd()) + cmd.AddCommand(it.NewListCmd()) + + return cmd +} + +func (it itemCLI) Install(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + downloadOnly, err := flags.GetBool("download-only") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + ignoreError, err := flags.GetBool("ignore") + if err != nil { + return err + } + + hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) + if err != nil { + return err + } + + for _, name := range args { + item := hub.GetItem(it.name, name) + if item == nil { + msg := suggestNearestMessage(hub, it.name, name) + if !ignoreError { + return fmt.Errorf(msg) + } + + log.Errorf(msg) + + continue + } + + if err := item.Install(force, downloadOnly); err != nil { + if !ignoreError { + return fmt.Errorf("error while installing '%s': %w", item.Name, err) + } + log.Errorf("Error while installing '%s': %s", item.Name, err) + } + } + + log.Infof(ReloadMessage()) + return nil +} + +func (it itemCLI) NewInstallCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: coalesce.String(it.installHelp.use, "install [item]..."), + Short: coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)), + Long: coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)), + Example: it.installHelp.example, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compAllItems(it.name, args, toComplete) + }, + RunE: it.Install, + } + + flags := cmd.Flags() + flags.BoolP("download-only", "d", false, "Only download packages, don't enable") + flags.Bool("force", false, "Force install: overwrite tainted and outdated files") + flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name)) + + return cmd +} + +// return the names of the installed parents of an item, used to check if we can remove it +func istalledParentNames(item *cwhub.Item) []string { + ret := make([]string, 0) + + for _, parent := range item.Ancestors() { + if parent.State.Installed { + ret = append(ret, parent.Name) + } + } + + return ret +} + +func (it itemCLI) Remove(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + purge, err := flags.GetBool("purge") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } + + if all { + getter := hub.GetInstalledItems + if purge { + getter = hub.GetAllItems + } + + items, err := getter(it.name) + if err != nil { + return err + } + + removed := 0 + + for _, item := range items { + didRemove, err := item.Remove(purge, force) + if err != nil { + return err + } + if didRemove { + log.Infof("Removed %s", item.Name) + removed++ + } + } + + log.Infof("Removed %d %s", removed, it.name) + if removed > 0 { + log.Infof(ReloadMessage()) + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular) + } + + removed := 0 + + for _, itemName := range args { + item := hub.GetItem(it.name, itemName) + if item == nil { + return fmt.Errorf("can't find '%s' in %s", itemName, it.name) + } + + parents := istalledParentNames(item) + + if !force && len(parents) > 0 { + log.Warningf("%s belongs to collections: %s", item.Name, parents) + log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular) + continue + } + + didRemove, err := item.Remove(purge, force) + if err != nil { + return err + } + + if didRemove { + log.Infof("Removed %s", item.Name) + removed++ + } + } + + log.Infof("Removed %d %s", removed, it.name) + if removed > 0 { + log.Infof(ReloadMessage()) + } + + return nil +} + +func (it itemCLI) NewRemoveCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: coalesce.String(it.removeHelp.use, "remove [item]..."), + Short: coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)), + Long: coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)), + Example: it.removeHelp.example, + Aliases: []string{"delete"}, + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(it.name, args, toComplete) + }, + RunE: it.Remove, + } + + flags := cmd.Flags() + flags.Bool("purge", false, "Delete source file too") + flags.Bool("force", false, "Force remove: remove tainted and outdated files") + flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name)) + + return cmd +} + +func (it itemCLI) Upgrade(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) + if err != nil { + return err + } + + if all { + items, err := hub.GetInstalledItems(it.name) + if err != nil { + return err + } + + updated := 0 + + for _, item := range items { + didUpdate, err := item.Upgrade(force) + if err != nil { + return err + } + if didUpdate { + updated++ + } + } + + log.Infof("Updated %d %s", updated, it.name) + + if updated > 0 { + log.Infof(ReloadMessage()) + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular) + } + + updated := 0 + + for _, itemName := range args { + item := hub.GetItem(it.name, itemName) + if item == nil { + return fmt.Errorf("can't find '%s' in %s", itemName, it.name) + } + + didUpdate, err := item.Upgrade(force) + if err != nil { + return err + } + + if didUpdate { + log.Infof("Updated %s", item.Name) + updated++ + } + } + if updated > 0 { + log.Infof(ReloadMessage()) + } + + return nil +} + +func (it itemCLI) NewUpgradeCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: coalesce.String(it.upgradeHelp.use, "upgrade [item]..."), + Short: coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)), + Long: coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)), + Example: it.upgradeHelp.example, + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(it.name, args, toComplete) + }, + RunE: it.Upgrade, + } + + flags := cmd.Flags() + flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name)) + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") + + return cmd +} + +func (it itemCLI) Inspect(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + url, err := flags.GetString("url") + if err != nil { + return err + } + + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } + + for _, name := range args { + item := hub.GetItem(it.name, name) + if item == nil { + return fmt.Errorf("can't find '%s' in %s", name, it.name) + } + if err = InspectItem(item, !noMetrics); err != nil { + return err + } + + if it.inspectDetail != nil { + if err = it.inspectDetail(item); err != nil { + return err + } + } + } + + return nil +} + +func (it itemCLI) NewInspectCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: coalesce.String(it.inspectHelp.use, "inspect [item]..."), + Short: coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)), + Long: coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)), + Example: it.inspectHelp.example, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(it.name, args, toComplete) + }, + RunE: it.Inspect, + } + + flags := cmd.Flags() + flags.StringP("url", "u", "", "Prometheus url") + flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") + + return cmd +} + +func (it itemCLI) List(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + hub, err := require.Hub(csConfig, nil) + if err != nil { + return err + } + + items := make(map[string][]*cwhub.Item) + + items[it.name], err = selectItems(hub, it.name, args, !all) + if err != nil { + return err + } + + if err = listItems(color.Output, []string{it.name}, items, false); err != nil { + return err + } + + return nil +} + +func (it itemCLI) NewListCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: coalesce.String(it.listHelp.use, "list [item... | -a]"), + Short: coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)), + Long: coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)), + Example: it.listHelp.example, + DisableAutoGenTag: true, + RunE: it.List, + } + + flags := cmd.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") + + return cmd +} diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go deleted file mode 100644 index 5c699fde9..000000000 --- a/cmd/crowdsec-cli/itemcommands.go +++ /dev/null @@ -1,609 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/fatih/color" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/crowdsecurity/go-cs-lib/coalesce" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" -) - -type cmdHelp struct { - // Example is required, the others have a default value - // generated from the item type - use string - short string - long string - example string -} - -type hubItemType struct { - name string // plural, as used in the hub index - singular string - oneOrMore string // parenthetical pluralizaion: "parser(s)" - help cmdHelp - installHelp cmdHelp - removeHelp cmdHelp - upgradeHelp cmdHelp - inspectHelp cmdHelp - listHelp cmdHelp -} - -var hubItemTypes = map[string]hubItemType{ - "parsers": { - name: cwhub.PARSERS, - singular: "parser", - oneOrMore: "parser(s)", - help: cmdHelp{ - example: `cscli parsers list -a -cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs -cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs -cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs -cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs -`, - }, - installHelp: cmdHelp{ - example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - }, - removeHelp: cmdHelp{ - example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - }, - upgradeHelp: cmdHelp{ - example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, - }, - inspectHelp: cmdHelp{ - example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`, - }, - listHelp: cmdHelp{ - example: `cscli parsers list -cscli parsers list -a -cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs - -List only enabled parsers unless "-a" or names are specified.`, - }, - }, - "postoverflows": { - name: cwhub.POSTOVERFLOWS, - singular: "postoverflow", - oneOrMore: "postoverflow(s)", - help: cmdHelp{ - example: `cscli postoverflows list -a -cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns -cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns -cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns -cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns -`, - }, - installHelp: cmdHelp{ - example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - }, - removeHelp: cmdHelp{ - example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - }, - upgradeHelp: cmdHelp{ - example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - }, - inspectHelp: cmdHelp{ - example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`, - }, - listHelp: cmdHelp{ - example: `cscli postoverflows list -cscli postoverflows list -a -cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns - -List only enabled postoverflows unless "-a" or names are specified.`, - }, - }, - "scenarios": { - name: cwhub.SCENARIOS, - singular: "scenario", - oneOrMore: "scenario(s)", - help: cmdHelp{ - example: `cscli scenarios list -a -cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing -cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing -cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing -cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing -`, - }, - installHelp: cmdHelp{ - example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`, - }, - removeHelp: cmdHelp{ - example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`, - }, - upgradeHelp: cmdHelp{ - example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`, - }, - inspectHelp: cmdHelp{ - example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`, - }, - listHelp: cmdHelp{ - example: `cscli scenarios list -cscli scenarios list -a -cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing - -List only enabled scenarios unless "-a" or names are specified.`, - }, - }, - "collections": { - name: cwhub.COLLECTIONS, - singular: "collection", - oneOrMore: "collection(s)", - help: cmdHelp{ - example: `cscli collections list -a -cscli collections install crowdsecurity/http-cve crowdsecurity/iptables -cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables -cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables -cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables -`, - }, - installHelp: cmdHelp{ - example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`, - }, - removeHelp: cmdHelp{ - example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`, - }, - upgradeHelp: cmdHelp{ - example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`, - }, - inspectHelp: cmdHelp{ - example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`, - }, - listHelp: cmdHelp{ - example: `cscli collections list -cscli collections list -a -cscli collections list crowdsecurity/http-cve crowdsecurity/iptables - -List only enabled collections unless "-a" or names are specified.`, - }, - }, -} - -func NewItemsCmd(typeName string) *cobra.Command { - it := hubItemTypes[typeName] - - cmd := &cobra.Command{ - Use: coalesce.String(it.help.use, fmt.Sprintf("%s [item]...", it.name)), - Short: coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)), - Long: it.help.long, - Example: it.help.example, - Args: cobra.MinimumNArgs(1), - Aliases: []string{it.singular}, - DisableAutoGenTag: true, - } - - cmd.AddCommand(NewItemsInstallCmd(typeName)) - cmd.AddCommand(NewItemsRemoveCmd(typeName)) - cmd.AddCommand(NewItemsUpgradeCmd(typeName)) - cmd.AddCommand(NewItemsInspectCmd(typeName)) - cmd.AddCommand(NewItemsListCmd(typeName)) - - return cmd -} - -func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { - run := func(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - downloadOnly, err := flags.GetBool("download-only") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - ignoreError, err := flags.GetBool("ignore") - if err != nil { - return err - } - - hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) - if err != nil { - return err - } - - for _, name := range args { - item := hub.GetItem(it.name, name) - if item == nil { - msg := suggestNearestMessage(hub, it.name, name) - if !ignoreError { - return fmt.Errorf(msg) - } - - log.Errorf(msg) - - continue - } - - if err := item.Install(force, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", item.Name, err) - } - log.Errorf("Error while installing '%s': %s", item.Name, err) - } - } - - log.Infof(ReloadMessage()) - return nil - } - - return run -} - -func NewItemsInstallCmd(typeName string) *cobra.Command { - it := hubItemTypes[typeName] - - cmd := &cobra.Command{ - Use: coalesce.String(it.installHelp.use, "install [item]..."), - Short: coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)), - Long: coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)), - Example: it.installHelp.example, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(typeName, args, toComplete) - }, - RunE: itemsInstallRunner(it), - } - - flags := cmd.Flags() - flags.BoolP("download-only", "d", false, "Only download packages, don't enable") - flags.Bool("force", false, "Force install: overwrite tainted and outdated files") - flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name)) - - return cmd -} - -// return the names of the installed parents of an item, used to check if we can remove it -func istalledParentNames(item *cwhub.Item) []string { - ret := make([]string, 0) - - for _, parent := range item.Ancestors() { - if parent.State.Installed { - ret = append(ret, parent.Name) - } - } - - return ret -} - -func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { - run := func(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - purge, err := flags.GetBool("purge") - if err != nil { - return err - } - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := require.Hub(csConfig, nil) - if err != nil { - return err - } - - if all { - getter := hub.GetInstalledItems - if purge { - getter = hub.GetAllItems - } - - items, err := getter(it.name) - if err != nil { - return err - } - - removed := 0 - - for _, item := range items { - didRemove, err := item.Remove(purge, force) - if err != nil { - return err - } - if didRemove { - log.Infof("Removed %s", item.Name) - removed++ - } - } - - log.Infof("Removed %d %s", removed, it.name) - if removed > 0 { - log.Infof(ReloadMessage()) - } - - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular) - } - - removed := 0 - - for _, itemName := range args { - item := hub.GetItem(it.name, itemName) - if item == nil { - return fmt.Errorf("can't find '%s' in %s", itemName, it.name) - } - - parents := istalledParentNames(item) - - if !force && len(parents) > 0 { - log.Warningf("%s belongs to collections: %s", item.Name, parents) - log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular) - continue - } - - didRemove, err := item.Remove(purge, force) - if err != nil { - return err - } - - if didRemove { - log.Infof("Removed %s", item.Name) - removed++ - } - } - - log.Infof("Removed %d %s", removed, it.name) - if removed > 0 { - log.Infof(ReloadMessage()) - } - - return nil - } - return run -} - -func NewItemsRemoveCmd(typeName string) *cobra.Command { - it := hubItemTypes[typeName] - - cmd := &cobra.Command{ - Use: coalesce.String(it.removeHelp.use, "remove [item]..."), - Short: coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)), - Long: coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)), - Example: it.removeHelp.example, - Aliases: []string{"delete"}, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(it.name, args, toComplete) - }, - RunE: itemsRemoveRunner(it), - } - - flags := cmd.Flags() - flags.Bool("purge", false, "Delete source file too") - flags.Bool("force", false, "Force remove: remove tainted and outdated files") - flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name)) - - return cmd -} - -func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { - run := func(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - force, err := flags.GetBool("force") - if err != nil { - return err - } - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) - if err != nil { - return err - } - - if all { - items, err := hub.GetInstalledItems(it.name) - if err != nil { - return err - } - - updated := 0 - - for _, item := range items { - didUpdate, err := item.Upgrade(force) - if err != nil { - return err - } - if didUpdate { - updated++ - } - } - - log.Infof("Updated %d %s", updated, it.name) - - if updated > 0 { - log.Infof(ReloadMessage()) - } - - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular) - } - - updated := 0 - - for _, itemName := range args { - item := hub.GetItem(it.name, itemName) - if item == nil { - return fmt.Errorf("can't find '%s' in %s", itemName, it.name) - } - - didUpdate, err := item.Upgrade(force) - if err != nil { - return err - } - - if didUpdate { - log.Infof("Updated %s", item.Name) - updated++ - } - } - if updated > 0 { - log.Infof(ReloadMessage()) - } - - return nil - } - - return run -} - -func NewItemsUpgradeCmd(typeName string) *cobra.Command { - it := hubItemTypes[typeName] - - cmd := &cobra.Command{ - Use: coalesce.String(it.upgradeHelp.use, "upgrade [item]..."), - Short: coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)), - Long: coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)), - Example: it.upgradeHelp.example, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(it.name, args, toComplete) - }, - RunE: itemsUpgradeRunner(it), - } - - flags := cmd.Flags() - flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name)) - flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") - - return cmd -} - -func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { - run := func(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - url, err := flags.GetString("url") - if err != nil { - return err - } - - if url != "" { - csConfig.Cscli.PrometheusUrl = url - } - - noMetrics, err := flags.GetBool("no-metrics") - if err != nil { - return err - } - - hub, err := require.Hub(csConfig, nil) - if err != nil { - return err - } - - for _, name := range args { - item := hub.GetItem(it.name, name) - if item == nil { - return fmt.Errorf("can't find '%s' in %s", name, it.name) - } - if err = InspectItem(item, !noMetrics); err != nil { - return err - } - } - - return nil - } - - return run -} - -func NewItemsInspectCmd(typeName string) *cobra.Command { - it := hubItemTypes[typeName] - - cmd := &cobra.Command{ - Use: coalesce.String(it.inspectHelp.use, "inspect [item]..."), - Short: coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)), - Long: coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)), - Example: it.inspectHelp.example, - Args: cobra.MinimumNArgs(1), - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(it.name, args, toComplete) - }, - RunE: itemsInspectRunner(it), - } - - flags := cmd.Flags() - flags.StringP("url", "u", "", "Prometheus url") - flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") - - return cmd -} - -func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) error { - run := func(cmd *cobra.Command, args []string) error { - flags := cmd.Flags() - - all, err := flags.GetBool("all") - if err != nil { - return err - } - - hub, err := require.Hub(csConfig, nil) - if err != nil { - return err - } - - items := make(map[string][]*cwhub.Item) - - items[it.name], err = selectItems(hub, it.name, args, !all) - if err != nil { - return err - } - - if err = listItems(color.Output, []string{it.name}, items, false); err != nil { - return err - } - - return nil - } - - return run -} - -func NewItemsListCmd(typeName string) *cobra.Command { - it := hubItemTypes[typeName] - - cmd := &cobra.Command{ - Use: coalesce.String(it.listHelp.use, "list [item... | -a]"), - Short: coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)), - Long: coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)), - Example: it.listHelp.example, - DisableAutoGenTag: true, - RunE: itemsListRunner(it), - } - - flags := cmd.Flags() - flags.BoolP("all", "a", false, "List disabled items as well") - - return cmd -} diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 1103a21a9..c9b7d7030 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -6,12 +6,13 @@ import ( "path/filepath" "strings" + "slices" + "github.com/fatih/color" cc "github.com/ivanpirog/coloredcobra" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/cobra/doc" - "slices" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwversion" @@ -240,10 +241,13 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewHubTestCmd()) rootCmd.AddCommand(NewNotificationsCmd()) rootCmd.AddCommand(NewSupportCmd()) - rootCmd.AddCommand(NewItemsCmd("collections")) - rootCmd.AddCommand(NewItemsCmd("parsers")) - rootCmd.AddCommand(NewItemsCmd("scenarios")) - rootCmd.AddCommand(NewItemsCmd("postoverflows")) + + rootCmd.AddCommand(NewCollectionCLI().NewCommand()) + rootCmd.AddCommand(NewParserCLI().NewCommand()) + rootCmd.AddCommand(NewScenarioCLI().NewCommand()) + rootCmd.AddCommand(NewPostOverflowCLI().NewCommand()) + rootCmd.AddCommand(NewAppsecConfigCLI().NewCommand()) + rootCmd.AddCommand(NewAppsecRuleCLI().NewCommand()) if fflag.CscliSetup.IsEnabled() { rootCmd.AddCommand(NewSetupCmd()) diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index a03614aae..5b24dc84c 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -63,6 +63,8 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error lapi_machine_stats := map[string]map[string]map[string]int{} lapi_bouncer_stats := map[string]map[string]map[string]int{} decisions_stats := map[string]map[string]map[string]int{} + appsec_engine_stats := map[string]map[string]int{} + appsec_rule_stats := map[string]map[string]map[string]int{} alerts_stats := map[string]int{} stash_stats := map[string]struct { Type string @@ -226,10 +228,30 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error Type string Count int }{Type: mtype, Count: ival} + case "cs_appsec_reqs_total": + if _, ok := appsec_engine_stats[metric.Labels["appsec_engine"]]; !ok { + appsec_engine_stats[metric.Labels["appsec_engine"]] = make(map[string]int, 0) + } + appsec_engine_stats[metric.Labels["appsec_engine"]]["processed"] = ival + case "cs_appsec_block_total": + if _, ok := appsec_engine_stats[metric.Labels["appsec_engine"]]; !ok { + appsec_engine_stats[metric.Labels["appsec_engine"]] = make(map[string]int, 0) + } + appsec_engine_stats[metric.Labels["appsec_engine"]]["blocked"] = ival + case "cs_appsec_rule_hits": + appsecEngine := metric.Labels["appsec_engine"] + ruleID := metric.Labels["rule_name"] + if _, ok := appsec_rule_stats[appsecEngine]; !ok { + appsec_rule_stats[appsecEngine] = make(map[string]map[string]int, 0) + } + if _, ok := appsec_rule_stats[appsecEngine][ruleID]; !ok { + appsec_rule_stats[appsecEngine][ruleID] = make(map[string]int, 0) + } + appsec_rule_stats[appsecEngine][ruleID]["triggered"] = ival default: + log.Debugf("unknown: %+v", fam.Name) continue } - } } @@ -244,6 +266,8 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error decisionStatsTable(out, decisions_stats) alertStatsTable(out, alerts_stats) stashStatsTable(out, stash_stats) + appsecMetricsToTable(out, appsec_engine_stats) + appsecRulesToTable(out, appsec_rule_stats) return nil } @@ -282,7 +306,6 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error var noUnit bool - func runMetrics(cmd *cobra.Command, args []string) error { flags := cmd.Flags() @@ -314,7 +337,6 @@ func runMetrics(cmd *cobra.Command, args []string) error { return nil } - func NewMetricsCmd() *cobra.Command { cmdMetrics := &cobra.Command{ Use: "metrics", @@ -322,7 +344,7 @@ func NewMetricsCmd() *cobra.Command { Long: `Fetch metrics from the prometheus server and display them in a human-friendly way`, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - RunE: runMetrics, + RunE: runMetrics, } flags := cmdMetrics.PersistentFlags() diff --git a/cmd/crowdsec-cli/metrics_table.go b/cmd/crowdsec-cli/metrics_table.go index 69706c7ac..80b9cb6e4 100644 --- a/cmd/crowdsec-cli/metrics_table.go +++ b/cmd/crowdsec-cli/metrics_table.go @@ -90,7 +90,7 @@ func bucketStatsTable(out io.Writer, stats map[string]map[string]int) { keys := []string{"curr_count", "overflow", "instantiation", "pour", "underflow"} if numRows, err := metricsToTable(t, stats, keys); err != nil { - log.Warningf("while collecting acquis stats: %s", err) + log.Warningf("while collecting bucket stats: %s", err) } else if numRows > 0 { renderTableTitle(out, "\nBucket Metrics:") t.Render() @@ -113,6 +113,37 @@ func acquisStatsTable(out io.Writer, stats map[string]map[string]int) { } } +func appsecMetricsToTable(out io.Writer, metrics map[string]map[string]int) { + t := newTable(out) + t.SetRowLines(false) + t.SetHeaders("Appsec Engine", "Processed", "Blocked") + t.SetAlignment(table.AlignLeft, table.AlignLeft) + keys := []string{"processed", "blocked"} + if numRows, err := metricsToTable(t, metrics, keys); err != nil { + log.Warningf("while collecting appsec stats: %s", err) + } else if numRows > 0 { + renderTableTitle(out, "\nAppsec Metrics:") + t.Render() + } +} + +func appsecRulesToTable(out io.Writer, metrics map[string]map[string]map[string]int) { + for appsecEngine, appsecEngineRulesStats := range metrics { + t := newTable(out) + t.SetRowLines(false) + t.SetHeaders("Rule ID", "Triggered") + t.SetAlignment(table.AlignLeft, table.AlignLeft) + keys := []string{"triggered"} + if numRows, err := metricsToTable(t, appsecEngineRulesStats, keys); err != nil { + log.Warningf("while collecting appsec rules stats: %s", err) + } else if numRows > 0 { + renderTableTitle(out, fmt.Sprintf("\nAppsec '%s' Rules Metrics:", appsecEngine)) + t.Render() + } + } + +} + func parserStatsTable(out io.Writer, stats map[string]map[string]int) { t := newTable(out) t.SetRowLines(false) @@ -122,7 +153,7 @@ func parserStatsTable(out io.Writer, stats map[string]map[string]int) { keys := []string{"hits", "parsed", "unparsed"} if numRows, err := metricsToTable(t, stats, keys); err != nil { - log.Warningf("while collecting acquis stats: %s", err) + log.Warningf("while collecting parsers stats: %s", err) } else if numRows > 0 { renderTableTitle(out, "\nParser Metrics:") t.Render() diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index fc1fdb946..1e0d54c07 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -13,6 +13,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/acquisition" + "github.com/crowdsecurity/crowdsec/pkg/appsec" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" @@ -33,6 +34,10 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, er return nil, fmt.Errorf("while loading scenarios: %w", err) } + if err := appsec.LoadAppsecRules(hub); err != nil { + return nil, fmt.Errorf("while loading appsec rules: %w", err) + } + if err := LoadAcquisition(cConfig); err != nil { return nil, fmt.Errorf("while loading acquisition config: %w", err) } diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index 6371a6046..ca893872e 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -161,7 +161,8 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, v1.LapiRouteHits, leaky.BucketsCurrentCount, - cache.CacheMetrics, exprhelpers.RegexpCacheMetrics) + cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, + ) } else { log.Infof("Loading prometheus collectors") prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo, @@ -170,7 +171,8 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions, v1.LapiResponseTime, leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount, globalActiveDecisions, globalAlerts, - cache.CacheMetrics, exprhelpers.RegexpCacheMetrics) + cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, + ) } } diff --git a/cmd/crowdsec/parse.go b/cmd/crowdsec/parse.go index aa93b6ec7..c62eeb586 100644 --- a/cmd/crowdsec/parse.go +++ b/cmd/crowdsec/parse.go @@ -22,6 +22,13 @@ LOOP: if !event.Process { continue } + /*Application security engine is going to generate 2 events: + - one that is treated as a log and can go to scenarios + - another one that will go directly to LAPI*/ + if event.Type == types.APPSEC { + outputEventChan <- event + continue + } if event.Line.Module == "" { log.Errorf("empty event.Line.Module field, the acquisition module must set it ! : %+v", event.Line) continue diff --git a/go.mod b/go.mod index 450926ca2..46bc74929 100644 --- a/go.mod +++ b/go.mod @@ -80,14 +80,19 @@ require ( github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 github.com/wasilibs/go-re2 v1.3.0 github.com/xhit/go-simple-mail/v2 v2.16.0 - golang.org/x/crypto v0.15.0 + golang.org/x/crypto v0.16.0 golang.org/x/mod v0.11.0 - golang.org/x/sys v0.14.0 + golang.org/x/sys v0.15.0 google.golang.org/grpc v1.56.3 google.golang.org/protobuf v1.31.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 gopkg.in/yaml.v2 v2.4.0 +) + +require ( + github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879 + golang.org/x/text v0.14.0 gopkg.in/yaml.v3 v3.0.1 gotest.tools/v3 v3.5.0 k8s.io/apiserver v0.28.4 @@ -103,6 +108,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/corazawaf/libinjection-go v0.1.2 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/creack/pty v1.1.18 // indirect @@ -149,7 +155,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/magefile/mage v1.14.0 // indirect + github.com/magefile/mage v1.15.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect @@ -168,6 +174,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/petar-dambovaliev/aho-corasick v0.0.0-20230725210150-fb29fc3c913e // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -181,7 +188,9 @@ require ( github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tetratelabs/wazero v1.2.1 // indirect - github.com/tidwall/gjson v1.13.0 // indirect + github.com/tidwall/gjson v1.17.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 // indirect @@ -192,10 +201,9 @@ require ( github.com/zclconf/go-cty v1.8.0 // indirect go.mongodb.org/mongo-driver v1.9.4 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/net v0.18.0 // indirect - golang.org/x/sync v0.2.0 // indirect - golang.org/x/term v0.14.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/sync v0.5.0 // indirect + golang.org/x/term v0.15.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect @@ -207,6 +215,7 @@ require ( k8s.io/apimachinery v0.28.4 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + rsc.io/binaryregexp v0.2.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/go.sum b/go.sum index 2703ac09f..835b288ac 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,8 @@ github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhD github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/corazawaf/libinjection-go v0.1.2 h1:oeiV9pc5rvJ+2oqOqXEAMJousPpGiup6f7Y3nZj5GoM= +github.com/corazawaf/libinjection-go v0.1.2/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nNZEmJNuWbNPOItn7aw= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= @@ -96,6 +98,16 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231114091225-b0f8bc435a75 h1:Kp1sY2PE1H5nbr7xgAQeEWDqDW/o3HNL1rHvcVqzWT4= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231114091225-b0f8bc435a75/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231204125126-35deffad7734 h1:THMSMkBW/DLG5NvMAr/Mdg/eQOrEnMJ9Y+UdFG4yV8k= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231204125126-35deffad7734/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135226-6c45fc2dedf9 h1:vFJiYtKOW5DwGQ9gxQi8+XDNc+YvuXXsJyWXXuiOn+M= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135226-6c45fc2dedf9/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135508-23eef9bf7f39 h1:vY0KZvoS4Xl9IfGucBA4l1CV1auRPPJtjZSTz/Rl6iQ= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231204135508-23eef9bf7f39/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879 h1:dhAc0AelASC3BbfuLURJeai1LYgFNgpMds0KPd9whbo= +github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8= @@ -125,6 +137,8 @@ github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkK github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= +github.com/foxcpp/go-mockdns v1.0.0/go.mod h1:lgRN6+KxQBawyIghpnl5CezHFGS9VLzvtVlwxvzXTQ4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -459,8 +473,8 @@ github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffkt github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= -github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= -github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= +github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -496,6 +510,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= +github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= @@ -548,6 +564,8 @@ github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUr github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/petar-dambovaliev/aho-corasick v0.0.0-20230725210150-fb29fc3c913e h1:POJco99aNgosh92lGqmx7L1ei+kCymivB/419SD15PQ= +github.com/petar-dambovaliev/aho-corasick v0.0.0-20230725210150-fb29fc3c913e/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -652,13 +670,14 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/tetratelabs/wazero v1.2.1 h1:J4X2hrGzJvt+wqltuvcSjHQ7ujQxA9gb6PeMs4qlUWs= github.com/tetratelabs/wazero v1.2.1/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M= -github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= +github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= @@ -748,6 +767,8 @@ golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -783,6 +804,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -792,8 +815,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -835,6 +858,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -844,6 +869,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -943,6 +970,8 @@ k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 65164b9c4..336029363 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -18,6 +18,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + appsecacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/appsec" cloudwatchacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/cloudwatch" dockeracquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/docker" fileacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file" @@ -76,6 +77,7 @@ var AcquisitionSources = map[string]func() DataSource{ "k8s-audit": func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{} }, "loki": func() DataSource { return &lokiacquisition.LokiSource{} }, "s3": func() DataSource { return &s3acquisition.S3Source{} }, + "appsec": func() DataSource { return &appsecacquisition.AppsecSource{} }, } var transformRuntimes = map[string]*vm.Program{} diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go new file mode 100644 index 000000000..2ae0ad939 --- /dev/null +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -0,0 +1,371 @@ +package appsecacquisition + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "sync" + "time" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/appsec" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/crowdsecurity/go-cs-lib/trace" + "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" + "gopkg.in/tomb.v2" + "gopkg.in/yaml.v2" +) + +const ( + InBand = "inband" + OutOfBand = "outofband" +) + +var ( + DefaultAuthCacheDuration = (1 * time.Minute) +) + +// configuration structure of the acquis for the application security engine +type AppsecSourceConfig struct { + ListenAddr string `yaml:"listen_addr"` + CertFilePath string `yaml:"cert_file"` + KeyFilePath string `yaml:"key_file"` + Path string `yaml:"path"` + Routines int `yaml:"routines"` + AppsecConfig string `yaml:"appsec_config"` + AppsecConfigPath string `yaml:"appsec_config_path"` + AuthCacheDuration *time.Duration `yaml:"auth_cache_duration"` + configuration.DataSourceCommonCfg `yaml:",inline"` +} + +// runtime structure of AppsecSourceConfig +type AppsecSource struct { + config AppsecSourceConfig + logger *log.Entry + mux *http.ServeMux + server *http.Server + outChan chan types.Event + InChan chan appsec.ParsedRequest + AppsecRuntime *appsec.AppsecRuntimeConfig + AppsecConfigs map[string]appsec.AppsecConfig + lapiURL string + AuthCache AuthCache + AppsecRunners []AppsecRunner //one for each go-routine +} + +// Struct to handle cache of authentication +type AuthCache struct { + APIKeys map[string]time.Time + mu sync.RWMutex +} + +func NewAuthCache() AuthCache { + return AuthCache{ + APIKeys: make(map[string]time.Time, 0), + mu: sync.RWMutex{}, + } +} + +func (ac *AuthCache) Set(apiKey string, expiration time.Time) { + ac.mu.Lock() + ac.APIKeys[apiKey] = expiration + ac.mu.Unlock() +} + +func (ac *AuthCache) Get(apiKey string) (time.Time, bool) { + ac.mu.RLock() + expiration, exists := ac.APIKeys[apiKey] + ac.mu.RUnlock() + return expiration, exists +} + +// @tko + @sbl : we might want to get rid of that or improve it +type BodyResponse struct { + Action string `json:"action"` +} + +func (w *AppsecSource) UnmarshalConfig(yamlConfig []byte) error { + + err := yaml.UnmarshalStrict(yamlConfig, &w.config) + if err != nil { + return errors.Wrap(err, "Cannot parse appsec configuration") + } + + if w.config.ListenAddr == "" { + w.config.ListenAddr = "127.0.0.1:7422" + } + + if w.config.Path == "" { + w.config.Path = "/" + } + + if w.config.Path[0] != '/' { + w.config.Path = "/" + w.config.Path + } + + if w.config.Mode == "" { + w.config.Mode = configuration.TAIL_MODE + } + + // always have at least one appsec routine + if w.config.Routines == 0 { + w.config.Routines = 1 + } + + if w.config.AppsecConfig == "" && w.config.AppsecConfigPath == "" { + return fmt.Errorf("appsec_config or appsec_config_path must be set") + } + + if w.config.Name == "" { + w.config.Name = fmt.Sprintf("%s%s", w.config.ListenAddr, w.config.Path) + } + + csConfig := csconfig.GetConfig() + w.lapiURL = fmt.Sprintf("%sv1/decisions/stream", csConfig.API.Client.Credentials.URL) + w.AuthCache = NewAuthCache() + + return nil +} + +func (w *AppsecSource) GetMetrics() []prometheus.Collector { + return []prometheus.Collector{AppsecReqCounter, AppsecBlockCounter, AppsecRuleHits, AppsecOutbandParsingHistogram, AppsecInbandParsingHistogram, AppsecGlobalParsingHistogram} +} + +func (w *AppsecSource) GetAggregMetrics() []prometheus.Collector { + return []prometheus.Collector{AppsecReqCounter, AppsecBlockCounter, AppsecRuleHits, AppsecOutbandParsingHistogram, AppsecInbandParsingHistogram, AppsecGlobalParsingHistogram} +} + +func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry) error { + err := w.UnmarshalConfig(yamlConfig) + if err != nil { + return errors.Wrap(err, "unable to parse appsec configuration") + } + w.logger = logger + + w.logger.Tracef("Appsec configuration: %+v", w.config) + + if w.config.AuthCacheDuration == nil { + w.config.AuthCacheDuration = &DefaultAuthCacheDuration + w.logger.Infof("Cache duration for auth not set, using default: %v", *w.config.AuthCacheDuration) + } + + w.mux = http.NewServeMux() + + w.server = &http.Server{ + Addr: w.config.ListenAddr, + Handler: w.mux, + } + + w.InChan = make(chan appsec.ParsedRequest) + appsecCfg := appsec.AppsecConfig{Logger: w.logger.WithField("component", "appsec_config")} + + //let's load the associated appsec_config: + if w.config.AppsecConfigPath != "" { + err := appsecCfg.LoadByPath(w.config.AppsecConfigPath) + if err != nil { + return fmt.Errorf("unable to load appsec_config : %s", err) + } + } else if w.config.AppsecConfig != "" { + err := appsecCfg.Load(w.config.AppsecConfig) + if err != nil { + return fmt.Errorf("unable to load appsec_config : %s", err) + } + } else { + return fmt.Errorf("no appsec_config provided") + } + + w.AppsecRuntime, err = appsecCfg.Build() + if err != nil { + return fmt.Errorf("unable to build appsec_config : %s", err) + } + + err = w.AppsecRuntime.ProcessOnLoadRules() + + if err != nil { + return fmt.Errorf("unable to process on load rules : %s", err) + } + + w.AppsecRunners = make([]AppsecRunner, w.config.Routines) + + for nbRoutine := 0; nbRoutine < w.config.Routines; nbRoutine++ { + appsecRunnerUUID := uuid.New().String() + //we copy AppsecRutime for each runner + wrt := *w.AppsecRuntime + wrt.Logger = w.logger.Dup().WithField("runner_uuid", appsecRunnerUUID) + runner := AppsecRunner{ + inChan: w.InChan, + UUID: appsecRunnerUUID, + logger: w.logger.WithFields(log.Fields{ + "runner_uuid": appsecRunnerUUID, + }), + AppsecRuntime: &wrt, + Labels: w.config.Labels, + } + err := runner.Init(appsecCfg.GetDataDir()) + if err != nil { + return fmt.Errorf("unable to initialize runner : %s", err) + } + w.AppsecRunners[nbRoutine] = runner + } + + w.logger.Infof("Created %d appsec runners", len(w.AppsecRunners)) + + //We don´t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec + w.mux.HandleFunc(w.config.Path, w.appsecHandler) + return nil +} + +func (w *AppsecSource) ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uuid string) error { + return fmt.Errorf("AppSec datasource does not support command line acquisition") +} + +func (w *AppsecSource) GetMode() string { + return w.config.Mode +} + +func (w *AppsecSource) GetName() string { + return "appsec" +} + +func (w *AppsecSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error { + return fmt.Errorf("AppSec datasource does not support command line acquisition") +} + +func (w *AppsecSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { + w.outChan = out + t.Go(func() error { + defer trace.CatchPanic("crowdsec/acquis/appsec/live") + + w.logger.Infof("%d appsec runner to start", len(w.AppsecRunners)) + for _, runner := range w.AppsecRunners { + runner := runner + runner.outChan = out + t.Go(func() error { + defer trace.CatchPanic("crowdsec/acquis/appsec/live/runner") + return runner.Run(t) + }) + } + + w.logger.Infof("Starting Appsec server on %s%s", w.config.ListenAddr, w.config.Path) + t.Go(func() error { + var err error + if w.config.CertFilePath != "" && w.config.KeyFilePath != "" { + err = w.server.ListenAndServeTLS(w.config.CertFilePath, w.config.KeyFilePath) + } else { + err = w.server.ListenAndServe() + } + + if err != nil && err != http.ErrServerClosed { + return errors.Wrap(err, "Appsec server failed") + } + return nil + }) + <-t.Dying() + w.logger.Infof("Stopping Appsec server on %s%s", w.config.ListenAddr, w.config.Path) + w.server.Shutdown(context.TODO()) + return nil + }) + return nil +} + +func (w *AppsecSource) CanRun() error { + return nil +} + +func (w *AppsecSource) GetUuid() string { + return w.config.UniqueId +} + +func (w *AppsecSource) Dump() interface{} { + return w +} + +func (w *AppsecSource) IsAuth(apiKey string) bool { + client := &http.Client{ + Timeout: 200 * time.Millisecond, + } + + req, err := http.NewRequest(http.MethodHead, w.lapiURL, nil) + if err != nil { + log.Errorf("Error creating request: %s", err) + return false + } + + req.Header.Add("X-Api-Key", apiKey) + resp, err := client.Do(req) + if err != nil { + log.Errorf("Error performing request: %s", err) + return false + } + defer resp.Body.Close() + + return resp.StatusCode == http.StatusOK + +} + +// should this be in the runner ? +func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { + w.logger.Debugf("Received request from '%s' on %s", r.RemoteAddr, r.URL.Path) + + apiKey := r.Header.Get(appsec.APIKeyHeaderName) + clientIP := r.Header.Get(appsec.IPHeaderName) + remoteIP := r.RemoteAddr + if apiKey == "" { + w.logger.Errorf("Unauthorized request from '%s' (real IP = %s)", remoteIP, clientIP) + rw.WriteHeader(http.StatusUnauthorized) + return + } + expiration, exists := w.AuthCache.Get(apiKey) + // if the apiKey is not in cache or has expired, just recheck the auth + if !exists || time.Now().After(expiration) { + if !w.IsAuth(apiKey) { + rw.WriteHeader(http.StatusUnauthorized) + w.logger.Errorf("Unauthorized request from '%s' (real IP = %s)", remoteIP, clientIP) + return + } + + // apiKey is valid, store it in cache + w.AuthCache.Set(apiKey, time.Now().Add(*w.config.AuthCacheDuration)) + } + + // parse the request only once + parsedRequest, err := appsec.NewParsedRequestFromRequest(r) + if err != nil { + log.Errorf("%s", err) + rw.WriteHeader(http.StatusInternalServerError) + return + } + parsedRequest.AppsecEngine = w.config.Name + + logger := w.logger.WithFields(log.Fields{ + "request_uuid": parsedRequest.UUID, + "client_ip": parsedRequest.ClientIP, + }) + + AppsecReqCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.AppsecEngine}).Inc() + + w.InChan <- parsedRequest + + response := <-parsedRequest.ResponseChannel + if response.InBandInterrupt { + AppsecBlockCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.AppsecEngine}).Inc() + } + + appsecResponse := w.AppsecRuntime.GenerateResponse(response, logger) + + rw.WriteHeader(appsecResponse.HTTPStatus) + body, err := json.Marshal(BodyResponse{Action: appsecResponse.Action}) + if err != nil { + logger.Errorf("unable to marshal response: %s", err) + rw.WriteHeader(http.StatusInternalServerError) + } else { + rw.Write(body) + } + +} diff --git a/pkg/acquisition/modules/appsec/appsec_runner.go b/pkg/acquisition/modules/appsec/appsec_runner.go new file mode 100644 index 000000000..086060b46 --- /dev/null +++ b/pkg/acquisition/modules/appsec/appsec_runner.go @@ -0,0 +1,350 @@ +package appsecacquisition + +import ( + "fmt" + "os" + "slices" + "time" + + "github.com/crowdsecurity/coraza/v3" + corazatypes "github.com/crowdsecurity/coraza/v3/types" + "github.com/crowdsecurity/crowdsec/pkg/appsec" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" + "gopkg.in/tomb.v2" +) + +// that's the runtime structure of the Application security engine as seen from the acquis +type AppsecRunner struct { + outChan chan types.Event + inChan chan appsec.ParsedRequest + UUID string + AppsecRuntime *appsec.AppsecRuntimeConfig //this holds the actual appsec runtime config, rules, remediations, hooks etc. + AppsecInbandEngine coraza.WAF + AppsecOutbandEngine coraza.WAF + Labels map[string]string + logger *log.Entry +} + +func (r *AppsecRunner) Init(datadir string) error { + var err error + fs := os.DirFS(datadir) + + inBandRules := "" + outOfBandRules := "" + + for _, collection := range r.AppsecRuntime.InBandRules { + inBandRules += collection.String() + } + + for _, collection := range r.AppsecRuntime.OutOfBandRules { + outOfBandRules += collection.String() + } + inBandLogger := r.logger.Dup().WithField("band", "inband") + outBandLogger := r.logger.Dup().WithField("band", "outband") + + //setting up inband engine + inbandCfg := coraza.NewWAFConfig().WithDirectives(inBandRules).WithRootFS(fs).WithDebugLogger(appsec.NewCrzLogger(inBandLogger)) + if !r.AppsecRuntime.Config.InbandOptions.DisableBodyInspection { + inbandCfg = inbandCfg.WithRequestBodyAccess() + } else { + log.Warningf("Disabling body inspection, Inband rules will not be able to match on body's content.") + } + if r.AppsecRuntime.Config.InbandOptions.RequestBodyInMemoryLimit != nil { + inbandCfg = inbandCfg.WithRequestBodyInMemoryLimit(*r.AppsecRuntime.Config.InbandOptions.RequestBodyInMemoryLimit) + } + r.AppsecInbandEngine, err = coraza.NewWAF(inbandCfg) + if err != nil { + return fmt.Errorf("unable to initialize inband engine : %w", err) + } + + //setting up outband engine + outbandCfg := coraza.NewWAFConfig().WithDirectives(outOfBandRules).WithRootFS(fs).WithDebugLogger(appsec.NewCrzLogger(outBandLogger)) + if !r.AppsecRuntime.Config.OutOfBandOptions.DisableBodyInspection { + outbandCfg = outbandCfg.WithRequestBodyAccess() + } else { + log.Warningf("Disabling body inspection, Out of band rules will not be able to match on body's content.") + } + if r.AppsecRuntime.Config.OutOfBandOptions.RequestBodyInMemoryLimit != nil { + outbandCfg = outbandCfg.WithRequestBodyInMemoryLimit(*r.AppsecRuntime.Config.OutOfBandOptions.RequestBodyInMemoryLimit) + } + r.AppsecOutbandEngine, err = coraza.NewWAF(outbandCfg) + + if r.AppsecRuntime.DisabledInBandRulesTags != nil { + for _, tag := range r.AppsecRuntime.DisabledInBandRulesTags { + r.AppsecInbandEngine.GetRuleGroup().DeleteByTag(tag) + } + } + + if r.AppsecRuntime.DisabledOutOfBandRulesTags != nil { + for _, tag := range r.AppsecRuntime.DisabledOutOfBandRulesTags { + r.AppsecOutbandEngine.GetRuleGroup().DeleteByTag(tag) + } + } + + if r.AppsecRuntime.DisabledInBandRuleIds != nil { + for _, id := range r.AppsecRuntime.DisabledInBandRuleIds { + r.AppsecInbandEngine.GetRuleGroup().DeleteByID(id) + } + } + + if r.AppsecRuntime.DisabledOutOfBandRuleIds != nil { + for _, id := range r.AppsecRuntime.DisabledOutOfBandRuleIds { + r.AppsecOutbandEngine.GetRuleGroup().DeleteByID(id) + } + } + + r.logger.Tracef("Loaded inband rules: %+v", r.AppsecInbandEngine.GetRuleGroup().GetRules()) + r.logger.Tracef("Loaded outband rules: %+v", r.AppsecOutbandEngine.GetRuleGroup().GetRules()) + + if err != nil { + return fmt.Errorf("unable to initialize outband engine : %w", err) + } + + return nil +} + +func (r *AppsecRunner) processRequest(tx appsec.ExtendedTransaction, request *appsec.ParsedRequest) error { + var in *corazatypes.Interruption + var err error + request.Tx = tx + + if request.Tx.IsRuleEngineOff() { + r.logger.Debugf("rule engine is off, skipping") + return nil + } + + defer func() { + request.Tx.ProcessLogging() + //We don't close the transaction here, as it will reset coraza internal state and break variable tracking + }() + + //pre eval (expr) rules + err = r.AppsecRuntime.ProcessPreEvalRules(request) + if err != nil { + r.logger.Errorf("unable to process PreEval rules: %s", err) + //FIXME: should we abort here ? + } + + request.Tx.Tx.ProcessConnection(request.RemoteAddr, 0, "", 0) + + for k, v := range request.Args { + for _, vv := range v { + request.Tx.AddGetRequestArgument(k, vv) + } + } + + request.Tx.ProcessURI(request.URI, request.Method, request.Proto) + + for k, vr := range request.Headers { + for _, v := range vr { + request.Tx.AddRequestHeader(k, v) + } + } + + if request.ClientHost != "" { + request.Tx.AddRequestHeader("Host", request.ClientHost) + request.Tx.SetServerName(request.ClientHost) + } + + if request.TransferEncoding != nil { + request.Tx.AddRequestHeader("Transfer-Encoding", request.TransferEncoding[0]) + } + + in = request.Tx.ProcessRequestHeaders() + + if in != nil { + r.logger.Infof("inband rules matched for headers : %s", in.Action) + return nil + } + + if request.Body != nil && len(request.Body) > 0 { + in, _, err = request.Tx.WriteRequestBody(request.Body) + if err != nil { + r.logger.Errorf("unable to write request body : %s", err) + return err + } + if in != nil { + return nil + } + } + + in, err = request.Tx.ProcessRequestBody() + + if err != nil { + r.logger.Errorf("unable to process request body : %s", err) + return err + } + + if in != nil { + r.logger.Debugf("rules matched for body : %d", in.RuleID) + } + + err = r.AppsecRuntime.ProcessPostEvalRules(request) + if err != nil { + r.logger.Errorf("unable to process PostEval rules: %s", err) + } + + return nil +} + +func (r *AppsecRunner) ProcessInBandRules(request *appsec.ParsedRequest) error { + tx := appsec.NewExtendedTransaction(r.AppsecInbandEngine, request.UUID) + r.AppsecRuntime.InBandTx = tx + err := r.processRequest(tx, request) + return err +} + +func (r *AppsecRunner) ProcessOutOfBandRules(request *appsec.ParsedRequest) error { + r.logger.Debugf("Processing out of band rules") + tx := appsec.NewExtendedTransaction(r.AppsecOutbandEngine, request.UUID) + r.AppsecRuntime.OutOfBandTx = tx + err := r.processRequest(tx, request) + return err +} + +func (r *AppsecRunner) handleInBandInterrupt(request *appsec.ParsedRequest) { + //create the associated event for crowdsec itself + evt, err := EventFromRequest(request, r.Labels) + if err != nil { + //let's not interrupt the pipeline for this + r.logger.Errorf("unable to create event from request : %s", err) + } + err = r.AccumulateTxToEvent(&evt, request) + if err != nil { + r.logger.Errorf("unable to accumulate tx to event : %s", err) + } + if in := request.Tx.Interruption(); in != nil { + r.logger.Debugf("inband rules matched : %d", in.RuleID) + r.AppsecRuntime.Response.InBandInterrupt = true + r.AppsecRuntime.Response.HTTPResponseCode = r.AppsecRuntime.Config.BlockedHTTPCode + r.AppsecRuntime.Response.Action = r.AppsecRuntime.DefaultRemediation + + if _, ok := r.AppsecRuntime.RemediationById[in.RuleID]; ok { + r.AppsecRuntime.Response.Action = r.AppsecRuntime.RemediationById[in.RuleID] + } + + for tag, remediation := range r.AppsecRuntime.RemediationByTag { + if slices.Contains[[]string, string](in.Tags, tag) { + r.AppsecRuntime.Response.Action = remediation + } + } + + err = r.AppsecRuntime.ProcessOnMatchRules(request, evt) + if err != nil { + r.logger.Errorf("unable to process OnMatch rules: %s", err) + return + } + // Should the in band match trigger an event ? + if r.AppsecRuntime.Response.SendEvent { + r.outChan <- evt + } + + // Should the in band match trigger an overflow ? + if r.AppsecRuntime.Response.SendAlert { + appsecOvlfw, err := AppsecEventGeneration(evt) + if err != nil { + r.logger.Errorf("unable to generate appsec event : %s", err) + return + } + r.outChan <- *appsecOvlfw + } + } +} + +func (r *AppsecRunner) handleOutBandInterrupt(request *appsec.ParsedRequest) { + evt, err := EventFromRequest(request, r.Labels) + if err != nil { + //let's not interrupt the pipeline for this + r.logger.Errorf("unable to create event from request : %s", err) + } + err = r.AccumulateTxToEvent(&evt, request) + if err != nil { + r.logger.Errorf("unable to accumulate tx to event : %s", err) + } + if in := request.Tx.Interruption(); in != nil { + r.logger.Debugf("inband rules matched : %d", in.RuleID) + r.AppsecRuntime.Response.OutOfBandInterrupt = true + + err = r.AppsecRuntime.ProcessOnMatchRules(request, evt) + if err != nil { + r.logger.Errorf("unable to process OnMatch rules: %s", err) + return + } + // Should the match trigger an event ? + if r.AppsecRuntime.Response.SendEvent { + r.outChan <- evt + } + + // Should the match trigger an overflow ? + if r.AppsecRuntime.Response.SendAlert { + appsecOvlfw, err := AppsecEventGeneration(evt) + if err != nil { + r.logger.Errorf("unable to generate appsec event : %s", err) + return + } + r.outChan <- *appsecOvlfw + } + } +} + +func (r *AppsecRunner) handleRequest(request *appsec.ParsedRequest) { + r.AppsecRuntime.Logger = r.AppsecRuntime.Logger.WithField("request_uuid", request.UUID) + logger := r.logger.WithField("request_uuid", request.UUID) + logger.Debug("Request received in runner") + r.AppsecRuntime.ClearResponse() + + request.IsInBand = true + request.IsOutBand = false + + //to measure the time spent in the Application Security Engine + startParsing := time.Now() + + //inband appsec rules + err := r.ProcessInBandRules(request) + if err != nil { + logger.Errorf("unable to process InBand rules: %s", err) + return + } + + if request.Tx.IsInterrupted() { + r.handleInBandInterrupt(request) + } + + elapsed := time.Since(startParsing) + AppsecInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) + + // send back the result to the HTTP handler for the InBand part + request.ResponseChannel <- r.AppsecRuntime.Response + + //Now let's process the out of band rules + + request.IsInBand = false + request.IsOutBand = true + r.AppsecRuntime.Response.SendAlert = false + r.AppsecRuntime.Response.SendEvent = true + + err = r.ProcessOutOfBandRules(request) + if err != nil { + logger.Errorf("unable to process OutOfBand rules: %s", err) + return + } + + if request.Tx.IsInterrupted() { + r.handleOutBandInterrupt(request) + } +} + +func (r *AppsecRunner) Run(t *tomb.Tomb) error { + r.logger.Infof("Appsec Runner ready to process event") + for { + select { + case <-t.Dying(): + r.logger.Infof("Appsec Runner is dying") + return nil + case request := <-r.inChan: + r.handleRequest(&request) + } + } +} diff --git a/pkg/acquisition/modules/appsec/metrics.go b/pkg/acquisition/modules/appsec/metrics.go new file mode 100644 index 000000000..9aa3c8bde --- /dev/null +++ b/pkg/acquisition/modules/appsec/metrics.go @@ -0,0 +1,54 @@ +package appsecacquisition + +import "github.com/prometheus/client_golang/prometheus" + +var AppsecGlobalParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the Application Security Engine.", + Name: "cs_appsec_parsing_time_seconds", + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"source"}, +) + +var AppsecInbandParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the inband Application Security Engine.", + Name: "cs_appsec_inband_parsing_time_seconds", + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"source"}, +) + +var AppsecOutbandParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the Application Security Engine.", + Name: "cs_appsec_outband_parsing_time_seconds", + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"source"}, +) + +var AppsecReqCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_appsec_reqs_total", + Help: "Total events processed by the Application Security Engine.", + }, + []string{"source", "appsec_engine"}, +) + +var AppsecBlockCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_appsec_block_total", + Help: "Total events blocked by the Application Security Engine.", + }, + []string{"source", "appsec_engine"}, +) + +var AppsecRuleHits = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_appsec_rule_hits", + Help: "Count of triggered rule, by rule_name, type (inband/outofband), appsec_engine and source", + }, + []string{"rule_name", "type", "appsec_engine", "source"}, +) diff --git a/pkg/acquisition/modules/appsec/rx_operator.go b/pkg/acquisition/modules/appsec/rx_operator.go new file mode 100644 index 000000000..43aaf9e94 --- /dev/null +++ b/pkg/acquisition/modules/appsec/rx_operator.go @@ -0,0 +1,94 @@ +package appsecacquisition + +import ( + "fmt" + "strconv" + "unicode/utf8" + + "github.com/crowdsecurity/coraza/v3/experimental/plugins" + "github.com/crowdsecurity/coraza/v3/experimental/plugins/plugintypes" + "github.com/wasilibs/go-re2" + "github.com/wasilibs/go-re2/experimental" +) + +type rx struct { + re *re2.Regexp +} + +var _ plugintypes.Operator = (*rx)(nil) + +func newRX(options plugintypes.OperatorOptions) (plugintypes.Operator, error) { + // (?sm) enables multiline mode which makes 942522-7 work, see + // - https://stackoverflow.com/a/27680233 + // - https://groups.google.com/g/golang-nuts/c/jiVdamGFU9E + data := fmt.Sprintf("(?sm)%s", options.Arguments) + + var re *re2.Regexp + var err error + + if matchesArbitraryBytes(data) { + re, err = experimental.CompileLatin1(data) + } else { + re, err = re2.Compile(data) + } + if err != nil { + return nil, err + } + return &rx{re: re}, nil +} + +func (o *rx) Evaluate(tx plugintypes.TransactionState, value string) bool { + if tx.Capturing() { + match := o.re.FindStringSubmatch(value) + if len(match) == 0 { + return false + } + for i, c := range match { + if i == 9 { + return true + } + tx.CaptureField(i, c) + } + return true + } else { + return o.re.MatchString(value) + } +} + +// RegisterRX registers the rx operator using a WASI implementation instead of Go. +func RegisterRX() { + plugins.RegisterOperator("rx", newRX) +} + +// matchesArbitraryBytes checks for control sequences for byte matches in the expression. +// If the sequences are not valid utf8, it returns true. +func matchesArbitraryBytes(expr string) bool { + decoded := make([]byte, 0, len(expr)) + for i := 0; i < len(expr); i++ { + c := expr[i] + if c != '\\' { + decoded = append(decoded, c) + continue + } + if i+3 >= len(expr) { + decoded = append(decoded, expr[i:]...) + break + } + if expr[i+1] != 'x' { + decoded = append(decoded, expr[i]) + continue + } + + v, mb, _, err := strconv.UnquoteChar(expr[i:], 0) + if err != nil || mb { + // Wasn't a byte escape sequence, shouldn't happen in practice. + decoded = append(decoded, expr[i]) + continue + } + + decoded = append(decoded, byte(v)) + i += 3 + } + + return !utf8.Valid(decoded) +} diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go new file mode 100644 index 000000000..e43313a19 --- /dev/null +++ b/pkg/acquisition/modules/appsec/utils.go @@ -0,0 +1,280 @@ +package appsecacquisition + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/crowdsecurity/coraza/v3/collection" + "github.com/crowdsecurity/coraza/v3/types/variables" + "github.com/crowdsecurity/crowdsec/pkg/appsec" + "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/crowdsecurity/go-cs-lib/ptr" + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" +) + +func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) { + //if the request didnd't trigger inband rules, we don't want to generate an event to LAPI/CAPI + if !inEvt.Appsec.HasInBandMatches { + return nil, nil + } + evt := types.Event{} + evt.Type = types.APPSEC + evt.Process = true + source := models.Source{ + Value: ptr.Of(inEvt.Parsed["source_ip"]), + IP: inEvt.Parsed["source_ip"], + Scope: ptr.Of(types.Ip), + } + + evt.Overflow.Sources = make(map[string]models.Source) + evt.Overflow.Sources["ip"] = source + + alert := models.Alert{} + alert.Capacity = ptr.Of(int32(1)) + alert.Events = make([]*models.Event, 0) + alert.Meta = make(models.Meta, 0) + for _, key := range []string{"target_uri", "method"} { + + valueByte, err := json.Marshal([]string{inEvt.Parsed[key]}) + if err != nil { + log.Debugf("unable to serialize key %s", key) + continue + } + + meta := models.MetaItems0{ + Key: key, + Value: string(valueByte), + } + alert.Meta = append(alert.Meta, &meta) + } + matchedZones := inEvt.Appsec.GetMatchedZones() + if matchedZones != nil { + valueByte, err := json.Marshal(matchedZones) + if err != nil { + log.Debugf("unable to serialize key matched_zones") + } else { + meta := models.MetaItems0{ + Key: "matched_zones", + Value: string(valueByte), + } + alert.Meta = append(alert.Meta, &meta) + } + } + for _, key := range evt.Appsec.MatchedRules.GetMatchedZones() { + valueByte, err := json.Marshal([]string{key}) + if err != nil { + log.Debugf("unable to serialize key %s", key) + continue + } + meta := models.MetaItems0{ + Key: "matched_zones", + Value: string(valueByte), + } + alert.Meta = append(alert.Meta, &meta) + } + alert.EventsCount = ptr.Of(int32(1)) + alert.Leakspeed = ptr.Of("") + alert.Scenario = ptr.Of(inEvt.Appsec.MatchedRules.GetName()) + alert.ScenarioHash = ptr.Of(inEvt.Appsec.MatchedRules.GetHash()) + alert.ScenarioVersion = ptr.Of(inEvt.Appsec.MatchedRules.GetVersion()) + alert.Simulated = ptr.Of(false) + alert.Source = &source + msg := fmt.Sprintf("AppSec block: %s from %s (%s)", inEvt.Appsec.MatchedRules.GetName(), + alert.Source.IP, inEvt.Parsed["remediation_cmpt_ip"]) + alert.Message = &msg + alert.StartAt = ptr.Of(time.Now().UTC().Format(time.RFC3339)) + alert.StopAt = ptr.Of(time.Now().UTC().Format(time.RFC3339)) + evt.Overflow.APIAlerts = []models.Alert{alert} + evt.Overflow.Alert = &alert + return &evt, nil +} + +func EventFromRequest(r *appsec.ParsedRequest, labels map[string]string) (types.Event, error) { + evt := types.Event{} + //we might want to change this based on in-band vs out-of-band ? + evt.Type = types.LOG + evt.ExpectMode = types.LIVE + //def needs fixing + evt.Stage = "s00-raw" + evt.Parsed = map[string]string{ + "source_ip": r.ClientIP, + "target_host": r.Host, + "target_uri": r.URI, + "method": r.Method, + "req_uuid": r.Tx.ID(), + "source": "crowdsec-appsec", + "remediation_cmpt_ip": r.RemoteAddrNormalized, + //TBD: + //http_status + //user_agent + + } + evt.Line = types.Line{ + Time: time.Now(), + //should we add some info like listen addr/port/path ? + Labels: labels, + Process: true, + Module: "appsec", + Src: "appsec", + Raw: "dummy-appsec-data", //we discard empty Line.Raw items :) + } + evt.Appsec = types.AppsecEvent{} + + return evt, nil +} + +func LogAppsecEvent(evt *types.Event, logger *log.Entry) { + req := evt.Parsed["target_uri"] + if len(req) > 12 { + req = req[:10] + ".." + } + + if evt.Meta["appsec_interrupted"] == "true" { + logger.WithFields(log.Fields{ + "module": "appsec", + "source": evt.Parsed["source_ip"], + "target_uri": req, + }).Infof("%s blocked on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Appsec.MatchedRules), evt.Appsec.GetRuleIDs()) + } else if evt.Parsed["outofband_interrupted"] == "true" { + logger.WithFields(log.Fields{ + "module": "appsec", + "source": evt.Parsed["source_ip"], + "target_uri": req, + }).Infof("%s out-of-band blocking rules on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Appsec.MatchedRules), evt.Appsec.GetRuleIDs()) + } else { + logger.WithFields(log.Fields{ + "module": "appsec", + "source": evt.Parsed["source_ip"], + "target_uri": req, + }).Debugf("%s triggered non-blocking rules on %s (%d rules) [%v]", evt.Parsed["source_ip"], req, len(evt.Appsec.MatchedRules), evt.Appsec.GetRuleIDs()) + } + +} + +func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedRequest) error { + + if evt == nil { + //an error was already emitted, let's not spam the logs + return nil + } + + if !req.Tx.IsInterrupted() { + //if the phase didn't generate an interruption, we don't have anything to add to the event + return nil + } + //if one interruption was generated, event is good for processing :) + evt.Process = true + + if evt.Meta == nil { + evt.Meta = map[string]string{} + } + if evt.Parsed == nil { + evt.Parsed = map[string]string{} + } + if req.IsInBand { + evt.Meta["appsec_interrupted"] = "true" + evt.Meta["appsec_action"] = req.Tx.Interruption().Action + evt.Parsed["inband_interrupted"] = "true" + evt.Parsed["inband_action"] = req.Tx.Interruption().Action + } else { + evt.Parsed["outofband_interrupted"] = "true" + evt.Parsed["outofband_action"] = req.Tx.Interruption().Action + } + + if evt.Appsec.Vars == nil { + evt.Appsec.Vars = map[string]string{} + } + + req.Tx.Variables().All(func(v variables.RuleVariable, col collection.Collection) bool { + for _, variable := range col.FindAll() { + key := "" + if variable.Key() == "" { + key = variable.Variable().Name() + } else { + key = variable.Variable().Name() + "." + variable.Key() + } + if variable.Value() == "" { + continue + } + for _, collectionToKeep := range r.AppsecRuntime.CompiledVariablesTracking { + match := collectionToKeep.MatchString(key) + if match { + evt.Appsec.Vars[key] = variable.Value() + r.logger.Debugf("%s.%s = %s", variable.Variable().Name(), variable.Key(), variable.Value()) + } else { + r.logger.Debugf("%s.%s != %s (%s) (not kept)", variable.Variable().Name(), variable.Key(), collectionToKeep, variable.Value()) + } + } + } + return true + }) + + for _, rule := range req.Tx.MatchedRules() { + if rule.Message() == "" { + r.logger.Tracef("discarding rule %d", rule.Rule().ID()) + continue + } + kind := "outofband" + if req.IsInBand { + kind = "inband" + evt.Appsec.HasInBandMatches = true + } else { + evt.Appsec.HasOutBandMatches = true + } + + name := "" + version := "" + hash := "" + ruleNameProm := fmt.Sprintf("%d", rule.Rule().ID()) + + if details, ok := appsec.AppsecRulesDetails[rule.Rule().ID()]; ok { + //Only set them for custom rules, not for rules written in seclang + name = details.Name + version = details.Version + hash = details.Hash + ruleNameProm = details.Name + r.logger.Debugf("custom rule for event, setting name: %s, version: %s, hash: %s", name, version, hash) + } else { + name = fmt.Sprintf("native_rule:%d", rule.Rule().ID()) + } + + AppsecRuleHits.With(prometheus.Labels{"rule_name": ruleNameProm, "type": kind, "source": req.RemoteAddrNormalized, "appsec_engine": req.AppsecEngine}).Inc() + + matchedZones := make([]string, 0) + for _, matchData := range rule.MatchedDatas() { + zone := matchData.Variable().Name() + varName := matchData.Key() + if varName != "" { + zone += "." + varName + } + matchedZones = append(matchedZones, zone) + } + + corazaRule := map[string]interface{}{ + "id": rule.Rule().ID(), + "uri": evt.Parsed["uri"], + "rule_type": kind, + "method": evt.Parsed["method"], + "disruptive": rule.Disruptive(), + "tags": rule.Rule().Tags(), + "file": rule.Rule().File(), + "file_line": rule.Rule().Line(), + "revision": rule.Rule().Revision(), + "secmark": rule.Rule().SecMark(), + "accuracy": rule.Rule().Accuracy(), + "msg": rule.Message(), + "severity": rule.Rule().Severity().String(), + "name": name, + "hash": hash, + "version": version, + "matched_zones": matchedZones, + } + evt.Appsec.MatchedRules = append(evt.Appsec.MatchedRules, corazaRule) + } + + return nil + +} diff --git a/pkg/appsec/appsec.go b/pkg/appsec/appsec.go new file mode 100644 index 000000000..4550990b2 --- /dev/null +++ b/pkg/appsec/appsec.go @@ -0,0 +1,579 @@ +package appsec + +import ( + "fmt" + "os" + "regexp" + + "github.com/antonmedv/expr" + "github.com/antonmedv/expr/vm" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +type Hook struct { + Filter string `yaml:"filter"` + FilterExpr *vm.Program `yaml:"-"` + + OnSuccess string `yaml:"on_success"` + Apply []string `yaml:"apply"` + ApplyExpr []*vm.Program `yaml:"-"` +} + +const ( + hookOnLoad = iota + hookPreEval + hookPostEval + hookOnMatch +) + +// @tko : todo - debug mode +func (h *Hook) Build(hookStage int) error { + + ctx := map[string]interface{}{} + switch hookStage { + case hookOnLoad: + ctx = GetOnLoadEnv(&AppsecRuntimeConfig{}) + case hookPreEval: + ctx = GetPreEvalEnv(&AppsecRuntimeConfig{}, &ParsedRequest{}) + case hookPostEval: + ctx = GetPostEvalEnv(&AppsecRuntimeConfig{}, &ParsedRequest{}) + case hookOnMatch: + ctx = GetOnMatchEnv(&AppsecRuntimeConfig{}, &ParsedRequest{}, types.Event{}) + } + opts := exprhelpers.GetExprOptions(ctx) + if h.Filter != "" { + program, err := expr.Compile(h.Filter, opts...) //FIXME: opts + if err != nil { + return fmt.Errorf("unable to compile filter %s : %w", h.Filter, err) + } + h.FilterExpr = program + } + for _, apply := range h.Apply { + program, err := expr.Compile(apply, opts...) + if err != nil { + return fmt.Errorf("unable to compile apply %s : %w", apply, err) + } + h.ApplyExpr = append(h.ApplyExpr, program) + } + return nil +} + +type AppsecTempResponse struct { + InBandInterrupt bool + OutOfBandInterrupt bool + Action string //allow, deny, captcha, log + HTTPResponseCode int + SendEvent bool //do we send an internal event on rule match + SendAlert bool //do we send an alert on rule match +} + +type AppsecSubEngineOpts struct { + DisableBodyInspection bool `yaml:"disable_body_inspection"` + RequestBodyInMemoryLimit *int `yaml:"request_body_in_memory_limit"` +} + +// runtime version of AppsecConfig +type AppsecRuntimeConfig struct { + Name string + OutOfBandRules []AppsecCollection + + InBandRules []AppsecCollection + + DefaultRemediation string + RemediationByTag map[string]string //Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME + RemediationById map[int]string + CompiledOnLoad []Hook + CompiledPreEval []Hook + CompiledPostEval []Hook + CompiledOnMatch []Hook + CompiledVariablesTracking []*regexp.Regexp + Config *AppsecConfig + //CorazaLogger debuglog.Logger + + //those are ephemeral, created/destroyed with every req + OutOfBandTx ExtendedTransaction //is it a good idea ? + InBandTx ExtendedTransaction //is it a good idea ? + Response AppsecTempResponse + //should we store matched rules here ? + + Logger *log.Entry + + //Set by on_load to ignore some rules on loading + DisabledInBandRuleIds []int + DisabledInBandRulesTags []string //Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME + + DisabledOutOfBandRuleIds []int + DisabledOutOfBandRulesTags []string //Also used for ByName, as the name (for modsec rules) is a tag crowdsec-NAME +} + +type AppsecConfig struct { + Name string `yaml:"name"` + OutOfBandRules []string `yaml:"outofband_rules"` + InBandRules []string `yaml:"inband_rules"` + DefaultRemediation string `yaml:"default_remediation"` + DefaultPassAction string `yaml:"default_pass_action"` + BlockedHTTPCode int `yaml:"blocked_http_code"` + PassedHTTPCode int `yaml:"passed_http_code"` + OnLoad []Hook `yaml:"on_load"` + PreEval []Hook `yaml:"pre_eval"` + PostEval []Hook `yaml:"post_eval"` + OnMatch []Hook `yaml:"on_match"` + VariablesTracking []string `yaml:"variables_tracking"` + InbandOptions AppsecSubEngineOpts `yaml:"inband_options"` + OutOfBandOptions AppsecSubEngineOpts `yaml:"outofband_options"` + + LogLevel *log.Level `yaml:"log_level"` + Logger *log.Entry `yaml:"-"` +} + +func (w *AppsecRuntimeConfig) ClearResponse() { + log.Debugf("#-> %p", w) + w.Response = AppsecTempResponse{} + log.Debugf("-> %p", w.Config) + w.Response.Action = w.Config.DefaultPassAction + w.Response.HTTPResponseCode = w.Config.PassedHTTPCode + w.Response.SendEvent = true + w.Response.SendAlert = true +} + +func (wc *AppsecConfig) LoadByPath(file string) error { + + wc.Logger.Debugf("loading config %s", file) + + yamlFile, err := os.ReadFile(file) + if err != nil { + return fmt.Errorf("unable to read file %s : %s", file, err) + } + err = yaml.UnmarshalStrict(yamlFile, wc) + if err != nil { + return fmt.Errorf("unable to parse yaml file %s : %s", file, err) + } + + if wc.Name == "" { + return fmt.Errorf("name cannot be empty") + } + if wc.LogLevel == nil { + lvl := wc.Logger.Logger.GetLevel() + wc.LogLevel = &lvl + } + wc.Logger = wc.Logger.Dup().WithField("name", wc.Name) + wc.Logger.Logger.SetLevel(*wc.LogLevel) + if wc.DefaultRemediation == "" { + return fmt.Errorf("default_remediation cannot be empty") + } + switch wc.DefaultRemediation { + case "ban", "captcha", "log": + //those are the officially supported remediation(s) + default: + wc.Logger.Warningf("default '%s' remediation of %s is none of [ban,captcha,log] ensure bouncer compatbility!", wc.DefaultRemediation, file) + } + if wc.BlockedHTTPCode == 0 { + wc.BlockedHTTPCode = 403 + } + if wc.PassedHTTPCode == 0 { + wc.PassedHTTPCode = 200 + } + if wc.DefaultPassAction == "" { + wc.DefaultPassAction = "allow" + } + return nil +} + +func (wc *AppsecConfig) Load(configName string) error { + appsecConfigs := hub.GetItemMap(cwhub.APPSEC_CONFIGS) + + for _, hubAppsecConfigItem := range appsecConfigs { + if !hubAppsecConfigItem.State.Installed { + continue + } + if hubAppsecConfigItem.Name != configName { + continue + } + wc.Logger.Infof("loading %s", hubAppsecConfigItem.State.LocalPath) + err := wc.LoadByPath(hubAppsecConfigItem.State.LocalPath) + if err != nil { + return fmt.Errorf("unable to load appsec-config %s : %s", hubAppsecConfigItem.State.LocalPath, err) + } + return nil + } + + return fmt.Errorf("no appsec-config found for %s", configName) +} + +func (wc *AppsecConfig) GetDataDir() string { + return hub.GetDataDir() +} + +func (wc *AppsecConfig) Build() (*AppsecRuntimeConfig, error) { + ret := &AppsecRuntimeConfig{Logger: wc.Logger.WithField("component", "appsec_runtime_config")} + ret.Name = wc.Name + ret.Config = wc + ret.DefaultRemediation = wc.DefaultRemediation + + wc.Logger.Tracef("Loading config %+v", wc) + //load rules + for _, rule := range wc.OutOfBandRules { + wc.Logger.Infof("loading outofband rule %s", rule) + collections, err := LoadCollection(rule, wc.Logger.WithField("component", "appsec_collection_loader")) + if err != nil { + return nil, fmt.Errorf("unable to load outofband rule %s : %s", rule, err) + } + ret.OutOfBandRules = append(ret.OutOfBandRules, collections...) + } + + wc.Logger.Infof("Loaded %d outofband rules", len(ret.OutOfBandRules)) + for _, rule := range wc.InBandRules { + wc.Logger.Infof("loading inband rule %s", rule) + collections, err := LoadCollection(rule, wc.Logger.WithField("component", "appsec_collection_loader")) + if err != nil { + return nil, fmt.Errorf("unable to load inband rule %s : %s", rule, err) + } + ret.InBandRules = append(ret.InBandRules, collections...) + } + + wc.Logger.Infof("Loaded %d inband rules", len(ret.InBandRules)) + + //load hooks + for _, hook := range wc.OnLoad { + err := hook.Build(hookOnLoad) + if err != nil { + return nil, fmt.Errorf("unable to build on_load hook : %s", err) + } + ret.CompiledOnLoad = append(ret.CompiledOnLoad, hook) + } + + for _, hook := range wc.PreEval { + err := hook.Build(hookPreEval) + if err != nil { + return nil, fmt.Errorf("unable to build pre_eval hook : %s", err) + } + ret.CompiledPreEval = append(ret.CompiledPreEval, hook) + } + + for _, hook := range wc.PostEval { + err := hook.Build(hookPostEval) + if err != nil { + return nil, fmt.Errorf("unable to build post_eval hook : %s", err) + } + ret.CompiledPostEval = append(ret.CompiledPostEval, hook) + } + + for _, hook := range wc.OnMatch { + err := hook.Build(hookOnMatch) + if err != nil { + return nil, fmt.Errorf("unable to build on_match hook : %s", err) + } + ret.CompiledOnMatch = append(ret.CompiledOnMatch, hook) + } + + //variable tracking + for _, variable := range wc.VariablesTracking { + compiledVariableRule, err := regexp.Compile(variable) + if err != nil { + return nil, fmt.Errorf("cannot compile variable regexp %s: %w", variable, err) + } + ret.CompiledVariablesTracking = append(ret.CompiledVariablesTracking, compiledVariableRule) + } + return ret, nil +} + +func (w *AppsecRuntimeConfig) ProcessOnLoadRules() error { + for _, rule := range w.CompiledOnLoad { + if rule.FilterExpr != nil { + output, err := exprhelpers.Run(rule.FilterExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel) + if err != nil { + return fmt.Errorf("unable to run appsec on_load filter %s : %w", rule.Filter, err) + } + switch t := output.(type) { + case bool: + if !t { + log.Debugf("filter didnt match") + continue + } + default: + log.Errorf("Filter must return a boolean, can't filter") + continue + } + } + for _, applyExpr := range rule.ApplyExpr { + _, err := exprhelpers.Run(applyExpr, GetOnLoadEnv(w), w.Logger, w.Logger.Level >= log.DebugLevel) + if err != nil { + log.Errorf("unable to apply appsec on_load expr: %s", err) + continue + } + } + } + return nil +} + +func (w *AppsecRuntimeConfig) ProcessOnMatchRules(request *ParsedRequest, evt types.Event) error { + + for _, rule := range w.CompiledOnMatch { + if rule.FilterExpr != nil { + output, err := exprhelpers.Run(rule.FilterExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel) + if err != nil { + return fmt.Errorf("unable to run appsec on_match filter %s : %w", rule.Filter, err) + } + switch t := output.(type) { + case bool: + if !t { + log.Debugf("filter didnt match") + continue + } + default: + log.Errorf("Filter must return a boolean, can't filter") + continue + } + } + for _, applyExpr := range rule.ApplyExpr { + _, err := exprhelpers.Run(applyExpr, GetOnMatchEnv(w, request, evt), w.Logger, w.Logger.Level >= log.DebugLevel) + if err != nil { + log.Errorf("unable to apply appsec on_match expr: %s", err) + continue + } + } + } + return nil +} + +func (w *AppsecRuntimeConfig) ProcessPreEvalRules(request *ParsedRequest) error { + for _, rule := range w.CompiledPreEval { + if rule.FilterExpr != nil { + output, err := exprhelpers.Run(rule.FilterExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) + if err != nil { + return fmt.Errorf("unable to run appsec pre_eval filter %s : %w", rule.Filter, err) + } + switch t := output.(type) { + case bool: + if !t { + log.Debugf("filter didnt match") + continue + } + default: + log.Errorf("Filter must return a boolean, can't filter") + continue + } + } + // here means there is no filter or the filter matched + for _, applyExpr := range rule.ApplyExpr { + _, err := exprhelpers.Run(applyExpr, GetPreEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) + if err != nil { + log.Errorf("unable to apply appsec pre_eval expr: %s", err) + continue + } + } + } + + return nil +} + +func (w *AppsecRuntimeConfig) ProcessPostEvalRules(request *ParsedRequest) error { + for _, rule := range w.CompiledPostEval { + if rule.FilterExpr != nil { + output, err := exprhelpers.Run(rule.FilterExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) + if err != nil { + return fmt.Errorf("unable to run appsec post_eval filter %s : %w", rule.Filter, err) + } + switch t := output.(type) { + case bool: + if !t { + log.Debugf("filter didnt match") + continue + } + default: + log.Errorf("Filter must return a boolean, can't filter") + continue + } + } + // here means there is no filter or the filter matched + for _, applyExpr := range rule.ApplyExpr { + _, err := exprhelpers.Run(applyExpr, GetPostEvalEnv(w, request), w.Logger, w.Logger.Level >= log.DebugLevel) + if err != nil { + log.Errorf("unable to apply appsec post_eval expr: %s", err) + continue + } + } + } + + return nil +} + +func (w *AppsecRuntimeConfig) RemoveInbandRuleByID(id int) error { + w.Logger.Debugf("removing inband rule %d", id) + return w.InBandTx.RemoveRuleByIDWithError(id) +} + +func (w *AppsecRuntimeConfig) RemoveOutbandRuleByID(id int) error { + w.Logger.Debugf("removing outband rule %d", id) + return w.OutOfBandTx.RemoveRuleByIDWithError(id) +} + +func (w *AppsecRuntimeConfig) RemoveInbandRuleByTag(tag string) error { + w.Logger.Debugf("removing inband rule with tag %s", tag) + return w.InBandTx.RemoveRuleByTagWithError(tag) +} + +func (w *AppsecRuntimeConfig) RemoveOutbandRuleByTag(tag string) error { + w.Logger.Debugf("removing outband rule with tag %s", tag) + return w.OutOfBandTx.RemoveRuleByTagWithError(tag) +} + +func (w *AppsecRuntimeConfig) RemoveInbandRuleByName(name string) error { + tag := fmt.Sprintf("crowdsec-%s", name) + w.Logger.Debugf("removing inband rule %s", tag) + return w.InBandTx.RemoveRuleByTagWithError(tag) +} + +func (w *AppsecRuntimeConfig) RemoveOutbandRuleByName(name string) error { + tag := fmt.Sprintf("crowdsec-%s", name) + w.Logger.Debugf("removing outband rule %s", tag) + return w.OutOfBandTx.RemoveRuleByTagWithError(tag) +} + +func (w *AppsecRuntimeConfig) CancelEvent() error { + w.Logger.Debugf("canceling event") + w.Response.SendEvent = false + return nil +} + +// Disable a rule at load time, meaning it will not run for any request +func (w *AppsecRuntimeConfig) DisableInBandRuleByID(id int) error { + w.DisabledInBandRuleIds = append(w.DisabledInBandRuleIds, id) + return nil +} + +// Disable a rule at load time, meaning it will not run for any request +func (w *AppsecRuntimeConfig) DisableInBandRuleByName(name string) error { + tagValue := fmt.Sprintf("crowdsec-%s", name) + w.DisabledInBandRulesTags = append(w.DisabledInBandRulesTags, tagValue) + return nil +} + +// Disable a rule at load time, meaning it will not run for any request +func (w *AppsecRuntimeConfig) DisableInBandRuleByTag(tag string) error { + w.DisabledInBandRulesTags = append(w.DisabledInBandRulesTags, tag) + return nil +} + +// Disable a rule at load time, meaning it will not run for any request +func (w *AppsecRuntimeConfig) DisableOutBandRuleByID(id int) error { + w.DisabledOutOfBandRuleIds = append(w.DisabledOutOfBandRuleIds, id) + return nil +} + +// Disable a rule at load time, meaning it will not run for any request +func (w *AppsecRuntimeConfig) DisableOutBandRuleByName(name string) error { + tagValue := fmt.Sprintf("crowdsec-%s", name) + w.DisabledOutOfBandRulesTags = append(w.DisabledOutOfBandRulesTags, tagValue) + return nil +} + +// Disable a rule at load time, meaning it will not run for any request +func (w *AppsecRuntimeConfig) DisableOutBandRuleByTag(tag string) error { + w.DisabledOutOfBandRulesTags = append(w.DisabledOutOfBandRulesTags, tag) + return nil +} + +func (w *AppsecRuntimeConfig) SendEvent() error { + w.Logger.Debugf("sending event") + w.Response.SendEvent = true + return nil +} + +func (w *AppsecRuntimeConfig) SendAlert() error { + w.Logger.Debugf("sending alert") + w.Response.SendAlert = true + return nil +} + +func (w *AppsecRuntimeConfig) CancelAlert() error { + w.Logger.Debugf("canceling alert") + w.Response.SendAlert = false + return nil +} + +func (w *AppsecRuntimeConfig) SetActionByTag(tag string, action string) error { + if w.RemediationByTag == nil { + w.RemediationByTag = make(map[string]string) + } + w.Logger.Debugf("setting action of %s to %s", tag, action) + w.RemediationByTag[tag] = action + return nil +} + +func (w *AppsecRuntimeConfig) SetActionByID(id int, action string) error { + if w.RemediationById == nil { + w.RemediationById = make(map[int]string) + } + w.Logger.Debugf("setting action of %d to %s", id, action) + w.RemediationById[id] = action + return nil +} + +func (w *AppsecRuntimeConfig) SetActionByName(name string, action string) error { + if w.RemediationByTag == nil { + w.RemediationByTag = make(map[string]string) + } + tag := fmt.Sprintf("crowdsec-%s", name) + w.Logger.Debugf("setting action of %s to %s", tag, action) + w.RemediationByTag[tag] = action + return nil +} + +func (w *AppsecRuntimeConfig) SetAction(action string) error { + //log.Infof("setting to %s", action) + w.Logger.Debugf("setting action to %s", action) + switch action { + case "allow": + w.Response.Action = action + w.Response.HTTPResponseCode = w.Config.PassedHTTPCode + //@tko how should we handle this ? it seems bouncer only understand bans, but it might be misleading ? + case "deny", "ban", "block": + w.Response.Action = "ban" + case "log": + w.Response.Action = action + w.Response.HTTPResponseCode = w.Config.PassedHTTPCode + case "captcha": + w.Response.Action = action + default: + return fmt.Errorf("unknown action %s", action) + } + return nil +} + +func (w *AppsecRuntimeConfig) SetHTTPCode(code int) error { + w.Logger.Debugf("setting http code to %d", code) + w.Response.HTTPResponseCode = code + return nil +} + +type BodyResponse struct { + Action string `json:"action"` + HTTPStatus int `json:"http_status"` +} + +func (w *AppsecRuntimeConfig) GenerateResponse(response AppsecTempResponse, logger *log.Entry) BodyResponse { + resp := BodyResponse{} + //if there is no interrupt, we should allow with default code + if !response.InBandInterrupt { + resp.Action = w.Config.DefaultPassAction + resp.HTTPStatus = w.Config.PassedHTTPCode + return resp + } + resp.Action = response.Action + if resp.Action == "" { + resp.Action = w.Config.DefaultRemediation + } + logger.Debugf("action is %s", resp.Action) + + resp.HTTPStatus = response.HTTPResponseCode + if resp.HTTPStatus == 0 { + resp.HTTPStatus = w.Config.BlockedHTTPCode + } + logger.Debugf("http status is %d", resp.HTTPStatus) + return resp +} diff --git a/pkg/appsec/appsec_rule/appsec_rule.go b/pkg/appsec/appsec_rule/appsec_rule.go new file mode 100644 index 000000000..c011e58fb --- /dev/null +++ b/pkg/appsec/appsec_rule/appsec_rule.go @@ -0,0 +1,67 @@ +package appsec_rule + +import ( + "fmt" +) + +/* +rules: + - name: "test" + and: + - zones: + - BODY_ARGS + variables: + - foo + - bar + transform: + - lowercase|uppercase|b64decode|... + match: + type: regex + value: "[^a-zA-Z]" + - zones: + - ARGS + variables: + - bla + +*/ + +type match struct { + Type string `yaml:"type"` + Value string `yaml:"value"` +} + +type CustomRule struct { + Name string `yaml:"name"` + + Zones []string `yaml:"zones"` + Variables []string `yaml:"variables"` + + Match match `yaml:"match"` + Transform []string `yaml:"transform"` //t:lowercase, t:uppercase, etc + And []CustomRule `yaml:"and,omitempty"` + Or []CustomRule `yaml:"or,omitempty"` + BodyType string `yaml:"body_type,omitempty"` +} + +func (v *CustomRule) Convert(ruleType string, appsecRuleName string) (string, []uint32, error) { + + if v.Zones == nil && v.And == nil && v.Or == nil { + return "", nil, fmt.Errorf("no zones defined") + } + + if v.Match.Type == "" && v.And == nil && v.Or == nil { + return "", nil, fmt.Errorf("no match type defined") + } + + if v.Match.Value == "" && v.And == nil && v.Or == nil { + return "", nil, fmt.Errorf("no match value defined") + } + + switch ruleType { + case ModsecurityRuleType: + r := ModsecurityRule{} + return r.Build(v, appsecRuleName) + default: + return "", nil, fmt.Errorf("unknown rule format '%s'", ruleType) + } +} diff --git a/pkg/appsec/appsec_rule/modsec_rule_test.go b/pkg/appsec/appsec_rule/modsec_rule_test.go new file mode 100644 index 000000000..d919dce25 --- /dev/null +++ b/pkg/appsec/appsec_rule/modsec_rule_test.go @@ -0,0 +1,118 @@ +package appsec_rule + +import "testing" + +func TestVPatchRuleString(t *testing.T) { + tests := []struct { + name string + rule CustomRule + expected string + }{ + { + name: "Base Rule", + rule: CustomRule{ + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:2203944045,phase:2,deny,log,msg:'Base Rule',tag:'crowdsec-Base Rule',t:lowercase"`, + }, + { + name: "Multiple Zones", + rule: CustomRule{ + Zones: []string{"ARGS", "BODY_ARGS"}, + Variables: []string{"foo"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + expected: `SecRule ARGS_GET:foo|ARGS_POST:foo "@rx [^a-zA-Z]" "id:3387135861,phase:2,deny,log,msg:'Multiple Zones',tag:'crowdsec-Multiple Zones',t:lowercase"`, + }, + { + name: "Basic AND", + rule: CustomRule{ + And: []CustomRule{ + { + + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + { + Zones: []string{"ARGS"}, + Variables: []string{"bar"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + }, + }, + expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:4145519614,phase:2,deny,log,msg:'Basic AND',tag:'crowdsec-Basic AND',t:lowercase,chain" +SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:1865217529,phase:2,deny,log,msg:'Basic AND',tag:'crowdsec-Basic AND',t:lowercase"`, + }, + { + name: "Basic OR", + rule: CustomRule{ + Or: []CustomRule{ + { + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + { + Zones: []string{"ARGS"}, + Variables: []string{"bar"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + }, + }, + expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:651140804,phase:2,deny,log,msg:'Basic OR',tag:'crowdsec-Basic OR',t:lowercase,skip:1" +SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:271441587,phase:2,deny,log,msg:'Basic OR',tag:'crowdsec-Basic OR',t:lowercase"`, + }, + { + name: "OR AND mix", + rule: CustomRule{ + And: []CustomRule{ + { + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + Or: []CustomRule{ + { + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + { + Zones: []string{"ARGS"}, + Variables: []string{"bar"}, + Match: match{Type: "regex", Value: "[^a-zA-Z]"}, + Transform: []string{"lowercase"}, + }, + }, + }, + }, + }, + expected: `SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:1714963250,phase:2,deny,log,msg:'OR AND mix',tag:'crowdsec-OR AND mix',t:lowercase,skip:1" +SecRule ARGS_GET:bar "@rx [^a-zA-Z]" "id:1519945803,phase:2,deny,log,msg:'OR AND mix',tag:'crowdsec-OR AND mix',t:lowercase" +SecRule ARGS_GET:foo "@rx [^a-zA-Z]" "id:1519945803,phase:2,deny,log,msg:'OR AND mix',tag:'crowdsec-OR AND mix',t:lowercase"`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual, _, err := tt.rule.Convert(ModsecurityRuleType, tt.name) + + if err != nil { + t.Errorf("Error converting rule: %s", err) + } + if actual != tt.expected { + t.Errorf("Expected:\n%s\nGot:\n%s", tt.expected, actual) + } + }) + } +} diff --git a/pkg/appsec/appsec_rule/modsecurity.go b/pkg/appsec/appsec_rule/modsecurity.go new file mode 100644 index 000000000..760c697cc --- /dev/null +++ b/pkg/appsec/appsec_rule/modsecurity.go @@ -0,0 +1,181 @@ +package appsec_rule + +import ( + "fmt" + "hash/fnv" + "strings" +) + +type ModsecurityRule struct { + ids []uint32 +} + +var zonesMap map[string]string = map[string]string{ + "ARGS": "ARGS_GET", + "ARGS_NAMES": "ARGS_GET_NAMES", + "BODY_ARGS": "ARGS_POST", + "BODY_ARGS_NAMES": "ARGS_POST_NAMES", + "HEADERS": "REQUEST_HEADERS", + "METHOD": "REQUEST_METHOD", + "PROTOCOL": "REQUEST_PROTOCOL", + "URI": "REQUEST_URI", +} + +var transformMap map[string]string = map[string]string{ + "lowercase": "t:lowercase", + "uppercase": "t:uppercase", + "b64decode": "t:base64Decode", + "hexdecode": "t:hexDecode", + "length": "t:length", +} + +var matchMap map[string]string = map[string]string{ + "regex": "@rx", + "equal": "@streq", + "startsWith": "@beginsWith", + "endsWith": "@endsWith", + "contains": "@contains", + "libinjectionSQL": "@detectSQLi", + "libinjectionXSS": "@detectXSS", + "gt": "@gt", + "lt": "@lt", + "ge": "@ge", + "le": "@le", +} + +var bodyTypeMatch map[string]string = map[string]string{ + "json": "JSON", + "xml": "XML", + "multipart": "MULTIPART", + "urlencoded": "URLENCODED", +} + +func (m *ModsecurityRule) Build(rule *CustomRule, appsecRuleName string) (string, []uint32, error) { + + rules, err := m.buildRules(rule, appsecRuleName, false, 0, 0) + + if err != nil { + return "", nil, err + } + + //We return the id of the first generated rule, as it's the interesting one in case of chain or skip + return strings.Join(rules, "\n"), m.ids, nil +} + +func (m *ModsecurityRule) generateRuleID(rule *CustomRule, appsecRuleName string, depth int) uint32 { + h := fnv.New32a() + h.Write([]byte(appsecRuleName)) + h.Write([]byte(rule.Match.Type)) + h.Write([]byte(rule.Match.Value)) + h.Write([]byte(fmt.Sprintf("%d", depth))) + for _, zone := range rule.Zones { + h.Write([]byte(zone)) + } + for _, transform := range rule.Transform { + h.Write([]byte(transform)) + } + id := h.Sum32() + m.ids = append(m.ids, id) + return id +} + +func (m *ModsecurityRule) buildRules(rule *CustomRule, appsecRuleName string, and bool, toSkip int, depth int) ([]string, error) { + ret := make([]string, 0) + + if len(rule.And) != 0 && len(rule.Or) != 0 { + return nil, fmt.Errorf("cannot have both 'and' and 'or' in the same rule") + } + + if rule.And != nil { + for c, andRule := range rule.And { + depth++ + lastRule := c == len(rule.And)-1 // || len(rule.Or) == 0 + rules, err := m.buildRules(&andRule, appsecRuleName, !lastRule, 0, depth) + if err != nil { + return nil, err + } + ret = append(ret, rules...) + } + } + + if rule.Or != nil { + for c, orRule := range rule.Or { + depth++ + skip := len(rule.Or) - c - 1 + rules, err := m.buildRules(&orRule, appsecRuleName, false, skip, depth) + if err != nil { + return nil, err + } + ret = append(ret, rules...) + } + } + + r := strings.Builder{} + + r.WriteString("SecRule ") + + if rule.Zones == nil { + return ret, nil + } + + for idx, zone := range rule.Zones { + mappedZone, ok := zonesMap[zone] + if !ok { + return nil, fmt.Errorf("unknown zone '%s'", zone) + } + if len(rule.Variables) == 0 { + r.WriteString(mappedZone) + } else { + for j, variable := range rule.Variables { + if idx > 0 || j > 0 { + r.WriteByte('|') + } + r.WriteString(fmt.Sprintf("%s:%s", mappedZone, variable)) + } + } + } + r.WriteByte(' ') + + if rule.Match.Type != "" { + if match, ok := matchMap[rule.Match.Type]; ok { + r.WriteString(fmt.Sprintf(`"%s %s"`, match, rule.Match.Value)) + } else { + return nil, fmt.Errorf("unknown match type '%s'", rule.Match.Type) + } + } + + //Should phase:2 be configurable? + r.WriteString(fmt.Sprintf(` "id:%d,phase:2,deny,log,msg:'%s',tag:'crowdsec-%s'`, m.generateRuleID(rule, appsecRuleName, depth), appsecRuleName, appsecRuleName)) + + if rule.Transform != nil { + for _, transform := range rule.Transform { + r.WriteByte(',') + if mappedTransform, ok := transformMap[transform]; ok { + r.WriteString(mappedTransform) + } else { + return nil, fmt.Errorf("unknown transform '%s'", transform) + } + } + } + + if rule.BodyType != "" { + if mappedBodyType, ok := bodyTypeMatch[rule.BodyType]; ok { + r.WriteString(fmt.Sprintf(",ctl:requestBodyProcessor=%s", mappedBodyType)) + } else { + return nil, fmt.Errorf("unknown body type '%s'", rule.BodyType) + } + } + + if and { + r.WriteString(",chain") + } + + if toSkip > 0 { + r.WriteString(fmt.Sprintf(",skip:%d", toSkip)) + } + + r.WriteByte('"') + + ret = append(ret, r.String()) + return ret, nil +} diff --git a/pkg/appsec/appsec_rule/types.go b/pkg/appsec/appsec_rule/types.go new file mode 100644 index 000000000..13716975a --- /dev/null +++ b/pkg/appsec/appsec_rule/types.go @@ -0,0 +1,9 @@ +package appsec_rule + +const ( + ModsecurityRuleType = "modsecurity" +) + +func SupportedTypes() []string { + return []string{ModsecurityRuleType} +} diff --git a/pkg/appsec/appsec_rules_collection.go b/pkg/appsec/appsec_rules_collection.go new file mode 100644 index 000000000..4ccc63989 --- /dev/null +++ b/pkg/appsec/appsec_rules_collection.go @@ -0,0 +1,144 @@ +package appsec + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + + log "github.com/sirupsen/logrus" +) + +type AppsecCollection struct { + collectionName string + Rules []string +} + +var APPSEC_RULE = "appsec-rule" + +// to be filled w/ seb update +type AppsecCollectionConfig struct { + Type string `yaml:"type"` + Name string `yaml:"name"` + Debug bool `yaml:"debug"` + Description string `yaml:"description"` + SecLangFilesRules []string `yaml:"seclang_files_rules"` + SecLangRules []string `yaml:"seclang_rules"` + Rules []appsec_rule.CustomRule `yaml:"rules"` + + Labels map[string]interface{} `yaml:"labels"` //Labels is K:V list aiming at providing context the overflow + + Data interface{} `yaml:"data"` //Ignore it + hash string `yaml:"-"` + version string `yaml:"-"` +} + +type RulesDetails struct { + LogLevel log.Level + Hash string + Version string + Name string +} + +// Should it be a global ? +// Is using the id is a good idea ? might be too specific to coraza and not easily reusable +var AppsecRulesDetails = make(map[int]RulesDetails) + +func LoadCollection(pattern string, logger *log.Entry) ([]AppsecCollection, error) { + ret := make([]AppsecCollection, 0) + + for _, appsecRule := range appsecRules { + + tmpMatch, err := exprhelpers.Match(pattern, appsecRule.Name) + + if err != nil { + logger.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err) + continue + } + + matched, ok := tmpMatch.(bool) + + if !ok { + logger.Errorf("unable to match %s with %s : %s", appsecRule.Name, pattern, err) + continue + } + + if !matched { + continue + } + + appsecCol := AppsecCollection{ + collectionName: appsecRule.Name, + } + + if appsecRule.SecLangFilesRules != nil { + for _, rulesFile := range appsecRule.SecLangFilesRules { + logger.Debugf("Adding rules from %s", rulesFile) + fullPath := filepath.Join(hub.GetDataDir(), rulesFile) + c, err := os.ReadFile(fullPath) + if err != nil { + logger.Errorf("unable to read file %s : %s", rulesFile, err) + continue + } + for _, line := range strings.Split(string(c), "\n") { + if strings.HasPrefix(line, "#") { + continue + } + if strings.TrimSpace(line) == "" { + continue + } + appsecCol.Rules = append(appsecCol.Rules, line) + } + } + } + + if appsecRule.SecLangRules != nil { + logger.Tracef("Adding inline rules %+v", appsecRule.SecLangRules) + appsecCol.Rules = append(appsecCol.Rules, appsecRule.SecLangRules...) + } + + if appsecRule.Rules != nil { + for _, rule := range appsecRule.Rules { + strRule, rulesId, err := rule.Convert(appsec_rule.ModsecurityRuleType, appsecRule.Name) + if err != nil { + logger.Errorf("unable to convert rule %s : %s", rule.Name, err) + return nil, err + } + logger.Debugf("Adding rule %s", strRule) + appsecCol.Rules = append(appsecCol.Rules, strRule) + + //We only take the first id, as it's the one of the "main" rule + if _, ok := AppsecRulesDetails[int(rulesId[0])]; !ok { + AppsecRulesDetails[int(rulesId[0])] = RulesDetails{ + LogLevel: log.InfoLevel, + Hash: appsecRule.hash, + Version: appsecRule.version, + Name: appsecRule.Name, + } + } else { + logger.Warnf("conflicting id %d for rule %s !", rulesId[0], rule.Name) + } + + for _, id := range rulesId { + SetRuleDebug(int(id), appsecRule.Debug) + } + } + } + ret = append(ret, appsecCol) + } + if len(ret) == 0 { + return nil, fmt.Errorf("no appsec-rules found for pattern %s", pattern) + } + return ret, nil +} + +func (w AppsecCollection) String() string { + ret := "" + for _, rule := range w.Rules { + ret += rule + "\n" + } + return ret +} diff --git a/pkg/appsec/coraza_logger.go b/pkg/appsec/coraza_logger.go new file mode 100644 index 000000000..372a0098e --- /dev/null +++ b/pkg/appsec/coraza_logger.go @@ -0,0 +1,194 @@ +package appsec + +import ( + "fmt" + "io" + + dbg "github.com/crowdsecurity/coraza/v3/debuglog" + log "github.com/sirupsen/logrus" +) + +var DebugRules map[int]bool = map[int]bool{} + +func SetRuleDebug(id int, debug bool) { + DebugRules[id] = debug +} + +func GetRuleDebug(id int) bool { + if val, ok := DebugRules[id]; ok { + return val + } + return false +} + +// type ContextField func(Event) Event + +type crzLogEvent struct { + fields log.Fields + logger *log.Entry + muted bool + level log.Level +} + +func (e *crzLogEvent) Msg(msg string) { + if e.muted { + return + } + + /*this is a hack. As we want to have per-level rule debug but it's not allowed by coraza/modsec, if a rule ID is flagged to be in debug mode, the + .Int("rule_id", ) call will set the log_level of the event to debug. However, given the logger is global to the appsec-runner, + we are switching forth and back the log level of the logger*/ + oldLvl := e.logger.Logger.GetLevel() + + if e.level != oldLvl { + e.logger.Logger.SetLevel(e.level) + } + + if len(e.fields) == 0 { + e.logger.Log(e.level, msg) + } else { + e.logger.WithFields(e.fields).Log(e.level, msg) + } + + if e.level != oldLvl { + e.logger.Logger.SetLevel(oldLvl) + e.level = oldLvl + } +} + +func (e *crzLogEvent) Str(key, val string) dbg.Event { + if e.muted { + return e + } + e.fields[key] = val + return e +} + +func (e *crzLogEvent) Err(err error) dbg.Event { + if e.muted { + return e + } + e.fields["error"] = err + return e +} + +func (e *crzLogEvent) Bool(key string, b bool) dbg.Event { + if e.muted { + return e + } + e.fields[key] = b + return e +} + +func (e *crzLogEvent) Int(key string, i int) dbg.Event { + if e.muted { + //this allows us to have per-rule debug logging + if key == "rule_id" && GetRuleDebug(i) { + e.muted = false + e.fields = map[string]interface{}{} + e.level = log.DebugLevel + } else { + return e + } + } + e.fields[key] = i + return e +} + +func (e *crzLogEvent) Uint(key string, i uint) dbg.Event { + if e.muted { + return e + } + e.fields[key] = i + return e +} + +func (e *crzLogEvent) Stringer(key string, val fmt.Stringer) dbg.Event { + if e.muted { + return e + } + e.fields[key] = val + return e +} + +func (e crzLogEvent) IsEnabled() bool { + return !e.muted +} + +type crzLogger struct { + logger *log.Entry + defaultFields log.Fields + logLevel log.Level +} + +func NewCrzLogger(logger *log.Entry) crzLogger { + return crzLogger{logger: logger, logLevel: logger.Logger.GetLevel()} +} + +func (c crzLogger) NewMutedEvt(lvl log.Level) dbg.Event { + return &crzLogEvent{muted: true, logger: c.logger, level: lvl} +} +func (c crzLogger) NewEvt(lvl log.Level) dbg.Event { + evt := &crzLogEvent{fields: map[string]interface{}{}, logger: c.logger, level: lvl} + if c.defaultFields != nil { + for k, v := range c.defaultFields { + evt.fields[k] = v + } + } + return evt +} + +func (c crzLogger) WithOutput(w io.Writer) dbg.Logger { + return c +} + +func (c crzLogger) WithLevel(lvl dbg.Level) dbg.Logger { + c.logLevel = log.Level(lvl) + c.logger.Logger.SetLevel(c.logLevel) + return c +} + +func (c crzLogger) With(fs ...dbg.ContextField) dbg.Logger { + var e dbg.Event = c.NewEvt(c.logLevel) + for _, f := range fs { + e = f(e) + } + c.defaultFields = e.(*crzLogEvent).fields + return c +} + +func (c crzLogger) Trace() dbg.Event { + if c.logLevel < log.TraceLevel { + return c.NewMutedEvt(log.TraceLevel) + } + return c.NewEvt(log.TraceLevel) +} + +func (c crzLogger) Debug() dbg.Event { + if c.logLevel < log.DebugLevel { + return c.NewMutedEvt(log.DebugLevel) + + } + return c.NewEvt(log.DebugLevel) +} + +func (c crzLogger) Info() dbg.Event { + if c.logLevel < log.InfoLevel { + return c.NewMutedEvt(log.InfoLevel) + } + return c.NewEvt(log.InfoLevel) +} + +func (c crzLogger) Warn() dbg.Event { + if c.logLevel < log.WarnLevel { + return c.NewMutedEvt(log.WarnLevel) + } + return c.NewEvt(log.WarnLevel) +} + +func (c crzLogger) Error() dbg.Event { + if c.logLevel < log.ErrorLevel { + return c.NewMutedEvt(log.ErrorLevel) + } + return c.NewEvt(log.ErrorLevel) +} diff --git a/pkg/appsec/loader.go b/pkg/appsec/loader.go new file mode 100644 index 000000000..fa13cb03d --- /dev/null +++ b/pkg/appsec/loader.go @@ -0,0 +1,52 @@ +package appsec + +import ( + "os" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +var appsecRules map[string]AppsecCollectionConfig = make(map[string]AppsecCollectionConfig) //FIXME: would probably be better to have a struct for this + +var hub *cwhub.Hub //FIXME: this is a temporary hack to make the hub available in the package + +func LoadAppsecRules(hubInstance *cwhub.Hub) error { + + hub = hubInstance + + for _, hubAppsecRuleItem := range hub.GetItemMap(cwhub.APPSEC_RULES) { + if !hubAppsecRuleItem.State.Installed { + continue + } + + content, err := os.ReadFile(hubAppsecRuleItem.State.LocalPath) + + if err != nil { + log.Warnf("unable to read file %s : %s", hubAppsecRuleItem.State.LocalPath, err) + continue + } + + var rule AppsecCollectionConfig + + err = yaml.UnmarshalStrict(content, &rule) + + if err != nil { + log.Warnf("unable to unmarshal file %s : %s", hubAppsecRuleItem.State.LocalPath, err) + continue + } + + rule.hash = hubAppsecRuleItem.State.LocalHash + rule.version = hubAppsecRuleItem.Version + + log.Infof("Adding %s to appsec rules", rule.Name) + + appsecRules[rule.Name] = rule + } + + if len(appsecRules) == 0 { + log.Debugf("No appsec rules found") + } + return nil +} diff --git a/pkg/appsec/request.go b/pkg/appsec/request.go new file mode 100644 index 000000000..9979caf90 --- /dev/null +++ b/pkg/appsec/request.go @@ -0,0 +1,345 @@ +package appsec + +import ( + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "regexp" + + "github.com/google/uuid" + log "github.com/sirupsen/logrus" +) + +const ( + URIHeaderName = "X-Crowdsec-Appsec-Uri" + VerbHeaderName = "X-Crowdsec-Appsec-Verb" + HostHeaderName = "X-Crowdsec-Appsec-Host" + IPHeaderName = "X-Crowdsec-Appsec-Ip" + APIKeyHeaderName = "X-Crowdsec-Appsec-Api-Key" +) + +type ParsedRequest struct { + RemoteAddr string `json:"remote_addr,omitempty"` + Host string `json:"host,omitempty"` + ClientIP string `json:"client_ip,omitempty"` + URI string `json:"uri,omitempty"` + Args url.Values `json:"args,omitempty"` + ClientHost string `json:"client_host,omitempty"` + Headers http.Header `json:"headers,omitempty"` + URL *url.URL `json:"url,omitempty"` + Method string `json:"method,omitempty"` + Proto string `json:"proto,omitempty"` + Body []byte `json:"body,omitempty"` + TransferEncoding []string `json:"transfer_encoding,omitempty"` + UUID string `json:"uuid,omitempty"` + Tx ExtendedTransaction `json:"transaction,omitempty"` + ResponseChannel chan AppsecTempResponse `json:"-"` + IsInBand bool `json:"-"` + IsOutBand bool `json:"-"` + AppsecEngine string `json:"appsec_engine,omitempty"` + RemoteAddrNormalized string `json:"normalized_remote_addr,omitempty"` +} + +type ReqDumpFilter struct { + req *ParsedRequest + HeadersContentFilters []string + HeadersNameFilters []string + HeadersDrop bool + + BodyDrop bool + //BodyContentFilters []string TBD + + ArgsContentFilters []string + ArgsNameFilters []string + ArgsDrop bool +} + +func (r *ParsedRequest) DumpRequest(params ...any) *ReqDumpFilter { + filter := ReqDumpFilter{} + filter.BodyDrop = true + filter.HeadersNameFilters = []string{"cookie", "authorization"} + filter.req = r + return &filter +} + +// clear filters +func (r *ReqDumpFilter) NoFilters() *ReqDumpFilter { + r2 := ReqDumpFilter{} + r2.req = r.req + return &r2 +} + +func (r *ReqDumpFilter) WithEmptyHeadersFilters() *ReqDumpFilter { + r.HeadersContentFilters = []string{} + return r +} + +func (r *ReqDumpFilter) WithHeadersContentFilters(filter string) *ReqDumpFilter { + r.HeadersContentFilters = append(r.HeadersContentFilters, filter) + return r +} + +func (r *ReqDumpFilter) WithHeadersNameFilter(filter string) *ReqDumpFilter { + r.HeadersNameFilters = append(r.HeadersNameFilters, filter) + return r +} + +func (r *ReqDumpFilter) WithNoHeaders() *ReqDumpFilter { + r.HeadersDrop = true + return r +} + +func (r *ReqDumpFilter) WithHeaders() *ReqDumpFilter { + r.HeadersDrop = false + r.HeadersNameFilters = []string{} + return r +} + +func (r *ReqDumpFilter) WithBody() *ReqDumpFilter { + r.BodyDrop = false + return r +} + +func (r *ReqDumpFilter) WithNoBody() *ReqDumpFilter { + r.BodyDrop = true + return r +} + +func (r *ReqDumpFilter) WithEmptyArgsFilters() *ReqDumpFilter { + r.ArgsContentFilters = []string{} + return r +} + +func (r *ReqDumpFilter) WithArgsContentFilters(filter string) *ReqDumpFilter { + r.ArgsContentFilters = append(r.ArgsContentFilters, filter) + return r +} + +func (r *ReqDumpFilter) WithArgsNameFilter(filter string) *ReqDumpFilter { + r.ArgsNameFilters = append(r.ArgsNameFilters, filter) + return r +} + +func (r *ReqDumpFilter) FilterBody(out *ParsedRequest) error { + if r.BodyDrop { + return nil + } + out.Body = r.req.Body + return nil +} + +func (r *ReqDumpFilter) FilterArgs(out *ParsedRequest) error { + if r.ArgsDrop { + return nil + } + if len(r.ArgsContentFilters) == 0 && len(r.ArgsNameFilters) == 0 { + out.Args = r.req.Args + return nil + } + out.Args = make(url.Values) + for k, vals := range r.req.Args { + reject := false + //exclude by match on name + for _, filter := range r.ArgsNameFilters { + ok, err := regexp.MatchString("(?i)"+filter, k) + if err != nil { + log.Debugf("error while matching string '%s' with '%s': %s", filter, k, err) + continue + } + if ok { + reject = true + break + } + } + + for _, v := range vals { + //exclude by content + for _, filter := range r.ArgsContentFilters { + ok, err := regexp.MatchString("(?i)"+filter, v) + if err != nil { + log.Debugf("error while matching string '%s' with '%s': %s", filter, v, err) + continue + } + if ok { + reject = true + break + } + + } + } + //if it was not rejected, let's add it + if !reject { + out.Args[k] = vals + } + } + return nil +} + +func (r *ReqDumpFilter) FilterHeaders(out *ParsedRequest) error { + if r.HeadersDrop { + return nil + } + + if len(r.HeadersContentFilters) == 0 && len(r.HeadersNameFilters) == 0 { + out.Headers = r.req.Headers + return nil + } + + out.Headers = make(http.Header) + for k, vals := range r.req.Headers { + reject := false + //exclude by match on name + for _, filter := range r.HeadersNameFilters { + ok, err := regexp.MatchString("(?i)"+filter, k) + if err != nil { + log.Debugf("error while matching string '%s' with '%s': %s", filter, k, err) + continue + } + if ok { + reject = true + break + } + } + + for _, v := range vals { + //exclude by content + for _, filter := range r.HeadersContentFilters { + ok, err := regexp.MatchString("(?i)"+filter, v) + if err != nil { + log.Debugf("error while matching string '%s' with '%s': %s", filter, v, err) + continue + } + if ok { + reject = true + break + } + + } + } + //if it was not rejected, let's add it + if !reject { + out.Headers[k] = vals + } + } + return nil +} + +func (r *ReqDumpFilter) GetFilteredRequest() *ParsedRequest { + //if there are no filters, we return the original request + if len(r.HeadersContentFilters) == 0 && + len(r.HeadersNameFilters) == 0 && + len(r.ArgsContentFilters) == 0 && + len(r.ArgsNameFilters) == 0 && + !r.BodyDrop && !r.HeadersDrop && !r.ArgsDrop { + log.Warningf("no filters, returning original request") + return r.req + } + + r2 := ParsedRequest{} + r.FilterHeaders(&r2) + r.FilterBody(&r2) + r.FilterArgs(&r2) + return &r2 +} + +func (r *ReqDumpFilter) ToJSON() error { + fd, err := os.CreateTemp("/tmp/", "crowdsec_req_dump_*.json") + if err != nil { + return fmt.Errorf("while creating temp file: %w", err) + } + defer fd.Close() + enc := json.NewEncoder(fd) + enc.SetIndent("", " ") + + req := r.GetFilteredRequest() + + log.Warningf("dumping : %+v", req) + + if err := enc.Encode(req); err != nil { + return fmt.Errorf("while encoding request: %w", err) + } + log.Warningf("request dumped to %s", fd.Name()) + return nil +} + +// Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the App security Engine +func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) { + var err error + body := make([]byte, 0) + + if r.Body != nil { + body, err = io.ReadAll(r.Body) + if err != nil { + return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err) + } + } + + // the real source of the request is set in 'x-client-ip' + clientIP := r.Header.Get(IPHeaderName) + if clientIP == "" { + return ParsedRequest{}, fmt.Errorf("missing '%s' header", IPHeaderName) + } + // the real target Host of the request is set in 'x-client-host' + clientHost := r.Header.Get(HostHeaderName) + if clientHost == "" { + return ParsedRequest{}, fmt.Errorf("missing '%s' header", HostHeaderName) + } + // the real URI of the request is set in 'x-client-uri' + clientURI := r.Header.Get(URIHeaderName) + if clientURI == "" { + return ParsedRequest{}, fmt.Errorf("missing '%s' header", URIHeaderName) + } + // the real VERB of the request is set in 'x-client-uri' + clientMethod := r.Header.Get(VerbHeaderName) + if clientMethod == "" { + return ParsedRequest{}, fmt.Errorf("missing '%s' header", VerbHeaderName) + } + + // delete those headers before coraza process the request + delete(r.Header, IPHeaderName) + delete(r.Header, HostHeaderName) + delete(r.Header, URIHeaderName) + delete(r.Header, VerbHeaderName) + + parsedURL, err := url.Parse(clientURI) + if err != nil { + return ParsedRequest{}, fmt.Errorf("unable to parse url '%s': %s", clientURI, err) + } + + remoteAddrNormalized := "" + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + log.Errorf("Invalid appsec remote IP source %v: %s", r.RemoteAddr, err.Error()) + remoteAddrNormalized = r.RemoteAddr + } else { + ip := net.ParseIP(host) + if ip == nil { + log.Errorf("Invalid appsec remote IP address source %v: %s", r.RemoteAddr, err.Error()) + remoteAddrNormalized = r.RemoteAddr + } else { + remoteAddrNormalized = ip.String() + } + } + + return ParsedRequest{ + RemoteAddr: r.RemoteAddr, + UUID: uuid.New().String(), + ClientHost: clientHost, + ClientIP: clientIP, + URI: parsedURL.Path, + Method: clientMethod, + Host: r.Host, + Headers: r.Header, + URL: r.URL, + Proto: r.Proto, + Body: body, + Args: parsedURL.Query(), //TODO: Check if there's not potential bypass as it excludes malformed args + TransferEncoding: r.TransferEncoding, + ResponseChannel: make(chan AppsecTempResponse), + RemoteAddrNormalized: remoteAddrNormalized, + }, nil +} diff --git a/pkg/appsec/request_test.go b/pkg/appsec/request_test.go new file mode 100644 index 000000000..b05ecbde6 --- /dev/null +++ b/pkg/appsec/request_test.go @@ -0,0 +1,181 @@ +package appsec + +import "testing" + +func TestBodyDumper(t *testing.T) { + + tests := []struct { + name string + req *ParsedRequest + expect *ParsedRequest + filter func(r *ReqDumpFilter) *ReqDumpFilter + }{ + { + name: "default filter (cookie+authorization stripped + no body)", + req: &ParsedRequest{ + Body: []byte("yo some body"), + Headers: map[string][]string{"cookie": {"toto"}, "authorization": {"tata"}, "foo": {"bar", "baz"}}, + }, + expect: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"foo": {"bar", "baz"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r + }, + }, + { + name: "explicit empty filter", + req: &ParsedRequest{ + Body: []byte("yo some body"), + Headers: map[string][]string{"cookie": {"toto"}, "authorization": {"tata"}, "foo": {"bar", "baz"}}, + }, + expect: &ParsedRequest{ + Body: []byte("yo some body"), + Headers: map[string][]string{"cookie": {"toto"}, "authorization": {"tata"}, "foo": {"bar", "baz"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.NoFilters() + }, + }, + { + name: "filter header", + req: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"test1": {"toto"}, "test2": {"tata"}}, + }, + expect: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"test1": {"toto"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithNoBody().WithHeadersNameFilter("test2") + }, + }, + { + name: "filter header content", + req: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"test1": {"toto"}, "test2": {"tata"}}, + }, + expect: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"test1": {"toto"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithHeadersContentFilters("tata") + }, + }, + { + name: "with headers", + req: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"cookie1": {"lol"}}, + }, + expect: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"cookie1": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithHeaders() + }, + }, + { + name: "drop headers", + req: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{"toto": {"lol"}}, + }, + expect: &ParsedRequest{ + Body: []byte{}, + Headers: map[string][]string{}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithNoHeaders() + }, + }, + { + name: "with body", + req: &ParsedRequest{ + Body: []byte("toto"), + Headers: map[string][]string{"toto": {"lol"}}, + }, + expect: &ParsedRequest{ + Body: []byte("toto"), + Headers: map[string][]string{"toto": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithBody() + }, + }, + { + name: "with empty args filter", + req: &ParsedRequest{ + Args: map[string][]string{"toto": {"lol"}}, + }, + expect: &ParsedRequest{ + Args: map[string][]string{"toto": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithEmptyArgsFilters() + }, + }, + { + name: "with args name filter", + req: &ParsedRequest{ + Args: map[string][]string{"toto": {"lol"}, "totolol": {"lol"}}, + }, + expect: &ParsedRequest{ + Args: map[string][]string{"totolol": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithArgsNameFilter("toto") + }, + }, + { + name: "WithEmptyHeadersFilters", + req: &ParsedRequest{ + Args: map[string][]string{"cookie": {"lol"}, "totolol": {"lol"}}, + }, + expect: &ParsedRequest{ + Args: map[string][]string{"cookie": {"lol"}, "totolol": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithEmptyHeadersFilters() + }, + }, + { + name: "WithArgsContentFilters", + req: &ParsedRequest{ + Args: map[string][]string{"test": {"lol"}, "test2": {"toto"}}, + }, + expect: &ParsedRequest{ + Args: map[string][]string{"test": {"lol"}}, + }, + filter: func(r *ReqDumpFilter) *ReqDumpFilter { + return r.WithArgsContentFilters("toto") + }, + }, + } + + for idx, test := range tests { + + t.Run(test.name, func(t *testing.T) { + orig_dr := test.req.DumpRequest() + result := test.filter(orig_dr).GetFilteredRequest() + + if len(result.Body) != len(test.expect.Body) { + t.Fatalf("test %d (%s) failed, got %d, expected %d", idx, test.name, len(test.req.Body), len(test.expect.Body)) + } + if len(result.Headers) != len(test.expect.Headers) { + t.Fatalf("test %d (%s) failed, got %d, expected %d", idx, test.name, len(test.req.Headers), len(test.expect.Headers)) + } + for k, v := range result.Headers { + if len(v) != len(test.expect.Headers[k]) { + t.Fatalf("test %d (%s) failed, got %d, expected %d", idx, test.name, len(v), len(test.expect.Headers[k])) + } + } + }) + } + +} diff --git a/pkg/appsec/tx.go b/pkg/appsec/tx.go new file mode 100644 index 000000000..47da19d15 --- /dev/null +++ b/pkg/appsec/tx.go @@ -0,0 +1,93 @@ +package appsec + +import ( + "github.com/crowdsecurity/coraza/v3" + "github.com/crowdsecurity/coraza/v3/experimental" + "github.com/crowdsecurity/coraza/v3/experimental/plugins/plugintypes" + "github.com/crowdsecurity/coraza/v3/types" +) + +type ExtendedTransaction struct { + Tx experimental.FullTransaction +} + +func NewExtendedTransaction(engine coraza.WAF, uuid string) ExtendedTransaction { + inBoundTx := engine.NewTransactionWithID(uuid) + expTx := inBoundTx.(experimental.FullTransaction) + tx := NewTransaction(expTx) + return tx +} + +func NewTransaction(tx experimental.FullTransaction) ExtendedTransaction { + return ExtendedTransaction{Tx: tx} +} + +func (t *ExtendedTransaction) RemoveRuleByIDWithError(id int) error { + t.Tx.RemoveRuleByID(id) + return nil +} + +func (t *ExtendedTransaction) RemoveRuleByTagWithError(tag string) error { + t.Tx.RemoveRuleByTag(tag) + return nil +} + +func (t *ExtendedTransaction) IsRuleEngineOff() bool { + return t.Tx.IsRuleEngineOff() +} + +func (t *ExtendedTransaction) ProcessLogging() { + t.Tx.ProcessLogging() +} + +func (t *ExtendedTransaction) ProcessConnection(client string, cPort int, server string, sPort int) { + t.Tx.ProcessConnection(client, cPort, server, sPort) +} + +func (t *ExtendedTransaction) AddGetRequestArgument(name string, value string) { + t.Tx.AddGetRequestArgument(name, value) +} + +func (t *ExtendedTransaction) ProcessURI(uri string, method string, httpVersion string) { + t.Tx.ProcessURI(uri, method, httpVersion) +} + +func (t *ExtendedTransaction) AddRequestHeader(name string, value string) { + t.Tx.AddRequestHeader(name, value) +} + +func (t *ExtendedTransaction) SetServerName(name string) { + t.Tx.SetServerName(name) +} + +func (t *ExtendedTransaction) ProcessRequestHeaders() *types.Interruption { + return t.Tx.ProcessRequestHeaders() +} + +func (t *ExtendedTransaction) ProcessRequestBody() (*types.Interruption, error) { + return t.Tx.ProcessRequestBody() +} + +func (t *ExtendedTransaction) WriteRequestBody(body []byte) (*types.Interruption, int, error) { + return t.Tx.WriteRequestBody(body) +} + +func (t *ExtendedTransaction) Interruption() *types.Interruption { + return t.Tx.Interruption() +} + +func (t *ExtendedTransaction) IsInterrupted() bool { + return t.Tx.IsInterrupted() +} + +func (t *ExtendedTransaction) Variables() plugintypes.TransactionVariables { + return t.Tx.Variables() +} + +func (t *ExtendedTransaction) MatchedRules() []types.MatchedRule { + return t.Tx.MatchedRules() +} + +func (t *ExtendedTransaction) ID() string { + return t.Tx.ID() +} diff --git a/pkg/appsec/waf_helpers.go b/pkg/appsec/waf_helpers.go new file mode 100644 index 000000000..605d42ccb --- /dev/null +++ b/pkg/appsec/waf_helpers.go @@ -0,0 +1,59 @@ +package appsec + +import ( + "github.com/crowdsecurity/crowdsec/pkg/types" +) + +func GetOnLoadEnv(w *AppsecRuntimeConfig) map[string]interface{} { + return map[string]interface{}{ + "RemoveInBandRuleByID": w.DisableInBandRuleByID, + "RemoveInBandRuleByTag": w.DisableInBandRuleByTag, + "RemoveInBandRuleByName": w.DisableInBandRuleByName, + "RemoveOutBandRuleByID": w.DisableOutBandRuleByID, + "RemoveOutBandRuleByTag": w.DisableOutBandRuleByTag, + "RemoveOutBandRuleByName": w.DisableOutBandRuleByName, + "SetRemediationByTag": w.SetActionByTag, + "SetRemediationByID": w.SetActionByID, + "SetRemediationByName": w.SetActionByName, + } +} + +func GetPreEvalEnv(w *AppsecRuntimeConfig, request *ParsedRequest) map[string]interface{} { + return map[string]interface{}{ + "IsInBand": request.IsInBand, + "IsOutBand": request.IsOutBand, + "RemoveInBandRuleByID": w.RemoveInbandRuleByID, + "RemoveInBandRuleByName": w.RemoveInbandRuleByName, + "RemoveInBandRuleByTag": w.RemoveInbandRuleByTag, + "RemoveOutBandRuleByID": w.RemoveOutbandRuleByID, + "RemoveOutBandRuleByTag": w.RemoveOutbandRuleByTag, + "RemoveOutBandRuleByName": w.RemoveOutbandRuleByName, + "SetRemediationByTag": w.SetActionByTag, + "SetRemediationByID": w.SetActionByID, + "SetRemediationByName": w.SetActionByName, + } +} + +func GetPostEvalEnv(w *AppsecRuntimeConfig, request *ParsedRequest) map[string]interface{} { + return map[string]interface{}{ + "IsInBand": request.IsInBand, + "IsOutBand": request.IsOutBand, + "DumpRequest": request.DumpRequest, + } +} + +func GetOnMatchEnv(w *AppsecRuntimeConfig, request *ParsedRequest, evt types.Event) map[string]interface{} { + return map[string]interface{}{ + "evt": evt, + "req": request, + "IsInBand": request.IsInBand, + "IsOutBand": request.IsOutBand, + "SetRemediation": w.SetAction, + "SetReturnCode": w.SetHTTPCode, + "CancelEvent": w.CancelEvent, + "SendEvent": w.SendEvent, + "CancelAlert": w.CancelAlert, + "SendAlert": w.SendAlert, + "DumpRequest": request.DumpRequest, + } +} diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index ccc0a1aaf..09f46e250 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -21,6 +21,8 @@ var defaultConfigDir = "/etc/crowdsec" // defaultDataDir is the base path to all data files, to be overridden in the Makefile */ var defaultDataDir = "/var/lib/crowdsec/data/" +var globalConfig = Config{} + // Config contains top-level defaults -> overridden by configuration file -> overridden by CLI flags type Config struct { //just a path to ourselves :p @@ -89,9 +91,15 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool return nil, "", err } + globalConfig = cfg + return &cfg, configData, nil } +func GetConfig() Config { + return globalConfig +} + func NewDefaultConfig() *Config { logLevel := log.InfoLevel commonCfg := CommonCfg{ diff --git a/pkg/cwhub/item.go b/pkg/cwhub/item.go index 9cf044396..4d588da3a 100644 --- a/pkg/cwhub/item.go +++ b/pkg/cwhub/item.go @@ -12,10 +12,12 @@ import ( const ( // managed item types. - COLLECTIONS = "collections" - PARSERS = "parsers" - POSTOVERFLOWS = "postoverflows" - SCENARIOS = "scenarios" + COLLECTIONS = "collections" + PARSERS = "parsers" + POSTOVERFLOWS = "postoverflows" + SCENARIOS = "scenarios" + APPSEC_CONFIGS = "appsec-configs" + APPSEC_RULES = "appsec-rules" ) const ( @@ -27,7 +29,7 @@ const ( var ( // The order is important, as it is used to range over sub-items in collections. - ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS} + ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, APPSEC_CONFIGS, APPSEC_RULES, COLLECTIONS} ) type HubItems map[string]map[string]*Item @@ -118,6 +120,8 @@ type Item struct { PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"` + AppsecConfigs []string `json:"appsec-configs,omitempty" yaml:"appsec-configs,omitempty"` + AppsecRules []string `json:"appsec-rules,omitempty" yaml:"appsec-rules,omitempty"` } // installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local @@ -227,6 +231,24 @@ func (i *Item) SubItems() []*Item { sub = append(sub, s) } + for _, name := range i.AppsecConfigs { + s := i.hub.GetItem(APPSEC_CONFIGS, name) + if s == nil { + continue + } + + sub = append(sub, s) + } + + for _, name := range i.AppsecRules { + s := i.hub.GetItem(APPSEC_RULES, name) + if s == nil { + continue + } + + sub = append(sub, s) + } + for _, name := range i.Collections { s := i.hub.GetItem(COLLECTIONS, name) if s == nil { @@ -262,6 +284,18 @@ func (i *Item) logMissingSubItems() { } } + for _, subName := range i.AppsecConfigs { + if i.hub.GetItem(APPSEC_CONFIGS, subName) == nil { + log.Errorf("can't find %s in %s, required by %s", subName, APPSEC_CONFIGS, i.Name) + } + } + + for _, subName := range i.AppsecRules { + if i.hub.GetItem(APPSEC_RULES, subName) == nil { + log.Errorf("can't find %s in %s, required by %s", subName, APPSEC_RULES, i.Name) + } + } + for _, subName := range i.Collections { if i.hub.GetItem(COLLECTIONS, subName) == nil { log.Errorf("can't find %s in %s, required by %s", subName, COLLECTIONS, i.Name) diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index ae62b9342..ed3abc2c5 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "slices" "sort" "strings" @@ -112,15 +113,13 @@ func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) { log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype) - if ret.stage == SCENARIOS { - ret.ftype = SCENARIOS + if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS { + if !slices.Contains(ItemTypes, ret.stage) { + return nil, fmt.Errorf("unknown configuration type for file '%s'", path) + } + + ret.ftype = ret.stage ret.stage = "" - } else if ret.stage == COLLECTIONS { - ret.ftype = COLLECTIONS - ret.stage = "" - } else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS { - // it's a PARSER / POSTOVERFLOW with a stage - return nil, fmt.Errorf("unknown configuration type for file '%s'", path) } log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype) diff --git a/pkg/exprhelpers/expr_lib.go b/pkg/exprhelpers/expr_lib.go index 0833420a2..db191b84a 100644 --- a/pkg/exprhelpers/expr_lib.go +++ b/pkg/exprhelpers/expr_lib.go @@ -20,6 +20,21 @@ var exprFuncs = []exprCustomFunc{ new(func(string) (*cticlient.SmokeItem, error)), }, }, + { + name: "Flatten", + function: Flatten, + signature: []interface{}{}, + }, + { + name: "Distinct", + function: Distinct, + signature: []interface{}{}, + }, + { + name: "FlattenDistinct", + function: FlattenDistinct, + signature: []interface{}{}, + }, { name: "Distance", function: Distance, diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index 1faa77536..79a621c7d 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -9,6 +9,7 @@ import ( "net/url" "os" "path/filepath" + "reflect" "regexp" "strconv" "strings" @@ -176,6 +177,54 @@ func FileInit(fileFolder string, filename string, fileType string) error { return nil } +// Expr helpers + +func Distinct(params ...any) (any, error) { + + if rt := reflect.TypeOf(params[0]).Kind(); rt != reflect.Slice && rt != reflect.Array { + return nil, nil + } + array := params[0].([]interface{}) + if array == nil { + return []interface{}{}, nil + } + + var exists map[any]bool = make(map[any]bool) + var ret []interface{} = make([]interface{}, 0) + + for _, val := range array { + if _, ok := exists[val]; !ok { + exists[val] = true + ret = append(ret, val) + } + } + return ret, nil + +} + +func FlattenDistinct(params ...any) (any, error) { + return Distinct(flatten(nil, reflect.ValueOf(params))) //nolint:asasalint +} + +func Flatten(params ...any) (any, error) { + return flatten(nil, reflect.ValueOf(params)), nil +} + +func flatten(args []interface{}, v reflect.Value) []interface{} { + if v.Kind() == reflect.Interface { + v = v.Elem() + } + + if v.Kind() == reflect.Array || v.Kind() == reflect.Slice { + for i := 0; i < v.Len(); i++ { + args = flatten(args, v.Index(i)) + } + } else { + args = append(args, v.Interface()) + } + + return args +} func existsInFileMaps(filename string, ftype string) (bool, error) { ok := false var err error diff --git a/pkg/hubtest/coverage.go b/pkg/hubtest/coverage.go index 8148603d7..edbe10454 100644 --- a/pkg/hubtest/coverage.go +++ b/pkg/hubtest/coverage.go @@ -7,9 +7,10 @@ import ( "path/filepath" "strings" - log "github.com/sirupsen/logrus" - + "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" ) type Coverage struct { @@ -18,6 +19,65 @@ type Coverage struct { PresentIn map[string]bool //poorman's set } +func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) { + if len(h.HubIndex.GetItemMap(cwhub.APPSEC_RULES)) == 0 { + return nil, fmt.Errorf("no appsec rules in hub index") + } + + // populate from hub, iterate in alphabetical order + pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.APPSEC_RULES)) + coverage := make([]Coverage, len(pkeys)) + + for i, name := range pkeys { + coverage[i] = Coverage{ + Name: name, + TestsCount: 0, + PresentIn: make(map[string]bool), + } + } + + // parser the expressions a-la-oneagain + appsecTestConfigs, err := filepath.Glob(".appsec-tests/*/config.yaml") + if err != nil { + return nil, fmt.Errorf("while find appsec-tests config: %s", err) + } + + for _, appsecTestConfigPath := range appsecTestConfigs { + configFileData := &HubTestItemConfig{} + yamlFile, err := os.ReadFile(appsecTestConfigPath) + if err != nil { + log.Printf("unable to open appsec test config file '%s': %s", appsecTestConfigPath, err) + continue + } + err = yaml.Unmarshal(yamlFile, configFileData) + if err != nil { + return nil, fmt.Errorf("unmarshal: %v", err) + } + + for _, appsecRulesFile := range configFileData.AppsecRules { + appsecRuleData := &appsec_rule.CustomRule{} + yamlFile, err := os.ReadFile(appsecRulesFile) + if err != nil { + log.Printf("unable to open appsec rule '%s': %s", appsecRulesFile, err) + } + err = yaml.Unmarshal(yamlFile, appsecRuleData) + if err != nil { + return nil, fmt.Errorf("unmarshal: %v", err) + } + appsecRuleName := appsecRuleData.Name + + for idx, cov := range coverage { + if cov.Name == appsecRuleName { + coverage[idx].TestsCount++ + coverage[idx].PresentIn[appsecTestConfigPath] = true + } + } + } + } + + return coverage, nil +} + func (h *HubTest) GetParsersCoverage() ([]Coverage, error) { if len(h.HubIndex.GetItemMap(cwhub.PARSERS)) == 0 { return nil, fmt.Errorf("no parsers in hub index") @@ -105,7 +165,7 @@ func (h *HubTest) GetParsersCoverage() ([]Coverage, error) { } func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) { - if len(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) == 0 { + if len(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) == 0 { return nil, fmt.Errorf("no scenarios in hub index") } @@ -127,7 +187,6 @@ func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) { return nil, fmt.Errorf("while find scenario asserts : %s", err) } - for _, assert := range passerts { file, err := os.Open(assert) if err != nil { diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index ec1f6ee5e..b9bde4e2c 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -11,25 +11,30 @@ import ( ) type HubTest struct { - CrowdSecPath string - CscliPath string - HubPath string - HubTestPath string - HubIndexFile string - TemplateConfigPath string - TemplateProfilePath string - TemplateSimulationPath string - HubIndex *cwhub.Hub - Tests []*HubTestItem + CrowdSecPath string + CscliPath string + HubPath string + HubTestPath string //generic parser/scenario tests .tests + HubAppsecTestPath string //dir specific to appsec tests .appsec-tests + HubIndexFile string + TemplateConfigPath string + TemplateProfilePath string + TemplateSimulationPath string + TemplateAcquisPath string + TemplateAppsecProfilePath string + HubIndex *cwhub.Hub + Tests []*HubTestItem } const ( - templateConfigFile = "template_config.yaml" - templateSimulationFile = "template_simulation.yaml" - templateProfileFile = "template_profiles.yaml" + templateConfigFile = "template_config.yaml" + templateSimulationFile = "template_simulation.yaml" + templateProfileFile = "template_profiles.yaml" + templateAcquisFile = "template_acquis.yaml" + templateAppsecProfilePath = "template_appsec-profile.yaml" ) -func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, error) { +func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecTest bool) (HubTest, error) { hubPath, err := filepath.Abs(hubPath) if err != nil { return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err) @@ -39,9 +44,6 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, if _, err = os.Stat(hubPath); os.IsNotExist(err) { return HubTest{}, fmt.Errorf("path to hub '%s' doesn't exist, can't run", hubPath) } - - HubTestPath := filepath.Join(hubPath, "./.tests/") - // we can't use hubtest without crowdsec binary if _, err = exec.LookPath(crowdsecPath); err != nil { if _, err = os.Stat(crowdsecPath); os.IsNotExist(err) { @@ -56,6 +58,39 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, } } + if isAppsecTest { + HubTestPath := filepath.Join(hubPath, "./.appsec-tests/") + hubIndexFile := filepath.Join(hubPath, ".index.json") + + local := &csconfig.LocalHubCfg{ + HubDir: hubPath, + HubIndexFile: hubIndexFile, + InstallDir: HubTestPath, + InstallDataDir: HubTestPath, + } + + hub, err := cwhub.NewHub(local, nil, false) + if err != nil { + return HubTest{}, fmt.Errorf("unable to load hub: %s", err) + } + + return HubTest{ + CrowdSecPath: crowdsecPath, + CscliPath: cscliPath, + HubPath: hubPath, + HubTestPath: HubTestPath, + HubIndexFile: hubIndexFile, + TemplateConfigPath: filepath.Join(HubTestPath, templateConfigFile), + TemplateProfilePath: filepath.Join(HubTestPath, templateProfileFile), + TemplateSimulationPath: filepath.Join(HubTestPath, templateSimulationFile), + TemplateAppsecProfilePath: filepath.Join(HubTestPath, templateAppsecProfilePath), + TemplateAcquisPath: filepath.Join(HubTestPath, templateAcquisFile), + HubIndex: hub, + }, nil + } + + HubTestPath := filepath.Join(hubPath, "./.tests/") + hubIndexFile := filepath.Join(hubPath, ".index.json") local := &csconfig.LocalHubCfg{ @@ -70,19 +105,15 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, return HubTest{}, fmt.Errorf("unable to load hub: %s", err) } - templateConfigFilePath := filepath.Join(HubTestPath, templateConfigFile) - templateProfilePath := filepath.Join(HubTestPath, templateProfileFile) - templateSimulationPath := filepath.Join(HubTestPath, templateSimulationFile) - return HubTest{ CrowdSecPath: crowdsecPath, CscliPath: cscliPath, HubPath: hubPath, HubTestPath: HubTestPath, HubIndexFile: hubIndexFile, - TemplateConfigPath: templateConfigFilePath, - TemplateProfilePath: templateProfilePath, - TemplateSimulationPath: templateSimulationPath, + TemplateConfigPath: filepath.Join(HubTestPath, templateConfigFile), + TemplateProfilePath: filepath.Join(HubTestPath, templateProfileFile), + TemplateSimulationPath: filepath.Join(HubTestPath, templateSimulationFile), HubIndex: hub, }, nil } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index dd84fc022..05be6803d 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -1,7 +1,9 @@ package hubtest import ( + "errors" "fmt" + "net/url" "os" "os/exec" "path/filepath" @@ -16,14 +18,17 @@ import ( ) type HubTestItemConfig struct { - Parsers []string `yaml:"parsers"` - Scenarios []string `yaml:"scenarios"` - PostOVerflows []string `yaml:"postoverflows"` - LogFile string `yaml:"log_file"` - LogType string `yaml:"log_type"` - Labels map[string]string `yaml:"labels"` - IgnoreParsers bool `yaml:"ignore_parsers"` // if we test a scenario, we don't want to assert on Parser - OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 + Parsers []string `yaml:"parsers,omitempty"` + Scenarios []string `yaml:"scenarios,omitempty"` + PostOverflows []string `yaml:"postoverflows,omitempty"` + AppsecRules []string `yaml:"appsec-rules,omitempty"` + NucleiTemplate string `yaml:"nuclei_template,omitempty"` + ExpectedNucleiFailure bool `yaml:"expect_failure,omitempty"` + LogFile string `yaml:"log_file,omitempty"` + LogType string `yaml:"log_type,omitempty"` + Labels map[string]string `yaml:"labels,omitempty"` + IgnoreParsers bool `yaml:"ignore_parsers,omitempty"` // if we test a scenario, we don't want to assert on Parser + OverrideStatics []parser.ExtraField `yaml:"override_statics,omitempty"` //Allow to override statics. Executed before s00 } type HubTestItem struct { @@ -40,6 +45,7 @@ type HubTestItem struct { RuntimeConfigFilePath string RuntimeProfileFilePath string RuntimeSimulationFilePath string + RuntimeAcquisFilePath string RuntimeHubConfig *csconfig.LocalHubCfg ResultsPath string @@ -47,13 +53,15 @@ type HubTestItem struct { ScenarioResultFile string BucketPourResultFile string - HubPath string - HubTestPath string - HubIndexFile string - TemplateConfigPath string - TemplateProfilePath string - TemplateSimulationPath string - HubIndex *cwhub.Hub + HubPath string + HubTestPath string + HubIndexFile string + TemplateConfigPath string + TemplateProfilePath string + TemplateSimulationPath string + TemplateAcquisPath string + TemplateAppsecProfilePath string + HubIndex *cwhub.Hub Config *HubTestItemConfig @@ -75,6 +83,11 @@ const ( ScenarioResultFileName = "bucket-dump.yaml" BucketPourResultFileName = "bucketpour-dump.yaml" + + TestBouncerApiKey = "this_is_a_bad_password" + + DefaultNucleiTarget = "http://127.0.0.1:80/" + DefaultAppsecHost = "127.0.0.1:4241" ) func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { @@ -115,6 +128,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { RuntimeConfigFilePath: filepath.Join(runtimeFolder, "config.yaml"), RuntimeProfileFilePath: filepath.Join(runtimeFolder, "profiles.yaml"), RuntimeSimulationFilePath: filepath.Join(runtimeFolder, "simulation.yaml"), + RuntimeAcquisFilePath: filepath.Join(runtimeFolder, "acquis.yaml"), ResultsPath: resultPath, ParserResultFile: filepath.Join(resultPath, ParserResultFileName), ScenarioResultFile: filepath.Join(resultPath, ScenarioResultFileName), @@ -125,17 +139,19 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { InstallDir: runtimeFolder, InstallDataDir: filepath.Join(runtimeFolder, "data"), }, - Config: configFileData, - HubPath: hubTest.HubPath, - HubTestPath: hubTest.HubTestPath, - HubIndexFile: hubTest.HubIndexFile, - TemplateConfigPath: hubTest.TemplateConfigPath, - TemplateProfilePath: hubTest.TemplateProfilePath, - TemplateSimulationPath: hubTest.TemplateSimulationPath, - HubIndex: hubTest.HubIndex, - ScenarioAssert: ScenarioAssert, - ParserAssert: ParserAssert, - CustomItemsLocation: []string{hubTest.HubPath, testPath}, + Config: configFileData, + HubPath: hubTest.HubPath, + HubTestPath: hubTest.HubTestPath, + HubIndexFile: hubTest.HubIndexFile, + TemplateConfigPath: hubTest.TemplateConfigPath, + TemplateProfilePath: hubTest.TemplateProfilePath, + TemplateSimulationPath: hubTest.TemplateSimulationPath, + TemplateAcquisPath: hubTest.TemplateAcquisPath, + TemplateAppsecProfilePath: hubTest.TemplateAppsecProfilePath, + HubIndex: hubTest.HubIndex, + ScenarioAssert: ScenarioAssert, + ParserAssert: ParserAssert, + CustomItemsLocation: []string{hubTest.HubPath, testPath}, }, nil } @@ -297,8 +313,81 @@ func (t *HubTestItem) InstallHub() error { } } + // install appsec-rules in runtime environment + for _, appsecrule := range t.Config.AppsecRules { + log.Infof("adding rule '%s'", appsecrule) + if appsecrule == "" { + continue + } + + if hubAppsecRule, ok := t.HubIndex.GetItemMap(cwhub.APPSEC_RULES)[appsecrule]; ok { + appsecRuleSource, err := filepath.Abs(filepath.Join(t.HubPath, hubAppsecRule.RemotePath)) + if err != nil { + return fmt.Errorf("can't get absolute path of '%s': %s", appsecRuleSource, err) + } + + appsecRuleFilename := filepath.Base(appsecRuleSource) + + // runtime/hub/appsec-rules/author/appsec-rule + hubDirAppsecRuleDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubAppsecRule.RemotePath)) + + // runtime/appsec-rules/ + appsecRuleDirDest := fmt.Sprintf("%s/appsec-rules/", t.RuntimePath) + + if err := os.MkdirAll(hubDirAppsecRuleDest, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %s", hubDirAppsecRuleDest, err) + } + + if err := os.MkdirAll(appsecRuleDirDest, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %s", appsecRuleDirDest, err) + } + + // runtime/hub/appsec-rules/crowdsecurity/rule.yaml + hubDirAppsecRulePath := filepath.Join(appsecRuleDirDest, appsecRuleFilename) + if err := Copy(appsecRuleSource, hubDirAppsecRulePath); err != nil { + return fmt.Errorf("unable to copy '%s' to '%s': %s", appsecRuleSource, hubDirAppsecRulePath, err) + } + + // runtime/appsec-rules/rule.yaml + appsecRulePath := filepath.Join(appsecRuleDirDest, appsecRuleFilename) + if err := os.Symlink(hubDirAppsecRulePath, appsecRulePath); err != nil { + if !os.IsExist(err) { + return fmt.Errorf("unable to symlink appsec-rule '%s' to '%s': %s", hubDirAppsecRulePath, appsecRulePath, err) + } + } + } else { + customAppsecRuleExist := false + for _, customPath := range t.CustomItemsLocation { + // we check if its a custom appsec-rule + customAppsecRulePath := filepath.Join(customPath, appsecrule) + if _, err := os.Stat(customAppsecRulePath); os.IsNotExist(err) { + continue + } + customAppsecRulePathSplit := strings.Split(customAppsecRulePath, "/") + customAppsecRuleName := customAppsecRulePathSplit[len(customAppsecRulePathSplit)-1] + + appsecRuleDirDest := fmt.Sprintf("%s/appsec-rules/", t.RuntimePath) + if err := os.MkdirAll(appsecRuleDirDest, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %s", appsecRuleDirDest, err) + } + + // runtime/appsec-rules/ + customAppsecRuleDest := fmt.Sprintf("%s/appsec-rules/%s", t.RuntimePath, customAppsecRuleName) + // if path to postoverflow exist, copy it + if err := Copy(customAppsecRulePath, customAppsecRuleDest); err != nil { + continue + } + customAppsecRuleExist = true + break + } + if !customAppsecRuleExist { + return fmt.Errorf("couldn't find custom appsec-rule '%s' in the following location: %+v", appsecrule, t.CustomItemsLocation) + } + } + } + // install postoverflows in runtime environment - for _, postoverflow := range t.Config.PostOVerflows { + for _, postoverflow := range t.Config.PostOverflows { if postoverflow == "" { continue } @@ -449,16 +538,114 @@ func (t *HubTestItem) Clean() error { return os.RemoveAll(t.RuntimePath) } -func (t *HubTestItem) Run() error { - t.Success = false - t.ErrorsList = make([]string, 0) +func (t *HubTestItem) RunWithNucleiTemplate() error { testPath := filepath.Join(t.HubTestPath, t.Name) if _, err := os.Stat(testPath); os.IsNotExist(err) { return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath) } - currentDir, err := os.Getwd() + if err := os.Chdir(testPath); err != nil { + return fmt.Errorf("can't 'cd' to '%s': %s", testPath, err) + } + + //machine add + cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"} + cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...) + + output, err := cscliRegisterCmd.CombinedOutput() + if err != nil { + if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") { + fmt.Println(string(output)) + return fmt.Errorf("fail to run '%s' for test '%s': %v", cscliRegisterCmd.String(), t.Name, err) + } + } + + //hardcode bouncer key + cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "bouncers", "add", "appsectests", "-k", TestBouncerApiKey} + cscliBouncerCmd := exec.Command(t.CscliPath, cmdArgs...) + + output, err = cscliBouncerCmd.CombinedOutput() + if err != nil { + if !strings.Contains(string(output), "unable to create bouncer: bouncer appsectests already exists") { + fmt.Println(string(output)) + return fmt.Errorf("fail to run '%s' for test '%s': %v", cscliRegisterCmd.String(), t.Name, err) + } + } + + //start crowdsec service + cmdArgs = []string{"-c", t.RuntimeConfigFilePath} + crowdsecDaemon := exec.Command(t.CrowdSecPath, cmdArgs...) + + crowdsecDaemon.Start() + + //wait for the appsec port to be available + if _, err := IsAlive(DefaultAppsecHost); err != nil { + return fmt.Errorf("appsec is down: %s", err) + } + + // check if the target is available + nucleiTargetParsedURL, err := url.Parse(DefaultNucleiTarget) + if err != nil { + return fmt.Errorf("unable to parse target '%s': %s", DefaultNucleiTarget, err) + } + nucleiTargetHost := nucleiTargetParsedURL.Host + if _, err := IsAlive(nucleiTargetHost); err != nil { + return fmt.Errorf("target is down: %s", err) + } + + nucleiConfig := NucleiConfig{ + Path: "nuclei", + OutputDir: t.RuntimePath, + CmdLineOptions: []string{"-ev", //allow variables from environment + "-nc", //no colors in output + "-dresp", //dump response + "-j", //json output + }, + } + + err = nucleiConfig.RunNucleiTemplate(t.Name, t.Config.NucleiTemplate, DefaultNucleiTarget) + crowdsecLogFile := fmt.Sprintf("%s/log/crowdsec.log", nucleiConfig.OutputDir) + if t.Config.ExpectedNucleiFailure { + if err != nil && errors.Is(err, NucleiTemplateFail) { + log.Infof("Appsec test %s failed as expected", t.Name) + t.Success = true + } else { + log.Errorf("Appsec test %s failed: %s", t.Name, err) + crowdsecLog, err := os.ReadFile(crowdsecLogFile) + if err != nil { + log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err) + } else { + log.Errorf("crowdsec log file '%s'", crowdsecLogFile) + log.Errorf("%s\n", string(crowdsecLog)) + } + } + } else { + if err == nil { + log.Infof("Appsec test %s succeeded", t.Name) + t.Success = true + } else { + log.Errorf("Appsec test %s failed: %s", t.Name, err) + crowdsecLog, err := os.ReadFile(crowdsecLogFile) + if err != nil { + log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err) + } else { + log.Errorf("crowdsec log file '%s'", crowdsecLogFile) + log.Errorf("%s\n", string(crowdsecLog)) + } + } + } + crowdsecDaemon.Process.Kill() + return nil +} + +func (t *HubTestItem) RunWithLogFile() error { + testPath := filepath.Join(t.HubTestPath, t.Name) + if _, err := os.Stat(testPath); os.IsNotExist(err) { + return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath) + } + + currentDir, err := os.Getwd() //xx if err != nil { return fmt.Errorf("can't get current directory: %+v", err) } @@ -650,3 +837,92 @@ func (t *HubTestItem) Run() error { return nil } + +func (t *HubTestItem) Run() error { + var err error + t.Success = false + t.ErrorsList = make([]string, 0) + + // create runtime folder + if err = os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err) + } + + // create runtime data folder + if err = os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeDataPath, err) + } + + // create runtime hub folder + if err = os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeHubPath, err) + } + + if err = Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil { + return fmt.Errorf("unable to copy .index.json file in '%s': %s", filepath.Join(t.RuntimeHubPath, ".index.json"), err) + } + + // create results folder + if err = os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %+v", t.ResultsPath, err) + } + + // copy template config file to runtime folder + if err = Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil { + return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err) + } + + // copy template profile file to runtime folder + if err = Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil { + return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err) + } + + // copy template simulation file to runtime folder + if err = Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil { + return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err) + } + + crowdsecPatternsFolder := csconfig.DefaultConfigPath("patterns") + + // copy template patterns folder to runtime folder + if err = CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil { + return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err) + } + + // create the appsec-configs dir + if err = os.MkdirAll(filepath.Join(t.RuntimePath, "appsec-configs"), os.ModePerm); err != nil { + return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err) + } + + //if it's an appsec rule test, we need acquis and appsec profile + if len(t.Config.AppsecRules) > 0 { + // copy template acquis file to runtime folder + log.Infof("copying %s to %s", t.TemplateAcquisPath, t.RuntimeAcquisFilePath) + if err = Copy(t.TemplateAcquisPath, t.RuntimeAcquisFilePath); err != nil { + return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateAcquisPath, t.RuntimeAcquisFilePath, err) + } + + log.Infof("copying %s to %s", t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml")) + // copy template appsec-config file to runtime folder + if err = Copy(t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml")); err != nil { + return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateAppsecProfilePath, filepath.Join(t.RuntimePath, "appsec-configs", "config.yaml"), err) + } + } else { //otherwise we drop a blank acquis file + if err = os.WriteFile(t.RuntimeAcquisFilePath, []byte(""), os.ModePerm); err != nil { + return fmt.Errorf("unable to write blank acquis file '%s': %s", t.RuntimeAcquisFilePath, err) + } + } + + // install the hub in the runtime folder + if err = t.InstallHub(); err != nil { + return fmt.Errorf("unable to install hub in '%s': %s", t.RuntimeHubPath, err) + } + + if t.Config.LogFile != "" { + return t.RunWithLogFile() + } else if t.Config.NucleiTemplate != "" { + return t.RunWithNucleiTemplate() + } else { + return fmt.Errorf("log file or nuclei template must be set in '%s'", t.Name) + } +} diff --git a/pkg/hubtest/nucleirunner.go b/pkg/hubtest/nucleirunner.go new file mode 100644 index 000000000..e3c6af73c --- /dev/null +++ b/pkg/hubtest/nucleirunner.go @@ -0,0 +1,66 @@ +package hubtest + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "time" + + log "github.com/sirupsen/logrus" +) + +type NucleiConfig struct { + Path string `yaml:"nuclei_path"` + OutputDir string `yaml:"output_dir"` + CmdLineOptions []string `yaml:"cmdline_options"` +} + +var NucleiTemplateFail = errors.New("Nuclei template failed") + +func (nc *NucleiConfig) RunNucleiTemplate(testName string, templatePath string, target string) error { + tstamp := time.Now().Unix() + //templatePath is the full path to the template, we just want the name ie. "sqli-random-test" + tmp := strings.Split(templatePath, "/") + template := strings.Split(tmp[len(tmp)-1], ".")[0] + + outputPrefix := fmt.Sprintf("%s/%s_%s-%d", nc.OutputDir, testName, template, tstamp) + + args := []string{ + "-u", target, + "-t", templatePath, + "-o", outputPrefix + ".json", + } + args = append(args, nc.CmdLineOptions...) + cmd := exec.Command(nc.Path, args...) + + var out bytes.Buffer + var outErr bytes.Buffer + + cmd.Stdout = &out + cmd.Stderr = &outErr + + err := cmd.Run() + + if err := os.WriteFile(outputPrefix+"_stdout.txt", out.Bytes(), 0644); err != nil { + log.Warningf("Error writing stdout: %s", err) + } + + if err := os.WriteFile(outputPrefix+"_stderr.txt", outErr.Bytes(), 0644); err != nil { + log.Warningf("Error writing stderr: %s", err) + } + + if err != nil { + log.Warningf("Error running nuclei: %s", err) + log.Warningf("Stdout saved to %s", outputPrefix+"_stdout.txt") + log.Warningf("Stderr saved to %s", outputPrefix+"_stderr.txt") + log.Warningf("Nuclei generated output saved to %s", outputPrefix+".json") + return err + } else if len(out.String()) == 0 { + //No stdout means no finding, it means our test failed + return NucleiTemplateFail + } + return nil +} diff --git a/pkg/hubtest/utils.go b/pkg/hubtest/utils.go index 090f1f85e..5f816fbe6 100644 --- a/pkg/hubtest/utils.go +++ b/pkg/hubtest/utils.go @@ -2,9 +2,13 @@ package hubtest import ( "fmt" + "net" "os" "path/filepath" "sort" + "time" + + log "github.com/sirupsen/logrus" ) func sortedMapKeys[V any](m map[string]V) []string { @@ -106,3 +110,19 @@ func CopyDir(src string, dest string) error { return nil } + +func IsAlive(target string) (bool, error) { + start := time.Now() + for { + conn, err := net.Dial("tcp", target) + if err == nil { + log.Debugf("appsec is up after %s", time.Since(start)) + conn.Close() + return true, nil + } + time.Sleep(500 * time.Millisecond) + if time.Since(start) > 10*time.Second { + return false, fmt.Errorf("took more than 10s for %s to be available", target) + } + } +} diff --git a/pkg/parser/node.go b/pkg/parser/node.go index f1d2253ad..23ed20511 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -214,6 +214,10 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri switch out := output.(type) { case string: gstr = out + case int: + gstr = fmt.Sprintf("%d", out) + case float64, float32: + gstr = fmt.Sprintf("%f", out) default: clog.Errorf("unexpected return type for RunTimeValue : %T", output) } diff --git a/pkg/parser/runtime.go b/pkg/parser/runtime.go index d2b85acd4..e1b33bc6e 100644 --- a/pkg/parser/runtime.go +++ b/pkg/parser/runtime.go @@ -127,6 +127,8 @@ func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error { value = out case int: value = strconv.Itoa(out) + case float64, float32: + value = fmt.Sprintf("%f", out) case map[string]interface{}: clog.Warnf("Expression '%s' returned a map, please use ToJsonString() to convert it to string if you want to keep it as is, or refine your expression to extract a string", static.ExpValue) case []interface{}: @@ -134,7 +136,7 @@ func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error { case nil: clog.Debugf("Expression '%s' returned nil, skipping", static.ExpValue) default: - clog.Errorf("unexpected return type for RunTimeValue : %T", output) + clog.Errorf("unexpected return type for '%s' : %T", static.ExpValue, output) return errors.New("unexpected return type for RunTimeValue") } } diff --git a/pkg/types/appsec_event.go b/pkg/types/appsec_event.go new file mode 100644 index 000000000..4cd5d8f58 --- /dev/null +++ b/pkg/types/appsec_event.go @@ -0,0 +1,240 @@ +package types + +import ( + "regexp" + + log "github.com/sirupsen/logrus" +) + +/* + 1. If user triggered a rule that is for a CVE, that has high confidence and that is blocking, ban + 2. If user triggered 3 distinct rules with medium confidence across 3 different requests, ban + + +any(evt.Waf.ByTag("CVE"), {.confidence == "high" && .action == "block"}) + +len(evt.Waf.ByTagRx("*CVE*").ByConfidence("high").ByAction("block")) > 1 + +*/ + +type MatchedRules []map[string]interface{} + +type AppsecEvent struct { + HasInBandMatches, HasOutBandMatches bool + MatchedRules + Vars map[string]string +} +type Field string + +func (f Field) String() string { + return string(f) +} + +const ( + ID Field = "id" + RuleType Field = "rule_type" + Tags Field = "tags" + File Field = "file" + Confidence Field = "confidence" + Revision Field = "revision" + SecMark Field = "secmark" + Accuracy Field = "accuracy" + Msg Field = "msg" + Severity Field = "severity" + Kind Field = "kind" +) + +func (w AppsecEvent) GetVar(varName string) string { + if w.Vars == nil { + return "" + } + if val, ok := w.Vars[varName]; ok { + return val + } + log.Infof("var %s not found. Available variables: %+v", varName, w.Vars) + return "" + +} + +// getters +func (w MatchedRules) GetField(field Field) []interface{} { + ret := make([]interface{}, 0) + for _, rule := range w { + ret = append(ret, rule[field.String()]) + } + return ret +} + +func (w MatchedRules) GetURI() string { + for _, rule := range w { + return rule["uri"].(string) + } + return "" +} + +func (w MatchedRules) GetHash() string { + for _, rule := range w { + //@sbl : let's fix this + return rule["hash"].(string) + } + return "" +} + +func (w MatchedRules) GetVersion() string { + for _, rule := range w { + //@sbl : let's fix this + return rule["version"].(string) + } + return "" +} + +func (w MatchedRules) GetName() string { + for _, rule := range w { + //@sbl : let's fix this + return rule["name"].(string) + } + return "" +} + +func (w MatchedRules) GetMethod() string { + for _, rule := range w { + return rule["method"].(string) + } + return "" +} + +func (w MatchedRules) GetRuleIDs() []int { + ret := make([]int, 0) + for _, rule := range w { + ret = append(ret, rule["id"].(int)) + } + return ret +} + +func (w MatchedRules) Kinds() []string { + ret := make([]string, 0) + for _, rule := range w { + exists := false + for _, val := range ret { + if val == rule["kind"] { + exists = true + break + } + } + if !exists { + ret = append(ret, rule["kind"].(string)) + } + } + return ret +} + +func (w MatchedRules) GetMatchedZones() []string { + ret := make([]string, 0) + + for _, rule := range w { + ret = append(ret, rule["matched_zones"].([]string)...) + } + return ret +} + +// filters +func (w MatchedRules) ByID(id int) MatchedRules { + ret := MatchedRules{} + + for _, rule := range w { + if rule["id"] == id { + ret = append(ret, rule) + } + } + return ret +} + +func (w MatchedRules) ByKind(kind string) MatchedRules { + ret := MatchedRules{} + for _, rule := range w { + if rule["kind"] == kind { + ret = append(ret, rule) + } + } + return ret +} + +func (w MatchedRules) ByTags(match []string) MatchedRules { + ret := MatchedRules{} + for _, rule := range w { + for _, tag := range rule["tags"].([]string) { + for _, match_tag := range match { + if tag == match_tag { + ret = append(ret, rule) + break + } + } + } + } + return ret +} + +func (w MatchedRules) ByTag(match string) MatchedRules { + ret := MatchedRules{} + for _, rule := range w { + for _, tag := range rule["tags"].([]string) { + if tag == match { + ret = append(ret, rule) + break + } + } + } + return ret +} + +func (w MatchedRules) ByTagRx(rx string) MatchedRules { + ret := MatchedRules{} + re := regexp.MustCompile(rx) + if re == nil { + return ret + } + for _, rule := range w { + for _, tag := range rule["tags"].([]string) { + log.Debugf("ByTagRx: %s = %s -> %t", rx, tag, re.MatchString(tag)) + if re.MatchString(tag) { + ret = append(ret, rule) + break + } + } + } + return ret +} + +func (w MatchedRules) ByDisruptiveness(is bool) MatchedRules { + ret := MatchedRules{} + for _, rule := range w { + if rule["disruptive"] == is { + ret = append(ret, rule) + } + } + log.Debugf("ByDisruptiveness(%t) -> %d", is, len(ret)) + + return ret +} + +func (w MatchedRules) BySeverity(severity string) MatchedRules { + ret := MatchedRules{} + for _, rule := range w { + if rule["severity"] == severity { + ret = append(ret, rule) + } + } + log.Debugf("BySeverity(%s) -> %d", severity, len(ret)) + return ret +} + +func (w MatchedRules) ByAccuracy(accuracy string) MatchedRules { + ret := MatchedRules{} + for _, rule := range w { + if rule["accuracy"] == accuracy { + ret = append(ret, rule) + } + } + log.Debugf("ByAccuracy(%s) -> %d", accuracy, len(ret)) + return ret +} diff --git a/pkg/types/event.go b/pkg/types/event.go index 622d1d8bc..074241918 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -13,6 +13,7 @@ import ( const ( LOG = iota OVFLW + APPSEC ) // Event is the structure representing a runtime event (log or overflow) @@ -40,6 +41,7 @@ type Event struct { StrTimeFormat string `yaml:"StrTimeFormat,omitempty" json:"StrTimeFormat,omitempty"` MarshaledTime string `yaml:"MarshaledTime,omitempty" json:"MarshaledTime,omitempty"` Process bool `yaml:"Process,omitempty" json:"Process,omitempty"` //can be set to false to avoid processing line + Appsec AppsecEvent `yaml:"Appsec,omitempty" json:"Appsec,omitempty"` /* Meta is the only part that will make it to the API - it should be normalized */ Meta map[string]string `yaml:"Meta,omitempty" json:"Meta,omitempty"` } diff --git a/test/bats/20_hub.bats b/test/bats/20_hub.bats index f64a64e4b..0b222dde0 100644 --- a/test/bats/20_hub.bats +++ b/test/bats/20_hub.bats @@ -36,7 +36,7 @@ teardown() { rune -0 cscli hub list assert_output "No items to display" rune -0 cscli hub list -o json - assert_json '{parsers:[],scenarios:[],collections:[],postoverflows:[]}' + assert_json '{"appsec-configs":[],"appsec-rules":[],parsers:[],scenarios:[],collections:[],postoverflows:[]}' rune -0 cscli hub list -o raw assert_output 'name,status,version,description,type' @@ -137,7 +137,7 @@ teardown() { assert_line "collections" rune -0 cscli hub types -o human rune -0 yq -o json <(output) - assert_json '["parsers","postoverflows","scenarios","collections"]' + assert_json '["parsers","postoverflows","scenarios","appsec-configs","appsec-rules","collections"]' rune -0 cscli hub types -o json - assert_json '["parsers","postoverflows","scenarios","collections"]' + assert_json '["parsers","postoverflows","scenarios","appsec-configs","appsec-rules","collections"]' } diff --git a/test/lib/bats-assert b/test/lib/bats-assert index 78fa631d1..44913ffe6 160000 --- a/test/lib/bats-assert +++ b/test/lib/bats-assert @@ -1 +1 @@ -Subproject commit 78fa631d1370562d2cd4a1390989e706158e7bf0 +Subproject commit 44913ffe6020d1561c4c4d1e26cda8e07a1f374f