Minor improvements to hubtest and appsec component (#2656)

cmd/crowdsec-cli/hubtest.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"text/template"
 
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/enescakir/emoji"
@@ -100,6 +101,10 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 				return fmt.Errorf("test '%s' already exists in '%s', exiting", testName, testPath)
 			}
 
+			if isAppsecTest {
+				logType = "appsec"
+			}
+
 			if logType == "" {
 				return fmt.Errorf("please provide a type (--type) for the test")
 			}
@@ -115,17 +120,24 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 				//create empty nuclei template file
 				nucleiFileName := fmt.Sprintf("%s.yaml", testName)
 				nucleiFilePath := filepath.Join(testPath, nucleiFileName)
-				nucleiFile, err := os.Create(nucleiFilePath)
+				nucleiFile, err := os.OpenFile(nucleiFilePath, os.O_RDWR|os.O_CREATE, 0755)
 				if err != nil {
 					return err
 				}
+
+				ntpl := template.Must(template.New("nuclei").Parse(hubtest.TemplateNucleiFile))
+				if ntpl == nil {
+					return fmt.Errorf("unable to parse nuclei template")
+				}
+				ntpl.ExecuteTemplate(nucleiFile, "nuclei", struct{ TestName string }{TestName: testName})
 				nucleiFile.Close()
-				configFileData.AppsecRules = []string{"your_rule_here.yaml"}
+				configFileData.AppsecRules = []string{"./appsec-rules/<author>/your_rule_here.yaml"}
 				configFileData.NucleiTemplate = nucleiFileName
 				fmt.Println()
 				fmt.Printf("  Test name                   :  %s\n", testName)
 				fmt.Printf("  Test path                   :  %s\n", testPath)
-				fmt.Printf("  Nuclei Template             :  %s\n", nucleiFileName)
+				fmt.Printf("  Config File                 :  %s\n", configFilePath)
+				fmt.Printf("  Nuclei Template             :  %s\n", nucleiFilePath)
 			} else {
 				// create empty log file
 				logFileName := fmt.Sprintf("%s.log", testName)

go.mod

@@ -91,7 +91,7 @@ require (
 )
 
 require (
-	github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879
+	github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f
 	golang.org/x/text v0.14.0
 	gopkg.in/yaml.v3 v3.0.1
 	gotest.tools/v3 v3.5.0

go.sum

@@ -100,6 +100,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879 h1:dhAc0AelASC3BbfuLURJeai1LYgFNgpMds0KPd9whbo=
 github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI=
+github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f h1:FkOB9aDw0xzDd14pTarGRLsUNAymONq3dc7zhvsXElg=
+github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f/go.mod h1:TrU7Li+z2RHNrPy0TKJ6R65V6Yzpan2sTIRryJJyJso=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
 github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8=

pkg/acquisition/modules/appsec/appsec.go

@@ -337,7 +337,7 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) {
 	// parse the request only once
 	parsedRequest, err := appsec.NewParsedRequestFromRequest(r)
 	if err != nil {
-		log.Errorf("%s", err)
+		w.logger.Errorf("%s", err)
 		rw.WriteHeader(http.StatusInternalServerError)
 		return
 	}
@@ -358,7 +358,7 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) {
 	}
 
 	appsecResponse := w.AppsecRuntime.GenerateResponse(response, logger)
-
+	logger.Debugf("Response: %+v", appsecResponse)
 	rw.WriteHeader(appsecResponse.HTTPStatus)
 	body, err := json.Marshal(BodyResponse{Action: appsecResponse.Action})
 	if err != nil {

pkg/acquisition/modules/appsec/appsec_runner.go

@@ -13,6 +13,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/tomb.v2"
+
+	_ "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/appsec/bodyprocessors"
 )
 
 // that's the runtime structure of the Application security engine as seen from the acquis
@@ -190,6 +192,9 @@ func (r *AppsecRunner) processRequest(tx appsec.ExtendedTransaction, request *ap
 }
 
 func (r *AppsecRunner) ProcessInBandRules(request *appsec.ParsedRequest) error {
+	if len(r.AppsecRuntime.InBandRules) == 0 {
+		return nil
+	}
 	tx := appsec.NewExtendedTransaction(r.AppsecInbandEngine, request.UUID)
 	r.AppsecRuntime.InBandTx = tx
 	err := r.processRequest(tx, request)
@@ -197,7 +202,9 @@ func (r *AppsecRunner) ProcessInBandRules(request *appsec.ParsedRequest) error {
 }
 
 func (r *AppsecRunner) ProcessOutOfBandRules(request *appsec.ParsedRequest) error {
-	r.logger.Debugf("Processing out of band rules")
+	if len(r.AppsecRuntime.OutOfBandRules) == 0 {
+		return nil
+	}
 	tx := appsec.NewExtendedTransaction(r.AppsecOutbandEngine, request.UUID)
 	r.AppsecRuntime.OutOfBandTx = tx
 	err := r.processRequest(tx, request)

pkg/acquisition/modules/appsec/bodyprocessors/raw.go

@@ -0,0 +1,45 @@
+package bodyprocessors
+
+import (
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/crowdsecurity/coraza/v3/experimental/plugins"
+	"github.com/crowdsecurity/coraza/v3/experimental/plugins/plugintypes"
+)
+
+type rawBodyProcessor struct {
+}
+
+type setterInterface interface {
+	Set(string)
+}
+
+func (*rawBodyProcessor) ProcessRequest(reader io.Reader, v plugintypes.TransactionVariables, options plugintypes.BodyProcessorOptions) error {
+	buf := new(strings.Builder)
+	if _, err := io.Copy(buf, reader); err != nil {
+		return err
+	}
+
+	b := buf.String()
+
+	v.RequestBody().(setterInterface).Set(b)
+	v.RequestBodyLength().(setterInterface).Set(strconv.Itoa(len(b)))
+	return nil
+}
+
+func (*rawBodyProcessor) ProcessResponse(reader io.Reader, v plugintypes.TransactionVariables, options plugintypes.BodyProcessorOptions) error {
+	return nil
+}
+
+var (
+	_ plugintypes.BodyProcessor = &rawBodyProcessor{}
+)
+
+//nolint:gochecknoinits //Coraza recommends to use init() for registering plugins
+func init() {
+	plugins.RegisterBodyProcessor("raw", func() plugintypes.BodyProcessor {
+		return &rawBodyProcessor{}
+	})
+}

pkg/appsec/appsec_rule/modsecurity.go

@@ -15,10 +15,12 @@ var zonesMap map[string]string = map[string]string{
 	"ARGS_NAMES":      "ARGS_GET_NAMES",
 	"BODY_ARGS":       "ARGS_POST",
 	"BODY_ARGS_NAMES": "ARGS_POST_NAMES",
+	"HEADERS_NAMES":   "REQUEST_HEADERS_NAMES",
 	"HEADERS":         "REQUEST_HEADERS",
 	"METHOD":          "REQUEST_METHOD",
 	"PROTOCOL":        "REQUEST_PROTOCOL",
 	"URI":             "REQUEST_URI",
+	"RAW_BODY":        "REQUEST_BODY",
 }
 
 var transformMap map[string]string = map[string]string{
@@ -31,7 +33,7 @@ var transformMap map[string]string = map[string]string{
 
 var matchMap map[string]string = map[string]string{
 	"regex":           "@rx",
-	"equal":           "@streq",
+	"equals":          "@streq",
 	"startsWith":      "@beginsWith",
 	"endsWith":        "@endsWith",
 	"contains":        "@contains",
@@ -39,8 +41,8 @@ var matchMap map[string]string = map[string]string{
 	"libinjectionXSS": "@detectXSS",
 	"gt":              "@gt",
 	"lt":              "@lt",
-	"ge":              "@ge",
-	"le":              "@le",
+	"gte":             "@ge",
+	"lte":             "@le",
 }
 
 var bodyTypeMatch map[string]string = map[string]string{

pkg/appsec/appsec_rules_collection.go

@@ -104,7 +104,7 @@ func LoadCollection(pattern string, logger *log.Entry) ([]AppsecCollection, erro
 			for _, rule := range appsecRule.Rules {
 				strRule, rulesId, err := rule.Convert(appsec_rule.ModsecurityRuleType, appsecRule.Name)
 				if err != nil {
-					logger.Errorf("unable to convert rule %s : %s", rule.Name, err)
+					logger.Errorf("unable to convert rule %s : %s", appsecRule.Name, err)
 					return nil, err
 				}
 				logger.Debugf("Adding rule %s", strRule)

pkg/appsec/request.go

@@ -269,10 +269,10 @@ func (r *ReqDumpFilter) ToJSON() error {
 // Generate a ParsedRequest from a http.Request. ParsedRequest can be consumed by the App security Engine
 func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) {
 	var err error
-	body := make([]byte, 0)
+	body := make([]byte, r.ContentLength)
 
 	if r.Body != nil {
-		body, err = io.ReadAll(r.Body)
+		_, err = io.ReadFull(r.Body, body)
 		if err != nil {
 			return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err)
 		}

pkg/hubtest/hubtest.go

@@ -32,6 +32,27 @@ const (
 	templateProfileFile       = "template_profiles.yaml"
 	templateAcquisFile        = "template_acquis.yaml"
 	templateAppsecProfilePath = "template_appsec-profile.yaml"
+	TemplateNucleiFile        = `id: {{.TestName}}
+info:
+  name: {{.TestName}}
+  author: crowdsec
+  severity: info
+  description: {{.TestName}} testing
+  tags: appsec-testing
+http:
+#this is a dummy request, edit the request(s) to match your needs
+  - raw:
+    - |
+      GET /test HTTP/1.1
+      Host: {{"{{"}}Hostname{{"}}"}}
+
+    cookie-reuse: true
+#test will fail because we won't match http status 
+    matchers:
+    - type: status
+      status:
+       - 403
+`
 )
 
 func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecTest bool) (HubTest, error) {

pkg/hubtest/hubtest_item.go

@@ -540,6 +540,8 @@ func (t *HubTestItem) Clean() error {
 
 func (t *HubTestItem) RunWithNucleiTemplate() error {
 
+	crowdsecLogFile := fmt.Sprintf("%s/log/crowdsec.log", t.RuntimePath)
+
 	testPath := filepath.Join(t.HubTestPath, t.Name)
 	if _, err := os.Stat(testPath); os.IsNotExist(err) {
 		return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath)
@@ -550,7 +552,7 @@ func (t *HubTestItem) RunWithNucleiTemplate() error {
 	}
 
 	//machine add
-	cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"}
+	cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--force", "--auto"}
 	cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...)
 
 	output, err := cscliRegisterCmd.CombinedOutput()
@@ -581,6 +583,13 @@ func (t *HubTestItem) RunWithNucleiTemplate() error {
 
 	//wait for the appsec port to be available
 	if _, err := IsAlive(DefaultAppsecHost); err != nil {
+		crowdsecLog, err2 := os.ReadFile(crowdsecLogFile)
+		if err2 != nil {
+			log.Errorf("unable to read crowdsec log file '%s': %s", crowdsecLogFile, err)
+		} else {
+			log.Errorf("crowdsec log file '%s'", crowdsecLogFile)
+			log.Errorf("%s\n", string(crowdsecLog))
+		}
 		return fmt.Errorf("appsec is down: %s", err)
 	}
 
@@ -605,7 +614,6 @@ func (t *HubTestItem) RunWithNucleiTemplate() error {
 	}
 
 	err = nucleiConfig.RunNucleiTemplate(t.Name, t.Config.NucleiTemplate, DefaultNucleiTarget)
-	crowdsecLogFile := fmt.Sprintf("%s/log/crowdsec.log", nucleiConfig.OutputDir)
 	if t.Config.ExpectedNucleiFailure {
 		if err != nil && errors.Is(err, NucleiTemplateFail) {
 			log.Infof("Appsec test %s failed as expected", t.Name)

pkg/hubtest/nucleirunner.go

@@ -36,6 +36,8 @@ func (nc *NucleiConfig) RunNucleiTemplate(testName string, templatePath string,
 	args = append(args, nc.CmdLineOptions...)
 	cmd := exec.Command(nc.Path, args...)
 
+	log.Debugf("Running Nuclei command: '%s'", cmd.String())
+
 	var out bytes.Buffer
 	var outErr bytes.Buffer
 
@@ -59,6 +61,9 @@ func (nc *NucleiConfig) RunNucleiTemplate(testName string, templatePath string,
 		log.Warningf("Nuclei generated output saved to %s", outputPrefix+".json")
 		return err
 	} else if len(out.String()) == 0 {
+		log.Warningf("Stdout saved to %s", outputPrefix+"_stdout.txt")
+		log.Warningf("Stderr saved to %s", outputPrefix+"_stderr.txt")
+		log.Warningf("Nuclei generated output saved to %s", outputPrefix+".json")
 		//No stdout means no finding, it means our test failed
 		return NucleiTemplateFail
 	}

wizard.sh

@@ -414,6 +414,8 @@ install_crowdsec() {
     mkdir -p "${CROWDSEC_CONFIG_PATH}/postoverflows" || exit
     mkdir -p "${CROWDSEC_CONFIG_PATH}/collections" || exit
     mkdir -p "${CROWDSEC_CONFIG_PATH}/patterns" || exit
+    mkdir -p "${CROWDSEC_CONFIG_PATH}/appsec-configs" || exit
+    mkdir -p "${CROWDSEC_CONFIG_PATH}/appsec-rules" || exit
     mkdir -p "${CROWDSEC_CONSOLE_DIR}" || exit
 
     #tmp