Aggregate WAF rules into a single event (#2350)

pkg/acquisition/modules/waf/utils.go

@@ -1,42 +1,18 @@
 package wafacquisition
 
 import (
-	"encoding/json"
 	"fmt"
-	"strings"
 	"time"
 
-	corazatypes "github.com/corazawaf/coraza/v3/types"
-	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/corazawaf/coraza/v3/experimental"
+	types "github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/crowdsecurity/crowdsec/pkg/waf"
-	"github.com/pkg/errors"
+	"github.com/davecgh/go-spew/spew"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 )
 
-func TxToEvents(r waf.ParsedRequest, kind string) ([]types.Event, error) {
-	evts := []types.Event{}
-	if r.Tx == nil {
-		return nil, fmt.Errorf("tx is nil")
-	}
-	for _, rule := range r.Tx.MatchedRules() {
-		//log.Printf("rule %d", idx)
-		if rule.Message() == "" {
-			continue
-		}
-		WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind}).Inc()
-		evt, err := RuleMatchToEvent(rule, r.Tx, r, kind)
-		if err != nil {
-			return nil, errors.Wrap(err, "Cannot convert rule match to event")
-		}
-		evts = append(evts, evt)
-	}
-
-	return evts, nil
-}
-
-// Transforms a coraza interruption to a crowdsec event
-func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r waf.ParsedRequest, kind string) (types.Event, error) {
+func EventFromRequest(r waf.ParsedRequest) (types.Event, error) {
 	evt := types.Event{}
 	//we might want to change this based on in-band vs out-of-band ?
 	evt.Type = types.LOG
@@ -44,39 +20,12 @@ func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction,
 	//def needs fixing
 	evt.Stage = "s00-raw"
 	evt.Process = true
-	log.WithFields(log.Fields{
-		"module": "waf",
-		"source": rule.ClientIPAddress(),
-		"id":     rule.Rule().ID(),
-	}).Infof("%s", rule.Message())
-	//we build a big-ass object that is going to be marshaled in line.raw and unmarshaled later.
-	//why ? because it's more consistent with the other data-sources etc. and it provides users with flexibility to alter our parsers
-	CorazaEvent := map[string]interface{}{
-		//core rule info
-		"rule_type": kind,
-		"rule_id":   rule.Rule().ID(),
-		//"rule_action":     tx.Interruption().Action,
-		"rule_disruptive": rule.Disruptive(),
-		"rule_tags":       rule.Rule().Tags(),
-		"rule_file":       rule.Rule().File(),
-		"rule_file_line":  rule.Rule().Line(),
-		"rule_revision":   rule.Rule().Revision(),
-		"rule_secmark":    rule.Rule().SecMark(),
-		"rule_accuracy":   rule.Rule().Accuracy(),
-
-		//http contextual infos
-		"upstream_addr": r.RemoteAddr,
-		"req_uuid":      tx.ID(),
-		"source_ip":     strings.Split(rule.ClientIPAddress(), ":")[0],
-		"uri":           rule.URI(),
-	}
-
-	if tx.Interruption() != nil {
-		CorazaEvent["rule_action"] = tx.Interruption().Action
-	}
-	corazaEventB, err := json.Marshal(CorazaEvent)
-	if err != nil {
-		return evt, fmt.Errorf("Unable to marshal coraza alert: %w", err)
+	evt.Parsed = map[string]string{
+		"source_ip":   r.ClientIP,
+		"target_host": r.Host,
+		"target_uri":  r.URI,
+		"method":      r.Method,
+		"req_uuid":    r.Tx.ID(),
 	}
 	evt.Line = types.Line{
 		Time: time.Now(),
@@ -85,8 +34,57 @@ func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction,
 		Process: true,
 		Module:  "waf",
 		Src:     "waf",
-		Raw:     string(corazaEventB),
+		Raw:     "dummy-waf-data", //we discard empty Line.Raw items :)
 	}
+	evt.Waap = []map[string]interface{}{}
 
 	return evt, nil
 }
+
+func LogWaapEvent(evt *types.Event) {
+	log.WithFields(log.Fields{
+		"module":     "waf",
+		"source":     evt.Parsed["source_ip"],
+		"target_uri": evt.Parsed["target_uri"],
+	}).Infof("%s triggered %d rules [%+v]", evt.Parsed["source_ip"], len(evt.Waap), evt.Waap.GetRuleIDs())
+	log.Infof("%s", evt.Waap)
+}
+
+func AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *types.Event) error {
+
+	if tx.IsInterrupted() {
+		log.Infof("interrupted() = %t", tx.IsInterrupted())
+		log.Infof("interrupted.action = %s", tx.Interruption().Action)
+		if evt.Meta == nil {
+			evt.Meta = map[string]string{}
+		}
+		evt.Meta["waap_interrupted"] = "1"
+		evt.Meta["waap_action"] = tx.Interruption().Action
+	}
+	log.Infof("TX %s", spew.Sdump(tx.MatchedRules()))
+	for _, rule := range tx.MatchedRules() {
+		if rule.Message() == "" {
+			continue
+		}
+		WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind}).Inc()
+
+		corazaRule := map[string]interface{}{
+			"id":         rule.Rule().ID(),
+			"uri":        evt.Parsed["uri"],
+			"rule_type":  kind,
+			"method":     evt.Parsed["method"],
+			"disruptive": rule.Disruptive(),
+			"tags":       rule.Rule().Tags(),
+			"file":       rule.Rule().File(),
+			"file_line":  rule.Rule().Line(),
+			"revision":   rule.Rule().Revision(),
+			"secmark":    rule.Rule().SecMark(),
+			"accuracy":   rule.Rule().Accuracy(),
+			"msg":        rule.Message(),
+			"severity":   rule.Rule().Severity().String(),
+		}
+		evt.Waap = append(evt.Waap, corazaRule)
+	}
+
+	return nil
+}

pkg/acquisition/modules/waf/waf.go

@@ -17,6 +17,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/crowdsecurity/crowdsec/pkg/waf"
 	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+	"github.com/davecgh/go-spew/spew"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
@@ -406,6 +407,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error {
 			log.Infof("Waf Runner is dying")
 			return nil
 		case request := <-r.inChan:
+			var evt *types.Event
 			WafReqCounter.With(prometheus.Labels{"source": request.RemoteAddr}).Inc()
 			//measure the time spent in the WAF
 			startParsing := time.Now()
@@ -458,6 +460,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error {
 
 			in, expTx, err := processReqWithEngine(expTx, request, InBand)
 			request.Tx = expTx
+			log.Infof("-> %s", spew.Sdump(in))
 
 			response := waf.NewResponseRequest(expTx, in, request.UUID, err)
 
@@ -510,16 +513,17 @@ func (r *WafRunner) Run(t *tomb.Tomb) error {
 			// send back the result to the HTTP handler for the InBand part
 			request.ResponseChannel <- response
 			if in != nil && response.SendEvents {
-				// Generate the events for InBand channel
-				events, err := TxToEvents(request, InBand)
+				evt = &types.Event{}
+				*evt, err = EventFromRequest(request)
 				if err != nil {
-					log.Errorf("Cannot convert transaction to events : %s", err)
-					continue
+					return fmt.Errorf("cannot create event from waap context : %w", err)
 				}
-
-				for _, evt := range events {
-					r.outChan <- evt
+				err = AccumulateTxToEvent(expTx, InBand, evt)
+				if err != nil {
+					return fmt.Errorf("cannot convert transaction to event : %w", err)
 				}
+				LogWaapEvent(evt)
+				r.outChan <- *evt
 			}
 
 			outBandStart := time.Now()
@@ -533,15 +537,21 @@ func (r *WafRunner) Run(t *tomb.Tomb) error {
 			}
 			request.Tx = expTx
 			if expTx != nil && len(expTx.MatchedRules()) > 0 {
-				events, err := TxToEvents(request, OutOfBand)
-				log.Infof("Request triggered by WAF, %d events to send", len(events))
-				for _, evt := range events {
-					r.outChan <- evt
+				//if event was not instantiated after inband processing, do it now
+				if evt == nil {
+					*evt, err = EventFromRequest(request)
+					if err != nil {
+						return fmt.Errorf("cannot create event from waap context : %w", err)
+					}
 				}
+
+				err = AccumulateTxToEvent(expTx, InBand, evt)
 				if err != nil {
-					log.Errorf("Cannot convert transaction to events : %s", err)
-					continue
+					return fmt.Errorf("cannot convert transaction to event : %w", err)
 				}
+				LogWaapEvent(evt)
+				r.outChan <- *evt
+
 			}
 			//measure the full time spent in the WAF
 			totalElapsed := time.Since(startParsing)

pkg/types/event.go

@@ -1,6 +1,7 @@
 package types
 
 import (
+	"regexp"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -14,6 +15,133 @@ const (
 	OVFLW
 )
 
+/*
+ 1. If user triggered a rule that is for a CVE, that has high confidence and that is blocking, ban
+ 2. If user triggered 3 distinct rules with medium confidence accross 3 different requests, ban
+
+
+any(evt.Waf.ByTag("CVE"), {.confidence == "high" && .action == "block"})
+
+len(evt.Waf.ByTagRx("*CVE*").ByConfidence("high").ByAction("block")) > 1
+
+*/
+
+type WaapEvent []map[string]interface{}
+
+func (w WaapEvent) ByID(id int) WaapEvent {
+	waap := WaapEvent{}
+
+	for _, rule := range w {
+		if rule["id"] == id {
+			waap = append(waap, rule)
+		}
+	}
+	return waap
+}
+
+func (w WaapEvent) GetURI() string {
+	for _, rule := range w {
+		return rule["uri"].(string)
+	}
+	return ""
+}
+
+func (w WaapEvent) GetMethod() string {
+	for _, rule := range w {
+		return rule["method"].(string)
+	}
+	return ""
+}
+
+func (w WaapEvent) GetRuleIDs() []int {
+	ret := make([]int, 0)
+	for _, rule := range w {
+		ret = append(ret, rule["id"].(int))
+	}
+	return ret
+}
+
+func (w WaapEvent) ByKind(kind string) WaapEvent {
+	waap := WaapEvent{}
+	for _, rule := range w {
+		if rule["kind"] == kind {
+			waap = append(waap, rule)
+		}
+	}
+	return waap
+}
+
+func (w WaapEvent) Kinds() []string {
+	ret := make([]string, 0)
+	for _, rule := range w {
+		exists := false
+		for _, val := range ret {
+			if val == rule["kind"] {
+				exists = true
+				break
+			}
+		}
+		if !exists {
+			ret = append(ret, rule["kind"].(string))
+		}
+	}
+	return ret
+}
+
+func (w WaapEvent) ByTag(match string) WaapEvent {
+	waap := WaapEvent{}
+	for _, rule := range w {
+		for _, tag := range rule["tags"].([]string) {
+			if tag == match {
+				waap = append(waap, rule)
+				break
+			}
+		}
+	}
+	return waap
+}
+
+func (w WaapEvent) ByTagRx(rx string) WaapEvent {
+	waap := WaapEvent{}
+	re := regexp.MustCompile(rx)
+	if re == nil {
+		return waap
+	}
+	for _, rule := range w {
+		for _, tag := range rule["tags"].([]string) {
+			if re.MatchString(tag) {
+				waap = append(waap, rule)
+				break
+			}
+		}
+	}
+	return waap
+}
+
+func (w WaapEvent) ByDisruptiveness(is bool) WaapEvent {
+	log.Infof("%s", w)
+	wap := WaapEvent{}
+	for _, rule := range w {
+		if rule["disruptive"] == is {
+			wap = append(wap, rule)
+		}
+	}
+	log.Infof("ByDisruptiveness(%t) -> %d", is, len(wap))
+
+	return wap
+}
+
+func (w WaapEvent) BySeverity(severity string) WaapEvent {
+	wap := WaapEvent{}
+	for _, rule := range w {
+		if rule["severity"] == severity {
+			wap = append(wap, rule)
+		}
+	}
+	log.Infof("BySeverity(%t) -> %d", severity, len(wap))
+	return wap
+}
+
 // Event is the structure representing a runtime event (log or overflow)
 type Event struct {
 	/* is it a log or an overflow */
@@ -39,6 +167,7 @@ type Event struct {
 	StrTimeFormat string       `yaml:"StrTimeFormat,omitempty" json:"StrTimeFormat,omitempty"`
 	MarshaledTime string       `yaml:"MarshaledTime,omitempty" json:"MarshaledTime,omitempty"`
 	Process       bool         `yaml:"Process,omitempty" json:"Process,omitempty"` //can be set to false to avoid processing line
+	Waap          WaapEvent    `yaml:"Waap,omitempty" json:"Waap,omitempty"`
 	/* Meta is the only part that will make it to the API - it should be normalized */
 	Meta map[string]string `yaml:"Meta,omitempty" json:"Meta,omitempty"`
 }