diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index a6a5b534f..988e92fa3 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -1,42 +1,18 @@ package wafacquisition import ( - "encoding/json" "fmt" - "strings" "time" - corazatypes "github.com/corazawaf/coraza/v3/types" - "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/corazawaf/coraza/v3/experimental" + types "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" - "github.com/pkg/errors" + "github.com/davecgh/go-spew/spew" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) -func TxToEvents(r waf.ParsedRequest, kind string) ([]types.Event, error) { - evts := []types.Event{} - if r.Tx == nil { - return nil, fmt.Errorf("tx is nil") - } - for _, rule := range r.Tx.MatchedRules() { - //log.Printf("rule %d", idx) - if rule.Message() == "" { - continue - } - WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind}).Inc() - evt, err := RuleMatchToEvent(rule, r.Tx, r, kind) - if err != nil { - return nil, errors.Wrap(err, "Cannot convert rule match to event") - } - evts = append(evts, evt) - } - - return evts, nil -} - -// Transforms a coraza interruption to a crowdsec event -func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r waf.ParsedRequest, kind string) (types.Event, error) { +func EventFromRequest(r waf.ParsedRequest) (types.Event, error) { evt := types.Event{} //we might want to change this based on in-band vs out-of-band ? evt.Type = types.LOG @@ -44,39 +20,12 @@ func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, //def needs fixing evt.Stage = "s00-raw" evt.Process = true - log.WithFields(log.Fields{ - "module": "waf", - "source": rule.ClientIPAddress(), - "id": rule.Rule().ID(), - }).Infof("%s", rule.Message()) - //we build a big-ass object that is going to be marshaled in line.raw and unmarshaled later. - //why ? because it's more consistent with the other data-sources etc. and it provides users with flexibility to alter our parsers - CorazaEvent := map[string]interface{}{ - //core rule info - "rule_type": kind, - "rule_id": rule.Rule().ID(), - //"rule_action": tx.Interruption().Action, - "rule_disruptive": rule.Disruptive(), - "rule_tags": rule.Rule().Tags(), - "rule_file": rule.Rule().File(), - "rule_file_line": rule.Rule().Line(), - "rule_revision": rule.Rule().Revision(), - "rule_secmark": rule.Rule().SecMark(), - "rule_accuracy": rule.Rule().Accuracy(), - - //http contextual infos - "upstream_addr": r.RemoteAddr, - "req_uuid": tx.ID(), - "source_ip": strings.Split(rule.ClientIPAddress(), ":")[0], - "uri": rule.URI(), - } - - if tx.Interruption() != nil { - CorazaEvent["rule_action"] = tx.Interruption().Action - } - corazaEventB, err := json.Marshal(CorazaEvent) - if err != nil { - return evt, fmt.Errorf("Unable to marshal coraza alert: %w", err) + evt.Parsed = map[string]string{ + "source_ip": r.ClientIP, + "target_host": r.Host, + "target_uri": r.URI, + "method": r.Method, + "req_uuid": r.Tx.ID(), } evt.Line = types.Line{ Time: time.Now(), @@ -85,8 +34,57 @@ func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, Process: true, Module: "waf", Src: "waf", - Raw: string(corazaEventB), + Raw: "dummy-waf-data", //we discard empty Line.Raw items :) } + evt.Waap = []map[string]interface{}{} return evt, nil } + +func LogWaapEvent(evt *types.Event) { + log.WithFields(log.Fields{ + "module": "waf", + "source": evt.Parsed["source_ip"], + "target_uri": evt.Parsed["target_uri"], + }).Infof("%s triggered %d rules [%+v]", evt.Parsed["source_ip"], len(evt.Waap), evt.Waap.GetRuleIDs()) + log.Infof("%s", evt.Waap) +} + +func AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *types.Event) error { + + if tx.IsInterrupted() { + log.Infof("interrupted() = %t", tx.IsInterrupted()) + log.Infof("interrupted.action = %s", tx.Interruption().Action) + if evt.Meta == nil { + evt.Meta = map[string]string{} + } + evt.Meta["waap_interrupted"] = "1" + evt.Meta["waap_action"] = tx.Interruption().Action + } + log.Infof("TX %s", spew.Sdump(tx.MatchedRules())) + for _, rule := range tx.MatchedRules() { + if rule.Message() == "" { + continue + } + WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind}).Inc() + + corazaRule := map[string]interface{}{ + "id": rule.Rule().ID(), + "uri": evt.Parsed["uri"], + "rule_type": kind, + "method": evt.Parsed["method"], + "disruptive": rule.Disruptive(), + "tags": rule.Rule().Tags(), + "file": rule.Rule().File(), + "file_line": rule.Rule().Line(), + "revision": rule.Rule().Revision(), + "secmark": rule.Rule().SecMark(), + "accuracy": rule.Rule().Accuracy(), + "msg": rule.Message(), + "severity": rule.Rule().Severity().String(), + } + evt.Waap = append(evt.Waap, corazaRule) + } + + return nil +} diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 51120d8cf..ea314b936 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -17,6 +17,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/davecgh/go-spew/spew" "github.com/google/uuid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -406,6 +407,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { log.Infof("Waf Runner is dying") return nil case request := <-r.inChan: + var evt *types.Event WafReqCounter.With(prometheus.Labels{"source": request.RemoteAddr}).Inc() //measure the time spent in the WAF startParsing := time.Now() @@ -458,6 +460,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { in, expTx, err := processReqWithEngine(expTx, request, InBand) request.Tx = expTx + log.Infof("-> %s", spew.Sdump(in)) response := waf.NewResponseRequest(expTx, in, request.UUID, err) @@ -510,16 +513,17 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { // send back the result to the HTTP handler for the InBand part request.ResponseChannel <- response if in != nil && response.SendEvents { - // Generate the events for InBand channel - events, err := TxToEvents(request, InBand) + evt = &types.Event{} + *evt, err = EventFromRequest(request) if err != nil { - log.Errorf("Cannot convert transaction to events : %s", err) - continue + return fmt.Errorf("cannot create event from waap context : %w", err) } - - for _, evt := range events { - r.outChan <- evt + err = AccumulateTxToEvent(expTx, InBand, evt) + if err != nil { + return fmt.Errorf("cannot convert transaction to event : %w", err) } + LogWaapEvent(evt) + r.outChan <- *evt } outBandStart := time.Now() @@ -533,15 +537,21 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { } request.Tx = expTx if expTx != nil && len(expTx.MatchedRules()) > 0 { - events, err := TxToEvents(request, OutOfBand) - log.Infof("Request triggered by WAF, %d events to send", len(events)) - for _, evt := range events { - r.outChan <- evt + //if event was not instantiated after inband processing, do it now + if evt == nil { + *evt, err = EventFromRequest(request) + if err != nil { + return fmt.Errorf("cannot create event from waap context : %w", err) + } } + + err = AccumulateTxToEvent(expTx, InBand, evt) if err != nil { - log.Errorf("Cannot convert transaction to events : %s", err) - continue + return fmt.Errorf("cannot convert transaction to event : %w", err) } + LogWaapEvent(evt) + r.outChan <- *evt + } //measure the full time spent in the WAF totalElapsed := time.Since(startParsing) diff --git a/pkg/types/event.go b/pkg/types/event.go index fc8d966ab..7dc957996 100644 --- a/pkg/types/event.go +++ b/pkg/types/event.go @@ -1,6 +1,7 @@ package types import ( + "regexp" "time" log "github.com/sirupsen/logrus" @@ -14,6 +15,133 @@ const ( OVFLW ) +/* + 1. If user triggered a rule that is for a CVE, that has high confidence and that is blocking, ban + 2. If user triggered 3 distinct rules with medium confidence accross 3 different requests, ban + + +any(evt.Waf.ByTag("CVE"), {.confidence == "high" && .action == "block"}) + +len(evt.Waf.ByTagRx("*CVE*").ByConfidence("high").ByAction("block")) > 1 + +*/ + +type WaapEvent []map[string]interface{} + +func (w WaapEvent) ByID(id int) WaapEvent { + waap := WaapEvent{} + + for _, rule := range w { + if rule["id"] == id { + waap = append(waap, rule) + } + } + return waap +} + +func (w WaapEvent) GetURI() string { + for _, rule := range w { + return rule["uri"].(string) + } + return "" +} + +func (w WaapEvent) GetMethod() string { + for _, rule := range w { + return rule["method"].(string) + } + return "" +} + +func (w WaapEvent) GetRuleIDs() []int { + ret := make([]int, 0) + for _, rule := range w { + ret = append(ret, rule["id"].(int)) + } + return ret +} + +func (w WaapEvent) ByKind(kind string) WaapEvent { + waap := WaapEvent{} + for _, rule := range w { + if rule["kind"] == kind { + waap = append(waap, rule) + } + } + return waap +} + +func (w WaapEvent) Kinds() []string { + ret := make([]string, 0) + for _, rule := range w { + exists := false + for _, val := range ret { + if val == rule["kind"] { + exists = true + break + } + } + if !exists { + ret = append(ret, rule["kind"].(string)) + } + } + return ret +} + +func (w WaapEvent) ByTag(match string) WaapEvent { + waap := WaapEvent{} + for _, rule := range w { + for _, tag := range rule["tags"].([]string) { + if tag == match { + waap = append(waap, rule) + break + } + } + } + return waap +} + +func (w WaapEvent) ByTagRx(rx string) WaapEvent { + waap := WaapEvent{} + re := regexp.MustCompile(rx) + if re == nil { + return waap + } + for _, rule := range w { + for _, tag := range rule["tags"].([]string) { + if re.MatchString(tag) { + waap = append(waap, rule) + break + } + } + } + return waap +} + +func (w WaapEvent) ByDisruptiveness(is bool) WaapEvent { + log.Infof("%s", w) + wap := WaapEvent{} + for _, rule := range w { + if rule["disruptive"] == is { + wap = append(wap, rule) + } + } + log.Infof("ByDisruptiveness(%t) -> %d", is, len(wap)) + + return wap +} + +func (w WaapEvent) BySeverity(severity string) WaapEvent { + wap := WaapEvent{} + for _, rule := range w { + if rule["severity"] == severity { + wap = append(wap, rule) + } + } + log.Infof("BySeverity(%t) -> %d", severity, len(wap)) + return wap +} + // Event is the structure representing a runtime event (log or overflow) type Event struct { /* is it a log or an overflow */ @@ -39,6 +167,7 @@ type Event struct { StrTimeFormat string `yaml:"StrTimeFormat,omitempty" json:"StrTimeFormat,omitempty"` MarshaledTime string `yaml:"MarshaledTime,omitempty" json:"MarshaledTime,omitempty"` Process bool `yaml:"Process,omitempty" json:"Process,omitempty"` //can be set to false to avoid processing line + Waap WaapEvent `yaml:"Waap,omitempty" json:"Waap,omitempty"` /* Meta is the only part that will make it to the API - it should be normalized */ Meta map[string]string `yaml:"Meta,omitempty" json:"Meta,omitempty"` }