0ct0pu5/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Aggregate WAF rules into a single event (#2350)

Browse source
This commit is contained in:
blotus 2023-07-13 16:20:04 +02:00 • committed by GitHub
parent a6ba0e869c
commit 57547c32c9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 212 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

122
pkg/acquisition/modules/waf/utils.go
View file

@ -1,42 +1,18 @@
package wafacquisition
import (
"encoding/json"
"fmt"
"strings"
"time"
corazatypes "github.com/corazawaf/coraza/v3/types"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/corazawaf/coraza/v3/experimental"
types "github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/pkg/waf"
"github.com/pkg/errors"
"github.com/davecgh/go-spew/spew"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
)
func TxToEvents(r waf.ParsedRequest, kind string) ([]types.Event, error) {
evts := []types.Event{}
if r.Tx == nil {
return nil, fmt.Errorf("tx is nil")
}
for _, rule := range r.Tx.MatchedRules() {
//log.Printf("rule %d", idx)
if rule.Message() == "" {
continue
}
WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind}).Inc()
evt, err := RuleMatchToEvent(rule, r.Tx, r, kind)
if err != nil {
return nil, errors.Wrap(err, "Cannot convert rule match to event")
}
evts = append(evts, evt)
}
return evts, nil
}
// Transforms a coraza interruption to a crowdsec event
func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r waf.ParsedRequest, kind string) (types.Event, error) {
func EventFromRequest(r waf.ParsedRequest) (types.Event, error) {
evt := types.Event{}
//we might want to change this based on in-band vs out-of-band ?
evt.Type = types.LOG
@ -44,39 +20,12 @@ func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction,
//def needs fixing
evt.Stage = "s00-raw"
evt.Process = true
log.WithFields(log.Fields{
"module": "waf",
"source": rule.ClientIPAddress(),
"id": rule.Rule().ID(),
}).Infof("%s", rule.Message())
//we build a big-ass object that is going to be marshaled in line.raw and unmarshaled later.
//why ? because it's more consistent with the other data-sources etc. and it provides users with flexibility to alter our parsers
CorazaEvent := map[string]interface{}{
//core rule info
"rule_type": kind,
"rule_id": rule.Rule().ID(),
//"rule_action": tx.Interruption().Action,
"rule_disruptive": rule.Disruptive(),
"rule_tags": rule.Rule().Tags(),
"rule_file": rule.Rule().File(),
"rule_file_line": rule.Rule().Line(),
"rule_revision": rule.Rule().Revision(),
"rule_secmark": rule.Rule().SecMark(),
"rule_accuracy": rule.Rule().Accuracy(),
//http contextual infos
"upstream_addr": r.RemoteAddr,
"req_uuid": tx.ID(),
"source_ip": strings.Split(rule.ClientIPAddress(), ":")[0],
"uri": rule.URI(),
}
if tx.Interruption() != nil {
CorazaEvent["rule_action"] = tx.Interruption().Action
}
corazaEventB, err := json.Marshal(CorazaEvent)
if err != nil {
return evt, fmt.Errorf("Unable to marshal coraza alert: %w", err)
evt.Parsed = map[string]string{
"source_ip": r.ClientIP,
"target_host": r.Host,
"target_uri": r.URI,
"method": r.Method,
"req_uuid": r.Tx.ID(),
}
evt.Line = types.Line{
Time: time.Now(),
@ -85,8 +34,57 @@ func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction,
Process: true,
Module: "waf",
Src: "waf",
Raw: string(corazaEventB),
Raw: "dummy-waf-data", //we discard empty Line.Raw items :)
}
evt.Waap = []map[string]interface{}{}
return evt, nil
}
func LogWaapEvent(evt *types.Event) {
log.WithFields(log.Fields{
"module": "waf",
"source": evt.Parsed["source_ip"],
"target_uri": evt.Parsed["target_uri"],
}).Infof("%s triggered %d rules [%+v]", evt.Parsed["source_ip"], len(evt.Waap), evt.Waap.GetRuleIDs())
log.Infof("%s", evt.Waap)
}
func AccumulateTxToEvent(tx experimental.FullTransaction, kind string, evt *types.Event) error {
if tx.IsInterrupted() {
log.Infof("interrupted() = %t", tx.IsInterrupted())
log.Infof("interrupted.action = %s", tx.Interruption().Action)
if evt.Meta == nil {
evt.Meta = map[string]string{}
}
evt.Meta["waap_interrupted"] = "1"
evt.Meta["waap_action"] = tx.Interruption().Action
}
log.Infof("TX %s", spew.Sdump(tx.MatchedRules()))
for _, rule := range tx.MatchedRules() {
if rule.Message() == "" {
continue
}
WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind}).Inc()
corazaRule := map[string]interface{}{
"id": rule.Rule().ID(),
"uri": evt.Parsed["uri"],
"rule_type": kind,
"method": evt.Parsed["method"],
"disruptive": rule.Disruptive(),
"tags": rule.Rule().Tags(),
"file": rule.Rule().File(),
"file_line": rule.Rule().Line(),
"revision": rule.Rule().Revision(),
"secmark": rule.Rule().SecMark(),
"accuracy": rule.Rule().Accuracy(),
"msg": rule.Message(),
"severity": rule.Rule().Severity().String(),
}
evt.Waap = append(evt.Waap, corazaRule)
}
return nil
}

36
pkg/acquisition/modules/waf/waf.go
View file

@ -17,6 +17,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/pkg/waf"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/davecgh/go-spew/spew"
"github.com/google/uuid"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
@ -406,6 +407,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error {
log.Infof("Waf Runner is dying")
return nil
case request := <-r.inChan:
var evt *types.Event
WafReqCounter.With(prometheus.Labels{"source": request.RemoteAddr}).Inc()
//measure the time spent in the WAF
startParsing := time.Now()
@ -458,6 +460,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error {
in, expTx, err := processReqWithEngine(expTx, request, InBand)
request.Tx = expTx
log.Infof("-> %s", spew.Sdump(in))
response := waf.NewResponseRequest(expTx, in, request.UUID, err)
@ -510,16 +513,17 @@ func (r *WafRunner) Run(t *tomb.Tomb) error {
// send back the result to the HTTP handler for the InBand part
request.ResponseChannel <- response
if in != nil && response.SendEvents {
// Generate the events for InBand channel
events, err := TxToEvents(request, InBand)
evt = &types.Event{}
*evt, err = EventFromRequest(request)
if err != nil {
log.Errorf("Cannot convert transaction to events : %s", err)
continue
return fmt.Errorf("cannot create event from waap context : %w", err)
}
for _, evt := range events {
r.outChan <- evt
err = AccumulateTxToEvent(expTx, InBand, evt)
if err != nil {
return fmt.Errorf("cannot convert transaction to event : %w", err)
}
LogWaapEvent(evt)
r.outChan <- *evt
}
outBandStart := time.Now()
@ -533,15 +537,21 @@ func (r *WafRunner) Run(t *tomb.Tomb) error {
}
request.Tx = expTx
if expTx != nil && len(expTx.MatchedRules()) > 0 {
events, err := TxToEvents(request, OutOfBand)
log.Infof("Request triggered by WAF, %d events to send", len(events))
for _, evt := range events {
r.outChan <- evt
//if event was not instantiated after inband processing, do it now
if evt == nil {
*evt, err = EventFromRequest(request)
if err != nil {
return fmt.Errorf("cannot create event from waap context : %w", err)
}
}
err = AccumulateTxToEvent(expTx, InBand, evt)
if err != nil {
log.Errorf("Cannot convert transaction to events : %s", err)
continue
return fmt.Errorf("cannot convert transaction to event : %w", err)
}
LogWaapEvent(evt)
r.outChan <- *evt
}
//measure the full time spent in the WAF
totalElapsed := time.Since(startParsing)

129
pkg/types/event.go
View file

@ -1,6 +1,7 @@
package types
import (
"regexp"
"time"
log "github.com/sirupsen/logrus"
@ -14,6 +15,133 @@ const (
OVFLW
)
/*
1. If user triggered a rule that is for a CVE, that has high confidence and that is blocking, ban
2. If user triggered 3 distinct rules with medium confidence accross 3 different requests, ban
any(evt.Waf.ByTag("CVE"), {.confidence == "high" && .action == "block"})
len(evt.Waf.ByTagRx("*CVE*").ByConfidence("high").ByAction("block")) > 1
*/
type WaapEvent []map[string]interface{}
func (w WaapEvent) ByID(id int) WaapEvent {
waap := WaapEvent{}
for _, rule := range w {
if rule["id"] == id {
waap = append(waap, rule)
}
}
return waap
}
func (w WaapEvent) GetURI() string {
for _, rule := range w {
return rule["uri"].(string)
}
return ""
}
func (w WaapEvent) GetMethod() string {
for _, rule := range w {
return rule["method"].(string)
}
return ""
}
func (w WaapEvent) GetRuleIDs() []int {
ret := make([]int, 0)
for _, rule := range w {
ret = append(ret, rule["id"].(int))
}
return ret
}
func (w WaapEvent) ByKind(kind string) WaapEvent {
waap := WaapEvent{}
for _, rule := range w {
if rule["kind"] == kind {
waap = append(waap, rule)
}
}
return waap
}
func (w WaapEvent) Kinds() []string {
ret := make([]string, 0)
for _, rule := range w {
exists := false
for _, val := range ret {
if val == rule["kind"] {
exists = true
break
}
}
if !exists {
ret = append(ret, rule["kind"].(string))
}
}
return ret
}
func (w WaapEvent) ByTag(match string) WaapEvent {
waap := WaapEvent{}
for _, rule := range w {
for _, tag := range rule["tags"].([]string) {
if tag == match {
waap = append(waap, rule)
break
}
}
}
return waap
}
func (w WaapEvent) ByTagRx(rx string) WaapEvent {
waap := WaapEvent{}
re := regexp.MustCompile(rx)
if re == nil {
return waap
}
for _, rule := range w {
for _, tag := range rule["tags"].([]string) {
if re.MatchString(tag) {
waap = append(waap, rule)
break
}
}
}
return waap
}
func (w WaapEvent) ByDisruptiveness(is bool) WaapEvent {
log.Infof("%s", w)
wap := WaapEvent{}
for _, rule := range w {
if rule["disruptive"] == is {
wap = append(wap, rule)
}
}
log.Infof("ByDisruptiveness(%t) -> %d", is, len(wap))
return wap
}
func (w WaapEvent) BySeverity(severity string) WaapEvent {
wap := WaapEvent{}
for _, rule := range w {
if rule["severity"] == severity {
wap = append(wap, rule)
}
}
log.Infof("BySeverity(%t) -> %d", severity, len(wap))
return wap
}
// Event is the structure representing a runtime event (log or overflow)
type Event struct {
/* is it a log or an overflow */
@ -39,6 +167,7 @@ type Event struct {
StrTimeFormat string `yaml:"StrTimeFormat,omitempty" json:"StrTimeFormat,omitempty"`
MarshaledTime string `yaml:"MarshaledTime,omitempty" json:"MarshaledTime,omitempty"`
Process bool `yaml:"Process,omitempty" json:"Process,omitempty"` //can be set to false to avoid processing line
Waap WaapEvent `yaml:"Waap,omitempty" json:"Waap,omitempty"`
/* Meta is the only part that will make it to the API - it should be normalized */
Meta map[string]string `yaml:"Meta,omitempty" json:"Meta,omitempty"`
}