add waf_routines

pkg/acquisition/modules/waf/utils.go

@@ -8,10 +8,12 @@ import (
 
 	corazatypes "github.com/corazawaf/coraza/v3/types"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/crowdsecurity/crowdsec/pkg/waf"
 	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
 )
 
-func TxToEvents(r ParsedRequest, kind string) ([]types.Event, error) {
+func TxToEvents(r waf.ParsedRequest, kind string) ([]types.Event, error) {
 	evts := []types.Event{}
 	if r.Tx == nil {
 		return nil, fmt.Errorf("tx is nil")
@@ -32,7 +34,7 @@ func TxToEvents(r ParsedRequest, kind string) ([]types.Event, error) {
 }
 
 // Transforms a coraza interruption to a crowdsec event
-func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r ParsedRequest, kind string) (types.Event, error) {
+func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r waf.ParsedRequest, kind string) (types.Event, error) {
 	evt := types.Event{}
 	//we might want to change this based on in-band vs out-of-band ?
 	evt.Type = types.LOG
@@ -40,7 +42,7 @@ func RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction,
 	//def needs fixing
 	evt.Stage = "s00-raw"
 	evt.Process = true
-
+	log.Infof("SOURCE IP: %+v", rule)
 	//we build a big-ass object that is going to be marshaled in line.raw and unmarshaled later.
 	//why ? because it's more consistent with the other data-sources etc. and it provides users with flexibility to alter our parsers
 	CorazaEvent := map[string]interface{}{

pkg/acquisition/modules/waf/waf.go

@@ -2,14 +2,15 @@ package wafacquisition
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
-	"net/url"
 	"os"
 	"strings"
 
+	"github.com/antonmedv/expr"
 	"github.com/corazawaf/coraza/v3"
+	"github.com/corazawaf/coraza/v3/experimental"
 	corazatypes "github.com/corazawaf/coraza/v3/types"
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
@@ -29,11 +30,20 @@ const (
 )
 
 type WafRunner struct {
-	outChan      chan types.Event
+	outChan          chan types.Event
-	inChan       chan ParsedRequest
+	inChan           chan waf.ParsedRequest
-	inBandWaf    coraza.WAF
+	inBandWaf        coraza.WAF
-	outOfBandWaf coraza.WAF
+	outOfBandWaf     coraza.WAF
-	UUID         string
+	UUID             string
+	RulesCollections []*waf.WafRulesCollection
+}
+
+type WafSourceConfig struct {
+	ListenAddr                        string `yaml:"listen_addr"`
+	ListenPort                        int    `yaml:"listen_port"`
+	Path                              string `yaml:"path"`
+	WafRoutines                       int    `yaml:"waf_routines"`
+	configuration.DataSourceCommonCfg `yaml:",inline"`
 }
 
 type WafSource struct {
@@ -43,47 +53,15 @@ type WafSource struct {
 	server  *http.Server
 	addr    string
 	outChan chan types.Event
-	InChan  chan ParsedRequest
+	InChan  chan waf.ParsedRequest
 
-	inBandWaf    coraza.WAF
+	inBandWaf        coraza.WAF
-	outOfBandWaf coraza.WAF
+	outOfBandWaf     coraza.WAF
+	RulesCollections []*waf.WafRulesCollection
 
 	WafRunners []WafRunner
 }
 
-type ParsedRequest struct {
-	RemoteAddr       string
-	Host             string
-	ClientIP         string
-	URI              string
-	ClientHost       string
-	Headers          http.Header
-	URL              *url.URL
-	Method           string
-	Proto            string
-	Body             []byte
-	TransferEncoding []string
-	UUID             string
-	Tx               corazatypes.Transaction
-	ResponseChannel  chan ResponseRequest
-}
-
-type ResponseRequest struct {
-	ResponseChannel chan ResponseRequest
-	UUID            string
-	Tx              corazatypes.Transaction
-	Interruption    *corazatypes.Interruption
-	Err             error
-}
-
-type WafSourceConfig struct {
-	ListenAddr                        string `yaml:"listen_addr"`
-	ListenPort                        int    `yaml:"listen_port"`
-	Path                              string `yaml:"path"`
-	WafRoutines                       int    `yaml:"waf_routines"`
-	configuration.DataSourceCommonCfg `yaml:",inline"`
-}
-
 func (w *WafSource) GetMetrics() []prometheus.Collector {
 	return nil
 }
@@ -156,6 +134,8 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error {
 		return fmt.Errorf("cannot load WAF rules: %w", err)
 	}
 
+	w.RulesCollections = rulesCollections
+
 	var inBandRules string
 	var outOfBandRules string
 
@@ -179,7 +159,7 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error {
 		w.config.WafRoutines = 1
 	}
 
-	w.InChan = make(chan ParsedRequest)
+	w.InChan = make(chan waf.ParsedRequest)
 	w.WafRunners = make([]WafRunner, w.config.WafRoutines)
 	for nbRoutine := 0; nbRoutine < w.config.WafRoutines; nbRoutine++ {
 		w.logger.Infof("Loading %d in-band rules", len(strings.Split(inBandRules, "\n")))
@@ -207,10 +187,11 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error {
 		}
 
 		runner := WafRunner{
-			outOfBandWaf: outofbandwaf,
+			outOfBandWaf:     outofbandwaf,
-			inBandWaf:    inbandwaf,
+			inBandWaf:        inbandwaf,
-			inChan:       w.InChan,
+			inChan:           w.InChan,
-			UUID:         uuid.New().String(),
+			UUID:             uuid.New().String(),
+			RulesCollections: rulesCollections,
 		}
 		w.WafRunners[nbRoutine] = runner
 	}
@@ -284,51 +265,8 @@ func (w *WafSource) Dump() interface{} {
 	return w
 }
 
-func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) {
+func processReqWithEngine(tx experimental.FullTransaction, r waf.ParsedRequest, wafType string) (*corazatypes.Interruption, experimental.FullTransaction, error) {
-	var body []byte
-	var err error
-
-	if r.Body != nil {
-		body = make([]byte, 0)
-		body, err = ioutil.ReadAll(r.Body)
-		if err != nil {
-			return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err)
-		}
-	}
-
-	// the real source of the request is set in 'x-client-ip'
-	clientIP := r.Header.Get("X-Client-Ip")
-	// the real target Host of the request is set in 'x-client-host'
-	clientHost := r.Header.Get("X-Client-Host")
-	// the real URI of the request is set in 'x-client-uri'
-	clientURI := r.Header.Get("X-Client-Uri")
-
-	// delete those headers before coraza process the request
-	delete(r.Header, "x-client-ip")
-	delete(r.Header, "x-client-host")
-	delete(r.Header, "x-client-uri")
-
-	return ParsedRequest{
-		RemoteAddr:       r.RemoteAddr,
-		UUID:             uuid.New().String(),
-		ClientHost:       clientHost,
-		ClientIP:         clientIP,
-		URI:              clientURI,
-		Host:             r.Host,
-		Headers:          r.Header,
-		URL:              r.URL,
-		Method:           r.Method,
-		Proto:            r.Proto,
-		Body:             body,
-		TransferEncoding: r.TransferEncoding,
-		ResponseChannel:  make(chan ResponseRequest),
-	}, nil
-}
-
-func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType string) (*corazatypes.Interruption, corazatypes.Transaction, error) {
 	var in *corazatypes.Interruption
-	tx := waf.NewTransactionWithID(uuid)
-
 	if tx.IsRuleEngineOff() {
 		log.Printf("engine is off")
 		return nil, nil, nil
@@ -346,7 +284,6 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType
 	//txx := experimental.ToFullInterface(tx)
 	//txx = tx.(experimental.FullTransaction)
 	//txx.RemoveRuleByID(1)
-
 	tx.ProcessConnection(r.ClientIP, 0, "", 0)
 
 	//tx.ProcessURI(r.URL.String(), r.Method, r.Proto) //FIXME: get it from the headers
@@ -369,12 +306,15 @@ func processReqWithEngine(waf coraza.WAF, r ParsedRequest, uuid string, wafType
 	}
 
 	in = tx.ProcessRequestHeaders()
+
 	//spew.Dump(in)
 	//spew.Dump(tx.MatchedRules())
 
-	/*for _, rule := range tx.MatchedRules() {
+	for _, rule := range tx.MatchedRules() {
-		spew.Dump(rule.Rule())
+		if rule.Message() == "" {
-	}*/
+			continue
+		}
+	}
 
 	//if we're inband, we should stop here, but for outofband go to the end
 	if in != nil && wafType == InBand {
@@ -422,17 +362,105 @@ func (r *WafRunner) Run(t *tomb.Tomb) error {
 			log.Infof("Waf Runner is dying")
 			return nil
 		case request := <-r.inChan:
-			in, tx, err := processReqWithEngine(r.inBandWaf, request, request.UUID, InBand)
+			inBoundTx := r.inBandWaf.NewTransactionWithID(request.UUID)
-			response := ResponseRequest{
+			expTx := inBoundTx.(experimental.FullTransaction)
-				Tx:           tx,
+			// we use this internal transaction for the expr helpers
-				Interruption: in,
+			tx := waf.NewTransaction(expTx)
-				Err:          err,
+
-				UUID:         request.UUID,
+			//Run the pre_eval hooks
+			for _, rules := range r.RulesCollections {
+				if len(rules.CompiledPreEval) == 0 {
+					continue
+				}
+				for _, compiledHook := range rules.CompiledPreEval {
+					if compiledHook.Filter != nil {
+						res, err := expr.Run(compiledHook.Filter, map[string]interface{}{
+							"rules": rules,
+							"req":   request,
+						})
+						if err != nil {
+							log.Errorf("unable to run PreEval filter: %s", err)
+							continue
+						}
+
+						switch t := res.(type) {
+						case bool:
+							if t == false {
+								log.Infof("filter didnt match")
+								continue
+							}
+						default:
+							log.Errorf("Filter must return a boolean, can't filter")
+							continue
+						}
+					}
+					// here means there is no filter or the filter matched
+					for _, applyExpr := range compiledHook.Apply {
+						_, err := expr.Run(applyExpr, map[string]interface{}{
+							"rules":          rules,
+							"req":            request,
+							"RemoveRuleByID": tx.RemoveRuleByIDWithError,
+						})
+						if err != nil {
+							log.Errorf("unable to apply filter: %s", err)
+							continue
+						}
+					}
+				}
 			}
+
+			in, expTx, err := processReqWithEngine(expTx, request, InBand)
+			request.Tx = expTx
+
+			response := waf.NewResponseRequest(expTx, in, request.UUID, err)
+
+			// run the on_match hooks
+			for _, rules := range r.RulesCollections {
+				if len(rules.CompiledOnMatch) == 0 {
+					continue
+				}
+				for _, compiledHook := range rules.CompiledOnMatch {
+					if compiledHook.Filter != nil {
+						res, err := expr.Run(compiledHook.Filter, map[string]interface{}{
+							"rules": rules,
+							"req":   request,
+						})
+						if err != nil {
+							log.Errorf("unable to run PreEval filter: %s", err)
+							continue
+						}
+
+						switch t := res.(type) {
+						case bool:
+							if t == false {
+								continue
+							}
+						default:
+							log.Errorf("Filter must return a boolean, can't filter")
+							continue
+						}
+					}
+					// here means there is no filter or the filter matched
+					for _, applyExpr := range compiledHook.Apply {
+						_, err := expr.Run(applyExpr, map[string]interface{}{
+							"rules":              rules,
+							"req":                request,
+							"RemoveRuleByID":     tx.RemoveRuleByIDWithError,
+							"SetRemediation":     response.SetRemediation,
+							"SetRemediationByID": response.SetRemediationByID,
+							"CancelEvent":        response.CancelEvent,
+						})
+						if err != nil {
+							log.Errorf("unable to apply filter: %s", err)
+							continue
+						}
+					}
+				}
+			}
+
 			// send back the result to the HTTP handler for the InBand part
 			request.ResponseChannel <- response
-			if in != nil {
+			if in != nil && response.SendEvents {
-				request.Tx = tx
 				// Generate the events for InBand channel
 				events, err := TxToEvents(request, InBand)
 				if err != nil {
@@ -446,13 +474,15 @@ func (r *WafRunner) Run(t *tomb.Tomb) error {
 			}
 
 			// Process outBand
-			in, tx, err = processReqWithEngine(r.outOfBandWaf, request, request.UUID, OutOfBand)
+			outBandTx := r.outOfBandWaf.NewTransactionWithID(request.UUID)
+			expTx = outBandTx.(experimental.FullTransaction)
+			in, expTx, err = processReqWithEngine(expTx, request, OutOfBand)
 			if err != nil { //things went south
 				log.Errorf("Error while processing request : %s", err)
 				continue
 			}
-			request.Tx = tx
+			request.Tx = expTx
-			if tx != nil && len(tx.MatchedRules()) > 0 {
+			if expTx != nil && len(expTx.MatchedRules()) > 0 {
 				events, err := TxToEvents(request, OutOfBand)
 				log.Infof("Request triggered by WAF, %d events to send", len(events))
 				for _, evt := range events {
@@ -467,9 +497,13 @@ func (r *WafRunner) Run(t *tomb.Tomb) error {
 	}
 }
 
+type BodyResponse struct {
+	Action string `json:"action"`
+}
+
 func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) {
 	// parse the request only once
-	parsedRequest, err := NewParsedRequestFromRequest(r)
+	parsedRequest, err := waf.NewParsedRequestFromRequest(r)
 	if err != nil {
 		log.Errorf("%s", err)
 		rw.WriteHeader(http.StatusForbidden)
@@ -487,10 +521,22 @@ func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) {
 
 	if message.Interruption != nil {
 		rw.WriteHeader(http.StatusForbidden)
+		body, err := json.Marshal(BodyResponse{Action: message.Interruption.Action})
+		if err != nil {
+			log.Errorf("unable to build response: %s", err)
+		} else {
+			rw.Write(body)
+		}
 		return
 	}
 
 	rw.WriteHeader(http.StatusOK)
+	body, err := json.Marshal(BodyResponse{Action: "allow"})
+	if err != nil {
+		log.Errorf("unable to build response: %s", err)
+	} else {
+		rw.Write(body)
+	}
 
 	return
 }

pkg/waf/env.go

@@ -0,0 +1,32 @@
+package waf
+
+import "github.com/corazawaf/coraza/v3/experimental"
+
+type Transaction struct {
+	Tx experimental.FullTransaction
+}
+
+func NewTransaction(tx experimental.FullTransaction) Transaction {
+	return Transaction{Tx: tx}
+}
+
+func (t *Transaction) RemoveRuleByIDWithError(id int) error {
+	t.Tx.RemoveRuleByID(id)
+	return nil
+}
+
+func GetEnv() map[string]interface{} {
+	ResponseRequest := ResponseRequest{}
+	ParsedRequest := ParsedRequest{}
+	Rules := &WafRulesCollection{}
+	Tx := Transaction{}
+
+	return map[string]interface{}{
+		"rules":              Rules,
+		"req":                ParsedRequest,
+		"SetRemediation":     ResponseRequest.SetRemediation,
+		"SetRemediationByID": ResponseRequest.SetRemediationByID,
+		"CancelEvent":        ResponseRequest.CancelEvent,
+		"RemoveRuleByID":     Tx.RemoveRuleByIDWithError,
+	}
+}

pkg/waf/request.go

@@ -0,0 +1,106 @@
+package waf
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+
+	"github.com/corazawaf/coraza/v3/experimental"
+	corazatypes "github.com/corazawaf/coraza/v3/types"
+	"github.com/google/uuid"
+)
+
+type ResponseRequest struct {
+	UUID         string
+	Tx           corazatypes.Transaction
+	Interruption *corazatypes.Interruption
+	Err          error
+	SendEvents   bool
+}
+
+func NewResponseRequest(Tx experimental.FullTransaction, in *corazatypes.Interruption, UUID string, err error) ResponseRequest {
+	return ResponseRequest{
+		UUID:         UUID,
+		Tx:           Tx,
+		Interruption: in,
+		Err:          err,
+		SendEvents:   true,
+	}
+}
+
+func (r *ResponseRequest) SetRemediation(remediation string) error {
+	r.Interruption.Action = remediation
+	return nil
+}
+
+func (r *ResponseRequest) SetRemediationByID(ID int, remediation string) error {
+	if r.Interruption.RuleID == ID {
+		r.Interruption.Action = remediation
+	}
+	return nil
+}
+
+func (r *ResponseRequest) CancelEvent() error {
+	// true by default
+	r.SendEvents = false
+	return nil
+}
+
+type ParsedRequest struct {
+	RemoteAddr       string
+	Host             string
+	ClientIP         string
+	URI              string
+	ClientHost       string
+	Headers          http.Header
+	URL              *url.URL
+	Method           string
+	Proto            string
+	Body             []byte
+	TransferEncoding []string
+	UUID             string
+	Tx               experimental.FullTransaction
+	ResponseChannel  chan ResponseRequest
+}
+
+func NewParsedRequestFromRequest(r *http.Request) (ParsedRequest, error) {
+	var body []byte
+	var err error
+
+	if r.Body != nil {
+		body = make([]byte, 0)
+		body, err = ioutil.ReadAll(r.Body)
+		if err != nil {
+			return ParsedRequest{}, fmt.Errorf("unable to read body: %s", err)
+		}
+	}
+
+	// the real source of the request is set in 'x-client-ip'
+	clientIP := r.Header.Get("X-Client-Ip")
+	// the real target Host of the request is set in 'x-client-host'
+	clientHost := r.Header.Get("X-Client-Host")
+	// the real URI of the request is set in 'x-client-uri'
+	clientURI := r.Header.Get("X-Client-Uri")
+
+	// delete those headers before coraza process the request
+	delete(r.Header, "x-client-ip")
+	delete(r.Header, "x-client-host")
+	delete(r.Header, "x-client-uri")
+
+	return ParsedRequest{
+		RemoteAddr:       r.RemoteAddr,
+		UUID:             uuid.New().String(),
+		ClientHost:       clientHost,
+		ClientIP:         clientIP,
+		URI:              clientURI,
+		Host:             r.Host,
+		Headers:          r.Header,
+		URL:              r.URL,
+		Method:           r.Method,
+		Proto:            r.Proto,
+		Body:             body,
+		TransferEncoding: r.TransferEncoding,
+		ResponseChannel:  make(chan ResponseRequest),
+	}, nil
+}

pkg/waf/waf.go

@@ -67,9 +67,7 @@ func buildHook(hook Hook) (CompiledHook, error) {
 		compiledHook.Filter = program
 	}
 	for _, apply := range hook.Apply {
-		program, err := expr.Compile(apply, GetExprWAFOptions(map[string]interface{}{
+		program, err := expr.Compile(apply, GetExprWAFOptions(GetEnv())...)
-			"rules": &WafRulesCollection{},
-		})...)
 		if err != nil {
 			return CompiledHook{}, fmt.Errorf("unable to compile apply %s : %w", apply, err)
 		}
@@ -173,9 +171,7 @@ func (w *WafRuleLoader) LoadWafRules() ([]*WafRulesCollection, error) {
 				//Ignore filter for on load ?
 				if onLoadHook.Apply != nil {
 					for exprIdx, applyExpr := range onLoadHook.Apply {
-						_, err := expr.Run(applyExpr, map[string]interface{}{
+						_, err := expr.Run(applyExpr, GetEnv())
-							"rules": collection,
-						})
 						if err != nil {
 							w.logger.Errorf("unable to run apply for on_load rule %s : %s", wafConfig.OnLoad[hookIdx].Apply[exprIdx], err)
 							continue

pkg/waf/waf_rules_collection.go

@@ -4,6 +4,8 @@ import "strings"
 
 type WafRule struct {
 	RawRule string
+	RuleID  string
+	InBand  bool
 }
 
 // This is the "compiled" state of a WafConfig