diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index 8e87eecd0..96db7290f 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -12,6 +12,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/pkg/trace" "github.com/crowdsecurity/go-cs-lib/pkg/version" + waf "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/waf" v1 "github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers/v1" "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/csconfig" @@ -169,7 +170,9 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, v1.LapiRouteHits, leaky.BucketsCurrentCount, - cache.CacheMetrics, exprhelpers.RegexpCacheMetrics) + cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, + waf.WafGlobalParsingHistogram, waf.WafReqCounter, waf.WafRuleHits, + ) } else { log.Infof("Loading prometheus collectors") prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo, @@ -178,7 +181,9 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions, v1.LapiResponseTime, leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount, globalActiveDecisions, globalAlerts, - cache.CacheMetrics, exprhelpers.RegexpCacheMetrics) + cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, + waf.WafGlobalParsingHistogram, waf.WafInbandParsingHistogram, waf.WafOutbandParsingHistogram, waf.WafReqCounter, waf.WafRuleHits, + ) } } diff --git a/pkg/acquisition/modules/waf/utils.go b/pkg/acquisition/modules/waf/utils.go index 6d2932bf9..879557477 100644 --- a/pkg/acquisition/modules/waf/utils.go +++ b/pkg/acquisition/modules/waf/utils.go @@ -10,6 +10,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) @@ -23,6 +24,7 @@ func TxToEvents(r waf.ParsedRequest, kind string) ([]types.Event, error) { if rule.Message() == "" { continue } + WafRuleHits.With(prometheus.Labels{"rule_id": fmt.Sprintf("%d", rule.Rule().ID()), "type": kind}).Inc() evt, err := RuleMatchToEvent(rule, r.Tx, r, kind) if err != nil { return nil, errors.Wrap(err, "Cannot convert rule match to event") diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index e46e875eb..51120d8cf 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "strings" + "time" "github.com/antonmedv/expr" "github.com/corazawaf/coraza/v3" @@ -62,6 +63,49 @@ type WafSource struct { WafRunners []WafRunner } +var WafGlobalParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the WAF.", + Name: "cs_waf_parsing_time_seconds", + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"source"}, +) + +var WafInbandParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the inband WAF.", + Name: "cs_waf_inband_parsing_time_seconds", + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"source"}, +) + +var WafOutbandParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the WAF.", + Name: "cs_waf_outband_parsing_time_seconds", + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"source"}, +) + +var WafReqCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_waf_reqs_total", + Help: "Total events processed by the WAF.", + }, + []string{"source"}, +) + +var WafRuleHits = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cs_waf_rule_hits", + Help: "Count of triggered rule, by rule_id and type (inband/outofband).", + }, + []string{"rule_id", "type"}, +) + func (w *WafSource) GetMetrics() []prometheus.Collector { return nil } @@ -362,6 +406,9 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { log.Infof("Waf Runner is dying") return nil case request := <-r.inChan: + WafReqCounter.With(prometheus.Labels{"source": request.RemoteAddr}).Inc() + //measure the time spent in the WAF + startParsing := time.Now() inBoundTx := r.inBandWaf.NewTransactionWithID(request.UUID) expTx := inBoundTx.(experimental.FullTransaction) // we use this internal transaction for the expr helpers @@ -457,7 +504,9 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { } } } - + //measure the full time spent in the WAF + elapsed := time.Since(startParsing) + WafInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) // send back the result to the HTTP handler for the InBand part request.ResponseChannel <- response if in != nil && response.SendEvents { @@ -473,6 +522,7 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { } } + outBandStart := time.Now() // Process outBand outBandTx := r.outOfBandWaf.NewTransactionWithID(request.UUID) expTx = outBandTx.(experimental.FullTransaction) @@ -493,6 +543,11 @@ func (r *WafRunner) Run(t *tomb.Tomb) error { continue } } + //measure the full time spent in the WAF + totalElapsed := time.Since(startParsing) + WafGlobalParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(totalElapsed.Seconds()) + elapsed = time.Since(outBandStart) + WafOutbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddr}).Observe(elapsed.Seconds()) } } }