refact cscli metric processing (#2816)
* typos * refact cscli metric processing * lint
This commit is contained in:
parent
3208a40ef3
commit
af1df0696b
3 changed files with 240 additions and 164 deletions
|
@ -22,7 +22,7 @@ linters-settings:
|
||||||
|
|
||||||
gocognit:
|
gocognit:
|
||||||
# lower this after refactoring
|
# lower this after refactoring
|
||||||
min-complexity: 150
|
min-complexity: 145
|
||||||
|
|
||||||
gocyclo:
|
gocyclo:
|
||||||
# lower this after refactoring
|
# lower this after refactoring
|
||||||
|
|
|
@ -2,6 +2,7 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
@ -42,8 +43,14 @@ type (
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrMissingConfig = errors.New("prometheus section missing, can't show metrics")
|
||||||
|
ErrMetricsDisabled = errors.New("prometheus is not enabled, can't show metrics")
|
||||||
|
|
||||||
|
)
|
||||||
|
|
||||||
type metricSection interface {
|
type metricSection interface {
|
||||||
Table(io.Writer, bool, bool)
|
Table(out io.Writer, noUnit bool, showEmpty bool)
|
||||||
Description() (string, string)
|
Description() (string, string)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -154,6 +161,9 @@ func (ms metricStore) Fetch(url string) error {
|
||||||
origin := metric.Labels["origin"]
|
origin := metric.Labels["origin"]
|
||||||
action := metric.Labels["action"]
|
action := metric.Labels["action"]
|
||||||
|
|
||||||
|
appsecEngine := metric.Labels["appsec_engine"]
|
||||||
|
appsecRule := metric.Labels["rule_name"]
|
||||||
|
|
||||||
mtype := metric.Labels["type"]
|
mtype := metric.Labels["type"]
|
||||||
|
|
||||||
fval, err := strconv.ParseFloat(value, 32)
|
fval, err := strconv.ParseFloat(value, 32)
|
||||||
|
@ -162,178 +172,78 @@ func (ms metricStore) Fetch(url string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
ival := int(fval)
|
ival := int(fval)
|
||||||
|
|
||||||
switch fam.Name {
|
switch fam.Name {
|
||||||
//
|
//
|
||||||
// buckets
|
// buckets
|
||||||
//
|
//
|
||||||
case "cs_bucket_created_total":
|
case "cs_bucket_created_total":
|
||||||
if _, ok := mBucket[name]; !ok {
|
mBucket.Process(name, "instantiation", ival)
|
||||||
mBucket[name] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mBucket[name]["instantiation"] += ival
|
|
||||||
case "cs_buckets":
|
case "cs_buckets":
|
||||||
if _, ok := mBucket[name]; !ok {
|
mBucket.Process(name, "curr_count", ival)
|
||||||
mBucket[name] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mBucket[name]["curr_count"] += ival
|
|
||||||
case "cs_bucket_overflowed_total":
|
case "cs_bucket_overflowed_total":
|
||||||
if _, ok := mBucket[name]; !ok {
|
mBucket.Process(name, "overflow", ival)
|
||||||
mBucket[name] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mBucket[name]["overflow"] += ival
|
|
||||||
case "cs_bucket_poured_total":
|
case "cs_bucket_poured_total":
|
||||||
if _, ok := mBucket[name]; !ok {
|
mBucket.Process(name, "pour", ival)
|
||||||
mBucket[name] = make(map[string]int)
|
mAcquis.Process(source, "pour", ival)
|
||||||
}
|
|
||||||
if _, ok := mAcquis[source]; !ok {
|
|
||||||
mAcquis[source] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mBucket[name]["pour"] += ival
|
|
||||||
mAcquis[source]["pour"] += ival
|
|
||||||
case "cs_bucket_underflowed_total":
|
case "cs_bucket_underflowed_total":
|
||||||
if _, ok := mBucket[name]; !ok {
|
mBucket.Process(name, "underflow", ival)
|
||||||
mBucket[name] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mBucket[name]["underflow"] += ival
|
|
||||||
//
|
//
|
||||||
// parsers
|
// parsers
|
||||||
//
|
//
|
||||||
case "cs_parser_hits_total":
|
case "cs_parser_hits_total":
|
||||||
if _, ok := mAcquis[source]; !ok {
|
mAcquis.Process(source, "reads", ival)
|
||||||
mAcquis[source] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mAcquis[source]["reads"] += ival
|
|
||||||
case "cs_parser_hits_ok_total":
|
case "cs_parser_hits_ok_total":
|
||||||
if _, ok := mAcquis[source]; !ok {
|
mAcquis.Process(source, "parsed", ival)
|
||||||
mAcquis[source] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mAcquis[source]["parsed"] += ival
|
|
||||||
case "cs_parser_hits_ko_total":
|
case "cs_parser_hits_ko_total":
|
||||||
if _, ok := mAcquis[source]; !ok {
|
mAcquis.Process(source, "unparsed", ival)
|
||||||
mAcquis[source] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mAcquis[source]["unparsed"] += ival
|
|
||||||
case "cs_node_hits_total":
|
case "cs_node_hits_total":
|
||||||
if _, ok := mParser[name]; !ok {
|
mParser.Process(name, "hits", ival)
|
||||||
mParser[name] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mParser[name]["hits"] += ival
|
|
||||||
case "cs_node_hits_ok_total":
|
case "cs_node_hits_ok_total":
|
||||||
if _, ok := mParser[name]; !ok {
|
mParser.Process(name, "parsed", ival)
|
||||||
mParser[name] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mParser[name]["parsed"] += ival
|
|
||||||
case "cs_node_hits_ko_total":
|
case "cs_node_hits_ko_total":
|
||||||
if _, ok := mParser[name]; !ok {
|
mParser.Process(name, "unparsed", ival)
|
||||||
mParser[name] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mParser[name]["unparsed"] += ival
|
|
||||||
//
|
//
|
||||||
// whitelists
|
// whitelists
|
||||||
//
|
//
|
||||||
case "cs_node_wl_hits_total":
|
case "cs_node_wl_hits_total":
|
||||||
if _, ok := mWhitelist[name]; !ok {
|
mWhitelist.Process(name, reason, "hits", ival)
|
||||||
mWhitelist[name] = make(map[string]map[string]int)
|
|
||||||
}
|
|
||||||
if _, ok := mWhitelist[name][reason]; !ok {
|
|
||||||
mWhitelist[name][reason] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mWhitelist[name][reason]["hits"] += ival
|
|
||||||
case "cs_node_wl_hits_ok_total":
|
case "cs_node_wl_hits_ok_total":
|
||||||
if _, ok := mWhitelist[name]; !ok {
|
mWhitelist.Process(name, reason, "whitelisted", ival)
|
||||||
mWhitelist[name] = make(map[string]map[string]int)
|
|
||||||
}
|
|
||||||
if _, ok := mWhitelist[name][reason]; !ok {
|
|
||||||
mWhitelist[name][reason] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mWhitelist[name][reason]["whitelisted"] += ival
|
|
||||||
// track as well whitelisted lines at acquis level
|
// track as well whitelisted lines at acquis level
|
||||||
if _, ok := mAcquis[source]; !ok {
|
mAcquis.Process(source, "whitelisted", ival)
|
||||||
mAcquis[source] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mAcquis[source]["whitelisted"] += ival
|
|
||||||
//
|
//
|
||||||
// lapi
|
// lapi
|
||||||
//
|
//
|
||||||
case "cs_lapi_route_requests_total":
|
case "cs_lapi_route_requests_total":
|
||||||
if _, ok := mLapi[route]; !ok {
|
mLapi.Process(route, method, ival)
|
||||||
mLapi[route] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mLapi[route][method] += ival
|
|
||||||
case "cs_lapi_machine_requests_total":
|
case "cs_lapi_machine_requests_total":
|
||||||
if _, ok := mLapiMachine[machine]; !ok {
|
mLapiMachine.Process(machine, route, method, ival)
|
||||||
mLapiMachine[machine] = make(map[string]map[string]int)
|
|
||||||
}
|
|
||||||
if _, ok := mLapiMachine[machine][route]; !ok {
|
|
||||||
mLapiMachine[machine][route] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mLapiMachine[machine][route][method] += ival
|
|
||||||
case "cs_lapi_bouncer_requests_total":
|
case "cs_lapi_bouncer_requests_total":
|
||||||
if _, ok := mLapiBouncer[bouncer]; !ok {
|
mLapiBouncer.Process(bouncer, route, method, ival)
|
||||||
mLapiBouncer[bouncer] = make(map[string]map[string]int)
|
|
||||||
}
|
|
||||||
if _, ok := mLapiBouncer[bouncer][route]; !ok {
|
|
||||||
mLapiBouncer[bouncer][route] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mLapiBouncer[bouncer][route][method] += ival
|
|
||||||
case "cs_lapi_decisions_ko_total", "cs_lapi_decisions_ok_total":
|
case "cs_lapi_decisions_ko_total", "cs_lapi_decisions_ok_total":
|
||||||
if _, ok := mLapiDecision[bouncer]; !ok {
|
mLapiDecision.Process(bouncer, fam.Name, ival)
|
||||||
mLapiDecision[bouncer] = struct {
|
|
||||||
NonEmpty int
|
|
||||||
Empty int
|
|
||||||
}{}
|
|
||||||
}
|
|
||||||
x := mLapiDecision[bouncer]
|
|
||||||
if fam.Name == "cs_lapi_decisions_ko_total" {
|
|
||||||
x.Empty += ival
|
|
||||||
} else if fam.Name == "cs_lapi_decisions_ok_total" {
|
|
||||||
x.NonEmpty += ival
|
|
||||||
}
|
|
||||||
mLapiDecision[bouncer] = x
|
|
||||||
//
|
//
|
||||||
// decisions
|
// decisions
|
||||||
//
|
//
|
||||||
case "cs_active_decisions":
|
case "cs_active_decisions":
|
||||||
if _, ok := mDecision[reason]; !ok {
|
mDecision.Process(reason, origin, action, ival)
|
||||||
mDecision[reason] = make(map[string]map[string]int)
|
|
||||||
}
|
|
||||||
if _, ok := mDecision[reason][origin]; !ok {
|
|
||||||
mDecision[reason][origin] = make(map[string]int)
|
|
||||||
}
|
|
||||||
mDecision[reason][origin][action] += ival
|
|
||||||
case "cs_alerts":
|
case "cs_alerts":
|
||||||
mAlert[reason] += ival
|
mAlert.Process(reason, ival)
|
||||||
//
|
//
|
||||||
// stash
|
// stash
|
||||||
//
|
//
|
||||||
case "cs_cache_size":
|
case "cs_cache_size":
|
||||||
mStash[name] = struct {
|
mStash.Process(name, mtype, ival)
|
||||||
Type string
|
|
||||||
Count int
|
|
||||||
}{Type: mtype, Count: ival}
|
|
||||||
//
|
//
|
||||||
// appsec
|
// appsec
|
||||||
//
|
//
|
||||||
case "cs_appsec_reqs_total":
|
case "cs_appsec_reqs_total":
|
||||||
if _, ok := mAppsecEngine[metric.Labels["appsec_engine"]]; !ok {
|
mAppsecEngine.Process(appsecEngine, "processed", ival)
|
||||||
mAppsecEngine[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
|
|
||||||
}
|
|
||||||
mAppsecEngine[metric.Labels["appsec_engine"]]["processed"] = ival
|
|
||||||
case "cs_appsec_block_total":
|
case "cs_appsec_block_total":
|
||||||
if _, ok := mAppsecEngine[metric.Labels["appsec_engine"]]; !ok {
|
mAppsecEngine.Process(appsecEngine, "blocked", ival)
|
||||||
mAppsecEngine[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
|
|
||||||
}
|
|
||||||
mAppsecEngine[metric.Labels["appsec_engine"]]["blocked"] = ival
|
|
||||||
case "cs_appsec_rule_hits":
|
case "cs_appsec_rule_hits":
|
||||||
appsecEngine := metric.Labels["appsec_engine"]
|
mAppsecRule.Process(appsecEngine, appsecRule, "triggered", ival)
|
||||||
ruleID := metric.Labels["rule_name"]
|
|
||||||
if _, ok := mAppsecRule[appsecEngine]; !ok {
|
|
||||||
mAppsecRule[appsecEngine] = make(map[string]map[string]int, 0)
|
|
||||||
}
|
|
||||||
if _, ok := mAppsecRule[appsecEngine][ruleID]; !ok {
|
|
||||||
mAppsecRule[appsecEngine][ruleID] = make(map[string]int, 0)
|
|
||||||
}
|
|
||||||
mAppsecRule[appsecEngine][ruleID]["triggered"] = ival
|
|
||||||
default:
|
default:
|
||||||
log.Debugf("unknown: %+v", fam.Name)
|
log.Debugf("unknown: %+v", fam.Name)
|
||||||
continue
|
continue
|
||||||
|
@ -380,13 +290,13 @@ func (ms metricStore) Format(out io.Writer, sections []string, formatType string
|
||||||
case "json":
|
case "json":
|
||||||
x, err := json.MarshalIndent(want, "", " ")
|
x, err := json.MarshalIndent(want, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to unmarshal metrics : %v", err)
|
return fmt.Errorf("failed to marshal metrics: %w", err)
|
||||||
}
|
}
|
||||||
out.Write(x)
|
out.Write(x)
|
||||||
case "raw":
|
case "raw":
|
||||||
x, err := yaml.Marshal(want)
|
x, err := yaml.Marshal(want)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to unmarshal metrics : %v", err)
|
return fmt.Errorf("failed to marshal metrics: %w", err)
|
||||||
}
|
}
|
||||||
out.Write(x)
|
out.Write(x)
|
||||||
default:
|
default:
|
||||||
|
@ -404,11 +314,11 @@ func (cli *cliMetrics) show(sections []string, url string, noUnit bool) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.Prometheus == nil {
|
if cfg.Prometheus == nil {
|
||||||
return fmt.Errorf("prometheus section missing, can't show metrics")
|
return ErrMissingConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
if !cfg.Prometheus.Enabled {
|
if !cfg.Prometheus.Enabled {
|
||||||
return fmt.Errorf("prometheus is not enabled, can't show metrics")
|
return ErrMetricsDisabled
|
||||||
}
|
}
|
||||||
|
|
||||||
ms := NewMetricStore()
|
ms := NewMetricStore()
|
||||||
|
@ -427,6 +337,7 @@ func (cli *cliMetrics) show(sections []string, url string, noUnit bool) error {
|
||||||
if err := ms.Format(color.Output, sections, cfg.Cscli.Output, noUnit); err != nil {
|
if err := ms.Format(color.Output, sections, cfg.Cscli.Output, noUnit); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -468,6 +379,7 @@ cscli metrics list`,
|
||||||
// expandAlias returns a list of sections. The input can be a list of sections or alias.
|
// expandAlias returns a list of sections. The input can be a list of sections or alias.
|
||||||
func (cli *cliMetrics) expandSectionGroups(args []string) []string {
|
func (cli *cliMetrics) expandSectionGroups(args []string) []string {
|
||||||
ret := []string{}
|
ret := []string{}
|
||||||
|
|
||||||
for _, section := range args {
|
for _, section := range args {
|
||||||
switch section {
|
switch section {
|
||||||
case "engine":
|
case "engine":
|
||||||
|
@ -522,8 +434,8 @@ cscli metrics show acquisition parsers buckets stash -o json`,
|
||||||
|
|
||||||
func (cli *cliMetrics) list() error {
|
func (cli *cliMetrics) list() error {
|
||||||
type metricType struct {
|
type metricType struct {
|
||||||
Type string `json:"type" yaml:"type"`
|
Type string `json:"type" yaml:"type"`
|
||||||
Title string `json:"title" yaml:"title"`
|
Title string `json:"title" yaml:"title"`
|
||||||
Description string `json:"description" yaml:"description"`
|
Description string `json:"description" yaml:"description"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -553,13 +465,13 @@ func (cli *cliMetrics) list() error {
|
||||||
case "json":
|
case "json":
|
||||||
x, err := json.MarshalIndent(allMetrics, "", " ")
|
x, err := json.MarshalIndent(allMetrics, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to unmarshal metrics: %w", err)
|
return fmt.Errorf("failed to marshal metric types: %w", err)
|
||||||
}
|
}
|
||||||
fmt.Println(string(x))
|
fmt.Println(string(x))
|
||||||
case "raw":
|
case "raw":
|
||||||
x, err := yaml.Marshal(allMetrics)
|
x, err := yaml.Marshal(allMetrics)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to unmarshal metrics: %w", err)
|
return fmt.Errorf("failed to marshal metric types: %w", err)
|
||||||
}
|
}
|
||||||
fmt.Println(string(x))
|
fmt.Println(string(x))
|
||||||
}
|
}
|
||||||
|
@ -575,8 +487,7 @@ func (cli *cliMetrics) newListCmd() *cobra.Command {
|
||||||
Args: cobra.ExactArgs(0),
|
Args: cobra.ExactArgs(0),
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
RunE: func(_ *cobra.Command, _ []string) error {
|
RunE: func(_ *cobra.Command, _ []string) error {
|
||||||
cli.list()
|
return cli.list()
|
||||||
return nil
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
"github.com/aquasecurity/table"
|
"github.com/aquasecurity/table"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
@ -11,17 +12,21 @@ import (
|
||||||
"github.com/crowdsecurity/go-cs-lib/maptools"
|
"github.com/crowdsecurity/go-cs-lib/maptools"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ErrNilTable means a nil pointer was passed instead of a table instance. This is a programming error.
|
||||||
|
var ErrNilTable = fmt.Errorf("nil table")
|
||||||
|
|
||||||
func lapiMetricsToTable(t *table.Table, stats map[string]map[string]map[string]int) int {
|
func lapiMetricsToTable(t *table.Table, stats map[string]map[string]map[string]int) int {
|
||||||
// stats: machine -> route -> method -> count
|
// stats: machine -> route -> method -> count
|
||||||
|
|
||||||
// sort keys to keep consistent order when printing
|
// sort keys to keep consistent order when printing
|
||||||
machineKeys := []string{}
|
machineKeys := []string{}
|
||||||
for k := range stats {
|
for k := range stats {
|
||||||
machineKeys = append(machineKeys, k)
|
machineKeys = append(machineKeys, k)
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Strings(machineKeys)
|
sort.Strings(machineKeys)
|
||||||
|
|
||||||
numRows := 0
|
numRows := 0
|
||||||
|
|
||||||
for _, machine := range machineKeys {
|
for _, machine := range machineKeys {
|
||||||
// oneRow: route -> method -> count
|
// oneRow: route -> method -> count
|
||||||
machineRow := stats[machine]
|
machineRow := stats[machine]
|
||||||
|
@ -33,53 +38,60 @@ func lapiMetricsToTable(t *table.Table, stats map[string]map[string]map[string]i
|
||||||
methodName,
|
methodName,
|
||||||
}
|
}
|
||||||
if count != 0 {
|
if count != 0 {
|
||||||
row = append(row, fmt.Sprintf("%d", count))
|
row = append(row, strconv.Itoa(count))
|
||||||
} else {
|
} else {
|
||||||
row = append(row, "-")
|
row = append(row, "-")
|
||||||
}
|
}
|
||||||
|
|
||||||
t.AddRow(row...)
|
t.AddRow(row...)
|
||||||
numRows++
|
numRows++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return numRows
|
return numRows
|
||||||
}
|
}
|
||||||
|
|
||||||
func wlMetricsToTable(t *table.Table, stats map[string]map[string]map[string]int, noUnit bool) (int, error) {
|
func wlMetricsToTable(t *table.Table, stats map[string]map[string]map[string]int, noUnit bool) (int, error) {
|
||||||
if t == nil {
|
if t == nil {
|
||||||
return 0, fmt.Errorf("nil table")
|
return 0, ErrNilTable
|
||||||
}
|
}
|
||||||
|
|
||||||
numRows := 0
|
numRows := 0
|
||||||
|
|
||||||
for _, name := range maptools.SortedKeys(stats) {
|
for _, name := range maptools.SortedKeys(stats) {
|
||||||
for _, reason := range maptools.SortedKeys(stats[name]) {
|
for _, reason := range maptools.SortedKeys(stats[name]) {
|
||||||
row := make([]string, 4)
|
row := []string{
|
||||||
row[0] = name
|
name,
|
||||||
row[1] = reason
|
reason,
|
||||||
row[2] = "-"
|
"-",
|
||||||
row[3] = "-"
|
"-",
|
||||||
|
}
|
||||||
|
|
||||||
for _, action := range maptools.SortedKeys(stats[name][reason]) {
|
for _, action := range maptools.SortedKeys(stats[name][reason]) {
|
||||||
value := stats[name][reason][action]
|
value := stats[name][reason][action]
|
||||||
if action == "whitelisted" {
|
|
||||||
row[3] = fmt.Sprintf("%d", value)
|
switch action {
|
||||||
} else if action == "hits" {
|
case "whitelisted":
|
||||||
row[2] = fmt.Sprintf("%d", value)
|
row[3] = strconv.Itoa(value)
|
||||||
} else {
|
case "hits":
|
||||||
|
row[2] = strconv.Itoa(value)
|
||||||
|
default:
|
||||||
log.Debugf("unexpected counter '%s' for whitelists = %d", action, value)
|
log.Debugf("unexpected counter '%s' for whitelists = %d", action, value)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
t.AddRow(row...)
|
t.AddRow(row...)
|
||||||
numRows++
|
numRows++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return numRows, nil
|
return numRows, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []string, noUnit bool) (int, error) {
|
func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []string, noUnit bool) (int, error) {
|
||||||
if t == nil {
|
if t == nil {
|
||||||
return 0, fmt.Errorf("nil table")
|
return 0, ErrNilTable
|
||||||
}
|
}
|
||||||
|
|
||||||
numRows := 0
|
numRows := 0
|
||||||
|
@ -89,12 +101,14 @@ func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []stri
|
||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
row := []string{
|
row := []string{
|
||||||
alabel,
|
alabel,
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, sl := range keys {
|
for _, sl := range keys {
|
||||||
if v, ok := astats[sl]; ok && v != 0 {
|
if v, ok := astats[sl]; ok && v != 0 {
|
||||||
numberToShow := fmt.Sprintf("%d", v)
|
numberToShow := strconv.Itoa(v)
|
||||||
if !noUnit {
|
if !noUnit {
|
||||||
numberToShow = formatNumber(v)
|
numberToShow = formatNumber(v)
|
||||||
}
|
}
|
||||||
|
@ -104,15 +118,26 @@ func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []stri
|
||||||
row = append(row, "-")
|
row = append(row, "-")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
t.AddRow(row...)
|
t.AddRow(row...)
|
||||||
numRows++
|
numRows++
|
||||||
}
|
}
|
||||||
|
|
||||||
return numRows, nil
|
return numRows, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s statBucket) Description() (string, string) {
|
func (s statBucket) Description() (string, string) {
|
||||||
return "Bucket Metrics",
|
return "Bucket Metrics",
|
||||||
`Measure events in different scenarios. Current count is the number of buckets during metrics collection. Overflows are past event-producing buckets, while Expired are the ones that didn’t receive enough events to Overflow.`
|
`Measure events in different scenarios. Current count is the number of buckets during metrics collection. ` +
|
||||||
|
`Overflows are past event-producing buckets, while Expired are the ones that didn’t receive enough events to Overflow.`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s statBucket) Process(bucket, metric string, val int) {
|
||||||
|
if _, ok := s[bucket]; !ok {
|
||||||
|
s[bucket] = make(map[string]int)
|
||||||
|
}
|
||||||
|
|
||||||
|
s[bucket][metric] += val
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s statBucket) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
func (s statBucket) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
|
@ -134,7 +159,18 @@ func (s statBucket) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
|
|
||||||
func (s statAcquis) Description() (string, string) {
|
func (s statAcquis) Description() (string, string) {
|
||||||
return "Acquisition Metrics",
|
return "Acquisition Metrics",
|
||||||
`Measures the lines read, parsed, and unparsed per datasource. Zero read lines indicate a misconfigured or inactive datasource. Zero parsed lines mean the parser(s) failed. Non-zero parsed lines are fine as crowdsec selects relevant lines.`
|
`Measures the lines read, parsed, and unparsed per datasource. ` +
|
||||||
|
`Zero read lines indicate a misconfigured or inactive datasource. ` +
|
||||||
|
`Zero parsed lines mean the parser(s) failed. ` +
|
||||||
|
`Non-zero parsed lines are fine as crowdsec selects relevant lines.`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s statAcquis) Process(source, metric string, val int) {
|
||||||
|
if _, ok := s[source]; !ok {
|
||||||
|
s[source] = make(map[string]int)
|
||||||
|
}
|
||||||
|
|
||||||
|
s[source][metric] += val
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s statAcquis) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
func (s statAcquis) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
|
@ -159,12 +195,22 @@ func (s statAppsecEngine) Description() (string, string) {
|
||||||
`Measures the number of parsed and blocked requests by the AppSec Component.`
|
`Measures the number of parsed and blocked requests by the AppSec Component.`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s statAppsecEngine) Process(appsecEngine, metric string, val int) {
|
||||||
|
if _, ok := s[appsecEngine]; !ok {
|
||||||
|
s[appsecEngine] = make(map[string]int)
|
||||||
|
}
|
||||||
|
|
||||||
|
s[appsecEngine][metric] += val
|
||||||
|
}
|
||||||
|
|
||||||
func (s statAppsecEngine) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
func (s statAppsecEngine) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
t := newTable(out)
|
t := newTable(out)
|
||||||
t.SetRowLines(false)
|
t.SetRowLines(false)
|
||||||
t.SetHeaders("Appsec Engine", "Processed", "Blocked")
|
t.SetHeaders("Appsec Engine", "Processed", "Blocked")
|
||||||
t.SetAlignment(table.AlignLeft, table.AlignLeft)
|
t.SetAlignment(table.AlignLeft, table.AlignLeft)
|
||||||
|
|
||||||
keys := []string{"processed", "blocked"}
|
keys := []string{"processed", "blocked"}
|
||||||
|
|
||||||
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
|
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
|
||||||
log.Warningf("while collecting appsec stats: %s", err)
|
log.Warningf("while collecting appsec stats: %s", err)
|
||||||
} else if numRows > 0 || showEmpty {
|
} else if numRows > 0 || showEmpty {
|
||||||
|
@ -179,13 +225,27 @@ func (s statAppsecRule) Description() (string, string) {
|
||||||
`Provides “per AppSec Component” information about the number of matches for loaded AppSec Rules.`
|
`Provides “per AppSec Component” information about the number of matches for loaded AppSec Rules.`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s statAppsecRule) Process(appsecEngine, appsecRule string, metric string, val int) {
|
||||||
|
if _, ok := s[appsecEngine]; !ok {
|
||||||
|
s[appsecEngine] = make(map[string]map[string]int)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := s[appsecEngine][appsecRule]; !ok {
|
||||||
|
s[appsecEngine][appsecRule] = make(map[string]int)
|
||||||
|
}
|
||||||
|
|
||||||
|
s[appsecEngine][appsecRule][metric] += val
|
||||||
|
}
|
||||||
|
|
||||||
func (s statAppsecRule) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
func (s statAppsecRule) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
for appsecEngine, appsecEngineRulesStats := range s {
|
for appsecEngine, appsecEngineRulesStats := range s {
|
||||||
t := newTable(out)
|
t := newTable(out)
|
||||||
t.SetRowLines(false)
|
t.SetRowLines(false)
|
||||||
t.SetHeaders("Rule ID", "Triggered")
|
t.SetHeaders("Rule ID", "Triggered")
|
||||||
t.SetAlignment(table.AlignLeft, table.AlignLeft)
|
t.SetAlignment(table.AlignLeft, table.AlignLeft)
|
||||||
|
|
||||||
keys := []string{"triggered"}
|
keys := []string{"triggered"}
|
||||||
|
|
||||||
if numRows, err := metricsToTable(t, appsecEngineRulesStats, keys, noUnit); err != nil {
|
if numRows, err := metricsToTable(t, appsecEngineRulesStats, keys, noUnit); err != nil {
|
||||||
log.Warningf("while collecting appsec rules stats: %s", err)
|
log.Warningf("while collecting appsec rules stats: %s", err)
|
||||||
} else if numRows > 0 || showEmpty {
|
} else if numRows > 0 || showEmpty {
|
||||||
|
@ -193,7 +253,6 @@ func (s statAppsecRule) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
t.Render()
|
t.Render()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s statWhitelist) Description() (string, string) {
|
func (s statWhitelist) Description() (string, string) {
|
||||||
|
@ -201,6 +260,18 @@ func (s statWhitelist) Description() (string, string) {
|
||||||
`Tracks the number of events processed and possibly whitelisted by each parser whitelist.`
|
`Tracks the number of events processed and possibly whitelisted by each parser whitelist.`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s statWhitelist) Process(whitelist, reason, metric string, val int) {
|
||||||
|
if _, ok := s[whitelist]; !ok {
|
||||||
|
s[whitelist] = make(map[string]map[string]int)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := s[whitelist][reason]; !ok {
|
||||||
|
s[whitelist][reason] = make(map[string]int)
|
||||||
|
}
|
||||||
|
|
||||||
|
s[whitelist][reason][metric] += val
|
||||||
|
}
|
||||||
|
|
||||||
func (s statWhitelist) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
func (s statWhitelist) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
t := newTable(out)
|
t := newTable(out)
|
||||||
t.SetRowLines(false)
|
t.SetRowLines(false)
|
||||||
|
@ -218,7 +289,17 @@ func (s statWhitelist) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
|
|
||||||
func (s statParser) Description() (string, string) {
|
func (s statParser) Description() (string, string) {
|
||||||
return "Parser Metrics",
|
return "Parser Metrics",
|
||||||
`Tracks the number of events processed by each parser and indicates success of failure. Zero parsed lines means the parer(s) failed. Non-zero unparsed lines are fine as crowdsec select relevant lines.`
|
`Tracks the number of events processed by each parser and indicates success of failure. ` +
|
||||||
|
`Zero parsed lines means the parer(s) failed. ` +
|
||||||
|
`Non-zero unparsed lines are fine as crowdsec select relevant lines.`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s statParser) Process(parser, metric string, val int) {
|
||||||
|
if _, ok := s[parser]; !ok {
|
||||||
|
s[parser] = make(map[string]int)
|
||||||
|
}
|
||||||
|
|
||||||
|
s[parser][metric] += val
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s statParser) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
func (s statParser) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
|
@ -243,6 +324,16 @@ func (s statStash) Description() (string, string) {
|
||||||
`Tracks the status of stashes that might be created by various parsers and scenarios.`
|
`Tracks the status of stashes that might be created by various parsers and scenarios.`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s statStash) Process(name, mtype string, val int) {
|
||||||
|
s[name] = struct {
|
||||||
|
Type string
|
||||||
|
Count int
|
||||||
|
}{
|
||||||
|
Type: mtype,
|
||||||
|
Count: val,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (s statStash) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
func (s statStash) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
t := newTable(out)
|
t := newTable(out)
|
||||||
t.SetRowLines(false)
|
t.SetRowLines(false)
|
||||||
|
@ -258,11 +349,12 @@ func (s statStash) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
row := []string{
|
row := []string{
|
||||||
alabel,
|
alabel,
|
||||||
astats.Type,
|
astats.Type,
|
||||||
fmt.Sprintf("%d", astats.Count),
|
strconv.Itoa(astats.Count),
|
||||||
}
|
}
|
||||||
t.AddRow(row...)
|
t.AddRow(row...)
|
||||||
numRows++
|
numRows++
|
||||||
}
|
}
|
||||||
|
|
||||||
if numRows > 0 || showEmpty {
|
if numRows > 0 || showEmpty {
|
||||||
title, _ := s.Description()
|
title, _ := s.Description()
|
||||||
renderTableTitle(out, "\n"+title+":")
|
renderTableTitle(out, "\n"+title+":")
|
||||||
|
@ -275,6 +367,14 @@ func (s statLapi) Description() (string, string) {
|
||||||
`Monitors the requests made to local API routes.`
|
`Monitors the requests made to local API routes.`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s statLapi) Process(route, method string, val int) {
|
||||||
|
if _, ok := s[route]; !ok {
|
||||||
|
s[route] = make(map[string]int)
|
||||||
|
}
|
||||||
|
|
||||||
|
s[route][method] += val
|
||||||
|
}
|
||||||
|
|
||||||
func (s statLapi) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
func (s statLapi) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
t := newTable(out)
|
t := newTable(out)
|
||||||
t.SetRowLines(false)
|
t.SetRowLines(false)
|
||||||
|
@ -291,13 +391,14 @@ func (s statLapi) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
for skey := range astats {
|
for skey := range astats {
|
||||||
subKeys = append(subKeys, skey)
|
subKeys = append(subKeys, skey)
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Strings(subKeys)
|
sort.Strings(subKeys)
|
||||||
|
|
||||||
for _, sl := range subKeys {
|
for _, sl := range subKeys {
|
||||||
row := []string{
|
row := []string{
|
||||||
alabel,
|
alabel,
|
||||||
sl,
|
sl,
|
||||||
fmt.Sprintf("%d", astats[sl]),
|
strconv.Itoa(astats[sl]),
|
||||||
}
|
}
|
||||||
t.AddRow(row...)
|
t.AddRow(row...)
|
||||||
numRows++
|
numRows++
|
||||||
|
@ -316,6 +417,18 @@ func (s statLapiMachine) Description() (string, string) {
|
||||||
`Tracks the number of calls to the local API from each registered machine.`
|
`Tracks the number of calls to the local API from each registered machine.`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s statLapiMachine) Process(machine, route, method string, val int) {
|
||||||
|
if _, ok := s[machine]; !ok {
|
||||||
|
s[machine] = make(map[string]map[string]int)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := s[machine][route]; !ok {
|
||||||
|
s[machine][route] = make(map[string]int)
|
||||||
|
}
|
||||||
|
|
||||||
|
s[machine][route][method] += val
|
||||||
|
}
|
||||||
|
|
||||||
func (s statLapiMachine) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
func (s statLapiMachine) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
t := newTable(out)
|
t := newTable(out)
|
||||||
t.SetRowLines(false)
|
t.SetRowLines(false)
|
||||||
|
@ -336,6 +449,18 @@ func (s statLapiBouncer) Description() (string, string) {
|
||||||
`Tracks total hits to remediation component related API routes.`
|
`Tracks total hits to remediation component related API routes.`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s statLapiBouncer) Process(bouncer, route, method string, val int) {
|
||||||
|
if _, ok := s[bouncer]; !ok {
|
||||||
|
s[bouncer] = make(map[string]map[string]int)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := s[bouncer][route]; !ok {
|
||||||
|
s[bouncer][route] = make(map[string]int)
|
||||||
|
}
|
||||||
|
|
||||||
|
s[bouncer][route][method] += val
|
||||||
|
}
|
||||||
|
|
||||||
func (s statLapiBouncer) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
func (s statLapiBouncer) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
t := newTable(out)
|
t := newTable(out)
|
||||||
t.SetRowLines(false)
|
t.SetRowLines(false)
|
||||||
|
@ -356,6 +481,26 @@ func (s statLapiDecision) Description() (string, string) {
|
||||||
`Tracks the number of empty/non-empty answers from LAPI to bouncers that are working in "live" mode.`
|
`Tracks the number of empty/non-empty answers from LAPI to bouncers that are working in "live" mode.`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s statLapiDecision) Process(bouncer, fam string, val int) {
|
||||||
|
if _, ok := s[bouncer]; !ok {
|
||||||
|
s[bouncer] = struct {
|
||||||
|
NonEmpty int
|
||||||
|
Empty int
|
||||||
|
}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
x := s[bouncer]
|
||||||
|
|
||||||
|
switch fam {
|
||||||
|
case "cs_lapi_decisions_ko_total":
|
||||||
|
x.Empty += val
|
||||||
|
case "cs_lapi_decisions_ok_total":
|
||||||
|
x.NonEmpty += val
|
||||||
|
}
|
||||||
|
|
||||||
|
s[bouncer] = x
|
||||||
|
}
|
||||||
|
|
||||||
func (s statLapiDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
func (s statLapiDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
t := newTable(out)
|
t := newTable(out)
|
||||||
t.SetRowLines(false)
|
t.SetRowLines(false)
|
||||||
|
@ -363,11 +508,12 @@ func (s statLapiDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
||||||
|
|
||||||
numRows := 0
|
numRows := 0
|
||||||
|
|
||||||
for bouncer, hits := range s {
|
for bouncer, hits := range s {
|
||||||
t.AddRow(
|
t.AddRow(
|
||||||
bouncer,
|
bouncer,
|
||||||
fmt.Sprintf("%d", hits.Empty),
|
strconv.Itoa(hits.Empty),
|
||||||
fmt.Sprintf("%d", hits.NonEmpty),
|
strconv.Itoa(hits.NonEmpty),
|
||||||
)
|
)
|
||||||
numRows++
|
numRows++
|
||||||
}
|
}
|
||||||
|
@ -381,7 +527,20 @@ func (s statLapiDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
|
|
||||||
func (s statDecision) Description() (string, string) {
|
func (s statDecision) Description() (string, string) {
|
||||||
return "Local API Decisions",
|
return "Local API Decisions",
|
||||||
`Provides information about all currently active decisions. Includes both local (crowdsec) and global decisions (CAPI), and lists subscriptions (lists).`
|
`Provides information about all currently active decisions. ` +
|
||||||
|
`Includes both local (crowdsec) and global decisions (CAPI), and lists subscriptions (lists).`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s statDecision) Process(reason, origin, action string, val int) {
|
||||||
|
if _, ok := s[reason]; !ok {
|
||||||
|
s[reason] = make(map[string]map[string]int)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := s[reason][origin]; !ok {
|
||||||
|
s[reason][origin] = make(map[string]int)
|
||||||
|
}
|
||||||
|
|
||||||
|
s[reason][origin][action] += val
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s statDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
func (s statDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
|
@ -391,6 +550,7 @@ func (s statDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
|
||||||
|
|
||||||
numRows := 0
|
numRows := 0
|
||||||
|
|
||||||
for reason, origins := range s {
|
for reason, origins := range s {
|
||||||
for origin, actions := range origins {
|
for origin, actions := range origins {
|
||||||
for action, hits := range actions {
|
for action, hits := range actions {
|
||||||
|
@ -398,7 +558,7 @@ func (s statDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
reason,
|
reason,
|
||||||
origin,
|
origin,
|
||||||
action,
|
action,
|
||||||
fmt.Sprintf("%d", hits),
|
strconv.Itoa(hits),
|
||||||
)
|
)
|
||||||
numRows++
|
numRows++
|
||||||
}
|
}
|
||||||
|
@ -417,6 +577,10 @@ func (s statAlert) Description() (string, string) {
|
||||||
`Tracks the total number of past and present alerts for the installed scenarios.`
|
`Tracks the total number of past and present alerts for the installed scenarios.`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s statAlert) Process(reason string, val int) {
|
||||||
|
s[reason] += val
|
||||||
|
}
|
||||||
|
|
||||||
func (s statAlert) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
func (s statAlert) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
t := newTable(out)
|
t := newTable(out)
|
||||||
t.SetRowLines(false)
|
t.SetRowLines(false)
|
||||||
|
@ -424,10 +588,11 @@ func (s statAlert) Table(out io.Writer, noUnit bool, showEmpty bool) {
|
||||||
t.SetAlignment(table.AlignLeft, table.AlignLeft)
|
t.SetAlignment(table.AlignLeft, table.AlignLeft)
|
||||||
|
|
||||||
numRows := 0
|
numRows := 0
|
||||||
|
|
||||||
for scenario, hits := range s {
|
for scenario, hits := range s {
|
||||||
t.AddRow(
|
t.AddRow(
|
||||||
scenario,
|
scenario,
|
||||||
fmt.Sprintf("%d", hits),
|
strconv.Itoa(hits),
|
||||||
)
|
)
|
||||||
numRows++
|
numRows++
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue