Improve explain (#1039)
* improve explain feature * nicer display for details, --verbose in favor of --debug for details
This commit is contained in:
Thibault "bui" Koechlin
GitHub
parent
cbada3d435
commit
d1ce543440
5 changed files with 96 additions and 17 deletions
|
|
@ -17,6 +17,7 @@ func NewExplainCmd() *cobra.Command {
|
|||
var dsn string
|
||||
var logLine string
|
||||
var logType string
|
||||
var details bool
|
||||
|
||||
var cmdExplain = &cobra.Command{
|
||||
Use: "explain",
|
||||
|
|
@ -95,7 +96,7 @@ cscli explain -dsn "file://myfile.log" --type nginx
|
|||
log.Fatalf("unable to load bucket dump result: %s", err)
|
||||
}
|
||||
|
||||
if err := cstest.DumpTree(*parserDump, *bucketStateDump); err != nil {
|
||||
if err := cstest.DumpTree(*parserDump, *bucketStateDump, details); err != nil {
|
||||
log.Fatalf(err.Error())
|
||||
}
|
||||
},
|
||||
|
|
@ -104,6 +105,7 @@ cscli explain -dsn "file://myfile.log" --type nginx
|
|||
cmdExplain.PersistentFlags().StringVarP(&dsn, "dsn", "d", "", "DSN to test")
|
||||
cmdExplain.PersistentFlags().StringVarP(&logLine, "log", "l", "", "Lgg line to test")
|
||||
cmdExplain.PersistentFlags().StringVarP(&logType, "type", "t", "", "Type of the acquisition to test")
|
||||
cmdExplain.PersistentFlags().BoolVarP(&details, "verbose", "v", false, "Display individual changes")
|
||||
|
||||
return cmdExplain
|
||||
}
|
||||
|
|
|
|||
|
|
@ -575,7 +575,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
|
|||
}
|
||||
}
|
||||
|
||||
cstest.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData)
|
||||
cstest.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, false)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
|
|
|||
3
go.mod
3
go.mod
|
|
@ -24,6 +24,7 @@ require (
|
|||
github.com/docker/docker v20.10.2+incompatible
|
||||
github.com/docker/go-connections v0.4.0
|
||||
github.com/enescakir/emoji v1.0.0
|
||||
github.com/fatih/color v1.13.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.4.9
|
||||
github.com/gin-gonic/gin v1.6.3
|
||||
github.com/go-co-op/gocron v1.9.0
|
||||
|
|
@ -44,7 +45,6 @@ require (
|
|||
github.com/influxdata/go-syslog/v3 v3.0.0
|
||||
github.com/leodido/go-urn v1.2.1 // indirect
|
||||
github.com/lib/pq v1.10.2
|
||||
github.com/mattn/go-colorable v0.1.8 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.10 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.8
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
|
||||
|
|
@ -62,6 +62,7 @@ require (
|
|||
github.com/prometheus/client_golang v1.10.0
|
||||
github.com/prometheus/client_model v0.2.0
|
||||
github.com/prometheus/prom2json v1.3.0
|
||||
github.com/r3labs/diff/v2 v2.14.1 // indirect
|
||||
github.com/rivo/uniseg v0.2.0 // indirect
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/sirupsen/logrus v1.8.1
|
||||
|
|
|
|||
13
go.sum
13
go.sum
|
|
@ -154,6 +154,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
|
|||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
|
||||
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
||||
github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
|
||||
github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
|
|
@ -482,12 +484,16 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
|
|||
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
|
||||
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U=
|
||||
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
|
||||
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
|
|
@ -627,6 +633,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
|
|||
github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y=
|
||||
github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/r3labs/diff/v2 v2.14.1 h1:wRZ3jB44Ny50DSXsoIcFQ27l2x+n5P31K/Pk+b9B0Ic=
|
||||
github.com/r3labs/diff/v2 v2.14.1/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rivo/tview v0.0.0-20200219210816-cd38d7432498/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84=
|
||||
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
|
|
@ -706,6 +714,8 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX
|
|||
github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
|
||||
github.com/vjeantet/grok v1.0.1 h1:2rhIR7J4gThTgcZ1m2JY4TrJZNgjn985U28kT2wQrJ4=
|
||||
github.com/vjeantet/grok v1.0.1/go.mod h1:ax1aAchzC6/QMXMcyzHQGZWaW1l195+uMYIkCWPCNIo=
|
||||
github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
|
||||
github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
|
||||
github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
|
|
@ -866,6 +876,7 @@ golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210921065528-437939a70204 h1:JJhkWtBuTQKyz2bd5WG9H8iUsJRU3En/KRfN8B2RnDs=
|
||||
golang.org/x/sys v0.0.0-20210921065528-437939a70204/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
|
|
@ -935,6 +946,8 @@ google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
|
|||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
|
||||
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
|
|
|
|||
|
|
@ -15,7 +15,9 @@ import (
|
|||
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||
"github.com/enescakir/emoji"
|
||||
"github.com/fatih/color"
|
||||
"github.com/pkg/errors"
|
||||
diff "github.com/r3labs/diff/v2"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
|
@ -153,9 +155,6 @@ func (p *ParserAssert) RunExpression(expression string) (interface{}, error) {
|
|||
if runtimeFilter, err = expr.Compile(expression, expr.Env(exprhelpers.GetExprEnv(env))); err != nil {
|
||||
return output, err
|
||||
}
|
||||
// if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(exprhelpers.GetExprEnv(env))); err != nil {
|
||||
// log.Warningf("Failed building debugher for %s : %s", assert, err)
|
||||
// }
|
||||
|
||||
//dump opcode in trace level
|
||||
log.Tracef("%s", runtimeFilter.Disassemble())
|
||||
|
|
@ -272,10 +271,10 @@ func LoadParserDump(filepath string) (*ParserResults, error) {
|
|||
return &pdump, nil
|
||||
}
|
||||
|
||||
func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo) error {
|
||||
func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, details bool) error {
|
||||
//note : we can use line -> time as the unique identifier (of acquisition)
|
||||
|
||||
state := make(map[time.Time]map[string]map[string]bool, 0)
|
||||
state := make(map[time.Time]map[string]map[string]ParserResult)
|
||||
assoc := make(map[time.Time]string, 0)
|
||||
|
||||
for stage, parsers := range parser_results {
|
||||
|
|
@ -283,14 +282,15 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo) error {
|
|||
for _, parser_res := range results {
|
||||
evt := parser_res.Evt
|
||||
if _, ok := state[evt.Line.Time]; !ok {
|
||||
state[evt.Line.Time] = make(map[string]map[string]bool)
|
||||
state[evt.Line.Time] = make(map[string]map[string]ParserResult)
|
||||
assoc[evt.Line.Time] = evt.Line.Raw
|
||||
}
|
||||
if _, ok := state[evt.Line.Time][stage]; !ok {
|
||||
state[evt.Line.Time][stage] = make(map[string]bool)
|
||||
state[evt.Line.Time][stage] = make(map[string]ParserResult)
|
||||
}
|
||||
state[evt.Line.Time][stage][parser] = parser_res.Success
|
||||
state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parser_res.Success}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -301,18 +301,20 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo) error {
|
|||
}
|
||||
//it might be bucket oveflow being reprocessed, skip this
|
||||
if _, ok := state[evt.Line.Time]; !ok {
|
||||
state[evt.Line.Time] = make(map[string]map[string]bool)
|
||||
state[evt.Line.Time] = make(map[string]map[string]ParserResult)
|
||||
assoc[evt.Line.Time] = evt.Line.Raw
|
||||
}
|
||||
//there is a trick : to know if an event succesfully exit the parsers, we check if it reached the pour() phase
|
||||
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
|
||||
if _, ok := state[evt.Line.Time]["buckets"]; !ok {
|
||||
state[evt.Line.Time]["buckets"] = make(map[string]bool)
|
||||
state[evt.Line.Time]["buckets"] = make(map[string]ParserResult)
|
||||
}
|
||||
state[evt.Line.Time]["buckets"][bname] = true
|
||||
state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true}
|
||||
}
|
||||
}
|
||||
|
||||
yellow := color.New(color.FgYellow).SprintFunc()
|
||||
red := color.New(color.FgRed).SprintFunc()
|
||||
green := color.New(color.FgGreen).SprintFunc()
|
||||
//get each line
|
||||
for tstamp, rawstr := range assoc {
|
||||
fmt.Printf("line: %s\n", rawstr)
|
||||
|
|
@ -327,6 +329,8 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo) error {
|
|||
}
|
||||
sort.Strings(skeys)
|
||||
//iterate stage
|
||||
var prev_item types.Event
|
||||
|
||||
for _, stage := range skeys {
|
||||
parsers := state[tstamp][stage]
|
||||
|
||||
|
|
@ -342,13 +346,72 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo) error {
|
|||
sort.Strings(pkeys)
|
||||
|
||||
for idx, parser := range pkeys {
|
||||
res := parsers[parser]
|
||||
res := parsers[parser].Success
|
||||
sep := "├"
|
||||
if idx == len(pkeys)-1 {
|
||||
sep = "└"
|
||||
}
|
||||
created := 0
|
||||
updated := 0
|
||||
deleted := 0
|
||||
whitelisted := false
|
||||
changeStr := ""
|
||||
detailsDisplay := ""
|
||||
|
||||
if res {
|
||||
fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.GreenCircle, parser)
|
||||
if prev_item.Stage == "" {
|
||||
changeStr = "first_parser"
|
||||
} else {
|
||||
changelog, _ := diff.Diff(prev_item, parsers[parser].Evt)
|
||||
for _, change := range changelog {
|
||||
switch change.Type {
|
||||
case "create":
|
||||
created++
|
||||
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s %s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To))
|
||||
case "update":
|
||||
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s %s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To))
|
||||
if change.Path[0] == "Whitelisted" && change.To == true {
|
||||
whitelisted = true
|
||||
}
|
||||
updated++
|
||||
case "delete":
|
||||
deleted++
|
||||
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s %s\n", presep, sep, change.Type, red(strings.Join(change.Path, ".")))
|
||||
}
|
||||
}
|
||||
}
|
||||
prev_item = parsers[parser].Evt
|
||||
}
|
||||
|
||||
if created > 0 {
|
||||
changeStr += green(fmt.Sprintf("+%d", created))
|
||||
}
|
||||
if updated > 0 {
|
||||
if len(changeStr) > 0 {
|
||||
changeStr += " "
|
||||
}
|
||||
changeStr += yellow(fmt.Sprintf("~%d", updated))
|
||||
}
|
||||
if deleted > 0 {
|
||||
if len(changeStr) > 0 {
|
||||
changeStr += " "
|
||||
}
|
||||
changeStr += red(fmt.Sprintf("-%d", deleted))
|
||||
}
|
||||
if whitelisted {
|
||||
if len(changeStr) > 0 {
|
||||
changeStr += " "
|
||||
}
|
||||
changeStr += red("[whitelisted]")
|
||||
}
|
||||
if changeStr == "" {
|
||||
changeStr = yellow("unchanged")
|
||||
}
|
||||
if res {
|
||||
fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr)
|
||||
if details {
|
||||
fmt.Print(detailsDisplay)
|
||||
}
|
||||
} else {
|
||||
fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue