From d1ce543440cea9f0f28a97060d1940bfe19ebe88 Mon Sep 17 00:00:00 2001 From: "Thibault \"bui\" Koechlin" Date: Tue, 2 Nov 2021 12:06:01 +0100 Subject: [PATCH] Improve explain (#1039) * improve explain feature * nicer display for details, --verbose in favor of --debug for details --- cmd/crowdsec-cli/explain.go | 4 +- cmd/crowdsec-cli/hubtest.go | 2 +- go.mod | 3 +- go.sum | 13 ++++++ pkg/cstest/parser_assert.go | 91 +++++++++++++++++++++++++++++++------ 5 files changed, 96 insertions(+), 17 deletions(-) diff --git a/cmd/crowdsec-cli/explain.go b/cmd/crowdsec-cli/explain.go index d95f5d61a..1eff44425 100644 --- a/cmd/crowdsec-cli/explain.go +++ b/cmd/crowdsec-cli/explain.go @@ -17,6 +17,7 @@ func NewExplainCmd() *cobra.Command { var dsn string var logLine string var logType string + var details bool var cmdExplain = &cobra.Command{ Use: "explain", @@ -95,7 +96,7 @@ cscli explain -dsn "file://myfile.log" --type nginx log.Fatalf("unable to load bucket dump result: %s", err) } - if err := cstest.DumpTree(*parserDump, *bucketStateDump); err != nil { + if err := cstest.DumpTree(*parserDump, *bucketStateDump, details); err != nil { log.Fatalf(err.Error()) } }, @@ -104,6 +105,7 @@ cscli explain -dsn "file://myfile.log" --type nginx cmdExplain.PersistentFlags().StringVarP(&dsn, "dsn", "d", "", "DSN to test") cmdExplain.PersistentFlags().StringVarP(&logLine, "log", "l", "", "Lgg line to test") cmdExplain.PersistentFlags().StringVarP(&logType, "type", "t", "", "Type of the acquisition to test") + cmdExplain.PersistentFlags().BoolVarP(&details, "verbose", "v", false, "Display individual changes") return cmdExplain } diff --git a/cmd/crowdsec-cli/hubtest.go b/cmd/crowdsec-cli/hubtest.go index 40384e35a..fbae42873 100644 --- a/cmd/crowdsec-cli/hubtest.go +++ b/cmd/crowdsec-cli/hubtest.go @@ -575,7 +575,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios } } - cstest.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData) + cstest.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, false) } }, } diff --git a/go.mod b/go.mod index 5995d5cf3..005c409e0 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/docker/docker v20.10.2+incompatible github.com/docker/go-connections v0.4.0 github.com/enescakir/emoji v1.0.0 + github.com/fatih/color v1.13.0 // indirect github.com/fsnotify/fsnotify v1.4.9 github.com/gin-gonic/gin v1.6.3 github.com/go-co-op/gocron v1.9.0 @@ -44,7 +45,6 @@ require ( github.com/influxdata/go-syslog/v3 v3.0.0 github.com/leodido/go-urn v1.2.1 // indirect github.com/lib/pq v1.10.2 - github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-runewidth v0.0.10 // indirect github.com/mattn/go-sqlite3 v1.14.8 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect @@ -62,6 +62,7 @@ require ( github.com/prometheus/client_golang v1.10.0 github.com/prometheus/client_model v0.2.0 github.com/prometheus/prom2json v1.3.0 + github.com/r3labs/diff/v2 v2.14.1 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.8.1 diff --git a/go.sum b/go.sum index 569a2455b..50e3a634f 100644 --- a/go.sum +++ b/go.sum @@ -154,6 +154,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -482,12 +484,16 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -627,6 +633,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y= github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/r3labs/diff/v2 v2.14.1 h1:wRZ3jB44Ny50DSXsoIcFQ27l2x+n5P31K/Pk+b9B0Ic= +github.com/r3labs/diff/v2 v2.14.1/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rivo/tview v0.0.0-20200219210816-cd38d7432498/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -706,6 +714,8 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vjeantet/grok v1.0.1 h1:2rhIR7J4gThTgcZ1m2JY4TrJZNgjn985U28kT2wQrJ4= github.com/vjeantet/grok v1.0.1/go.mod h1:ax1aAchzC6/QMXMcyzHQGZWaW1l195+uMYIkCWPCNIo= +github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= +github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -866,6 +876,7 @@ golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210921065528-437939a70204 h1:JJhkWtBuTQKyz2bd5WG9H8iUsJRU3En/KRfN8B2RnDs= golang.org/x/sys v0.0.0-20210921065528-437939a70204/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -935,6 +946,8 @@ google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= diff --git a/pkg/cstest/parser_assert.go b/pkg/cstest/parser_assert.go index 02639af8a..b8696ef5e 100644 --- a/pkg/cstest/parser_assert.go +++ b/pkg/cstest/parser_assert.go @@ -15,7 +15,9 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/enescakir/emoji" + "github.com/fatih/color" "github.com/pkg/errors" + diff "github.com/r3labs/diff/v2" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) @@ -153,9 +155,6 @@ func (p *ParserAssert) RunExpression(expression string) (interface{}, error) { if runtimeFilter, err = expr.Compile(expression, expr.Env(exprhelpers.GetExprEnv(env))); err != nil { return output, err } - // if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(exprhelpers.GetExprEnv(env))); err != nil { - // log.Warningf("Failed building debugher for %s : %s", assert, err) - // } //dump opcode in trace level log.Tracef("%s", runtimeFilter.Disassemble()) @@ -272,10 +271,10 @@ func LoadParserDump(filepath string) (*ParserResults, error) { return &pdump, nil } -func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo) error { +func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, details bool) error { //note : we can use line -> time as the unique identifier (of acquisition) - state := make(map[time.Time]map[string]map[string]bool, 0) + state := make(map[time.Time]map[string]map[string]ParserResult) assoc := make(map[time.Time]string, 0) for stage, parsers := range parser_results { @@ -283,14 +282,15 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo) error { for _, parser_res := range results { evt := parser_res.Evt if _, ok := state[evt.Line.Time]; !ok { - state[evt.Line.Time] = make(map[string]map[string]bool) + state[evt.Line.Time] = make(map[string]map[string]ParserResult) assoc[evt.Line.Time] = evt.Line.Raw } if _, ok := state[evt.Line.Time][stage]; !ok { - state[evt.Line.Time][stage] = make(map[string]bool) + state[evt.Line.Time][stage] = make(map[string]ParserResult) } - state[evt.Line.Time][stage][parser] = parser_res.Success + state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parser_res.Success} } + } } @@ -301,18 +301,20 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo) error { } //it might be bucket oveflow being reprocessed, skip this if _, ok := state[evt.Line.Time]; !ok { - state[evt.Line.Time] = make(map[string]map[string]bool) + state[evt.Line.Time] = make(map[string]map[string]ParserResult) assoc[evt.Line.Time] = evt.Line.Raw } //there is a trick : to know if an event succesfully exit the parsers, we check if it reached the pour() phase //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered if _, ok := state[evt.Line.Time]["buckets"]; !ok { - state[evt.Line.Time]["buckets"] = make(map[string]bool) + state[evt.Line.Time]["buckets"] = make(map[string]ParserResult) } - state[evt.Line.Time]["buckets"][bname] = true + state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true} } } - + yellow := color.New(color.FgYellow).SprintFunc() + red := color.New(color.FgRed).SprintFunc() + green := color.New(color.FgGreen).SprintFunc() //get each line for tstamp, rawstr := range assoc { fmt.Printf("line: %s\n", rawstr) @@ -327,6 +329,8 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo) error { } sort.Strings(skeys) //iterate stage + var prev_item types.Event + for _, stage := range skeys { parsers := state[tstamp][stage] @@ -342,13 +346,72 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo) error { sort.Strings(pkeys) for idx, parser := range pkeys { - res := parsers[parser] + res := parsers[parser].Success sep := "├" if idx == len(pkeys)-1 { sep = "└" } + created := 0 + updated := 0 + deleted := 0 + whitelisted := false + changeStr := "" + detailsDisplay := "" + if res { - fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.GreenCircle, parser) + if prev_item.Stage == "" { + changeStr = "first_parser" + } else { + changelog, _ := diff.Diff(prev_item, parsers[parser].Evt) + for _, change := range changelog { + switch change.Type { + case "create": + created++ + detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s %s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To)) + case "update": + detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s %s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To)) + if change.Path[0] == "Whitelisted" && change.To == true { + whitelisted = true + } + updated++ + case "delete": + deleted++ + detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s %s\n", presep, sep, change.Type, red(strings.Join(change.Path, "."))) + } + } + } + prev_item = parsers[parser].Evt + } + + if created > 0 { + changeStr += green(fmt.Sprintf("+%d", created)) + } + if updated > 0 { + if len(changeStr) > 0 { + changeStr += " " + } + changeStr += yellow(fmt.Sprintf("~%d", updated)) + } + if deleted > 0 { + if len(changeStr) > 0 { + changeStr += " " + } + changeStr += red(fmt.Sprintf("-%d", deleted)) + } + if whitelisted { + if len(changeStr) > 0 { + changeStr += " " + } + changeStr += red("[whitelisted]") + } + if changeStr == "" { + changeStr = yellow("unchanged") + } + if res { + fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr) + if details { + fmt.Print(detailsDisplay) + } } else { fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser)