From 6ca053ca67e2c60d9595df5e28f5f50b64669d54 Mon Sep 17 00:00:00 2001 From: "Thibault \"bui\" Koechlin" Date: Mon, 15 Jan 2024 09:16:03 +0100 Subject: [PATCH] fix #2720 #2719 (#2724) * fix order of display of parsers * add a --no-clean opt --- cmd/crowdsec-cli/explain.go | 20 ++- cmd/crowdsec-cli/hubtest.go | 5 +- go.mod | 2 +- go.sum | 2 + pkg/dumps/bucket_dump.go | 32 ++++ pkg/dumps/parser_dump.go | 319 +++++++++++++++++++++++++++++++++ pkg/hubtest/coverage.go | 7 +- pkg/hubtest/parser_assert.go | 319 ++------------------------------- pkg/hubtest/scenario_assert.go | 29 +-- pkg/hubtest/utils.go | 38 ++-- pkg/parser/runtime.go | 26 +-- 11 files changed, 418 insertions(+), 381 deletions(-) create mode 100644 pkg/dumps/bucket_dump.go create mode 100644 pkg/dumps/parser_dump.go diff --git a/cmd/crowdsec-cli/explain.go b/cmd/crowdsec-cli/explain.go index 3e7f48fa0..e6dd3598f 100644 --- a/cmd/crowdsec-cli/explain.go +++ b/cmd/crowdsec-cli/explain.go @@ -12,6 +12,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/crowdsecurity/crowdsec/pkg/dumps" "github.com/crowdsecurity/crowdsec/pkg/hubtest" ) @@ -35,7 +36,7 @@ func GetLineCountForFile(filepath string) (int, error) { return lc, nil } -type cliExplain struct {} +type cliExplain struct{} func NewCLIExplain() *cliExplain { return &cliExplain{} @@ -109,6 +110,7 @@ tail -n 5 myfile.log | cscli explain --type nginx -f - flags.Bool("failures", false, "Only show failed lines") flags.Bool("only-successful-parsers", false, "Only show successful parsers") flags.String("crowdsec", "crowdsec", "Path to crowdsec") + flags.Bool("no-clean", false, "Don't clean runtime environment after tests") return cmd } @@ -136,13 +138,18 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error { return err } - opts := hubtest.DumpOpts{} + opts := dumps.DumpOpts{} opts.Details, err = flags.GetBool("verbose") if err != nil { return err } + no_clean, err := flags.GetBool("no-clean") + if err != nil { + return err + } + opts.SkipOk, err = flags.GetBool("failures") if err != nil { return err @@ -172,6 +179,9 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error { return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %s", err) } defer func() { + if no_clean { + return + } if _, err := os.Stat(dir); !os.IsNotExist(err) { if err := os.RemoveAll(dir); err != nil { log.Errorf("unable to delete temporary directory '%s': %s", dir, err) @@ -254,17 +264,17 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error { parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName) bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName) - parserDump, err := hubtest.LoadParserDump(parserDumpFile) + parserDump, err := dumps.LoadParserDump(parserDumpFile) if err != nil { return fmt.Errorf("unable to load parser dump result: %s", err) } - bucketStateDump, err := hubtest.LoadBucketPourDump(bucketStateDumpFile) + bucketStateDump, err := dumps.LoadBucketPourDump(bucketStateDumpFile) if err != nil { return fmt.Errorf("unable to load bucket dump result: %s", err) } - hubtest.DumpTree(*parserDump, *bucketStateDump, opts) + dumps.DumpTree(*parserDump, *bucketStateDump, opts) return nil } diff --git a/cmd/crowdsec-cli/hubtest.go b/cmd/crowdsec-cli/hubtest.go index daf22fb5c..1860540e7 100644 --- a/cmd/crowdsec-cli/hubtest.go +++ b/cmd/crowdsec-cli/hubtest.go @@ -16,6 +16,7 @@ import ( "github.com/spf13/cobra" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/crowdsec/pkg/dumps" "github.com/crowdsecurity/crowdsec/pkg/hubtest" ) @@ -679,8 +680,8 @@ func (cli cliHubTest) NewExplainCmd() *cobra.Command { return fmt.Errorf("unable to load scenario result after run: %s", err) } } - opts := hubtest.DumpOpts{} - hubtest.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts) + opts := dumps.DumpOpts{} + dumps.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts) } return nil diff --git a/go.mod b/go.mod index b0fe9368f..d61c191c1 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 - github.com/crowdsecurity/go-cs-lib v0.0.5 + github.com/crowdsecurity/go-cs-lib v0.0.6 github.com/crowdsecurity/grokky v0.2.1 github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index 2feb45fb9..f5f61594e 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,8 @@ github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8= github.com/crowdsecurity/go-cs-lib v0.0.5/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k= +github.com/crowdsecurity/go-cs-lib v0.0.6 h1:Ef6MylXe0GaJE9vrfvxEdbHb31+JUP1os+murPz7Pos= +github.com/crowdsecurity/go-cs-lib v0.0.6/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k= github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4= github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= diff --git a/pkg/dumps/bucket_dump.go b/pkg/dumps/bucket_dump.go new file mode 100644 index 000000000..5f5ce1c40 --- /dev/null +++ b/pkg/dumps/bucket_dump.go @@ -0,0 +1,32 @@ +package dumps + +import ( + "io" + "os" + + "github.com/crowdsecurity/crowdsec/pkg/types" + "gopkg.in/yaml.v2" +) + +type BucketPourInfo map[string][]types.Event + +func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) { + dumpData, err := os.Open(filepath) + if err != nil { + return nil, err + } + defer dumpData.Close() + + results, err := io.ReadAll(dumpData) + if err != nil { + return nil, err + } + + var bucketDump BucketPourInfo + + if err := yaml.Unmarshal(results, &bucketDump); err != nil { + return nil, err + } + + return &bucketDump, nil +} diff --git a/pkg/dumps/parser_dump.go b/pkg/dumps/parser_dump.go new file mode 100644 index 000000000..566b87a08 --- /dev/null +++ b/pkg/dumps/parser_dump.go @@ -0,0 +1,319 @@ +package dumps + +import ( + "fmt" + "io" + "os" + "sort" + "strings" + "time" + + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/crowdsecurity/go-cs-lib/maptools" + "github.com/enescakir/emoji" + "github.com/fatih/color" + diff "github.com/r3labs/diff/v2" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +type ParserResult struct { + Idx int + Evt types.Event + Success bool +} + +type ParserResults map[string]map[string][]ParserResult + +type DumpOpts struct { + Details bool + SkipOk bool + ShowNotOkParsers bool +} + +func LoadParserDump(filepath string) (*ParserResults, error) { + dumpData, err := os.Open(filepath) + if err != nil { + return nil, err + } + defer dumpData.Close() + + results, err := io.ReadAll(dumpData) + if err != nil { + return nil, err + } + + pdump := ParserResults{} + + if err := yaml.Unmarshal(results, &pdump); err != nil { + return nil, err + } + + /* we know that some variables should always be set, + let's check if they're present in last parser output of last stage */ + + stages := maptools.SortedKeys(pdump) + + var lastStage string + + //Loop over stages to find last successful one with at least one parser + for i := len(stages) - 2; i >= 0; i-- { + if len(pdump[stages[i]]) != 0 { + lastStage = stages[i] + break + } + } + + parsers := make([]string, 0, len(pdump[lastStage])) + + for k := range pdump[lastStage] { + parsers = append(parsers, k) + } + + sort.Strings(parsers) + + if len(parsers) == 0 { + return nil, fmt.Errorf("no parser found. Please install the appropriate parser and retry") + } + + lastParser := parsers[len(parsers)-1] + + for idx, result := range pdump[lastStage][lastParser] { + if result.Evt.StrTime == "" { + log.Warningf("Line %d/%d is missing evt.StrTime. It is most likely a mistake as it will prevent your logs to be processed in time-machine/forensic mode.", idx, len(pdump[lastStage][lastParser])) + } else { + log.Debugf("Line %d/%d has evt.StrTime set to '%s'", idx, len(pdump[lastStage][lastParser]), result.Evt.StrTime) + } + } + + return &pdump, nil +} + +func DumpTree(parserResults ParserResults, bucketPour BucketPourInfo, opts DumpOpts) { + //note : we can use line -> time as the unique identifier (of acquisition) + state := make(map[time.Time]map[string]map[string]ParserResult) + assoc := make(map[time.Time]string, 0) + parser_order := make(map[string][]string) + + for stage, parsers := range parserResults { + //let's process parsers in the order according to idx + parser_order[stage] = make([]string, len(parsers)) + for pname, parser := range parsers { + if len(parser) > 0 { + parser_order[stage][parser[0].Idx-1] = pname + } + } + + for _, parser := range parser_order[stage] { + results := parsers[parser] + for _, parserRes := range results { + evt := parserRes.Evt + if _, ok := state[evt.Line.Time]; !ok { + state[evt.Line.Time] = make(map[string]map[string]ParserResult) + assoc[evt.Line.Time] = evt.Line.Raw + } + + if _, ok := state[evt.Line.Time][stage]; !ok { + state[evt.Line.Time][stage] = make(map[string]ParserResult) + } + + state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parserRes.Success} + } + } + } + + for bname, evtlist := range bucketPour { + for _, evt := range evtlist { + if evt.Line.Raw == "" { + continue + } + + //it might be bucket overflow being reprocessed, skip this + if _, ok := state[evt.Line.Time]; !ok { + state[evt.Line.Time] = make(map[string]map[string]ParserResult) + assoc[evt.Line.Time] = evt.Line.Raw + } + + //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase + //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered + if _, ok := state[evt.Line.Time]["buckets"]; !ok { + state[evt.Line.Time]["buckets"] = make(map[string]ParserResult) + } + + state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true} + } + } + + yellow := color.New(color.FgYellow).SprintFunc() + red := color.New(color.FgRed).SprintFunc() + green := color.New(color.FgGreen).SprintFunc() + whitelistReason := "" + //get each line + for tstamp, rawstr := range assoc { + if opts.SkipOk { + if _, ok := state[tstamp]["buckets"]["OK"]; ok { + continue + } + } + + fmt.Printf("line: %s\n", rawstr) + + skeys := make([]string, 0, len(state[tstamp])) + + for k := range state[tstamp] { + //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase + //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered + if k == "buckets" { + continue + } + + skeys = append(skeys, k) + } + + sort.Strings(skeys) + + // iterate stage + var prevItem types.Event + + for _, stage := range skeys { + parsers := state[tstamp][stage] + + sep := "├" + presep := "|" + + fmt.Printf("\t%s %s\n", sep, stage) + + for idx, parser := range parser_order[stage] { + res := parsers[parser].Success + sep := "├" + + if idx == len(parser_order[stage])-1 { + sep = "└" + } + + created := 0 + updated := 0 + deleted := 0 + whitelisted := false + changeStr := "" + detailsDisplay := "" + + if res { + changelog, _ := diff.Diff(prevItem, parsers[parser].Evt) + for _, change := range changelog { + switch change.Type { + case "create": + created++ + + detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To)) + case "update": + detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To)) + + if change.Path[0] == "Whitelisted" && change.To == true { + whitelisted = true + + if whitelistReason == "" { + whitelistReason = parsers[parser].Evt.WhitelistReason + } + } + updated++ + case "delete": + deleted++ + + detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, "."))) + } + } + + prevItem = parsers[parser].Evt + } + + if created > 0 { + changeStr += green(fmt.Sprintf("+%d", created)) + } + + if updated > 0 { + if len(changeStr) > 0 { + changeStr += " " + } + + changeStr += yellow(fmt.Sprintf("~%d", updated)) + } + + if deleted > 0 { + if len(changeStr) > 0 { + changeStr += " " + } + + changeStr += red(fmt.Sprintf("-%d", deleted)) + } + + if whitelisted { + if len(changeStr) > 0 { + changeStr += " " + } + + changeStr += red("[whitelisted]") + } + + if changeStr == "" { + changeStr = yellow("unchanged") + } + + if res { + fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr) + + if opts.Details { + fmt.Print(detailsDisplay) + } + } else if opts.ShowNotOkParsers { + fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser) + } + } + } + + sep := "└" + + if len(state[tstamp]["buckets"]) > 0 { + sep = "├" + } + + //did the event enter the bucket pour phase ? + if _, ok := state[tstamp]["buckets"]["OK"]; ok { + fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle) + } else if whitelistReason != "" { + fmt.Printf("\t%s-------- parser success, ignored by whitelist (%s) %s\n", sep, whitelistReason, emoji.GreenCircle) + } else { + fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle) + } + + //now print bucket info + if len(state[tstamp]["buckets"]) > 0 { + fmt.Printf("\t├ Scenarios\n") + } + + bnames := make([]string, 0, len(state[tstamp]["buckets"])) + + for k := range state[tstamp]["buckets"] { + //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase + //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered + if k == "OK" { + continue + } + + bnames = append(bnames, k) + } + + sort.Strings(bnames) + + for idx, bname := range bnames { + sep := "├" + if idx == len(bnames)-1 { + sep = "└" + } + + fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname) + } + + fmt.Println() + } +} diff --git a/pkg/hubtest/coverage.go b/pkg/hubtest/coverage.go index edbe10454..dc3d1d13a 100644 --- a/pkg/hubtest/coverage.go +++ b/pkg/hubtest/coverage.go @@ -9,6 +9,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/go-cs-lib/maptools" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) @@ -25,7 +26,7 @@ func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) { } // populate from hub, iterate in alphabetical order - pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.APPSEC_RULES)) + pkeys := maptools.SortedKeys(h.HubIndex.GetItemMap(cwhub.APPSEC_RULES)) coverage := make([]Coverage, len(pkeys)) for i, name := range pkeys { @@ -84,7 +85,7 @@ func (h *HubTest) GetParsersCoverage() ([]Coverage, error) { } // populate from hub, iterate in alphabetical order - pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.PARSERS)) + pkeys := maptools.SortedKeys(h.HubIndex.GetItemMap(cwhub.PARSERS)) coverage := make([]Coverage, len(pkeys)) for i, name := range pkeys { @@ -170,7 +171,7 @@ func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) { } // populate from hub, iterate in alphabetical order - pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) + pkeys := maptools.SortedKeys(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) coverage := make([]Coverage, len(pkeys)) for i, name := range pkeys { diff --git a/pkg/hubtest/parser_assert.go b/pkg/hubtest/parser_assert.go index aadf16af7..db27f710e 100644 --- a/pkg/hubtest/parser_assert.go +++ b/pkg/hubtest/parser_assert.go @@ -3,21 +3,16 @@ package hubtest import ( "bufio" "fmt" - "io" "os" - "sort" "strings" - "time" "github.com/antonmedv/expr" - "github.com/enescakir/emoji" - "github.com/fatih/color" - diff "github.com/r3labs/diff/v2" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/crowdsec/pkg/dumps" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/crowdsecurity/go-cs-lib/maptools" ) type AssertFail struct { @@ -34,16 +29,9 @@ type ParserAssert struct { NbAssert int Fails []AssertFail Success bool - TestData *ParserResults + TestData *dumps.ParserResults } -type ParserResult struct { - Evt types.Event - Success bool -} - -type ParserResults map[string]map[string][]ParserResult - func NewParserAssert(file string) *ParserAssert { ParserAssert := &ParserAssert{ File: file, @@ -51,7 +39,7 @@ func NewParserAssert(file string) *ParserAssert { Success: false, Fails: make([]AssertFail, 0), AutoGenAssert: false, - TestData: &ParserResults{}, + TestData: &dumps.ParserResults{}, } return ParserAssert @@ -69,7 +57,7 @@ func (p *ParserAssert) AutoGenFromFile(filename string) (string, error) { } func (p *ParserAssert) LoadTest(filename string) error { - parserDump, err := LoadParserDump(filename) + parserDump, err := dumps.LoadParserDump(filename) if err != nil { return fmt.Errorf("loading parser dump file: %+v", err) } @@ -229,13 +217,13 @@ func (p *ParserAssert) AutoGenParserAssert() string { ret := fmt.Sprintf("len(results) == %d\n", len(*p.TestData)) //sort map keys for consistent order - stages := sortedMapKeys(*p.TestData) + stages := maptools.SortedKeys(*p.TestData) for _, stage := range stages { parsers := (*p.TestData)[stage] //sort map keys for consistent order - pnames := sortedMapKeys(parsers) + pnames := maptools.SortedKeys(parsers) for _, parser := range pnames { presults := parsers[parser] @@ -248,7 +236,7 @@ func (p *ParserAssert) AutoGenParserAssert() string { continue } - for _, pkey := range sortedMapKeys(result.Evt.Parsed) { + for _, pkey := range maptools.SortedKeys(result.Evt.Parsed) { pval := result.Evt.Parsed[pkey] if pval == "" { continue @@ -257,7 +245,7 @@ func (p *ParserAssert) AutoGenParserAssert() string { ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Parsed["%s"] == "%s"`+"\n", stage, parser, pidx, pkey, Escape(pval)) } - for _, mkey := range sortedMapKeys(result.Evt.Meta) { + for _, mkey := range maptools.SortedKeys(result.Evt.Meta) { mval := result.Evt.Meta[mkey] if mval == "" { continue @@ -266,7 +254,7 @@ func (p *ParserAssert) AutoGenParserAssert() string { ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval)) } - for _, ekey := range sortedMapKeys(result.Evt.Enriched) { + for _, ekey := range maptools.SortedKeys(result.Evt.Enriched) { eval := result.Evt.Enriched[ekey] if eval == "" { continue @@ -275,7 +263,7 @@ func (p *ParserAssert) AutoGenParserAssert() string { ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Enriched["%s"] == "%s"`+"\n", stage, parser, pidx, ekey, Escape(eval)) } - for _, ukey := range sortedMapKeys(result.Evt.Unmarshaled) { + for _, ukey := range maptools.SortedKeys(result.Evt.Unmarshaled) { uval := result.Evt.Unmarshaled[ukey] if uval == "" { continue @@ -328,288 +316,3 @@ func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []s return ret } - -func LoadParserDump(filepath string) (*ParserResults, error) { - dumpData, err := os.Open(filepath) - if err != nil { - return nil, err - } - defer dumpData.Close() - - results, err := io.ReadAll(dumpData) - if err != nil { - return nil, err - } - - pdump := ParserResults{} - - if err := yaml.Unmarshal(results, &pdump); err != nil { - return nil, err - } - - /* we know that some variables should always be set, - let's check if they're present in last parser output of last stage */ - - stages := sortedMapKeys(pdump) - - var lastStage string - - //Loop over stages to find last successful one with at least one parser - for i := len(stages) - 2; i >= 0; i-- { - if len(pdump[stages[i]]) != 0 { - lastStage = stages[i] - break - } - } - - parsers := make([]string, 0, len(pdump[lastStage])) - - for k := range pdump[lastStage] { - parsers = append(parsers, k) - } - - sort.Strings(parsers) - - if len(parsers) == 0 { - return nil, fmt.Errorf("no parser found. Please install the appropriate parser and retry") - } - - lastParser := parsers[len(parsers)-1] - - for idx, result := range pdump[lastStage][lastParser] { - if result.Evt.StrTime == "" { - log.Warningf("Line %d/%d is missing evt.StrTime. It is most likely a mistake as it will prevent your logs to be processed in time-machine/forensic mode.", idx, len(pdump[lastStage][lastParser])) - } else { - log.Debugf("Line %d/%d has evt.StrTime set to '%s'", idx, len(pdump[lastStage][lastParser]), result.Evt.StrTime) - } - } - - return &pdump, nil -} - -type DumpOpts struct { - Details bool - SkipOk bool - ShowNotOkParsers bool -} - -func DumpTree(parserResults ParserResults, bucketPour BucketPourInfo, opts DumpOpts) { - //note : we can use line -> time as the unique identifier (of acquisition) - state := make(map[time.Time]map[string]map[string]ParserResult) - assoc := make(map[time.Time]string, 0) - - for stage, parsers := range parserResults { - for parser, results := range parsers { - for _, parserRes := range results { - evt := parserRes.Evt - if _, ok := state[evt.Line.Time]; !ok { - state[evt.Line.Time] = make(map[string]map[string]ParserResult) - assoc[evt.Line.Time] = evt.Line.Raw - } - - if _, ok := state[evt.Line.Time][stage]; !ok { - state[evt.Line.Time][stage] = make(map[string]ParserResult) - } - - state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parserRes.Success} - } - } - } - - for bname, evtlist := range bucketPour { - for _, evt := range evtlist { - if evt.Line.Raw == "" { - continue - } - - //it might be bucket overflow being reprocessed, skip this - if _, ok := state[evt.Line.Time]; !ok { - state[evt.Line.Time] = make(map[string]map[string]ParserResult) - assoc[evt.Line.Time] = evt.Line.Raw - } - - //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase - //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered - if _, ok := state[evt.Line.Time]["buckets"]; !ok { - state[evt.Line.Time]["buckets"] = make(map[string]ParserResult) - } - - state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true} - } - } - - yellow := color.New(color.FgYellow).SprintFunc() - red := color.New(color.FgRed).SprintFunc() - green := color.New(color.FgGreen).SprintFunc() - whitelistReason := "" - //get each line - for tstamp, rawstr := range assoc { - if opts.SkipOk { - if _, ok := state[tstamp]["buckets"]["OK"]; ok { - continue - } - } - - fmt.Printf("line: %s\n", rawstr) - - skeys := make([]string, 0, len(state[tstamp])) - - for k := range state[tstamp] { - //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase - //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered - if k == "buckets" { - continue - } - - skeys = append(skeys, k) - } - - sort.Strings(skeys) - - // iterate stage - var prevItem types.Event - - for _, stage := range skeys { - parsers := state[tstamp][stage] - - sep := "├" - presep := "|" - - fmt.Printf("\t%s %s\n", sep, stage) - - pkeys := sortedMapKeys(parsers) - - for idx, parser := range pkeys { - res := parsers[parser].Success - sep := "├" - - if idx == len(pkeys)-1 { - sep = "└" - } - - created := 0 - updated := 0 - deleted := 0 - whitelisted := false - changeStr := "" - detailsDisplay := "" - - if res { - changelog, _ := diff.Diff(prevItem, parsers[parser].Evt) - for _, change := range changelog { - switch change.Type { - case "create": - created++ - - detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To)) - case "update": - detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To)) - - if change.Path[0] == "Whitelisted" && change.To == true { - whitelisted = true - - if whitelistReason == "" { - whitelistReason = parsers[parser].Evt.WhitelistReason - } - } - updated++ - case "delete": - deleted++ - - detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, "."))) - } - } - - prevItem = parsers[parser].Evt - } - - if created > 0 { - changeStr += green(fmt.Sprintf("+%d", created)) - } - - if updated > 0 { - if len(changeStr) > 0 { - changeStr += " " - } - - changeStr += yellow(fmt.Sprintf("~%d", updated)) - } - - if deleted > 0 { - if len(changeStr) > 0 { - changeStr += " " - } - - changeStr += red(fmt.Sprintf("-%d", deleted)) - } - - if whitelisted { - if len(changeStr) > 0 { - changeStr += " " - } - - changeStr += red("[whitelisted]") - } - - if changeStr == "" { - changeStr = yellow("unchanged") - } - - if res { - fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr) - - if opts.Details { - fmt.Print(detailsDisplay) - } - } else if opts.ShowNotOkParsers { - fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser) - } - } - } - - sep := "└" - - if len(state[tstamp]["buckets"]) > 0 { - sep = "├" - } - - //did the event enter the bucket pour phase ? - if _, ok := state[tstamp]["buckets"]["OK"]; ok { - fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle) - } else if whitelistReason != "" { - fmt.Printf("\t%s-------- parser success, ignored by whitelist (%s) %s\n", sep, whitelistReason, emoji.GreenCircle) - } else { - fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle) - } - - //now print bucket info - if len(state[tstamp]["buckets"]) > 0 { - fmt.Printf("\t├ Scenarios\n") - } - - bnames := make([]string, 0, len(state[tstamp]["buckets"])) - - for k := range state[tstamp]["buckets"] { - //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase - //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered - if k == "OK" { - continue - } - - bnames = append(bnames, k) - } - - sort.Strings(bnames) - - for idx, bname := range bnames { - sep := "├" - if idx == len(bnames)-1 { - sep = "└" - } - - fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname) - } - - fmt.Println() - } -} diff --git a/pkg/hubtest/scenario_assert.go b/pkg/hubtest/scenario_assert.go index 011d3dcfb..5195b814e 100644 --- a/pkg/hubtest/scenario_assert.go +++ b/pkg/hubtest/scenario_assert.go @@ -12,6 +12,7 @@ import ( log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/crowdsec/pkg/dumps" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -24,11 +25,10 @@ type ScenarioAssert struct { Fails []AssertFail Success bool TestData *BucketResults - PourData *BucketPourInfo + PourData *dumps.BucketPourInfo } type BucketResults []types.Event -type BucketPourInfo map[string][]types.Event func NewScenarioAssert(file string) *ScenarioAssert { ScenarioAssert := &ScenarioAssert{ @@ -38,7 +38,7 @@ func NewScenarioAssert(file string) *ScenarioAssert { Fails: make([]AssertFail, 0), AutoGenAssert: false, TestData: &BucketResults{}, - PourData: &BucketPourInfo{}, + PourData: &dumps.BucketPourInfo{}, } return ScenarioAssert @@ -64,7 +64,7 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error { s.TestData = bucketDump if bucketpour != "" { - pourDump, err := LoadBucketPourDump(bucketpour) + pourDump, err := dumps.LoadBucketPourDump(bucketpour) if err != nil { return fmt.Errorf("loading bucket pour dump file '%s': %+v", filename, err) } @@ -252,27 +252,6 @@ func (b BucketResults) Swap(i, j int) { b[i], b[j] = b[j], b[i] } -func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) { - dumpData, err := os.Open(filepath) - if err != nil { - return nil, err - } - defer dumpData.Close() - - results, err := io.ReadAll(dumpData) - if err != nil { - return nil, err - } - - var bucketDump BucketPourInfo - - if err := yaml.Unmarshal(results, &bucketDump); err != nil { - return nil, err - } - - return &bucketDump, nil -} - func LoadScenarioDump(filepath string) (*BucketResults, error) { dumpData, err := os.Open(filepath) if err != nil { diff --git a/pkg/hubtest/utils.go b/pkg/hubtest/utils.go index 6c48cb3a6..9009d0ddd 100644 --- a/pkg/hubtest/utils.go +++ b/pkg/hubtest/utils.go @@ -5,21 +5,25 @@ import ( "net" "os" "path/filepath" - "sort" "time" log "github.com/sirupsen/logrus" ) -func sortedMapKeys[V any](m map[string]V) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) +func IsAlive(target string) (bool, error) { + start := time.Now() + for { + conn, err := net.Dial("tcp", target) + if err == nil { + log.Debugf("'%s' is up after %s", target, time.Since(start)) + conn.Close() + return true, nil + } + time.Sleep(500 * time.Millisecond) + if time.Since(start) > 10*time.Second { + return false, fmt.Errorf("took more than 10s for %s to be available", target) + } } - - sort.Strings(keys) - - return keys } func Copy(src string, dst string) error { @@ -110,19 +114,3 @@ func CopyDir(src string, dest string) error { return nil } - -func IsAlive(target string) (bool, error) { - start := time.Now() - for { - conn, err := net.Dial("tcp", target) - if err == nil { - log.Debugf("'%s' is up after %s", target, time.Since(start)) - conn.Close() - return true, nil - } - time.Sleep(500 * time.Millisecond) - if time.Since(start) > 10*time.Second { - return false, fmt.Errorf("took more than 10s for %s to be available", target) - } - } -} diff --git a/pkg/parser/runtime.go b/pkg/parser/runtime.go index e1b33bc6e..693fb1e7d 100644 --- a/pkg/parser/runtime.go +++ b/pkg/parser/runtime.go @@ -18,6 +18,7 @@ import ( "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/crowdsec/pkg/dumps" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -229,14 +230,10 @@ func stageidx(stage string, stages []string) int { return -1 } -type ParserResult struct { - Evt types.Event - Success bool -} - var ParseDump bool var DumpFolder string -var StageParseCache map[string]map[string][]ParserResult + +var StageParseCache dumps.ParserResults var StageParseMutex sync.Mutex func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error) { @@ -271,9 +268,9 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error) if ParseDump { if StageParseCache == nil { StageParseMutex.Lock() - StageParseCache = make(map[string]map[string][]ParserResult) - StageParseCache["success"] = make(map[string][]ParserResult) - StageParseCache["success"][""] = make([]ParserResult, 0) + StageParseCache = make(dumps.ParserResults) + StageParseCache["success"] = make(map[string][]dumps.ParserResult) + StageParseCache["success"][""] = make([]dumps.ParserResult, 0) StageParseMutex.Unlock() } } @@ -282,7 +279,7 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error) if ParseDump { StageParseMutex.Lock() if _, ok := StageParseCache[stage]; !ok { - StageParseCache[stage] = make(map[string][]ParserResult) + StageParseCache[stage] = make(map[string][]dumps.ParserResult) } StageParseMutex.Unlock() } @@ -322,13 +319,18 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error) } clog.Tracef("node (%s) ret : %v", node.rn, ret) if ParseDump { + parserIdxInStage := 0 StageParseMutex.Lock() if len(StageParseCache[stage][node.Name]) == 0 { - StageParseCache[stage][node.Name] = make([]ParserResult, 0) + StageParseCache[stage][node.Name] = make([]dumps.ParserResult, 0) + parserIdxInStage = len(StageParseCache[stage]) + } else { + parserIdxInStage = StageParseCache[stage][node.Name][0].Idx } StageParseMutex.Unlock() + evtcopy := deepcopy.Copy(event) - parserInfo := ParserResult{Evt: evtcopy.(types.Event), Success: ret} + parserInfo := dumps.ParserResult{Evt: evtcopy.(types.Event), Success: ret, Idx: parserIdxInStage} StageParseMutex.Lock() StageParseCache[stage][node.Name] = append(StageParseCache[stage][node.Name], parserInfo) StageParseMutex.Unlock()