fix #2720 #2719 (#2724)

* fix order of display of parsers

* add a --no-clean opt

cmd/crowdsec-cli/explain.go

@@ -12,6 +12,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/pkg/dumps"
 	"github.com/crowdsecurity/crowdsec/pkg/hubtest"
 )
 
@@ -109,6 +110,7 @@ tail -n 5 myfile.log | cscli explain --type nginx -f -
 	flags.Bool("failures", false, "Only show failed lines")
 	flags.Bool("only-successful-parsers", false, "Only show successful parsers")
 	flags.String("crowdsec", "crowdsec", "Path to crowdsec")
+	flags.Bool("no-clean", false, "Don't clean runtime environment after tests")
 
 	return cmd
 }
@@ -136,13 +138,18 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	opts := hubtest.DumpOpts{}
+	opts := dumps.DumpOpts{}
 
 	opts.Details, err = flags.GetBool("verbose")
 	if err != nil {
 		return err
 	}
 
+	no_clean, err := flags.GetBool("no-clean")
+	if err != nil {
+		return err
+	}
+
 	opts.SkipOk, err = flags.GetBool("failures")
 	if err != nil {
 		return err
@@ -172,6 +179,9 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %s", err)
 	}
 	defer func() {
+		if no_clean {
+			return
+		}
 		if _, err := os.Stat(dir); !os.IsNotExist(err) {
 			if err := os.RemoveAll(dir); err != nil {
 				log.Errorf("unable to delete temporary directory '%s': %s", dir, err)
@@ -254,17 +264,17 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error {
 	parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName)
 	bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName)
 
-	parserDump, err := hubtest.LoadParserDump(parserDumpFile)
+	parserDump, err := dumps.LoadParserDump(parserDumpFile)
 	if err != nil {
 		return fmt.Errorf("unable to load parser dump result: %s", err)
 	}
 
-	bucketStateDump, err := hubtest.LoadBucketPourDump(bucketStateDumpFile)
+	bucketStateDump, err := dumps.LoadBucketPourDump(bucketStateDumpFile)
 	if err != nil {
 		return fmt.Errorf("unable to load bucket dump result: %s", err)
 	}
 
-	hubtest.DumpTree(*parserDump, *bucketStateDump, opts)
+	dumps.DumpTree(*parserDump, *bucketStateDump, opts)
 
 	return nil
 }

cmd/crowdsec-cli/hubtest.go

@@ -16,6 +16,7 @@ import (
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
 
+	"github.com/crowdsecurity/crowdsec/pkg/dumps"
 	"github.com/crowdsecurity/crowdsec/pkg/hubtest"
 )
 
@@ -679,8 +680,8 @@ func (cli cliHubTest) NewExplainCmd() *cobra.Command {
 						return fmt.Errorf("unable to load scenario result after run: %s", err)
 					}
 				}
-				opts := hubtest.DumpOpts{}
+				opts := dumps.DumpOpts{}
-				hubtest.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts)
+				dumps.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts)
 			}
 
 			return nil

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607
 	github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
-	github.com/crowdsecurity/go-cs-lib v0.0.5
+	github.com/crowdsecurity/go-cs-lib v0.0.6
 	github.com/crowdsecurity/grokky v0.2.1
 	github.com/crowdsecurity/machineid v1.0.2
 	github.com/davecgh/go-spew v1.1.1

go.sum

@@ -106,6 +106,8 @@ github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
 github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8=
 github.com/crowdsecurity/go-cs-lib v0.0.5/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
+github.com/crowdsecurity/go-cs-lib v0.0.6 h1:Ef6MylXe0GaJE9vrfvxEdbHb31+JUP1os+murPz7Pos=
+github.com/crowdsecurity/go-cs-lib v0.0.6/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
 github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
 github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
 github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=

pkg/dumps/bucket_dump.go

@@ -0,0 +1,32 @@
+package dumps
+
+import (
+	"io"
+	"os"
+
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"gopkg.in/yaml.v2"
+)
+
+type BucketPourInfo map[string][]types.Event
+
+func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
+	dumpData, err := os.Open(filepath)
+	if err != nil {
+		return nil, err
+	}
+	defer dumpData.Close()
+
+	results, err := io.ReadAll(dumpData)
+	if err != nil {
+		return nil, err
+	}
+
+	var bucketDump BucketPourInfo
+
+	if err := yaml.Unmarshal(results, &bucketDump); err != nil {
+		return nil, err
+	}
+
+	return &bucketDump, nil
+}

pkg/dumps/parser_dump.go

@@ -0,0 +1,319 @@
+package dumps
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/crowdsecurity/go-cs-lib/maptools"
+	"github.com/enescakir/emoji"
+	"github.com/fatih/color"
+	diff "github.com/r3labs/diff/v2"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+)
+
+type ParserResult struct {
+	Idx     int
+	Evt     types.Event
+	Success bool
+}
+
+type ParserResults map[string]map[string][]ParserResult
+
+type DumpOpts struct {
+	Details          bool
+	SkipOk           bool
+	ShowNotOkParsers bool
+}
+
+func LoadParserDump(filepath string) (*ParserResults, error) {
+	dumpData, err := os.Open(filepath)
+	if err != nil {
+		return nil, err
+	}
+	defer dumpData.Close()
+
+	results, err := io.ReadAll(dumpData)
+	if err != nil {
+		return nil, err
+	}
+
+	pdump := ParserResults{}
+
+	if err := yaml.Unmarshal(results, &pdump); err != nil {
+		return nil, err
+	}
+
+	/* we know that some variables should always be set,
+	let's check if they're present in last parser output of last stage */
+
+	stages := maptools.SortedKeys(pdump)
+
+	var lastStage string
+
+	//Loop over stages to find last successful one with at least one parser
+	for i := len(stages) - 2; i >= 0; i-- {
+		if len(pdump[stages[i]]) != 0 {
+			lastStage = stages[i]
+			break
+		}
+	}
+
+	parsers := make([]string, 0, len(pdump[lastStage]))
+
+	for k := range pdump[lastStage] {
+		parsers = append(parsers, k)
+	}
+
+	sort.Strings(parsers)
+
+	if len(parsers) == 0 {
+		return nil, fmt.Errorf("no parser found. Please install the appropriate parser and retry")
+	}
+
+	lastParser := parsers[len(parsers)-1]
+
+	for idx, result := range pdump[lastStage][lastParser] {
+		if result.Evt.StrTime == "" {
+			log.Warningf("Line %d/%d is missing evt.StrTime. It is most likely a mistake as it will prevent your logs to be processed in time-machine/forensic mode.", idx, len(pdump[lastStage][lastParser]))
+		} else {
+			log.Debugf("Line %d/%d has evt.StrTime set to '%s'", idx, len(pdump[lastStage][lastParser]), result.Evt.StrTime)
+		}
+	}
+
+	return &pdump, nil
+}
+
+func DumpTree(parserResults ParserResults, bucketPour BucketPourInfo, opts DumpOpts) {
+	//note : we can use line -> time as the unique identifier (of acquisition)
+	state := make(map[time.Time]map[string]map[string]ParserResult)
+	assoc := make(map[time.Time]string, 0)
+	parser_order := make(map[string][]string)
+
+	for stage, parsers := range parserResults {
+		//let's process parsers in the order according to idx
+		parser_order[stage] = make([]string, len(parsers))
+		for pname, parser := range parsers {
+			if len(parser) > 0 {
+				parser_order[stage][parser[0].Idx-1] = pname
+			}
+		}
+
+		for _, parser := range parser_order[stage] {
+			results := parsers[parser]
+			for _, parserRes := range results {
+				evt := parserRes.Evt
+				if _, ok := state[evt.Line.Time]; !ok {
+					state[evt.Line.Time] = make(map[string]map[string]ParserResult)
+					assoc[evt.Line.Time] = evt.Line.Raw
+				}
+
+				if _, ok := state[evt.Line.Time][stage]; !ok {
+					state[evt.Line.Time][stage] = make(map[string]ParserResult)
+				}
+
+				state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parserRes.Success}
+			}
+		}
+	}
+
+	for bname, evtlist := range bucketPour {
+		for _, evt := range evtlist {
+			if evt.Line.Raw == "" {
+				continue
+			}
+
+			//it might be bucket overflow being reprocessed, skip this
+			if _, ok := state[evt.Line.Time]; !ok {
+				state[evt.Line.Time] = make(map[string]map[string]ParserResult)
+				assoc[evt.Line.Time] = evt.Line.Raw
+			}
+
+			//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
+			//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
+			if _, ok := state[evt.Line.Time]["buckets"]; !ok {
+				state[evt.Line.Time]["buckets"] = make(map[string]ParserResult)
+			}
+
+			state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true}
+		}
+	}
+
+	yellow := color.New(color.FgYellow).SprintFunc()
+	red := color.New(color.FgRed).SprintFunc()
+	green := color.New(color.FgGreen).SprintFunc()
+	whitelistReason := ""
+	//get each line
+	for tstamp, rawstr := range assoc {
+		if opts.SkipOk {
+			if _, ok := state[tstamp]["buckets"]["OK"]; ok {
+				continue
+			}
+		}
+
+		fmt.Printf("line: %s\n", rawstr)
+
+		skeys := make([]string, 0, len(state[tstamp]))
+
+		for k := range state[tstamp] {
+			//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
+			//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
+			if k == "buckets" {
+				continue
+			}
+
+			skeys = append(skeys, k)
+		}
+
+		sort.Strings(skeys)
+
+		// iterate stage
+		var prevItem types.Event
+
+		for _, stage := range skeys {
+			parsers := state[tstamp][stage]
+
+			sep := "├"
+			presep := "|"
+
+			fmt.Printf("\t%s %s\n", sep, stage)
+
+			for idx, parser := range parser_order[stage] {
+				res := parsers[parser].Success
+				sep := "├"
+
+				if idx == len(parser_order[stage])-1 {
+					sep = "└"
+				}
+
+				created := 0
+				updated := 0
+				deleted := 0
+				whitelisted := false
+				changeStr := ""
+				detailsDisplay := ""
+
+				if res {
+					changelog, _ := diff.Diff(prevItem, parsers[parser].Evt)
+					for _, change := range changelog {
+						switch change.Type {
+						case "create":
+							created++
+
+							detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To))
+						case "update":
+							detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To))
+
+							if change.Path[0] == "Whitelisted" && change.To == true {
+								whitelisted = true
+
+								if whitelistReason == "" {
+									whitelistReason = parsers[parser].Evt.WhitelistReason
+								}
+							}
+							updated++
+						case "delete":
+							deleted++
+
+							detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, ".")))
+						}
+					}
+
+					prevItem = parsers[parser].Evt
+				}
+
+				if created > 0 {
+					changeStr += green(fmt.Sprintf("+%d", created))
+				}
+
+				if updated > 0 {
+					if len(changeStr) > 0 {
+						changeStr += " "
+					}
+
+					changeStr += yellow(fmt.Sprintf("~%d", updated))
+				}
+
+				if deleted > 0 {
+					if len(changeStr) > 0 {
+						changeStr += " "
+					}
+
+					changeStr += red(fmt.Sprintf("-%d", deleted))
+				}
+
+				if whitelisted {
+					if len(changeStr) > 0 {
+						changeStr += " "
+					}
+
+					changeStr += red("[whitelisted]")
+				}
+
+				if changeStr == "" {
+					changeStr = yellow("unchanged")
+				}
+
+				if res {
+					fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr)
+
+					if opts.Details {
+						fmt.Print(detailsDisplay)
+					}
+				} else if opts.ShowNotOkParsers {
+					fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser)
+				}
+			}
+		}
+
+		sep := "└"
+
+		if len(state[tstamp]["buckets"]) > 0 {
+			sep = "├"
+		}
+
+		//did the event enter the bucket pour phase ?
+		if _, ok := state[tstamp]["buckets"]["OK"]; ok {
+			fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle)
+		} else if whitelistReason != "" {
+			fmt.Printf("\t%s-------- parser success, ignored by whitelist (%s) %s\n", sep, whitelistReason, emoji.GreenCircle)
+		} else {
+			fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle)
+		}
+
+		//now print bucket info
+		if len(state[tstamp]["buckets"]) > 0 {
+			fmt.Printf("\t├ Scenarios\n")
+		}
+
+		bnames := make([]string, 0, len(state[tstamp]["buckets"]))
+
+		for k := range state[tstamp]["buckets"] {
+			//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
+			//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
+			if k == "OK" {
+				continue
+			}
+
+			bnames = append(bnames, k)
+		}
+
+		sort.Strings(bnames)
+
+		for idx, bname := range bnames {
+			sep := "├"
+			if idx == len(bnames)-1 {
+				sep = "└"
+			}
+
+			fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname)
+		}
+
+		fmt.Println()
+	}
+}

pkg/hubtest/coverage.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/go-cs-lib/maptools"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 )
@@ -25,7 +26,7 @@ func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) {
 	}
 
 	// populate from hub, iterate in alphabetical order
-	pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.APPSEC_RULES))
+	pkeys := maptools.SortedKeys(h.HubIndex.GetItemMap(cwhub.APPSEC_RULES))
 	coverage := make([]Coverage, len(pkeys))
 
 	for i, name := range pkeys {
@@ -84,7 +85,7 @@ func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
 	}
 
 	// populate from hub, iterate in alphabetical order
-	pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.PARSERS))
+	pkeys := maptools.SortedKeys(h.HubIndex.GetItemMap(cwhub.PARSERS))
 	coverage := make([]Coverage, len(pkeys))
 
 	for i, name := range pkeys {
@@ -170,7 +171,7 @@ func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) {
 	}
 
 	// populate from hub, iterate in alphabetical order
-	pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.SCENARIOS))
+	pkeys := maptools.SortedKeys(h.HubIndex.GetItemMap(cwhub.SCENARIOS))
 	coverage := make([]Coverage, len(pkeys))
 
 	for i, name := range pkeys {

pkg/hubtest/parser_assert.go

@@ -3,21 +3,16 @@ package hubtest
 import (
 	"bufio"
 	"fmt"
-	"io"
 	"os"
-	"sort"
 	"strings"
-	"time"
 
 	"github.com/antonmedv/expr"
-	"github.com/enescakir/emoji"
-	"github.com/fatih/color"
-	diff "github.com/r3labs/diff/v2"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 
+	"github.com/crowdsecurity/crowdsec/pkg/dumps"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
-	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/crowdsecurity/go-cs-lib/maptools"
 )
 
 type AssertFail struct {
@@ -34,16 +29,9 @@ type ParserAssert struct {
 	NbAssert          int
 	Fails             []AssertFail
 	Success           bool
-	TestData          *ParserResults
+	TestData          *dumps.ParserResults
 }
 
-type ParserResult struct {
-	Evt     types.Event
-	Success bool
-}
-
-type ParserResults map[string]map[string][]ParserResult
-
 func NewParserAssert(file string) *ParserAssert {
 	ParserAssert := &ParserAssert{
 		File:          file,
@@ -51,7 +39,7 @@ func NewParserAssert(file string) *ParserAssert {
 		Success:       false,
 		Fails:         make([]AssertFail, 0),
 		AutoGenAssert: false,
-		TestData:      &ParserResults{},
+		TestData:      &dumps.ParserResults{},
 	}
 
 	return ParserAssert
@@ -69,7 +57,7 @@ func (p *ParserAssert) AutoGenFromFile(filename string) (string, error) {
 }
 
 func (p *ParserAssert) LoadTest(filename string) error {
-	parserDump, err := LoadParserDump(filename)
+	parserDump, err := dumps.LoadParserDump(filename)
 	if err != nil {
 		return fmt.Errorf("loading parser dump file: %+v", err)
 	}
@@ -229,13 +217,13 @@ func (p *ParserAssert) AutoGenParserAssert() string {
 	ret := fmt.Sprintf("len(results) == %d\n", len(*p.TestData))
 
 	//sort map keys for consistent order
-	stages := sortedMapKeys(*p.TestData)
+	stages := maptools.SortedKeys(*p.TestData)
 
 	for _, stage := range stages {
 		parsers := (*p.TestData)[stage]
 
 		//sort map keys for consistent order
-		pnames := sortedMapKeys(parsers)
+		pnames := maptools.SortedKeys(parsers)
 
 		for _, parser := range pnames {
 			presults := parsers[parser]
@@ -248,7 +236,7 @@ func (p *ParserAssert) AutoGenParserAssert() string {
 					continue
 				}
 
-				for _, pkey := range sortedMapKeys(result.Evt.Parsed) {
+				for _, pkey := range maptools.SortedKeys(result.Evt.Parsed) {
 					pval := result.Evt.Parsed[pkey]
 					if pval == "" {
 						continue
@@ -257,7 +245,7 @@ func (p *ParserAssert) AutoGenParserAssert() string {
 					ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Parsed["%s"] == "%s"`+"\n", stage, parser, pidx, pkey, Escape(pval))
 				}
 
-				for _, mkey := range sortedMapKeys(result.Evt.Meta) {
+				for _, mkey := range maptools.SortedKeys(result.Evt.Meta) {
 					mval := result.Evt.Meta[mkey]
 					if mval == "" {
 						continue
@@ -266,7 +254,7 @@ func (p *ParserAssert) AutoGenParserAssert() string {
 					ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval))
 				}
 
-				for _, ekey := range sortedMapKeys(result.Evt.Enriched) {
+				for _, ekey := range maptools.SortedKeys(result.Evt.Enriched) {
 					eval := result.Evt.Enriched[ekey]
 					if eval == "" {
 						continue
@@ -275,7 +263,7 @@ func (p *ParserAssert) AutoGenParserAssert() string {
 					ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Enriched["%s"] == "%s"`+"\n", stage, parser, pidx, ekey, Escape(eval))
 				}
 
-				for _, ukey := range sortedMapKeys(result.Evt.Unmarshaled) {
+				for _, ukey := range maptools.SortedKeys(result.Evt.Unmarshaled) {
 					uval := result.Evt.Unmarshaled[ukey]
 					if uval == "" {
 						continue
@@ -328,288 +316,3 @@ func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []s
 
 	return ret
 }
-
-func LoadParserDump(filepath string) (*ParserResults, error) {
-	dumpData, err := os.Open(filepath)
-	if err != nil {
-		return nil, err
-	}
-	defer dumpData.Close()
-
-	results, err := io.ReadAll(dumpData)
-	if err != nil {
-		return nil, err
-	}
-
-	pdump := ParserResults{}
-
-	if err := yaml.Unmarshal(results, &pdump); err != nil {
-		return nil, err
-	}
-
-	/* we know that some variables should always be set,
-	let's check if they're present in last parser output of last stage */
-
-	stages := sortedMapKeys(pdump)
-
-	var lastStage string
-
-	//Loop over stages to find last successful one with at least one parser
-	for i := len(stages) - 2; i >= 0; i-- {
-		if len(pdump[stages[i]]) != 0 {
-			lastStage = stages[i]
-			break
-		}
-	}
-
-	parsers := make([]string, 0, len(pdump[lastStage]))
-
-	for k := range pdump[lastStage] {
-		parsers = append(parsers, k)
-	}
-
-	sort.Strings(parsers)
-
-	if len(parsers) == 0 {
-		return nil, fmt.Errorf("no parser found. Please install the appropriate parser and retry")
-	}
-
-	lastParser := parsers[len(parsers)-1]
-
-	for idx, result := range pdump[lastStage][lastParser] {
-		if result.Evt.StrTime == "" {
-			log.Warningf("Line %d/%d is missing evt.StrTime. It is most likely a mistake as it will prevent your logs to be processed in time-machine/forensic mode.", idx, len(pdump[lastStage][lastParser]))
-		} else {
-			log.Debugf("Line %d/%d has evt.StrTime set to '%s'", idx, len(pdump[lastStage][lastParser]), result.Evt.StrTime)
-		}
-	}
-
-	return &pdump, nil
-}
-
-type DumpOpts struct {
-	Details          bool
-	SkipOk           bool
-	ShowNotOkParsers bool
-}
-
-func DumpTree(parserResults ParserResults, bucketPour BucketPourInfo, opts DumpOpts) {
-	//note : we can use line -> time as the unique identifier (of acquisition)
-	state := make(map[time.Time]map[string]map[string]ParserResult)
-	assoc := make(map[time.Time]string, 0)
-
-	for stage, parsers := range parserResults {
-		for parser, results := range parsers {
-			for _, parserRes := range results {
-				evt := parserRes.Evt
-				if _, ok := state[evt.Line.Time]; !ok {
-					state[evt.Line.Time] = make(map[string]map[string]ParserResult)
-					assoc[evt.Line.Time] = evt.Line.Raw
-				}
-
-				if _, ok := state[evt.Line.Time][stage]; !ok {
-					state[evt.Line.Time][stage] = make(map[string]ParserResult)
-				}
-
-				state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parserRes.Success}
-			}
-		}
-	}
-
-	for bname, evtlist := range bucketPour {
-		for _, evt := range evtlist {
-			if evt.Line.Raw == "" {
-				continue
-			}
-
-			//it might be bucket overflow being reprocessed, skip this
-			if _, ok := state[evt.Line.Time]; !ok {
-				state[evt.Line.Time] = make(map[string]map[string]ParserResult)
-				assoc[evt.Line.Time] = evt.Line.Raw
-			}
-
-			//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
-			//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
-			if _, ok := state[evt.Line.Time]["buckets"]; !ok {
-				state[evt.Line.Time]["buckets"] = make(map[string]ParserResult)
-			}
-
-			state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true}
-		}
-	}
-
-	yellow := color.New(color.FgYellow).SprintFunc()
-	red := color.New(color.FgRed).SprintFunc()
-	green := color.New(color.FgGreen).SprintFunc()
-	whitelistReason := ""
-	//get each line
-	for tstamp, rawstr := range assoc {
-		if opts.SkipOk {
-			if _, ok := state[tstamp]["buckets"]["OK"]; ok {
-				continue
-			}
-		}
-
-		fmt.Printf("line: %s\n", rawstr)
-
-		skeys := make([]string, 0, len(state[tstamp]))
-
-		for k := range state[tstamp] {
-			//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
-			//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
-			if k == "buckets" {
-				continue
-			}
-
-			skeys = append(skeys, k)
-		}
-
-		sort.Strings(skeys)
-
-		// iterate stage
-		var prevItem types.Event
-
-		for _, stage := range skeys {
-			parsers := state[tstamp][stage]
-
-			sep := "├"
-			presep := "|"
-
-			fmt.Printf("\t%s %s\n", sep, stage)
-
-			pkeys := sortedMapKeys(parsers)
-
-			for idx, parser := range pkeys {
-				res := parsers[parser].Success
-				sep := "├"
-
-				if idx == len(pkeys)-1 {
-					sep = "└"
-				}
-
-				created := 0
-				updated := 0
-				deleted := 0
-				whitelisted := false
-				changeStr := ""
-				detailsDisplay := ""
-
-				if res {
-					changelog, _ := diff.Diff(prevItem, parsers[parser].Evt)
-					for _, change := range changelog {
-						switch change.Type {
-						case "create":
-							created++
-
-							detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To))
-						case "update":
-							detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To))
-
-							if change.Path[0] == "Whitelisted" && change.To == true {
-								whitelisted = true
-
-								if whitelistReason == "" {
-									whitelistReason = parsers[parser].Evt.WhitelistReason
-								}
-							}
-							updated++
-						case "delete":
-							deleted++
-
-							detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, ".")))
-						}
-					}
-
-					prevItem = parsers[parser].Evt
-				}
-
-				if created > 0 {
-					changeStr += green(fmt.Sprintf("+%d", created))
-				}
-
-				if updated > 0 {
-					if len(changeStr) > 0 {
-						changeStr += " "
-					}
-
-					changeStr += yellow(fmt.Sprintf("~%d", updated))
-				}
-
-				if deleted > 0 {
-					if len(changeStr) > 0 {
-						changeStr += " "
-					}
-
-					changeStr += red(fmt.Sprintf("-%d", deleted))
-				}
-
-				if whitelisted {
-					if len(changeStr) > 0 {
-						changeStr += " "
-					}
-
-					changeStr += red("[whitelisted]")
-				}
-
-				if changeStr == "" {
-					changeStr = yellow("unchanged")
-				}
-
-				if res {
-					fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr)
-
-					if opts.Details {
-						fmt.Print(detailsDisplay)
-					}
-				} else if opts.ShowNotOkParsers {
-					fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser)
-				}
-			}
-		}
-
-		sep := "└"
-
-		if len(state[tstamp]["buckets"]) > 0 {
-			sep = "├"
-		}
-
-		//did the event enter the bucket pour phase ?
-		if _, ok := state[tstamp]["buckets"]["OK"]; ok {
-			fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle)
-		} else if whitelistReason != "" {
-			fmt.Printf("\t%s-------- parser success, ignored by whitelist (%s) %s\n", sep, whitelistReason, emoji.GreenCircle)
-		} else {
-			fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle)
-		}
-
-		//now print bucket info
-		if len(state[tstamp]["buckets"]) > 0 {
-			fmt.Printf("\t├ Scenarios\n")
-		}
-
-		bnames := make([]string, 0, len(state[tstamp]["buckets"]))
-
-		for k := range state[tstamp]["buckets"] {
-			//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
-			//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
-			if k == "OK" {
-				continue
-			}
-
-			bnames = append(bnames, k)
-		}
-
-		sort.Strings(bnames)
-
-		for idx, bname := range bnames {
-			sep := "├"
-			if idx == len(bnames)-1 {
-				sep = "└"
-			}
-
-			fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname)
-		}
-
-		fmt.Println()
-	}
-}

pkg/hubtest/scenario_assert.go

@@ -12,6 +12,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 
+	"github.com/crowdsecurity/crowdsec/pkg/dumps"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
@@ -24,11 +25,10 @@ type ScenarioAssert struct {
 	Fails             []AssertFail
 	Success           bool
 	TestData          *BucketResults
-	PourData          *BucketPourInfo
+	PourData          *dumps.BucketPourInfo
 }
 
 type BucketResults []types.Event
-type BucketPourInfo map[string][]types.Event
 
 func NewScenarioAssert(file string) *ScenarioAssert {
 	ScenarioAssert := &ScenarioAssert{
@@ -38,7 +38,7 @@ func NewScenarioAssert(file string) *ScenarioAssert {
 		Fails:         make([]AssertFail, 0),
 		AutoGenAssert: false,
 		TestData:      &BucketResults{},
-		PourData:      &BucketPourInfo{},
+		PourData:      &dumps.BucketPourInfo{},
 	}
 
 	return ScenarioAssert
@@ -64,7 +64,7 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error {
 	s.TestData = bucketDump
 
 	if bucketpour != "" {
-		pourDump, err := LoadBucketPourDump(bucketpour)
+		pourDump, err := dumps.LoadBucketPourDump(bucketpour)
 		if err != nil {
 			return fmt.Errorf("loading bucket pour dump file '%s': %+v", filename, err)
 		}
@@ -252,27 +252,6 @@ func (b BucketResults) Swap(i, j int) {
 	b[i], b[j] = b[j], b[i]
 }
 
-func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
-	dumpData, err := os.Open(filepath)
-	if err != nil {
-		return nil, err
-	}
-	defer dumpData.Close()
-
-	results, err := io.ReadAll(dumpData)
-	if err != nil {
-		return nil, err
-	}
-
-	var bucketDump BucketPourInfo
-
-	if err := yaml.Unmarshal(results, &bucketDump); err != nil {
-		return nil, err
-	}
-
-	return &bucketDump, nil
-}
-
 func LoadScenarioDump(filepath string) (*BucketResults, error) {
 	dumpData, err := os.Open(filepath)
 	if err != nil {

pkg/hubtest/utils.go

@@ -5,21 +5,25 @@ import (
 	"net"
 	"os"
 	"path/filepath"
-	"sort"
 	"time"
 
 	log "github.com/sirupsen/logrus"
 )
 
-func sortedMapKeys[V any](m map[string]V) []string {
+func IsAlive(target string) (bool, error) {
-	keys := make([]string, 0, len(m))
+	start := time.Now()
-	for k := range m {
+	for {
-		keys = append(keys, k)
+		conn, err := net.Dial("tcp", target)
+		if err == nil {
+			log.Debugf("'%s' is up after %s", target, time.Since(start))
+			conn.Close()
+			return true, nil
+		}
+		time.Sleep(500 * time.Millisecond)
+		if time.Since(start) > 10*time.Second {
+			return false, fmt.Errorf("took more than 10s for %s to be available", target)
+		}
 	}
-
-	sort.Strings(keys)
-
-	return keys
 }
 
 func Copy(src string, dst string) error {
@@ -110,19 +114,3 @@ func CopyDir(src string, dest string) error {
 
 	return nil
 }
-
-func IsAlive(target string) (bool, error) {
-	start := time.Now()
-	for {
-		conn, err := net.Dial("tcp", target)
-		if err == nil {
-			log.Debugf("'%s' is up after %s", target, time.Since(start))
-			conn.Close()
-			return true, nil
-		}
-		time.Sleep(500 * time.Millisecond)
-		if time.Since(start) > 10*time.Second {
-			return false, fmt.Errorf("took more than 10s for %s to be available", target)
-		}
-	}
-}

pkg/parser/runtime.go

@@ -18,6 +18,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 
+	"github.com/crowdsecurity/crowdsec/pkg/dumps"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
@@ -229,14 +230,10 @@ func stageidx(stage string, stages []string) int {
 	return -1
 }
 
-type ParserResult struct {
-	Evt     types.Event
-	Success bool
-}
-
 var ParseDump bool
 var DumpFolder string
-var StageParseCache map[string]map[string][]ParserResult
+
+var StageParseCache dumps.ParserResults
 var StageParseMutex sync.Mutex
 
 func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error) {
@@ -271,9 +268,9 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error)
 	if ParseDump {
 		if StageParseCache == nil {
 			StageParseMutex.Lock()
-			StageParseCache = make(map[string]map[string][]ParserResult)
+			StageParseCache = make(dumps.ParserResults)
-			StageParseCache["success"] = make(map[string][]ParserResult)
+			StageParseCache["success"] = make(map[string][]dumps.ParserResult)
-			StageParseCache["success"][""] = make([]ParserResult, 0)
+			StageParseCache["success"][""] = make([]dumps.ParserResult, 0)
 			StageParseMutex.Unlock()
 		}
 	}
@@ -282,7 +279,7 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error)
 		if ParseDump {
 			StageParseMutex.Lock()
 			if _, ok := StageParseCache[stage]; !ok {
-				StageParseCache[stage] = make(map[string][]ParserResult)
+				StageParseCache[stage] = make(map[string][]dumps.ParserResult)
 			}
 			StageParseMutex.Unlock()
 		}
@@ -322,13 +319,18 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error)
 			}
 			clog.Tracef("node (%s) ret : %v", node.rn, ret)
 			if ParseDump {
+				parserIdxInStage := 0
 				StageParseMutex.Lock()
 				if len(StageParseCache[stage][node.Name]) == 0 {
-					StageParseCache[stage][node.Name] = make([]ParserResult, 0)
+					StageParseCache[stage][node.Name] = make([]dumps.ParserResult, 0)
+					parserIdxInStage = len(StageParseCache[stage])
+				} else {
+					parserIdxInStage = StageParseCache[stage][node.Name][0].Idx
 				}
 				StageParseMutex.Unlock()
+
 				evtcopy := deepcopy.Copy(event)
-				parserInfo := ParserResult{Evt: evtcopy.(types.Event), Success: ret}
+				parserInfo := dumps.ParserResult{Evt: evtcopy.(types.Event), Success: ret, Idx: parserIdxInStage}
 				StageParseMutex.Lock()
 				StageParseCache[stage][node.Name] = append(StageParseCache[stage][node.Name], parserInfo)
 				StageParseMutex.Unlock()