beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix #2720 #2719 (#2724)

Browse source 
* fix order of display of parsers

* add a --no-clean opt
This commit is contained in:
Thibault "bui" Koechlin 2024-01-15 09:16:03 +01:00 committed by GitHub
parent 1e0bcedef5
commit 6ca053ca67
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 418 additions and 381 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
cmd/crowdsec-cli/explain.go
View file

@ -12,6 +12,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/dumps"
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
)
@ -109,6 +110,7 @@ tail -n 5 myfile.log | cscli explain --type nginx -f -
flags.Bool("failures", false, "Only show failed lines")
flags.Bool("only-successful-parsers", false, "Only show successful parsers")
flags.String("crowdsec", "crowdsec", "Path to crowdsec")
flags.Bool("no-clean", false, "Don't clean runtime environment after tests")
return cmd
}
@ -136,13 +138,18 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error {
return err
}
opts := hubtest.DumpOpts{}
opts := dumps.DumpOpts{}
opts.Details, err = flags.GetBool("verbose")
if err != nil {
return err
}
no_clean, err := flags.GetBool("no-clean")
if err != nil {
return err
}
opts.SkipOk, err = flags.GetBool("failures")
if err != nil {
return err
@ -172,6 +179,9 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error {
return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %s", err)
}
defer func() {
if no_clean {
return
}
if _, err := os.Stat(dir); !os.IsNotExist(err) {
if err := os.RemoveAll(dir); err != nil {
log.Errorf("unable to delete temporary directory '%s': %s", dir, err)
@ -254,17 +264,17 @@ func (cli cliExplain) run(cmd *cobra.Command, args []string) error {
parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName)
bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName)
parserDump, err := hubtest.LoadParserDump(parserDumpFile)
parserDump, err := dumps.LoadParserDump(parserDumpFile)
if err != nil {
return fmt.Errorf("unable to load parser dump result: %s", err)
}
bucketStateDump, err := hubtest.LoadBucketPourDump(bucketStateDumpFile)
bucketStateDump, err := dumps.LoadBucketPourDump(bucketStateDumpFile)
if err != nil {
return fmt.Errorf("unable to load bucket dump result: %s", err)
}
hubtest.DumpTree(*parserDump, *bucketStateDump, opts)
dumps.DumpTree(*parserDump, *bucketStateDump, opts)
return nil
}

5
cmd/crowdsec-cli/hubtest.go
View file

@ -16,6 +16,7 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/dumps"
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
)
@ -679,8 +680,8 @@ func (cli cliHubTest) NewExplainCmd() *cobra.Command {
return fmt.Errorf("unable to load scenario result after run: %s", err)
}
}
opts := hubtest.DumpOpts{}
hubtest.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts)
opts := dumps.DumpOpts{}
dumps.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts)
}
return nil

2
go.mod
View file

@ -26,7 +26,7 @@ require (
github.com/cespare/xxhash/v2 v2.2.0
github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
github.com/crowdsecurity/go-cs-lib v0.0.5
github.com/crowdsecurity/go-cs-lib v0.0.6
github.com/crowdsecurity/grokky v0.2.1
github.com/crowdsecurity/machineid v1.0.2
github.com/davecgh/go-spew v1.1.1

2
go.sum
View file

@ -106,6 +106,8 @@ github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8=
github.com/crowdsecurity/go-cs-lib v0.0.5/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
github.com/crowdsecurity/go-cs-lib v0.0.6 h1:Ef6MylXe0GaJE9vrfvxEdbHb31+JUP1os+murPz7Pos=
github.com/crowdsecurity/go-cs-lib v0.0.6/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=

32
pkg/dumps/bucket_dump.go Normal file
View file

@ -0,0 +1,32 @@
package dumps
import (
"io"
"os"
"github.com/crowdsecurity/crowdsec/pkg/types"
"gopkg.in/yaml.v2"
)
type BucketPourInfo map[string][]types.Event
func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
dumpData, err := os.Open(filepath)
if err != nil {
return nil, err
}
defer dumpData.Close()
results, err := io.ReadAll(dumpData)
if err != nil {
return nil, err
}
var bucketDump BucketPourInfo
if err := yaml.Unmarshal(results, &bucketDump); err != nil {
return nil, err
}
return &bucketDump, nil
}

319
pkg/dumps/parser_dump.go Normal file
View file

@ -0,0 +1,319 @@
package dumps
import (
"fmt"
"io"
"os"
"sort"
"strings"
"time"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/go-cs-lib/maptools"
"github.com/enescakir/emoji"
"github.com/fatih/color"
diff "github.com/r3labs/diff/v2"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
type ParserResult struct {
Idx int
Evt types.Event
Success bool
}
type ParserResults map[string]map[string][]ParserResult
type DumpOpts struct {
Details bool
SkipOk bool
ShowNotOkParsers bool
}
func LoadParserDump(filepath string) (*ParserResults, error) {
dumpData, err := os.Open(filepath)
if err != nil {
return nil, err
}
defer dumpData.Close()
results, err := io.ReadAll(dumpData)
if err != nil {
return nil, err
}
pdump := ParserResults{}
if err := yaml.Unmarshal(results, &pdump); err != nil {
return nil, err
}
/* we know that some variables should always be set,
let's check if they're present in last parser output of last stage */
stages := maptools.SortedKeys(pdump)
var lastStage string
//Loop over stages to find last successful one with at least one parser
for i := len(stages) - 2; i >= 0; i-- {
if len(pdump[stages[i]]) != 0 {
lastStage = stages[i]
break
}
}
parsers := make([]string, 0, len(pdump[lastStage]))
for k := range pdump[lastStage] {
parsers = append(parsers, k)
}
sort.Strings(parsers)
if len(parsers) == 0 {
return nil, fmt.Errorf("no parser found. Please install the appropriate parser and retry")
}
lastParser := parsers[len(parsers)-1]
for idx, result := range pdump[lastStage][lastParser] {
if result.Evt.StrTime == "" {
log.Warningf("Line %d/%d is missing evt.StrTime. It is most likely a mistake as it will prevent your logs to be processed in time-machine/forensic mode.", idx, len(pdump[lastStage][lastParser]))
} else {
log.Debugf("Line %d/%d has evt.StrTime set to '%s'", idx, len(pdump[lastStage][lastParser]), result.Evt.StrTime)
}
}
return &pdump, nil
}
func DumpTree(parserResults ParserResults, bucketPour BucketPourInfo, opts DumpOpts) {
//note : we can use line -> time as the unique identifier (of acquisition)
state := make(map[time.Time]map[string]map[string]ParserResult)
assoc := make(map[time.Time]string, 0)
parser_order := make(map[string][]string)
for stage, parsers := range parserResults {
//let's process parsers in the order according to idx
parser_order[stage] = make([]string, len(parsers))
for pname, parser := range parsers {
if len(parser) > 0 {
parser_order[stage][parser[0].Idx-1] = pname
}
}
for _, parser := range parser_order[stage] {
results := parsers[parser]
for _, parserRes := range results {
evt := parserRes.Evt
if _, ok := state[evt.Line.Time]; !ok {
state[evt.Line.Time] = make(map[string]map[string]ParserResult)
assoc[evt.Line.Time] = evt.Line.Raw
}
if _, ok := state[evt.Line.Time][stage]; !ok {
state[evt.Line.Time][stage] = make(map[string]ParserResult)
}
state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parserRes.Success}
}
}
}
for bname, evtlist := range bucketPour {
for _, evt := range evtlist {
if evt.Line.Raw == "" {
continue
}
//it might be bucket overflow being reprocessed, skip this
if _, ok := state[evt.Line.Time]; !ok {
state[evt.Line.Time] = make(map[string]map[string]ParserResult)
assoc[evt.Line.Time] = evt.Line.Raw
}
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
if _, ok := state[evt.Line.Time]["buckets"]; !ok {
state[evt.Line.Time]["buckets"] = make(map[string]ParserResult)
}
state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true}
}
}
yellow := color.New(color.FgYellow).SprintFunc()
red := color.New(color.FgRed).SprintFunc()
green := color.New(color.FgGreen).SprintFunc()
whitelistReason := ""
//get each line
for tstamp, rawstr := range assoc {
if opts.SkipOk {
if _, ok := state[tstamp]["buckets"]["OK"]; ok {
continue
}
}
fmt.Printf("line: %s\n", rawstr)
skeys := make([]string, 0, len(state[tstamp]))
for k := range state[tstamp] {
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
if k == "buckets" {
continue
}
skeys = append(skeys, k)
}
sort.Strings(skeys)
// iterate stage
var prevItem types.Event
for _, stage := range skeys {
parsers := state[tstamp][stage]
sep := "├"
presep := "|"
fmt.Printf("\t%s %s\n", sep, stage)
for idx, parser := range parser_order[stage] {
res := parsers[parser].Success
sep := "├"
if idx == len(parser_order[stage])-1 {
sep = "└"
}
created := 0
updated := 0
deleted := 0
whitelisted := false
changeStr := ""
detailsDisplay := ""
if res {
changelog, _ := diff.Diff(prevItem, parsers[parser].Evt)
for _, change := range changelog {
switch change.Type {
case "create":
created++
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To))
case "update":
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To))
if change.Path[0] == "Whitelisted" && change.To == true {
whitelisted = true
if whitelistReason == "" {
whitelistReason = parsers[parser].Evt.WhitelistReason
}
}
updated++
case "delete":
deleted++
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, ".")))
}
}
prevItem = parsers[parser].Evt
}
if created > 0 {
changeStr += green(fmt.Sprintf("+%d", created))
}
if updated > 0 {
if len(changeStr) > 0 {
changeStr += " "
}
changeStr += yellow(fmt.Sprintf("~%d", updated))
}
if deleted > 0 {
if len(changeStr) > 0 {
changeStr += " "
}
changeStr += red(fmt.Sprintf("-%d", deleted))
}
if whitelisted {
if len(changeStr) > 0 {
changeStr += " "
}
changeStr += red("[whitelisted]")
}
if changeStr == "" {
changeStr = yellow("unchanged")
}
if res {
fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr)
if opts.Details {
fmt.Print(detailsDisplay)
}
} else if opts.ShowNotOkParsers {
fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser)
}
}
}
sep := "└"
if len(state[tstamp]["buckets"]) > 0 {
sep = "├"
}
//did the event enter the bucket pour phase ?
if _, ok := state[tstamp]["buckets"]["OK"]; ok {
fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle)
} else if whitelistReason != "" {
fmt.Printf("\t%s-------- parser success, ignored by whitelist (%s) %s\n", sep, whitelistReason, emoji.GreenCircle)
} else {
fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle)
}
//now print bucket info
if len(state[tstamp]["buckets"]) > 0 {
fmt.Printf("\t├ Scenarios\n")
}
bnames := make([]string, 0, len(state[tstamp]["buckets"]))
for k := range state[tstamp]["buckets"] {
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
if k == "OK" {
continue
}
bnames = append(bnames, k)
}
sort.Strings(bnames)
for idx, bname := range bnames {
sep := "├"
if idx == len(bnames)-1 {
sep = "└"
}
fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname)
}
fmt.Println()
}
}

7
pkg/hubtest/coverage.go
View file

@ -9,6 +9,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/go-cs-lib/maptools"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
@ -25,7 +26,7 @@ func (h *HubTest) GetAppsecCoverage() ([]Coverage, error) {
}
// populate from hub, iterate in alphabetical order
pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.APPSEC_RULES))
pkeys := maptools.SortedKeys(h.HubIndex.GetItemMap(cwhub.APPSEC_RULES))
coverage := make([]Coverage, len(pkeys))
for i, name := range pkeys {
@ -84,7 +85,7 @@ func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
}
// populate from hub, iterate in alphabetical order
pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.PARSERS))
pkeys := maptools.SortedKeys(h.HubIndex.GetItemMap(cwhub.PARSERS))
coverage := make([]Coverage, len(pkeys))
for i, name := range pkeys {
@ -170,7 +171,7 @@ func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) {
}
// populate from hub, iterate in alphabetical order
pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.SCENARIOS))
pkeys := maptools.SortedKeys(h.HubIndex.GetItemMap(cwhub.SCENARIOS))
coverage := make([]Coverage, len(pkeys))
for i, name := range pkeys {

319
pkg/hubtest/parser_assert.go
View file

@ -3,21 +3,16 @@ package hubtest
import (
"bufio"
"fmt"
"io"
"os"
"sort"
"strings"
"time"
"github.com/antonmedv/expr"
"github.com/enescakir/emoji"
"github.com/fatih/color"
diff "github.com/r3labs/diff/v2"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/dumps"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/go-cs-lib/maptools"
)
type AssertFail struct {
@ -34,16 +29,9 @@ type ParserAssert struct {
NbAssert int
Fails []AssertFail
Success bool
TestData *ParserResults
TestData *dumps.ParserResults
}
type ParserResult struct {
Evt types.Event
Success bool
}
type ParserResults map[string]map[string][]ParserResult
func NewParserAssert(file string) *ParserAssert {
ParserAssert := &ParserAssert{
File: file,
@ -51,7 +39,7 @@ func NewParserAssert(file string) *ParserAssert {
Success: false,
Fails: make([]AssertFail, 0),
AutoGenAssert: false,
TestData: &ParserResults{},
TestData: &dumps.ParserResults{},
}
return ParserAssert
@ -69,7 +57,7 @@ func (p *ParserAssert) AutoGenFromFile(filename string) (string, error) {
}
func (p *ParserAssert) LoadTest(filename string) error {
parserDump, err := LoadParserDump(filename)
parserDump, err := dumps.LoadParserDump(filename)
if err != nil {
return fmt.Errorf("loading parser dump file: %+v", err)
}
@ -229,13 +217,13 @@ func (p *ParserAssert) AutoGenParserAssert() string {
ret := fmt.Sprintf("len(results) == %d\n", len(*p.TestData))
//sort map keys for consistent order
stages := sortedMapKeys(*p.TestData)
stages := maptools.SortedKeys(*p.TestData)
for _, stage := range stages {
parsers := (*p.TestData)[stage]
//sort map keys for consistent order
pnames := sortedMapKeys(parsers)
pnames := maptools.SortedKeys(parsers)
for _, parser := range pnames {
presults := parsers[parser]
@ -248,7 +236,7 @@ func (p *ParserAssert) AutoGenParserAssert() string {
continue
}
for _, pkey := range sortedMapKeys(result.Evt.Parsed) {
for _, pkey := range maptools.SortedKeys(result.Evt.Parsed) {
pval := result.Evt.Parsed[pkey]
if pval == "" {
continue
@ -257,7 +245,7 @@ func (p *ParserAssert) AutoGenParserAssert() string {
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Parsed["%s"] == "%s"`+"\n", stage, parser, pidx, pkey, Escape(pval))
}
for _, mkey := range sortedMapKeys(result.Evt.Meta) {
for _, mkey := range maptools.SortedKeys(result.Evt.Meta) {
mval := result.Evt.Meta[mkey]
if mval == "" {
continue
@ -266,7 +254,7 @@ func (p *ParserAssert) AutoGenParserAssert() string {
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval))
}
for _, ekey := range sortedMapKeys(result.Evt.Enriched) {
for _, ekey := range maptools.SortedKeys(result.Evt.Enriched) {
eval := result.Evt.Enriched[ekey]
if eval == "" {
continue
@ -275,7 +263,7 @@ func (p *ParserAssert) AutoGenParserAssert() string {
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Enriched["%s"] == "%s"`+"\n", stage, parser, pidx, ekey, Escape(eval))
}
for _, ukey := range sortedMapKeys(result.Evt.Unmarshaled) {
for _, ukey := range maptools.SortedKeys(result.Evt.Unmarshaled) {
uval := result.Evt.Unmarshaled[ukey]
if uval == "" {
continue
@ -328,288 +316,3 @@ func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []s
return ret
}
func LoadParserDump(filepath string) (*ParserResults, error) {
dumpData, err := os.Open(filepath)
if err != nil {
return nil, err
}
defer dumpData.Close()
results, err := io.ReadAll(dumpData)
if err != nil {
return nil, err
}
pdump := ParserResults{}
if err := yaml.Unmarshal(results, &pdump); err != nil {
return nil, err
}
/* we know that some variables should always be set,
let's check if they're present in last parser output of last stage */
stages := sortedMapKeys(pdump)
var lastStage string
//Loop over stages to find last successful one with at least one parser
for i := len(stages) - 2; i >= 0; i-- {
if len(pdump[stages[i]]) != 0 {
lastStage = stages[i]
break
}
}
parsers := make([]string, 0, len(pdump[lastStage]))
for k := range pdump[lastStage] {
parsers = append(parsers, k)
}
sort.Strings(parsers)
if len(parsers) == 0 {
return nil, fmt.Errorf("no parser found. Please install the appropriate parser and retry")
}
lastParser := parsers[len(parsers)-1]
for idx, result := range pdump[lastStage][lastParser] {
if result.Evt.StrTime == "" {
log.Warningf("Line %d/%d is missing evt.StrTime. It is most likely a mistake as it will prevent your logs to be processed in time-machine/forensic mode.", idx, len(pdump[lastStage][lastParser]))
} else {
log.Debugf("Line %d/%d has evt.StrTime set to '%s'", idx, len(pdump[lastStage][lastParser]), result.Evt.StrTime)
}
}
return &pdump, nil
}
type DumpOpts struct {
Details bool
SkipOk bool
ShowNotOkParsers bool
}
func DumpTree(parserResults ParserResults, bucketPour BucketPourInfo, opts DumpOpts) {
//note : we can use line -> time as the unique identifier (of acquisition)
state := make(map[time.Time]map[string]map[string]ParserResult)
assoc := make(map[time.Time]string, 0)
for stage, parsers := range parserResults {
for parser, results := range parsers {
for _, parserRes := range results {
evt := parserRes.Evt
if _, ok := state[evt.Line.Time]; !ok {
state[evt.Line.Time] = make(map[string]map[string]ParserResult)
assoc[evt.Line.Time] = evt.Line.Raw
}
if _, ok := state[evt.Line.Time][stage]; !ok {
state[evt.Line.Time][stage] = make(map[string]ParserResult)
}
state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parserRes.Success}
}
}
}
for bname, evtlist := range bucketPour {
for _, evt := range evtlist {
if evt.Line.Raw == "" {
continue
}
//it might be bucket overflow being reprocessed, skip this
if _, ok := state[evt.Line.Time]; !ok {
state[evt.Line.Time] = make(map[string]map[string]ParserResult)
assoc[evt.Line.Time] = evt.Line.Raw
}
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
if _, ok := state[evt.Line.Time]["buckets"]; !ok {
state[evt.Line.Time]["buckets"] = make(map[string]ParserResult)
}
state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true}
}
}
yellow := color.New(color.FgYellow).SprintFunc()
red := color.New(color.FgRed).SprintFunc()
green := color.New(color.FgGreen).SprintFunc()
whitelistReason := ""
//get each line
for tstamp, rawstr := range assoc {
if opts.SkipOk {
if _, ok := state[tstamp]["buckets"]["OK"]; ok {
continue
}
}
fmt.Printf("line: %s\n", rawstr)
skeys := make([]string, 0, len(state[tstamp]))
for k := range state[tstamp] {
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
if k == "buckets" {
continue
}
skeys = append(skeys, k)
}
sort.Strings(skeys)
// iterate stage
var prevItem types.Event
for _, stage := range skeys {
parsers := state[tstamp][stage]
sep := "├"
presep := "|"
fmt.Printf("\t%s %s\n", sep, stage)
pkeys := sortedMapKeys(parsers)
for idx, parser := range pkeys {
res := parsers[parser].Success
sep := "├"
if idx == len(pkeys)-1 {
sep = "└"
}
created := 0
updated := 0
deleted := 0
whitelisted := false
changeStr := ""
detailsDisplay := ""
if res {
changelog, _ := diff.Diff(prevItem, parsers[parser].Evt)
for _, change := range changelog {
switch change.Type {
case "create":
created++
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To))
case "update":
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To))
if change.Path[0] == "Whitelisted" && change.To == true {
whitelisted = true
if whitelistReason == "" {
whitelistReason = parsers[parser].Evt.WhitelistReason
}
}
updated++
case "delete":
deleted++
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, ".")))
}
}
prevItem = parsers[parser].Evt
}
if created > 0 {
changeStr += green(fmt.Sprintf("+%d", created))
}
if updated > 0 {
if len(changeStr) > 0 {
changeStr += " "
}
changeStr += yellow(fmt.Sprintf("~%d", updated))
}
if deleted > 0 {
if len(changeStr) > 0 {
changeStr += " "
}
changeStr += red(fmt.Sprintf("-%d", deleted))
}
if whitelisted {
if len(changeStr) > 0 {
changeStr += " "
}
changeStr += red("[whitelisted]")
}
if changeStr == "" {
changeStr = yellow("unchanged")
}
if res {
fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr)
if opts.Details {
fmt.Print(detailsDisplay)
}
} else if opts.ShowNotOkParsers {
fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser)
}
}
}
sep := "└"
if len(state[tstamp]["buckets"]) > 0 {
sep = "├"
}
//did the event enter the bucket pour phase ?
if _, ok := state[tstamp]["buckets"]["OK"]; ok {
fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle)
} else if whitelistReason != "" {
fmt.Printf("\t%s-------- parser success, ignored by whitelist (%s) %s\n", sep, whitelistReason, emoji.GreenCircle)
} else {
fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle)
}
//now print bucket info
if len(state[tstamp]["buckets"]) > 0 {
fmt.Printf("\t├ Scenarios\n")
}
bnames := make([]string, 0, len(state[tstamp]["buckets"]))
for k := range state[tstamp]["buckets"] {
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
if k == "OK" {
continue
}
bnames = append(bnames, k)
}
sort.Strings(bnames)
for idx, bname := range bnames {
sep := "├"
if idx == len(bnames)-1 {
sep = "└"
}
fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname)
}
fmt.Println()
}
}

29
pkg/hubtest/scenario_assert.go
View file

@ -12,6 +12,7 @@ import (
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/dumps"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
@ -24,11 +25,10 @@ type ScenarioAssert struct {
Fails []AssertFail
Success bool
TestData *BucketResults
PourData *BucketPourInfo
PourData *dumps.BucketPourInfo
}
type BucketResults []types.Event
type BucketPourInfo map[string][]types.Event
func NewScenarioAssert(file string) *ScenarioAssert {
ScenarioAssert := &ScenarioAssert{
@ -38,7 +38,7 @@ func NewScenarioAssert(file string) *ScenarioAssert {
Fails: make([]AssertFail, 0),
AutoGenAssert: false,
TestData: &BucketResults{},
PourData: &BucketPourInfo{},
PourData: &dumps.BucketPourInfo{},
}
return ScenarioAssert
@ -64,7 +64,7 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error {
s.TestData = bucketDump
if bucketpour != "" {
pourDump, err := LoadBucketPourDump(bucketpour)
pourDump, err := dumps.LoadBucketPourDump(bucketpour)
if err != nil {
return fmt.Errorf("loading bucket pour dump file '%s': %+v", filename, err)
}
@ -252,27 +252,6 @@ func (b BucketResults) Swap(i, j int) {
b[i], b[j] = b[j], b[i]
}
func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
dumpData, err := os.Open(filepath)
if err != nil {
return nil, err
}
defer dumpData.Close()
results, err := io.ReadAll(dumpData)
if err != nil {
return nil, err
}
var bucketDump BucketPourInfo
if err := yaml.Unmarshal(results, &bucketDump); err != nil {
return nil, err
}
return &bucketDump, nil
}
func LoadScenarioDump(filepath string) (*BucketResults, error) {
dumpData, err := os.Open(filepath)
if err != nil {

38
pkg/hubtest/utils.go
View file

@ -5,21 +5,25 @@ import (
"net"
"os"
"path/filepath"
"sort"
"time"
log "github.com/sirupsen/logrus"
)
func sortedMapKeys[V any](m map[string]V) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
func IsAlive(target string) (bool, error) {
start := time.Now()
for {
conn, err := net.Dial("tcp", target)
if err == nil {
log.Debugf("'%s' is up after %s", target, time.Since(start))
conn.Close()
return true, nil
}
time.Sleep(500 * time.Millisecond)
if time.Since(start) > 10*time.Second {
return false, fmt.Errorf("took more than 10s for %s to be available", target)
}
}
sort.Strings(keys)
return keys
}
func Copy(src string, dst string) error {
@ -110,19 +114,3 @@ func CopyDir(src string, dest string) error {
return nil
}
func IsAlive(target string) (bool, error) {
start := time.Now()
for {
conn, err := net.Dial("tcp", target)
if err == nil {
log.Debugf("'%s' is up after %s", target, time.Since(start))
conn.Close()
return true, nil
}
time.Sleep(500 * time.Millisecond)
if time.Since(start) > 10*time.Second {
return false, fmt.Errorf("took more than 10s for %s to be available", target)
}
}
}

26
pkg/parser/runtime.go
View file

@ -18,6 +18,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/dumps"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
@ -229,14 +230,10 @@ func stageidx(stage string, stages []string) int {
return -1
}
type ParserResult struct {
Evt types.Event
Success bool
}
var ParseDump bool
var DumpFolder string
var StageParseCache map[string]map[string][]ParserResult
var StageParseCache dumps.ParserResults
var StageParseMutex sync.Mutex
func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error) {
@ -271,9 +268,9 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error)
if ParseDump {
if StageParseCache == nil {
StageParseMutex.Lock()
StageParseCache = make(map[string]map[string][]ParserResult)
StageParseCache["success"] = make(map[string][]ParserResult)
StageParseCache["success"][""] = make([]ParserResult, 0)
StageParseCache = make(dumps.ParserResults)
StageParseCache["success"] = make(map[string][]dumps.ParserResult)
StageParseCache["success"][""] = make([]dumps.ParserResult, 0)
StageParseMutex.Unlock()
}
}
@ -282,7 +279,7 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error)
if ParseDump {
StageParseMutex.Lock()
if _, ok := StageParseCache[stage]; !ok {
StageParseCache[stage] = make(map[string][]ParserResult)
StageParseCache[stage] = make(map[string][]dumps.ParserResult)
}
StageParseMutex.Unlock()
}
@ -322,13 +319,18 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error)
}
clog.Tracef("node (%s) ret : %v", node.rn, ret)
if ParseDump {
parserIdxInStage := 0
StageParseMutex.Lock()
if len(StageParseCache[stage][node.Name]) == 0 {
StageParseCache[stage][node.Name] = make([]ParserResult, 0)
StageParseCache[stage][node.Name] = make([]dumps.ParserResult, 0)
parserIdxInStage = len(StageParseCache[stage])
} else {
parserIdxInStage = StageParseCache[stage][node.Name][0].Idx
}
StageParseMutex.Unlock()
evtcopy := deepcopy.Copy(event)
parserInfo := ParserResult{Evt: evtcopy.(types.Event), Success: ret}
parserInfo := dumps.ParserResult{Evt: evtcopy.(types.Event), Success: ret, Idx: parserIdxInStage}
StageParseMutex.Lock()
StageParseCache[stage][node.Name] = append(StageParseCache[stage][node.Name], parserInfo)
StageParseMutex.Unlock()