diff --git a/cmd/crowdsec-cli/hubtest.go b/cmd/crowdsec-cli/hubtest.go index 97bb8c8dd..5052c1332 100644 --- a/cmd/crowdsec-cli/hubtest.go +++ b/cmd/crowdsec-cli/hubtest.go @@ -18,9 +18,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/hubtest" ) -var ( - HubTest hubtest.HubTest -) +var HubTest hubtest.HubTest func NewHubTestCmd() *cobra.Command { var hubPath string @@ -43,6 +41,7 @@ func NewHubTestCmd() *cobra.Command { return nil }, } + cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder") cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec") cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli") @@ -59,7 +58,6 @@ func NewHubTestCmd() *cobra.Command { return cmdHubTest } - func NewHubTestCreateCmd() *cobra.Command { parsers := []string{} postoverflows := []string{} @@ -164,6 +162,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios return nil }, } + cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test") cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test") cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test") @@ -173,7 +172,6 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios return cmdHubTestCreate } - func NewHubTestRunCmd() *cobra.Command { var noClean bool var runAll bool @@ -186,7 +184,7 @@ func NewHubTestRunCmd() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { if !runAll && len(args) == 0 { printHelp(cmd) - return fmt.Errorf("Please provide test to run or --all flag") + return fmt.Errorf("please provide test to run or --all flag") } if runAll { @@ -202,6 +200,9 @@ func NewHubTestRunCmd() *cobra.Command { } } + // set timezone to avoid DST issues + os.Setenv("TZ", "UTC") + for _, test := range HubTest.Tests { if csConfig.Cscli.Output == "human" { log.Infof("Running test '%s'", test.Name) @@ -293,9 +294,11 @@ func NewHubTestRunCmd() *cobra.Command { } } } - if csConfig.Cscli.Output == "human" { + + switch csConfig.Cscli.Output { + case "human": hubTestResultTable(color.Output, testResult) - } else if csConfig.Cscli.Output == "json" { + case "json": jsonResult := make(map[string][]string, 0) jsonResult["success"] = make([]string, 0) jsonResult["fail"] = make([]string, 0) @@ -311,6 +314,8 @@ func NewHubTestRunCmd() *cobra.Command { return fmt.Errorf("unable to json test result: %s", err) } fmt.Println(string(jsonStr)) + default: + return fmt.Errorf("only human/json output modes are supported") } if !success { @@ -320,6 +325,7 @@ func NewHubTestRunCmd() *cobra.Command { return nil }, } + cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed") cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail") cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests") @@ -327,7 +333,6 @@ func NewHubTestRunCmd() *cobra.Command { return cmdHubTestRun } - func NewHubTestCleanCmd() *cobra.Command { var cmdHubTestClean = &cobra.Command{ Use: "clean", @@ -352,7 +357,6 @@ func NewHubTestCleanCmd() *cobra.Command { return cmdHubTestClean } - func NewHubTestInfoCmd() *cobra.Command { var cmdHubTestInfo = &cobra.Command{ Use: "info", @@ -381,7 +385,6 @@ func NewHubTestInfoCmd() *cobra.Command { return cmdHubTestInfo } - func NewHubTestListCmd() *cobra.Command { var cmdHubTestList = &cobra.Command{ Use: "list", @@ -412,7 +415,6 @@ func NewHubTestListCmd() *cobra.Command { return cmdHubTestList } - func NewHubTestCoverageCmd() *cobra.Command { var showParserCov bool var showScenarioCov bool @@ -427,8 +429,8 @@ func NewHubTestCoverageCmd() *cobra.Command { return fmt.Errorf("unable to load all tests: %+v", err) } var err error - scenarioCoverage := []hubtest.ScenarioCoverage{} - parserCoverage := []hubtest.ParserCoverage{} + scenarioCoverage := []hubtest.Coverage{} + parserCoverage := []hubtest.Coverage{} scenarioCoveragePercent := 0 parserCoveragePercent := 0 @@ -443,7 +445,7 @@ func NewHubTestCoverageCmd() *cobra.Command { parserTested := 0 for _, test := range parserCoverage { if test.TestsCount > 0 { - parserTested += 1 + parserTested++ } } parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100))) @@ -454,12 +456,14 @@ func NewHubTestCoverageCmd() *cobra.Command { if err != nil { return fmt.Errorf("while getting scenario coverage: %s", err) } + scenarioTested := 0 for _, test := range scenarioCoverage { if test.TestsCount > 0 { - scenarioTested += 1 + scenarioTested++ } } + scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100))) } @@ -474,7 +478,8 @@ func NewHubTestCoverageCmd() *cobra.Command { os.Exit(0) } - if csConfig.Cscli.Output == "human" { + switch csConfig.Cscli.Output { + case "human": if showParserCov || showAll { hubTestParserCoverageTable(color.Output, parserCoverage) } @@ -489,7 +494,7 @@ func NewHubTestCoverageCmd() *cobra.Command { if showScenarioCov || showAll { fmt.Printf("SCENARIOS : %d%% of coverage\n", scenarioCoveragePercent) } - } else if csConfig.Cscli.Output == "json" { + case "json": dump, err := json.MarshalIndent(parserCoverage, "", " ") if err != nil { return err @@ -500,13 +505,14 @@ func NewHubTestCoverageCmd() *cobra.Command { return err } fmt.Printf("%s", dump) - } else { + default: return fmt.Errorf("only human/json output modes are supported") } return nil }, } + cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage") cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage") cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage") @@ -514,7 +520,6 @@ func NewHubTestCoverageCmd() *cobra.Command { return cmdHubTestCoverage } - func NewHubTestEvalCmd() *cobra.Command { var evalExpression string var cmdHubTestEval = &cobra.Command{ @@ -528,26 +533,29 @@ func NewHubTestEvalCmd() *cobra.Command { if err != nil { return fmt.Errorf("can't load test: %+v", err) } + err = test.ParserAssert.LoadTest(test.ParserResultFile) if err != nil { return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err) } + output, err := test.ParserAssert.EvalExpression(evalExpression) if err != nil { return err } + fmt.Print(output) } return nil }, } + cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval") return cmdHubTestEval } - func NewHubTestExplainCmd() *cobra.Command { var cmdHubTestExplain = &cobra.Command{ Use: "explain", @@ -562,24 +570,22 @@ func NewHubTestExplainCmd() *cobra.Command { } err = test.ParserAssert.LoadTest(test.ParserResultFile) if err != nil { - err := test.Run() - if err != nil { + if err = test.Run(); err != nil { return fmt.Errorf("running test '%s' failed: %+v", test.Name, err) } - err = test.ParserAssert.LoadTest(test.ParserResultFile) - if err != nil { + + if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil { return fmt.Errorf("unable to load parser result after run: %s", err) } } err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile) if err != nil { - err := test.Run() - if err != nil { + if err = test.Run(); err != nil { return fmt.Errorf("running test '%s' failed: %+v", test.Name, err) } - err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile) - if err != nil { + + if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil { return fmt.Errorf("unable to load scenario result after run: %s", err) } } diff --git a/cmd/crowdsec-cli/hubtest_table.go b/cmd/crowdsec-cli/hubtest_table.go index 9f28c3699..9b31a79a2 100644 --- a/cmd/crowdsec-cli/hubtest_table.go +++ b/cmd/crowdsec-cli/hubtest_table.go @@ -41,39 +41,41 @@ func hubTestListTable(out io.Writer, tests []*hubtest.HubTestItem) { t.Render() } -func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.ParserCoverage) { +func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) { t := newLightTable(out) t.SetHeaders("Parser", "Status", "Number of tests") t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) parserTested := 0 + for _, test := range coverage { status := emoji.RedCircle.String() if test.TestsCount > 0 { status = emoji.GreenCircle.String() parserTested++ } - t.AddRow(test.Parser, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) + t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) } t.Render() } -func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.ScenarioCoverage) { +func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) { t := newLightTable(out) t.SetHeaders("Scenario", "Status", "Number of tests") t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) parserTested := 0 + for _, test := range coverage { status := emoji.RedCircle.String() if test.TestsCount > 0 { status = emoji.GreenCircle.String() parserTested++ } - t.AddRow(test.Scenario, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) + t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) } t.Render() diff --git a/pkg/hubtest/coverage.go b/pkg/hubtest/coverage.go index 29db52715..5d3b79fe1 100644 --- a/pkg/hubtest/coverage.go +++ b/pkg/hubtest/coverage.go @@ -5,173 +5,194 @@ import ( "fmt" "os" "path/filepath" - "regexp" - "sort" "strings" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -type ParserCoverage struct { - Parser string +type Coverage struct { + Name string TestsCount int PresentIn map[string]bool //poorman's set } -type ScenarioCoverage struct { - Scenario string - TestsCount int - PresentIn map[string]bool -} - -func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) { - var coverage []ParserCoverage +func (h *HubTest) GetParsersCoverage() ([]Coverage, error) { if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok { - return coverage, fmt.Errorf("no parsers in hub index") + return nil, fmt.Errorf("no parsers in hub index") } - //populate from hub, iterate in alphabetical order - var pkeys []string - for pname := range h.HubIndex.Items[cwhub.PARSERS] { - pkeys = append(pkeys, pname) - } - sort.Strings(pkeys) - for _, pname := range pkeys { - coverage = append(coverage, ParserCoverage{ - Parser: pname, + + // populate from hub, iterate in alphabetical order + pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.PARSERS]) + coverage := make([]Coverage, len(pkeys)) + + for i, name := range pkeys { + coverage[i] = Coverage{ + Name: name, TestsCount: 0, PresentIn: make(map[string]bool), - }) + } } - //parser the expressions a-la-oneagain + // parser the expressions a-la-oneagain passerts, err := filepath.Glob(".tests/*/parser.assert") if err != nil { - return coverage, fmt.Errorf("while find parser asserts : %s", err) + return nil, fmt.Errorf("while find parser asserts : %s", err) } + for _, assert := range passerts { file, err := os.Open(assert) if err != nil { - return coverage, fmt.Errorf("while reading %s : %s", assert, err) + return nil, fmt.Errorf("while reading %s : %s", assert, err) } + scanner := bufio.NewScanner(file) for scanner.Scan() { - assertLine := regexp.MustCompile(`^results\["[^"]+"\]\["(?P[^"]+)"\]\[[0-9]+\]\.Evt\..*`) line := scanner.Text() log.Debugf("assert line : %s", line) - match := assertLine.FindStringSubmatch(line) + + match := parserResultRE.FindStringSubmatch(line) if len(match) == 0 { log.Debugf("%s doesn't match", line) continue } - sidx := assertLine.SubexpIndex("parser") + + sidx := parserResultRE.SubexpIndex("parser") capturedParser := match[sidx] + for idx, pcover := range coverage { - if pcover.Parser == capturedParser { + if pcover.Name == capturedParser { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } - parserNameSplit := strings.Split(pcover.Parser, "/") + + parserNameSplit := strings.Split(pcover.Name, "/") parserNameOnly := parserNameSplit[len(parserNameSplit)-1] + if parserNameOnly == capturedParser { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } + capturedParserSplit := strings.Split(capturedParser, "/") capturedParserName := capturedParserSplit[len(capturedParserSplit)-1] + if capturedParserName == parserNameOnly { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } + if capturedParserName == parserNameOnly+"-logs" { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } } } + file.Close() } + return coverage, nil } -func (h *HubTest) GetScenariosCoverage() ([]ScenarioCoverage, error) { - var coverage []ScenarioCoverage +func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) { if _, ok := h.HubIndex.Items[cwhub.SCENARIOS]; !ok { - return coverage, fmt.Errorf("no scenarios in hub index") - } - //populate from hub, iterate in alphabetical order - var pkeys []string - for scenarioName := range h.HubIndex.Items[cwhub.SCENARIOS] { - pkeys = append(pkeys, scenarioName) - } - sort.Strings(pkeys) - for _, scenarioName := range pkeys { - coverage = append(coverage, ScenarioCoverage{ - Scenario: scenarioName, - TestsCount: 0, - PresentIn: make(map[string]bool), - }) + return nil, fmt.Errorf("no scenarios in hub index") } - //parser the expressions a-la-oneagain + // populate from hub, iterate in alphabetical order + pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.SCENARIOS]) + coverage := make([]Coverage, len(pkeys)) + + for i, name := range pkeys { + coverage[i] = Coverage{ + Name: name, + TestsCount: 0, + PresentIn: make(map[string]bool), + } + } + + // parser the expressions a-la-oneagain passerts, err := filepath.Glob(".tests/*/scenario.assert") if err != nil { - return coverage, fmt.Errorf("while find scenario asserts : %s", err) + return nil, fmt.Errorf("while find scenario asserts : %s", err) } + + for _, assert := range passerts { file, err := os.Open(assert) if err != nil { - return coverage, fmt.Errorf("while reading %s : %s", assert, err) + return nil, fmt.Errorf("while reading %s : %s", assert, err) } + scanner := bufio.NewScanner(file) for scanner.Scan() { - assertLine := regexp.MustCompile(`^results\[[0-9]+\].Overflow.Alert.GetScenario\(\) == "(?P[^"]+)"`) line := scanner.Text() log.Debugf("assert line : %s", line) - match := assertLine.FindStringSubmatch(line) + match := scenarioResultRE.FindStringSubmatch(line) + if len(match) == 0 { log.Debugf("%s doesn't match", line) continue } - sidx := assertLine.SubexpIndex("scenario") - scanner_name := match[sidx] + + sidx := scenarioResultRE.SubexpIndex("scenario") + scannerName := match[sidx] + for idx, pcover := range coverage { - if pcover.Scenario == scanner_name { + if pcover.Name == scannerName { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } - scenarioNameSplit := strings.Split(pcover.Scenario, "/") + + scenarioNameSplit := strings.Split(pcover.Name, "/") scenarioNameOnly := scenarioNameSplit[len(scenarioNameSplit)-1] - if scenarioNameOnly == scanner_name { + + if scenarioNameOnly == scannerName { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } - fixedProbingWord := strings.ReplaceAll(pcover.Scenario, "probbing", "probing") - fixedProbingAssert := strings.ReplaceAll(scanner_name, "probbing", "probing") + + fixedProbingWord := strings.ReplaceAll(pcover.Name, "probbing", "probing") + fixedProbingAssert := strings.ReplaceAll(scannerName, "probbing", "probing") + if fixedProbingWord == fixedProbingAssert { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } - if fmt.Sprintf("%s-detection", pcover.Scenario) == scanner_name { + + if fmt.Sprintf("%s-detection", pcover.Name) == scannerName { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } + if fmt.Sprintf("%s-detection", fixedProbingWord) == fixedProbingAssert { coverage[idx].TestsCount++ coverage[idx].PresentIn[assert] = true + continue } } } file.Close() } + return coverage, nil } diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index b110369ff..ec1f6ee5e 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -30,28 +30,28 @@ const ( ) func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, error) { - var err error - - hubPath, err = filepath.Abs(hubPath) + hubPath, err := filepath.Abs(hubPath) if err != nil { return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err) } + // we can't use hubtest without the hub - if _, err := os.Stat(hubPath); os.IsNotExist(err) { + if _, err = os.Stat(hubPath); os.IsNotExist(err) { return HubTest{}, fmt.Errorf("path to hub '%s' doesn't exist, can't run", hubPath) } + HubTestPath := filepath.Join(hubPath, "./.tests/") // we can't use hubtest without crowdsec binary - if _, err := exec.LookPath(crowdsecPath); err != nil { - if _, err := os.Stat(crowdsecPath); os.IsNotExist(err) { + if _, err = exec.LookPath(crowdsecPath); err != nil { + if _, err = os.Stat(crowdsecPath); os.IsNotExist(err) { return HubTest{}, fmt.Errorf("path to crowdsec binary '%s' doesn't exist or is not in $PATH, can't run", crowdsecPath) } } // we can't use hubtest without cscli binary - if _, err := exec.LookPath(cscliPath); err != nil { - if _, err := os.Stat(cscliPath); os.IsNotExist(err) { + if _, err = exec.LookPath(cscliPath); err != nil { + if _, err = os.Stat(cscliPath); os.IsNotExist(err) { return HubTest{}, fmt.Errorf("path to cscli binary '%s' doesn't exist or is not in $PATH, can't run", cscliPath) } } @@ -59,9 +59,9 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, hubIndexFile := filepath.Join(hubPath, ".index.json") local := &csconfig.LocalHubCfg{ - HubDir: hubPath, - HubIndexFile: hubIndexFile, - InstallDir: HubTestPath, + HubDir: hubPath, + HubIndexFile: hubIndexFile, + InstallDir: HubTestPath, InstallDataDir: HubTestPath, } @@ -89,10 +89,12 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, func (h *HubTest) LoadTestItem(name string) (*HubTestItem, error) { HubTestItem := &HubTestItem{} + testItem, err := NewTest(name, h) if err != nil { return HubTestItem, err } + h.Tests = append(h.Tests, testItem) return testItem, nil @@ -111,5 +113,6 @@ func (h *HubTest) LoadAllTests() error { } } } + return nil } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 44018a14a..329e74a5d 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -7,11 +7,12 @@ import ( "path/filepath" "strings" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/parser" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" ) type HubTestItemConfig struct { @@ -76,8 +77,6 @@ const ( BucketPourResultFileName = "bucketpour-dump.yaml" ) -var crowdsecPatternsFolder = csconfig.DefaultConfigPath("patterns") - func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { testPath := filepath.Join(hubTest.HubTestPath, name) runtimeFolder := filepath.Join(testPath, "runtime") @@ -87,10 +86,12 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { // read test configuration file configFileData := &HubTestItemConfig{} + yamlFile, err := os.ReadFile(configFilePath) if err != nil { log.Printf("no config file found in '%s': %v", testPath, err) } + err = yaml.Unmarshal(yamlFile, configFileData) if err != nil { return nil, fmt.Errorf("unmarshal: %v", err) @@ -101,6 +102,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { scenarioAssertFilePath := filepath.Join(testPath, ScenarioAssertFileName) ScenarioAssert := NewScenarioAssert(scenarioAssertFilePath) + return &HubTestItem{ Name: name, Path: testPath, @@ -143,23 +145,25 @@ func (t *HubTestItem) InstallHub() error { if parser == "" { continue } - var parserDirDest string + if hubParser, ok := t.HubIndex.Items[cwhub.PARSERS][parser]; ok { parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err) } + parserFileName := filepath.Base(parserSource) // runtime/hub/parsers/s00-raw/crowdsecurity/ hubDirParserDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubParser.RemotePath)) // runtime/parsers/s00-raw/ - parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage) + parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage) if err := os.MkdirAll(hubDirParserDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", hubDirParserDest, err) } + if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err) } @@ -200,7 +204,7 @@ func (t *HubTestItem) InstallHub() error { //return fmt.Errorf("stage '%s' extracted from '%s' doesn't exist in the hub", customParserStage, hubStagePath) } - parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage) + parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage) if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil { continue //return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err) @@ -227,23 +231,25 @@ func (t *HubTestItem) InstallHub() error { if scenario == "" { continue } - var scenarioDirDest string + if hubScenario, ok := t.HubIndex.Items[cwhub.SCENARIOS][scenario]; ok { scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path to: %s", scenarioSource) } + scenarioFileName := filepath.Base(scenarioSource) // runtime/hub/scenarios/crowdsecurity/ hubDirScenarioDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubScenario.RemotePath)) // runtime/parsers/scenarios/ - scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath) + scenarioDirDest := fmt.Sprintf("%s/scenarios/", t.RuntimePath) if err := os.MkdirAll(hubDirScenarioDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", hubDirScenarioDest, err) } + if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err) } @@ -271,7 +277,7 @@ func (t *HubTestItem) InstallHub() error { //return fmt.Errorf("scenarios '%s' doesn't exist in the hub and doesn't appear to be a custom one.", scenario) } - scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath) + scenarioDirDest := fmt.Sprintf("%s/scenarios/", t.RuntimePath) if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err) } @@ -296,23 +302,25 @@ func (t *HubTestItem) InstallHub() error { if postoverflow == "" { continue } - var postoverflowDirDest string + if hubPostOverflow, ok := t.HubIndex.Items[cwhub.POSTOVERFLOWS][postoverflow]; ok { postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err) } + postoverflowFileName := filepath.Base(postoverflowSource) // runtime/hub/postoverflows/s00-enrich/crowdsecurity/ hubDirPostoverflowDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubPostOverflow.RemotePath)) // runtime/postoverflows/s00-enrich - postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage) + postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage) if err := os.MkdirAll(hubDirPostoverflowDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", hubDirPostoverflowDest, err) } + if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil { return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err) } @@ -353,7 +361,7 @@ func (t *HubTestItem) InstallHub() error { //return fmt.Errorf("stage '%s' from extracted '%s' doesn't exist in the hub", customPostoverflowStage, hubStagePath) } - postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage) + postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage) if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil { continue //return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err) @@ -380,10 +388,12 @@ func (t *HubTestItem) InstallHub() error { Filter: "1==1", Statics: t.Config.OverrideStatics, } + b, err := yaml.Marshal(n) if err != nil { return fmt.Errorf("unable to marshal overrides: %s", err) } + tgtFilename := fmt.Sprintf("%s/parsers/s00-raw/00_overrides.yaml", t.RuntimePath) if err := os.WriteFile(tgtFilename, b, os.ModePerm); err != nil { return fmt.Errorf("unable to write overrides to '%s': %s", tgtFilename, err) @@ -403,6 +413,7 @@ func (t *HubTestItem) InstallHub() error { if err := hub.DownloadDataIfNeeded(item, true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err) } + log.Debugf("parser '%s' installed successfully in runtime environment", parserName) } } @@ -414,6 +425,7 @@ func (t *HubTestItem) InstallHub() error { if err := hub.DownloadDataIfNeeded(item, true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err) } + log.Debugf("scenario '%s' installed successfully in runtime environment", scenarioName) } } @@ -425,6 +437,7 @@ func (t *HubTestItem) InstallHub() error { if err := hub.DownloadDataIfNeeded(item, true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err) } + log.Debugf("postoverflow '%s' installed successfully in runtime environment", postoverflowName) } } @@ -489,6 +502,8 @@ func (t *HubTestItem) Run() error { return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err) } + crowdsecPatternsFolder := csconfig.DefaultConfigPath("patterns") + // copy template patterns folder to runtime folder if err = CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil { return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err) @@ -511,6 +526,7 @@ func (t *HubTestItem) Run() error { if err != nil { return fmt.Errorf("unable to stat log file '%s': %s", logFile, err) } + if logFileStat.Size() == 0 { return fmt.Errorf("log file '%s' is empty, please fill it with log", logFile) } @@ -518,6 +534,7 @@ func (t *HubTestItem) Run() error { cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"} cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...) log.Debugf("%s", cscliRegisterCmd.String()) + output, err := cscliRegisterCmd.CombinedOutput() if err != nil { if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") { @@ -527,16 +544,20 @@ func (t *HubTestItem) Run() error { } cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", t.ResultsPath, "-order-event"} + for labelKey, labelValue := range t.Config.Labels { arg := fmt.Sprintf("%s:%s", labelKey, labelValue) cmdArgs = append(cmdArgs, "-label", arg) } + crowdsecCmd := exec.Command(t.CrowdSecPath, cmdArgs...) log.Debugf("%s", crowdsecCmd.String()) output, err = crowdsecCmd.CombinedOutput() + if log.GetLevel() >= log.DebugLevel || err != nil { fmt.Println(string(output)) } + if err != nil { return fmt.Errorf("fail to run '%s' for test '%s': %v", crowdsecCmd.String(), t.Name, err) } @@ -553,8 +574,10 @@ func (t *HubTestItem) Run() error { if err != nil { return err } + parserAssertFile.Close() } + assertFileStat, err := os.Stat(t.ParserAssert.File) if err != nil { return fmt.Errorf("error while stats '%s': %s", t.ParserAssert.File, err) @@ -565,6 +588,7 @@ func (t *HubTestItem) Run() error { if err != nil { return fmt.Errorf("couldn't generate assertion: %s", err) } + t.ParserAssert.AutoGenAssertData = assertData t.ParserAssert.AutoGenAssert = true } else { @@ -576,12 +600,15 @@ func (t *HubTestItem) Run() error { // assert scenarios nbScenario := 0 + for _, scenario := range t.Config.Scenarios { if scenario == "" { continue } - nbScenario += 1 + + nbScenario++ } + if nbScenario > 0 { _, err := os.Stat(t.ScenarioAssert.File) if os.IsNotExist(err) { @@ -589,8 +616,10 @@ func (t *HubTestItem) Run() error { if err != nil { return err } + scenarioAssertFile.Close() } + assertFileStat, err := os.Stat(t.ScenarioAssert.File) if err != nil { return fmt.Errorf("error while stats '%s': %s", t.ScenarioAssert.File, err) @@ -601,6 +630,7 @@ func (t *HubTestItem) Run() error { if err != nil { return fmt.Errorf("couldn't generate assertion: %s", err) } + t.ScenarioAssert.AutoGenAssertData = assertData t.ScenarioAssert.AutoGenAssert = true } else { diff --git a/pkg/hubtest/parser_assert.go b/pkg/hubtest/parser_assert.go index c9a183336..620e2645e 100644 --- a/pkg/hubtest/parser_assert.go +++ b/pkg/hubtest/parser_assert.go @@ -5,13 +5,11 @@ import ( "fmt" "io" "os" - "regexp" "sort" "strings" "time" "github.com/antonmedv/expr" - "github.com/antonmedv/expr/vm" "github.com/enescakir/emoji" "github.com/fatih/color" diff "github.com/r3labs/diff/v2" @@ -43,10 +41,10 @@ type ParserResult struct { Evt types.Event Success bool } + type ParserResults map[string]map[string][]ParserResult func NewParserAssert(file string) *ParserAssert { - ParserAssert := &ParserAssert{ File: file, NbAssert: 0, @@ -55,6 +53,7 @@ func NewParserAssert(file string) *ParserAssert { AutoGenAssert: false, TestData: &ParserResults{}, } + return ParserAssert } @@ -63,22 +62,24 @@ func (p *ParserAssert) AutoGenFromFile(filename string) (string, error) { if err != nil { return "", err } + ret := p.AutoGenParserAssert() + return ret, nil } func (p *ParserAssert) LoadTest(filename string) error { - var err error parserDump, err := LoadParserDump(filename) if err != nil { return fmt.Errorf("loading parser dump file: %+v", err) } + p.TestData = parserDump + return nil } func (p *ParserAssert) AssertFile(testFile string) error { - file, err := os.Open(p.File) if err != nil { @@ -88,19 +89,26 @@ func (p *ParserAssert) AssertFile(testFile string) error { if err := p.LoadTest(testFile); err != nil { return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err) } + scanner := bufio.NewScanner(file) scanner.Split(bufio.ScanLines) + nbLine := 0 + for scanner.Scan() { - nbLine += 1 + nbLine++ + if scanner.Text() == "" { continue } + ok, err := p.Run(scanner.Text()) if err != nil { return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err) } - p.NbAssert += 1 + + p.NbAssert++ + if !ok { log.Debugf("%s is FALSE", scanner.Text()) failedAssert := &AssertFail{ @@ -109,37 +117,43 @@ func (p *ParserAssert) AssertFile(testFile string) error { Expression: scanner.Text(), Debug: make(map[string]string), } - variableRE := regexp.MustCompile(`(?P[^ =]+) == .*`) + match := variableRE.FindStringSubmatch(scanner.Text()) variable := "" + if len(match) == 0 { log.Infof("Couldn't get variable of line '%s'", scanner.Text()) variable = scanner.Text() } else { variable = match[1] } + result, err := p.EvalExpression(variable) if err != nil { log.Errorf("unable to evaluate variable '%s': %s", variable, err) continue } + failedAssert.Debug[variable] = result p.Fails = append(p.Fails, *failedAssert) continue } //fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text()) - } + file.Close() + if p.NbAssert == 0 { assertData, err := p.AutoGenFromFile(testFile) if err != nil { return fmt.Errorf("couldn't generate assertion: %s", err) } + p.AutoGenAssertData = assertData p.AutoGenAssert = true } + if len(p.Fails) == 0 { p.Success = true } @@ -148,15 +162,14 @@ func (p *ParserAssert) AssertFile(testFile string) error { } func (p *ParserAssert) RunExpression(expression string) (interface{}, error) { - var err error //debug doesn't make much sense with the ability to evaluate "on the fly" //var debugFilter *exprhelpers.ExprDebugger - var runtimeFilter *vm.Program var output interface{} env := map[string]interface{}{"results": *p.TestData} - if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil { + runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...) + if err != nil { log.Errorf("failed to compile '%s' : %s", expression, err) return output, err } @@ -168,8 +181,10 @@ func (p *ParserAssert) RunExpression(expression string) (interface{}, error) { if err != nil { log.Warningf("running : %s", expression) log.Warningf("runtime error : %s", err) + return output, fmt.Errorf("while running expression %s: %w", expression, err) } + return output, nil } @@ -178,10 +193,13 @@ func (p *ParserAssert) EvalExpression(expression string) (string, error) { if err != nil { return "", err } + ret, err := yaml.Marshal(output) + if err != nil { return "", err } + return string(ret), nil } @@ -190,6 +208,7 @@ func (p *ParserAssert) Run(assert string) (bool, error) { if err != nil { return false, err } + switch out := output.(type) { case bool: return out, nil @@ -201,80 +220,89 @@ func (p *ParserAssert) Run(assert string) (bool, error) { func Escape(val string) string { val = strings.ReplaceAll(val, `\`, `\\`) val = strings.ReplaceAll(val, `"`, `\"`) + return val } func (p *ParserAssert) AutoGenParserAssert() string { //attempt to autogen parser asserts - var ret string + ret := fmt.Sprintf("len(results) == %d\n", len(*p.TestData)) + + //sort map keys for consistent order + stages := sortedMapKeys(*p.TestData) - //sort map keys for consistent ordre - var stages []string - for stage := range *p.TestData { - stages = append(stages, stage) - } - sort.Strings(stages) - ret += fmt.Sprintf("len(results) == %d\n", len(*p.TestData)) for _, stage := range stages { parsers := (*p.TestData)[stage] - //sort map keys for consistent ordre - var pnames []string - for pname := range parsers { - pnames = append(pnames, pname) - } - sort.Strings(pnames) + + //sort map keys for consistent order + pnames := sortedMapKeys(parsers) + for _, parser := range pnames { presults := parsers[parser] ret += fmt.Sprintf(`len(results["%s"]["%s"]) == %d`+"\n", stage, parser, len(presults)) + for pidx, result := range presults { ret += fmt.Sprintf(`results["%s"]["%s"][%d].Success == %t`+"\n", stage, parser, pidx, result.Success) if !result.Success { continue } + for _, pkey := range sortedMapKeys(result.Evt.Parsed) { pval := result.Evt.Parsed[pkey] if pval == "" { continue } + ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Parsed["%s"] == "%s"`+"\n", stage, parser, pidx, pkey, Escape(pval)) } + for _, mkey := range sortedMapKeys(result.Evt.Meta) { mval := result.Evt.Meta[mkey] if mval == "" { continue } + ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval)) } + for _, ekey := range sortedMapKeys(result.Evt.Enriched) { eval := result.Evt.Enriched[ekey] if eval == "" { continue } + ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Enriched["%s"] == "%s"`+"\n", stage, parser, pidx, ekey, Escape(eval)) } + for _, ukey := range sortedMapKeys(result.Evt.Unmarshaled) { uval := result.Evt.Unmarshaled[ukey] if uval == "" { continue } + base := fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Unmarshaled["%s"]`, stage, parser, pidx, ukey) + for _, line := range p.buildUnmarshaledAssert(base, uval) { ret += line } } + ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Whitelisted == %t`+"\n", stage, parser, pidx, result.Evt.Whitelisted) + if result.Evt.WhitelistReason != "" { ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.WhitelistReason == "%s"`+"\n", stage, parser, pidx, Escape(result.Evt.WhitelistReason)) } } } } + return ret } func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []string { ret := make([]string, 0) + switch val := eval.(type) { case map[string]interface{}: for k, v := range val { @@ -297,12 +325,11 @@ func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []s default: log.Warningf("unknown type '%T' for key '%s'", val, ekey) } + return ret } func LoadParserDump(filepath string) (*ParserResults, error) { - var pdump ParserResults - dumpData, err := os.Open(filepath) if err != nil { return nil, err @@ -314,18 +341,19 @@ func LoadParserDump(filepath string) (*ParserResults, error) { return nil, err } + pdump := ParserResults{} + if err := yaml.Unmarshal(results, &pdump); err != nil { return nil, err } /* we know that some variables should always be set, let's check if they're present in last parser output of last stage */ - stages := make([]string, 0, len(pdump)) - for k := range pdump { - stages = append(stages, k) - } - sort.Strings(stages) + + stages := sortedMapKeys(pdump) + var lastStage string + //Loop over stages to find last successful one with at least one parser for i := len(stages) - 2; i >= 0; i-- { if len(pdump[stages[i]]) != 0 { @@ -333,10 +361,13 @@ func LoadParserDump(filepath string) (*ParserResults, error) { break } } + parsers := make([]string, 0, len(pdump[lastStage])) + for k := range pdump[lastStage] { parsers = append(parsers, k) } + sort.Strings(parsers) lastParser := parsers[len(parsers)-1] @@ -357,47 +388,51 @@ type DumpOpts struct { ShowNotOkParsers bool } -func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts DumpOpts) { +func DumpTree(parserResults ParserResults, bucketPour BucketPourInfo, opts DumpOpts) { //note : we can use line -> time as the unique identifier (of acquisition) - state := make(map[time.Time]map[string]map[string]ParserResult) assoc := make(map[time.Time]string, 0) - for stage, parsers := range parser_results { + for stage, parsers := range parserResults { for parser, results := range parsers { - for _, parser_res := range results { - evt := parser_res.Evt + for _, parserRes := range results { + evt := parserRes.Evt if _, ok := state[evt.Line.Time]; !ok { state[evt.Line.Time] = make(map[string]map[string]ParserResult) assoc[evt.Line.Time] = evt.Line.Raw } + if _, ok := state[evt.Line.Time][stage]; !ok { state[evt.Line.Time][stage] = make(map[string]ParserResult) } - state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parser_res.Success} - } + state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parserRes.Success} + } } } - for bname, evtlist := range bucket_pour { + for bname, evtlist := range bucketPour { for _, evt := range evtlist { if evt.Line.Raw == "" { continue } + //it might be bucket overflow being reprocessed, skip this if _, ok := state[evt.Line.Time]; !ok { state[evt.Line.Time] = make(map[string]map[string]ParserResult) assoc[evt.Line.Time] = evt.Line.Raw } + //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered if _, ok := state[evt.Line.Time]["buckets"]; !ok { state[evt.Line.Time]["buckets"] = make(map[string]ParserResult) } + state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true} } } + yellow := color.New(color.FgYellow).SprintFunc() red := color.New(color.FgRed).SprintFunc() green := color.New(color.FgGreen).SprintFunc() @@ -409,19 +444,25 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum continue } } + fmt.Printf("line: %s\n", rawstr) + skeys := make([]string, 0, len(state[tstamp])) + for k := range state[tstamp] { //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered if k == "buckets" { continue } + skeys = append(skeys, k) } + sort.Strings(skeys) - //iterate stage - var prev_item types.Event + + // iterate stage + var prevItem types.Event for _, stage := range skeys { parsers := state[tstamp][stage] @@ -431,18 +472,16 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum fmt.Printf("\t%s %s\n", sep, stage) - pkeys := make([]string, 0, len(parsers)) - for k := range parsers { - pkeys = append(pkeys, k) - } - sort.Strings(pkeys) + pkeys := sortedMapKeys(parsers) for idx, parser := range pkeys { res := parsers[parser].Success sep := "├" + if idx == len(pkeys)-1 { sep = "└" } + created := 0 updated := 0 deleted := 0 @@ -451,16 +490,19 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum detailsDisplay := "" if res { - changelog, _ := diff.Diff(prev_item, parsers[parser].Evt) + changelog, _ := diff.Diff(prevItem, parsers[parser].Evt) for _, change := range changelog { switch change.Type { case "create": created++ + detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To)) case "update": detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To)) + if change.Path[0] == "Whitelisted" && change.To == true { whitelisted = true + if whitelistReason == "" { whitelistReason = parsers[parser].Evt.WhitelistReason } @@ -468,51 +510,64 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum updated++ case "delete": deleted++ + detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, "."))) } } - prev_item = parsers[parser].Evt + + prevItem = parsers[parser].Evt } if created > 0 { changeStr += green(fmt.Sprintf("+%d", created)) } + if updated > 0 { if len(changeStr) > 0 { changeStr += " " } + changeStr += yellow(fmt.Sprintf("~%d", updated)) } + if deleted > 0 { if len(changeStr) > 0 { changeStr += " " } + changeStr += red(fmt.Sprintf("-%d", deleted)) } + if whitelisted { if len(changeStr) > 0 { changeStr += " " } + changeStr += red("[whitelisted]") } + if changeStr == "" { changeStr = yellow("unchanged") } + if res { fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr) + if opts.Details { fmt.Print(detailsDisplay) } } else if opts.ShowNotOkParsers { fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser) - } } } + sep := "└" + if len(state[tstamp]["buckets"]) > 0 { sep = "├" } + //did the event enter the bucket pour phase ? if _, ok := state[tstamp]["buckets"]["OK"]; ok { fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle) @@ -521,27 +576,35 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum } else { fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle) } + //now print bucket info if len(state[tstamp]["buckets"]) > 0 { fmt.Printf("\t├ Scenarios\n") } + bnames := make([]string, 0, len(state[tstamp]["buckets"])) + for k := range state[tstamp]["buckets"] { //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered if k == "OK" { continue } + bnames = append(bnames, k) } + sort.Strings(bnames) + for idx, bname := range bnames { sep := "├" if idx == len(bnames)-1 { sep = "└" } + fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname) } + fmt.Println() } } diff --git a/pkg/hubtest/regexp.go b/pkg/hubtest/regexp.go new file mode 100644 index 000000000..f9165eae3 --- /dev/null +++ b/pkg/hubtest/regexp.go @@ -0,0 +1,11 @@ +package hubtest + +import ( + "regexp" +) + +var ( + variableRE = regexp.MustCompile(`(?P[^ =]+) == .*`) + parserResultRE = regexp.MustCompile(`^results\["[^"]+"\]\["(?P[^"]+)"\]\[[0-9]+\]\.Evt\..*`) + scenarioResultRE = regexp.MustCompile(`^results\[[0-9]+\].Overflow.Alert.GetScenario\(\) == "(?P[^"]+)"`) +) diff --git a/pkg/hubtest/scenario_assert.go b/pkg/hubtest/scenario_assert.go index f5517c350..011d3dcfb 100644 --- a/pkg/hubtest/scenario_assert.go +++ b/pkg/hubtest/scenario_assert.go @@ -5,12 +5,10 @@ import ( "fmt" "io" "os" - "regexp" "sort" "strings" "github.com/antonmedv/expr" - "github.com/antonmedv/expr/vm" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" @@ -42,6 +40,7 @@ func NewScenarioAssert(file string) *ScenarioAssert { TestData: &BucketResults{}, PourData: &BucketPourInfo{}, } + return ScenarioAssert } @@ -50,7 +49,9 @@ func (s *ScenarioAssert) AutoGenFromFile(filename string) (string, error) { if err != nil { return "", err } + ret := s.AutoGenScenarioAssert() + return ret, nil } @@ -59,6 +60,7 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error { if err != nil { return fmt.Errorf("loading scenario dump file '%s': %+v", filename, err) } + s.TestData = bucketDump if bucketpour != "" { @@ -66,8 +68,10 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error { if err != nil { return fmt.Errorf("loading bucket pour dump file '%s': %+v", filename, err) } + s.PourData = pourDump } + return nil } @@ -81,19 +85,26 @@ func (s *ScenarioAssert) AssertFile(testFile string) error { if err := s.LoadTest(testFile, ""); err != nil { return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err) } + scanner := bufio.NewScanner(file) scanner.Split(bufio.ScanLines) + nbLine := 0 + for scanner.Scan() { - nbLine += 1 + nbLine++ + if scanner.Text() == "" { continue } + ok, err := s.Run(scanner.Text()) if err != nil { return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err) } - s.NbAssert += 1 + + s.NbAssert++ + if !ok { log.Debugf("%s is FALSE", scanner.Text()) failedAssert := &AssertFail{ @@ -102,31 +113,38 @@ func (s *ScenarioAssert) AssertFile(testFile string) error { Expression: scanner.Text(), Debug: make(map[string]string), } - variableRE := regexp.MustCompile(`(?P[^ ]+) == .*`) + match := variableRE.FindStringSubmatch(scanner.Text()) + if len(match) == 0 { log.Infof("Couldn't get variable of line '%s'", scanner.Text()) continue } + variable := match[1] + result, err := s.EvalExpression(variable) if err != nil { log.Errorf("unable to evaluate variable '%s': %s", variable, err) continue } + failedAssert.Debug[variable] = result s.Fails = append(s.Fails, *failedAssert) + continue } //fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text()) - } + file.Close() + if s.NbAssert == 0 { assertData, err := s.AutoGenFromFile(testFile) if err != nil { return fmt.Errorf("couldn't generate assertion: %s", err) } + s.AutoGenAssertData = assertData s.AutoGenAssert = true } @@ -139,15 +157,14 @@ func (s *ScenarioAssert) AssertFile(testFile string) error { } func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { - var err error //debug doesn't make much sense with the ability to evaluate "on the fly" //var debugFilter *exprhelpers.ExprDebugger - var runtimeFilter *vm.Program var output interface{} env := map[string]interface{}{"results": *s.TestData} - if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil { + runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...) + if err != nil { return nil, err } // if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil { @@ -161,8 +178,10 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { if err != nil { log.Warningf("running : %s", expression) log.Warningf("runtime error : %s", err) + return nil, fmt.Errorf("while running expression %s: %w", expression, err) } + return output, nil } @@ -171,10 +190,12 @@ func (s *ScenarioAssert) EvalExpression(expression string) (string, error) { if err != nil { return "", err } + ret, err := yaml.Marshal(output) if err != nil { return "", err } + return string(ret), nil } @@ -183,6 +204,7 @@ func (s *ScenarioAssert) Run(assert string) (bool, error) { if err != nil { return false, err } + switch out := output.(type) { case bool: return out, nil @@ -192,9 +214,9 @@ func (s *ScenarioAssert) Run(assert string) (bool, error) { } func (s *ScenarioAssert) AutoGenScenarioAssert() string { - //attempt to autogen parser asserts - var ret string - ret += fmt.Sprintf(`len(results) == %d`+"\n", len(*s.TestData)) + // attempt to autogen scenario asserts + ret := fmt.Sprintf(`len(results) == %d`+"\n", len(*s.TestData)) + for eventIndex, event := range *s.TestData { for ipSrc, source := range event.Overflow.Sources { ret += fmt.Sprintf(`"%s" in results[%d].Overflow.GetSources()`+"\n", ipSrc, eventIndex) @@ -203,15 +225,18 @@ func (s *ScenarioAssert) AutoGenScenarioAssert() string { ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetScope() == "%s"`+"\n", eventIndex, ipSrc, *source.Scope) ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetValue() == "%s"`+"\n", eventIndex, ipSrc, *source.Value) } + for evtIndex, evt := range event.Overflow.Alert.Events { for _, meta := range evt.Meta { ret += fmt.Sprintf(`results[%d].Overflow.Alert.Events[%d].GetMeta("%s") == "%s"`+"\n", eventIndex, evtIndex, meta.Key, Escape(meta.Value)) } } + ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetScenario() == "%s"`+"\n", eventIndex, *event.Overflow.Alert.Scenario) ret += fmt.Sprintf(`results[%d].Overflow.Alert.Remediation == %t`+"\n", eventIndex, event.Overflow.Alert.Remediation) ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetEventsCount() == %d`+"\n", eventIndex, *event.Overflow.Alert.EventsCount) } + return ret } @@ -228,8 +253,6 @@ func (b BucketResults) Swap(i, j int) { } func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) { - var bucketDump BucketPourInfo - dumpData, err := os.Open(filepath) if err != nil { return nil, err @@ -241,6 +264,8 @@ func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) { return nil, err } + var bucketDump BucketPourInfo + if err := yaml.Unmarshal(results, &bucketDump); err != nil { return nil, err } @@ -249,8 +274,6 @@ func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) { } func LoadScenarioDump(filepath string) (*BucketResults, error) { - var bucketDump BucketResults - dumpData, err := os.Open(filepath) if err != nil { return nil, err @@ -262,6 +285,8 @@ func LoadScenarioDump(filepath string) (*BucketResults, error) { return nil, err } + var bucketDump BucketResults + if err := yaml.Unmarshal(results, &bucketDump); err != nil { return nil, err } diff --git a/pkg/hubtest/utils.go b/pkg/hubtest/utils.go index 489b324c4..090f1f85e 100644 --- a/pkg/hubtest/utils.go +++ b/pkg/hubtest/utils.go @@ -12,7 +12,9 @@ func sortedMapKeys[V any](m map[string]V) []string { for k := range m { keys = append(keys, k) } + sort.Strings(keys) + return keys } @@ -43,16 +45,20 @@ func checkPathNotContained(path string, subpath string) error { } current := absSubPath + for { if current == absPath { return fmt.Errorf("cannot copy a folder onto itself") } + up := filepath.Dir(current) if current == up { break } + current = up } + return nil } diff --git a/pkg/hubtest/utils_test.go b/pkg/hubtest/utils_test.go index de4f1aac3..ce86785af 100644 --- a/pkg/hubtest/utils_test.go +++ b/pkg/hubtest/utils_test.go @@ -3,16 +3,16 @@ package hubtest import ( "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestCheckPathNotContained(t *testing.T) { - assert.Nil(t, checkPathNotContained("/foo", "/bar")) - assert.Nil(t, checkPathNotContained("/foo/bar", "/foo")) - assert.Nil(t, checkPathNotContained("/foo/bar", "/")) - assert.Nil(t, checkPathNotContained("/path/to/somewhere", "/path/to/somewhere-else")) - assert.Nil(t, checkPathNotContained("~/.local/path/to/somewhere", "~/.local/path/to/somewhere-else")) - assert.NotNil(t, checkPathNotContained("/foo", "/foo/bar")) - assert.NotNil(t, checkPathNotContained("/", "/foo")) - assert.NotNil(t, checkPathNotContained("/", "/foo/bar/baz")) + require.NoError(t, checkPathNotContained("/foo", "/bar")) + require.NoError(t, checkPathNotContained("/foo/bar", "/foo")) + require.NoError(t, checkPathNotContained("/foo/bar", "/")) + require.NoError(t, checkPathNotContained("/path/to/somewhere", "/path/to/somewhere-else")) + require.NoError(t, checkPathNotContained("~/.local/path/to/somewhere", "~/.local/path/to/somewhere-else")) + require.Error(t, checkPathNotContained("/foo", "/foo/bar")) + require.Error(t, checkPathNotContained("/", "/foo")) + require.Error(t, checkPathNotContained("/", "/foo/bar/baz")) }