hubtests revamp + cscli explain (#988)

 * New hubtest CI for scenarios/parsers from the hub
 * New `cscli explain` command to visualize parsers/scenarios pipeline

Co-authored-by: alteredCoder <kevin@crowdsec.net>
Co-authored-by: Sebastien Blot <sebastien@crowdsec.net>
Co-authored-by: he2ss <hamza.essahely@gmail.com>
Co-authored-by: Cristian Nitescu <cristian@crowdsec.net>

.github/workflows/ci_hubtest.yml

@@ -0,0 +1,74 @@
+name: Hub Tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+    name: Hub tests
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go 1.16
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.16
+      id: go
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+    - id: keydb
+      uses: pozetroninc/github-action-get-latest-release@master
+      with:
+        owner: crowdsecurity
+        repo: crowdsec
+        excludes: draft
+    - name: Build release
+      run: BUILD_VERSION=${{ steps.keydb.outputs.release }} make release
+    - name: "Force machineid"
+      run: |
+          sudo chmod +w /etc/machine-id
+          echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
+    - name: Install release
+      run: |
+        cd crowdsec-${{ steps.keydb.outputs.release }}
+        sudo ./wizard.sh --unattended
+    - name: "Clone CrowdSec Hub"
+      run: |
+          git clone https://github.com/crowdsecurity/hub.git
+    - name: "Run tests"
+      run: |
+          cd hub/
+          git checkout hub_tests
+          cscli hubtest run --all --clean
+          echo "PARSERS_COV=$(cscli hubtest coverage --parsers --percent | cut -d '=' -f2)" >> $GITHUB_ENV
+          echo "SCENARIOS_COV=$(cscli hubtest coverage --scenarios --percent | cut -d '=' -f2)" >> $GITHUB_ENV
+          echo "PARSERS_COV_NUMBER=$(cscli hubtest coverage --parsers --percent | cut -d '=' -f2 | tr -d '%')"
+          echo "SCENARIOS_COV_NUMBER=$(cscli hubtest coverage --scenarios --percent | cut -d '=' -f2 | tr -d '%')"
+          echo "PARSER_BADGE_COLOR=$(if [ $PARSERS_COV_NUMBER -lt '70' ]; then echo 'red'; else echo 'green'; fi)" >> $GITHUB_ENV
+          echo "SCENARIO_BADGE_COLOR=$(if [ $SCENARIOS_COV_NUMBER -lt '70' ]; then echo 'red'; else echo 'green'; fi)" >> $GITHUB_ENV
+    - name: Create Parsers badge
+      if: github.ref == 'ref/head/master'
+      uses: schneegans/dynamic-badges-action@v1.1.0
+      with:
+        auth: ${{ secrets.GIST_BADGES_SECRET }}
+        gistID: ${{ secrets.GIST_BADGES_ID }}
+        filename: crowdsec_parsers_badge.json
+        label: Hub Parsers
+        message: ${{ env.PARSERS_COV }}
+        color: ${{ env.SCENARIO_BADGE_COLOR }}
+    - name: Create Scenarios badge
+      if: github.ref == 'ref/head/master'
+      uses: schneegans/dynamic-badges-action@v1.1.0
+      with:
+        auth: ${{ secrets.GIST_BADGES_SECRET }}
+        gistID: ${{ secrets.GIST_BADGES_ID }}
+        filename: crowdsec_scenarios_badge.json
+        label: Hub Scenarios
+        message: ${{ env.SCENARIOS_COV }}
+        color: ${{ env.SCENARIO_BADGE_COLOR }}
+
+

.github/workflows/dispatch_create_tag_hub-tests.yaml

@@ -1,24 +0,0 @@
-name: Dispatch to hub-tests when creating pre-release
-
-on: 
-  release:
-    types: prereleased
-
-jobs:
-  dispatch:
-    name: dispatch to hub-tests
-    runs-on: ubuntu-latest
-    steps:
-      - id: keydb
-        uses: pozetroninc/github-action-get-latest-release@master
-        with:
-          owner: crowdsecurity
-          repo: crowdsec
-          excludes: prerelease, draft
-      - name: Repository Dispatch
-        uses: peter-evans/repository-dispatch@v1
-        with:
-          token: ${{ secrets.DISPATCH_TOKEN }}
-          event-type: create_tag
-          repository: crowdsecurity/hub-tests
-          client-payload: '{"version": "${{ steps.keydb.outputs.release }}"}'

README.md

@@ -11,7 +11,8 @@
 <a href='https://coveralls.io/github/crowdsecurity/crowdsec?branch=master'><img src='https://coveralls.io/repos/github/crowdsecurity/crowdsec/badge.svg?branch=master' alt='Coverage Status' /></a>
 <img src="https://goreportcard.com/badge/github.com/crowdsecurity/crowdsec">
 <img src="https://img.shields.io/github/license/crowdsecurity/crowdsec">
-<img src="https://github.com/crowdsecurity/crowdsec/workflows/Hub-CI/badge.svg">
+<img src="https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/AlteredCoder/ed74e50c43e3b17bdfc4d93149f23d37/raw/a473458a57da096789e79c4538b432ceeac2d853/crowdsec_parsers_badge.json">
+<img src="https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/AlteredCoder/ed74e50c43e3b17bdfc4d93149f23d37/raw/a473458a57da096789e79c4538b432ceeac2d853/crowdsec_scenarios_badge.json">
 </p>
 
 <p align="center">

cmd/crowdsec-cli/explain.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cstest"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func NewExplainCmd() *cobra.Command {
+	/* ---- HUB COMMAND */
+	var logFile string
+	var dsn string
+	var logLine string
+	var logType string
+
+	var cmdExplain = &cobra.Command{
+		Use:   "explain",
+		Short: "Explain log pipeline",
+		Long: `
+Explain log pipeline 
+		`,
+		Example: `
+cscli explain --file ./myfile.log --type nginx 
+cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog
+cscli explain -dsn "file://myfile.log" --type nginx
+		`,
+		Args:              cobra.ExactArgs(0),
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+
+			if logType == "" || (logLine == "" && logFile == "" && dsn == "") {
+				cmd.Help()
+				fmt.Println()
+				fmt.Printf("Please provide --type flag\n")
+				os.Exit(1)
+			}
+
+			// we create a temporary log file if a log line has been provided
+			if logLine != "" {
+				logFile = "./cscli_test_tmp.log"
+				f, err := os.Create(logFile)
+				if err != nil {
+					log.Fatal(err)
+				}
+				defer f.Close()
+
+				_, err = f.WriteString(logLine)
+
+				if err != nil {
+					log.Fatal(err)
+				}
+			}
+
+			if logFile != "" {
+				absolutePath, err := filepath.Abs(logFile)
+				if err != nil {
+					log.Fatalf("unable to get absolue path of '%s', exiting", logFile)
+				}
+				dsn = fmt.Sprintf("file://%s", absolutePath)
+			}
+
+			if dsn == "" {
+				log.Fatal("no acquisition (--file or --dsn) provided, can't run cscli test.")
+			}
+
+			cmdArgs := []string{"-c", ConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", "./", "-no-api"}
+			crowdsecCmd := exec.Command("crowdsec", cmdArgs...)
+			output, err := crowdsecCmd.CombinedOutput()
+			if err != nil {
+				fmt.Println(string(output))
+				log.Fatalf("fail to run crowdsec for test: %v", err)
+			}
+
+			// rm the temporary log file if only a log line was provided
+			if logLine != "" {
+				if err := os.Remove(logFile); err != nil {
+					log.Fatalf("unable to remove tmp log file '%s': %+v", logFile, err)
+				}
+			}
+			parserDumpFile := filepath.Join("./", cstest.ParserResultFileName)
+			bucketStateDumpFile := filepath.Join("./", cstest.BucketPourResultFileName)
+
+			parserDump, err := cstest.LoadParserDump(parserDumpFile)
+			if err != nil {
+				log.Fatalf("unable to load parser dump result: %s", err)
+			}
+
+			bucketStateDump, err := cstest.LoadBucketPourDump(bucketStateDumpFile)
+			if err != nil {
+				log.Fatalf("unable to load bucket dump result: %s", err)
+			}
+
+			if err := cstest.DumpTree(*parserDump, *bucketStateDump); err != nil {
+				log.Fatalf(err.Error())
+			}
+		},
+	}
+	cmdExplain.PersistentFlags().StringVarP(&logFile, "file", "f", "", "Log file to test")
+	cmdExplain.PersistentFlags().StringVarP(&dsn, "dsn", "d", "", "DSN to test")
+	cmdExplain.PersistentFlags().StringVarP(&logLine, "log", "l", "", "Lgg line to test")
+	cmdExplain.PersistentFlags().StringVarP(&logType, "type", "t", "", "Type of the acquisition to test")
+
+	return cmdExplain
+}

cmd/crowdsec-cli/hubtest.go

@@ -0,0 +1,585 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/crowdsecurity/crowdsec/pkg/cstest"
+	"github.com/enescakir/emoji"
+	"github.com/olekukonko/tablewriter"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
+)
+
+var (
+	HubTest cstest.HubTest
+)
+
+func NewHubTestCmd() *cobra.Command {
+	/* ---- HUB COMMAND */
+	var hubPath string
+	var logType string
+	var crowdsecPath string
+	var cscliPath string
+
+	var cmdHubTest = &cobra.Command{
+		Use:   "hubtest",
+		Short: "Run fonctionnals tests on hub configurations",
+		Long: `
+		Run fonctionnals tests on hub configurations (parsers, scenarios, collections...)
+		`,
+		Args:              cobra.ExactArgs(0),
+		DisableAutoGenTag: true,
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			var err error
+			HubTest, err = cstest.NewHubTest(hubPath, crowdsecPath, cscliPath)
+			if err != nil {
+				log.Fatalf("unable to load hubtest: %+v", err)
+			}
+		},
+	}
+	cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
+	cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
+	cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
+
+	parsers := []string{}
+	postoverflows := []string{}
+	scenarios := []string{}
+	var ignoreParsers bool
+
+	var cmdHubTestCreate = &cobra.Command{
+		Use:   "create",
+		Short: "create [test_name]",
+		Example: `cscli hubtest create my-awesome-test --type syslog
+cscli hubtest create my-nginx-custom-test --type nginx
+cscli hubtest create my-scenario-test --parser crowdsecurity/nginx --scenario crowdsecurity/http-probing`,
+		Args:              cobra.ExactArgs(1),
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			testName := args[0]
+			testPath := filepath.Join(HubTest.HubTestPath, testName)
+			if _, err := os.Stat(testPath); os.IsExist(err) {
+				log.Fatalf("test '%s' already exists in '%s', exiting", testName, testPath)
+			}
+
+			if logType == "" {
+				log.Fatalf("please provid a type (--type) for the test")
+			}
+
+			if err := os.MkdirAll(testPath, os.ModePerm); err != nil {
+				log.Fatalf("unable to create folder '%s': %+v", testPath, err)
+			}
+
+			// create empty log file
+			logFileName := fmt.Sprintf("%s.log", testName)
+			logFilePath := filepath.Join(testPath, logFileName)
+			logFile, err := os.Create(logFilePath)
+			if err != nil {
+				log.Fatal(err)
+			}
+			logFile.Close()
+
+			// create empty parser assertion file
+			parserAssertFilePath := filepath.Join(testPath, cstest.ParserAssertFileName)
+			parserAssertFile, err := os.Create(parserAssertFilePath)
+			if err != nil {
+				log.Fatal(err)
+			}
+			parserAssertFile.Close()
+
+			// create empty scenario assertion file
+			scenarioAssertFilePath := filepath.Join(testPath, cstest.ScenarioAssertFileName)
+			scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
+			if err != nil {
+				log.Fatal(err)
+			}
+			scenarioAssertFile.Close()
+
+			parsers = append(parsers, "crowdsecurity/syslog-logs")
+			parsers = append(parsers, "crowdsecurity/dateparse-enrich")
+
+			if len(scenarios) == 0 {
+				scenarios = append(scenarios, "")
+			}
+
+			if len(postoverflows) == 0 {
+				postoverflows = append(postoverflows, "")
+			}
+
+			configFileData := &cstest.HubTestItemConfig{
+				Parsers:       parsers,
+				Scenarios:     scenarios,
+				PostOVerflows: postoverflows,
+				LogFile:       logFileName,
+				LogType:       logType,
+				IgnoreParsers: ignoreParsers,
+			}
+
+			configFilePath := filepath.Join(testPath, "config.yaml")
+			fd, err := os.OpenFile(configFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
+			if err != nil {
+				log.Fatalf("open: %s", err)
+			}
+			data, err := yaml.Marshal(configFileData)
+			if err != nil {
+				log.Fatalf("marshal: %s", err)
+			}
+			_, err = fd.Write(data)
+			if err != nil {
+				log.Fatalf("write: %s", err)
+			}
+			if err := fd.Close(); err != nil {
+				log.Fatalf(" close: %s", err)
+			}
+			fmt.Println()
+			fmt.Printf("  Test name                   :  %s\n", testName)
+			fmt.Printf("  Test path                   :  %s\n", testPath)
+			fmt.Printf("  Log file                    :  %s (please fill it with logs)\n", logFilePath)
+			fmt.Printf("  Parser assertion file       :  %s (please fill it with assertion)\n", parserAssertFilePath)
+			fmt.Printf("  Scenario assertion file     :  %s (please fill it with assertion)\n", scenarioAssertFilePath)
+			fmt.Printf("  Configuration File          :  %s (please fill it with parsers, scenarios...)\n", configFilePath)
+
+		},
+	}
+	cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
+	cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
+	cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
+	cmdHubTestCreate.Flags().StringSliceVarP(&scenarios, "scenarios", "s", scenarios, "Scenarios to add to test")
+	cmdHubTestCreate.PersistentFlags().BoolVar(&ignoreParsers, "ignore-parsers", false, "Don't run test on parsers")
+	cmdHubTest.AddCommand(cmdHubTestCreate)
+
+	var noClean bool
+	var runAll bool
+	var forceClean bool
+	var cmdHubTestRun = &cobra.Command{
+		Use:               "run",
+		Short:             "run [test_name]",
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			if !runAll && len(args) == 0 {
+				cmd.Help()
+				fmt.Println("Please provide test to run or --all flag")
+				os.Exit(1)
+			}
+
+			if runAll {
+				if err := HubTest.LoadAllTests(); err != nil {
+					log.Fatalf("unable to load all tests: %+v", err)
+				}
+			} else {
+				for _, testName := range args {
+					_, err := HubTest.LoadTestItem(testName)
+					if err != nil {
+						log.Fatalf("unable to load test '%s': %s", testName, err)
+					}
+				}
+			}
+
+			for _, test := range HubTest.Tests {
+				if csConfig.Cscli.Output == "human" {
+					log.Infof("Running test '%s'", test.Name)
+				}
+				err := test.Run()
+				if err != nil {
+					log.Errorf("running test '%s' failed: %+v", test.Name, err)
+				}
+			}
+
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			success := true
+			testResult := make(map[string]bool)
+			for _, test := range HubTest.Tests {
+				if test.AutoGen {
+					if test.ParserAssert.AutoGenAssert {
+						log.Warningf("Assert file '%s' is empty, generating assertion:", test.ParserAssert.File)
+						fmt.Println()
+						fmt.Println(test.ParserAssert.AutoGenAssertData)
+					}
+					if test.ScenarioAssert.AutoGenAssert {
+						log.Warningf("Assert file '%s' is empty, generating assertion:", test.ScenarioAssert.File)
+						fmt.Println()
+						fmt.Println(test.ScenarioAssert.AutoGenAssertData)
+					}
+					if !noClean {
+						if err := test.Clean(); err != nil {
+							log.Fatalf("unable to clean test '%s' env: %s", test.Name, err)
+						}
+					}
+					fmt.Printf("\nPlease fill your assert file(s) for test '%s', exiting\n", test.Name)
+					os.Exit(1)
+				}
+				testResult[test.Name] = test.Success
+				if test.Success {
+					if csConfig.Cscli.Output == "human" {
+						log.Infof("Test '%s' passed successfully (%d assertions)\n", test.Name, test.ParserAssert.NbAssert+test.ScenarioAssert.NbAssert)
+					}
+					if !noClean {
+						if err := test.Clean(); err != nil {
+							log.Fatalf("unable to clean test '%s' env: %s", test.Name, err)
+						}
+					}
+				} else {
+					success = false
+					cleanTestEnv := false
+					if csConfig.Cscli.Output == "human" {
+						if len(test.ParserAssert.Fails) > 0 {
+							fmt.Println()
+							log.Errorf("Parser test '%s' failed (%d errors)\n", test.Name, len(test.ParserAssert.Fails))
+							for _, fail := range test.ParserAssert.Fails {
+								fmt.Printf("(L.%d)  %s  => %s\n", fail.Line, emoji.RedCircle, fail.Expression)
+								fmt.Printf("        Actual expression values:\n")
+								for key, value := range fail.Debug {
+									fmt.Printf("            %s = '%s'\n", key, strings.TrimSuffix(value, "\n"))
+								}
+								fmt.Println()
+							}
+						}
+						if len(test.ScenarioAssert.Fails) > 0 {
+							fmt.Println()
+							log.Errorf("Scenario test '%s' failed (%d errors)\n", test.Name, len(test.ScenarioAssert.Fails))
+							for _, fail := range test.ScenarioAssert.Fails {
+								fmt.Printf("(L.%d)  %s  => %s\n", fail.Line, emoji.RedCircle, fail.Expression)
+								fmt.Printf("        Actual expression values:\n")
+								for key, value := range fail.Debug {
+									fmt.Printf("            %s = '%s'\n", key, strings.TrimSuffix(value, "\n"))
+								}
+								fmt.Println()
+							}
+						}
+						if !forceClean {
+							prompt := &survey.Confirm{
+								Message: fmt.Sprintf("\nDo you want to remove runtime folder for test '%s'? (default: Yes)", test.Name),
+								Default: true,
+							}
+							if err := survey.AskOne(prompt, &cleanTestEnv); err != nil {
+								log.Fatalf("unable to ask to remove runtime folder: %s", err)
+							}
+						}
+					}
+
+					if cleanTestEnv || forceClean {
+						if err := test.Clean(); err != nil {
+							log.Fatalf("unable to clean test '%s' env: %s", test.Name, err)
+						}
+					}
+				}
+			}
+			if csConfig.Cscli.Output == "human" {
+				table := tablewriter.NewWriter(os.Stdout)
+				table.SetCenterSeparator("")
+				table.SetColumnSeparator("")
+
+				table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
+				table.SetAlignment(tablewriter.ALIGN_LEFT)
+
+				table.SetHeader([]string{"Test", "Result"})
+				for testName, success := range testResult {
+					status := emoji.CheckMarkButton.String()
+					if !success {
+						status = emoji.CrossMark.String()
+					}
+					table.Append([]string{testName, status})
+				}
+				table.Render()
+			} else if csConfig.Cscli.Output == "json" {
+				jsonResult := make(map[string][]string, 0)
+				jsonResult["success"] = make([]string, 0)
+				jsonResult["fail"] = make([]string, 0)
+				for testName, success := range testResult {
+					if success {
+						jsonResult["success"] = append(jsonResult["success"], testName)
+					} else {
+						jsonResult["fail"] = append(jsonResult["fail"], testName)
+					}
+				}
+				jsonStr, err := json.Marshal(jsonResult)
+				if err != nil {
+					log.Fatalf("unable to json test result: %s", err.Error())
+				}
+				fmt.Println(string(jsonStr))
+			}
+
+			if !success {
+				os.Exit(1)
+			}
+		},
+	}
+	cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
+	cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
+	cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests")
+	cmdHubTest.AddCommand(cmdHubTestRun)
+
+	var cmdHubTestClean = &cobra.Command{
+		Use:               "clean",
+		Short:             "clean [test_name]",
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			for _, testName := range args {
+				test, err := HubTest.LoadTestItem(testName)
+				if err != nil {
+					log.Fatalf("unable to load test '%s': %s", testName, err)
+				}
+				if err := test.Clean(); err != nil {
+					log.Fatalf("unable to clean test '%s' env: %s", test.Name, err)
+				}
+			}
+		},
+	}
+	cmdHubTest.AddCommand(cmdHubTestClean)
+
+	var cmdHubTestInfo = &cobra.Command{
+		Use:               "info",
+		Short:             "info [test_name]",
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			for _, testName := range args {
+				test, err := HubTest.LoadTestItem(testName)
+				if err != nil {
+					log.Fatalf("unable to load test '%s': %s", testName, err)
+				}
+				fmt.Println()
+				fmt.Printf("  Test name                   :  %s\n", test.Name)
+				fmt.Printf("  Test path                   :  %s\n", test.Path)
+				fmt.Printf("  Log file                    :  %s\n", filepath.Join(test.Path, test.Config.LogFile))
+				fmt.Printf("  Parser assertion file       :  %s\n", filepath.Join(test.Path, cstest.ParserAssertFileName))
+				fmt.Printf("  Scenario assertion file     :  %s\n", filepath.Join(test.Path, cstest.ScenarioAssertFileName))
+				fmt.Printf("  Configuration File          :  %s\n", filepath.Join(test.Path, "config.yaml"))
+			}
+		},
+	}
+	cmdHubTest.AddCommand(cmdHubTestInfo)
+
+	var cmdHubTestList = &cobra.Command{
+		Use:               "list",
+		Short:             "list",
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := HubTest.LoadAllTests(); err != nil {
+				log.Fatalf("unable to load all tests: %+v", err)
+			}
+
+			table := tablewriter.NewWriter(os.Stdout)
+			table.SetCenterSeparator("")
+			table.SetColumnSeparator("")
+
+			table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
+			table.SetAlignment(tablewriter.ALIGN_LEFT)
+			table.SetHeader([]string{"Name", "Path"})
+			for _, test := range HubTest.Tests {
+				table.Append([]string{test.Name, test.Path})
+			}
+			table.Render()
+
+		},
+	}
+	cmdHubTest.AddCommand(cmdHubTestList)
+
+	var showParserCov bool
+	var showScenarioCov bool
+	var showOnlyPercent bool
+	var cmdHubTestCoverage = &cobra.Command{
+		Use:               "coverage",
+		Short:             "coverage",
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := HubTest.LoadAllTests(); err != nil {
+				log.Fatalf("unable to load all tests: %+v", err)
+			}
+			var err error
+			scenarioCoverage := []cstest.ScenarioCoverage{}
+			parserCoverage := []cstest.ParserCoverage{}
+			scenarioCoveragePercent := 0
+			parserCoveragePercent := 0
+			showAll := false
+
+			if !showScenarioCov && !showParserCov { // if both are false (flag by default), show both
+				showAll = true
+			}
+
+			if showParserCov || showAll {
+				parserCoverage, err = HubTest.GetParsersCoverage()
+				if err != nil {
+					log.Fatalf("while getting parser coverage : %s", err)
+				}
+				parserTested := 0
+				for _, test := range parserCoverage {
+					if test.TestsCount > 0 {
+						parserTested += 1
+					}
+				}
+				parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100)))
+			}
+
+			if showScenarioCov || showAll {
+				scenarioCoverage, err = HubTest.GetScenariosCoverage()
+				if err != nil {
+					log.Fatalf("while getting scenario coverage: %s", err)
+				}
+				scenarioTested := 0
+				for _, test := range scenarioCoverage {
+					if test.TestsCount > 0 {
+						scenarioTested += 1
+					}
+				}
+				scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100)))
+			}
+
+			if showOnlyPercent {
+				if showAll {
+					fmt.Printf("parsers=%d%%\nscenarios=%d%%", parserCoveragePercent, scenarioCoveragePercent)
+				} else if showParserCov {
+					fmt.Printf("parsers=%d%%", parserCoveragePercent)
+				} else if showScenarioCov {
+					fmt.Printf("scenarios=%d%%", scenarioCoveragePercent)
+				}
+				os.Exit(0)
+			}
+
+			if csConfig.Cscli.Output == "human" {
+				if showParserCov || showAll {
+					table := tablewriter.NewWriter(os.Stdout)
+					table.SetCenterSeparator("")
+					table.SetColumnSeparator("")
+
+					table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
+					table.SetAlignment(tablewriter.ALIGN_LEFT)
+
+					table.SetHeader([]string{"Parser", "Status", "Number of tests"})
+					parserTested := 0
+					for _, test := range parserCoverage {
+						status := emoji.RedCircle.String()
+						if test.TestsCount > 0 {
+							status = emoji.GreenCircle.String()
+							parserTested += 1
+						}
+						table.Append([]string{test.Parser, status, fmt.Sprintf("%d times (accross %d tests)", test.TestsCount, len(test.PresentIn))})
+					}
+					table.Render()
+				}
+
+				if showScenarioCov || showAll {
+					table := tablewriter.NewWriter(os.Stdout)
+					table.SetCenterSeparator("")
+					table.SetColumnSeparator("")
+
+					table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
+					table.SetAlignment(tablewriter.ALIGN_LEFT)
+
+					table.SetHeader([]string{"Scenario", "Status", "Number of tests"})
+					for _, test := range scenarioCoverage {
+						status := emoji.RedCircle.String()
+						if test.TestsCount > 0 {
+							status = emoji.GreenCircle.String()
+						}
+						table.Append([]string{test.Scenario, status, fmt.Sprintf("%d times (accross %d tests)", test.TestsCount, len(test.PresentIn))})
+					}
+					table.Render()
+				}
+				fmt.Println()
+				if showParserCov || showAll {
+					fmt.Printf("PARSERS    : %d%% of coverage\n", parserCoveragePercent)
+				}
+				if showScenarioCov || showAll {
+					fmt.Printf("SCENARIOS  : %d%% of coverage\n", scenarioCoveragePercent)
+				}
+			} else if csConfig.Cscli.Output == "json" {
+				dump, err := json.MarshalIndent(parserCoverage, "", " ")
+				if err != nil {
+					log.Fatal(err)
+				}
+				fmt.Printf("%s", dump)
+				dump, err = json.MarshalIndent(scenarioCoverage, "", " ")
+				if err != nil {
+					log.Fatal(err)
+				}
+				fmt.Printf("%s", dump)
+			} else {
+				log.Fatalf("only human/json output modes are supported")
+			}
+
+		},
+	}
+	cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
+	cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
+	cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
+	cmdHubTest.AddCommand(cmdHubTestCoverage)
+
+	var evalExpression string
+	var cmdHubTestEval = &cobra.Command{
+		Use:               "eval",
+		Short:             "eval [test_name]",
+		Args:              cobra.ExactArgs(1),
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			for _, testName := range args {
+				test, err := HubTest.LoadTestItem(testName)
+				if err != nil {
+					log.Fatalf("can't load test: %+v", err)
+				}
+				err = test.ParserAssert.LoadTest(test.ParserResultFile)
+				if err != nil {
+					log.Fatalf("can't load test results from '%s': %+v", test.ParserResultFile, err)
+				}
+				output, err := test.ParserAssert.EvalExpression(evalExpression)
+				if err != nil {
+					log.Fatalf(err.Error())
+				}
+				fmt.Printf(output)
+			}
+		},
+	}
+	cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
+	cmdHubTest.AddCommand(cmdHubTestEval)
+
+	var cmdHubTestExplain = &cobra.Command{
+		Use:               "explain",
+		Short:             "explain [test_name]",
+		Args:              cobra.ExactArgs(1),
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			for _, testName := range args {
+				test, err := HubTest.LoadTestItem(testName)
+				if err != nil {
+					log.Fatalf("can't load test: %+v", err)
+				}
+				err = test.ParserAssert.LoadTest(test.ParserResultFile)
+				if err != nil {
+					err := test.Run()
+					if err != nil {
+						log.Fatalf("running test '%s' failed: %+v", test.Name, err)
+					}
+					err = test.ParserAssert.LoadTest(test.ParserResultFile)
+					if err != nil {
+						log.Fatalf("unable to load parser result after run: %s", err)
+					}
+				}
+
+				err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
+				if err != nil {
+					err := test.Run()
+					if err != nil {
+						log.Fatalf("running test '%s' failed: %+v", test.Name, err)
+					}
+					err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
+					if err != nil {
+						log.Fatalf("unable to load scenario result after run: %s", err)
+					}
+				}
+
+				cstest.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData)
+			}
+		},
+	}
+	cmdHubTest.AddCommand(cmdHubTestExplain)
+
+	return cmdHubTest
+}

cmd/crowdsec-cli/machines.go

@@ -88,6 +88,9 @@ Note: This command requires database direct access, so is intended to be run on
 		DisableAutoGenTag: true,
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
+				if err != nil {
+					log.Errorf("local api : %s", err)
+				}
 				log.Fatal("Local API is disabled, please run this command on the local API machine")
 			}
 			if err := csConfig.LoadDBConfig(); err != nil {

cmd/crowdsec-cli/main.go

@@ -174,6 +174,8 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	rootCmd.AddCommand(NewLapiCmd())
 	rootCmd.AddCommand(NewCompletionCmd())
 	rootCmd.AddCommand(NewConsoleCmd())
+	rootCmd.AddCommand(NewExplainCmd())
+	rootCmd.AddCommand(NewHubTestCmd())
 	if err := rootCmd.Execute(); err != nil {
 		log.Fatalf("While executing root command : %s", err)
 	}

cmd/crowdsec/crowdsec.go

@@ -2,8 +2,11 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"sync"
 
+	"path/filepath"
+
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
@@ -13,6 +16,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
 )
 
 func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
@@ -149,10 +153,75 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config) {
 			log.Fatalf("unable to shutdown crowdsec routines: %s", err)
 		}
 		log.Debugf("everything is dead, return crowdsecTomb")
+		if dumpStates {
+			dumpParserState()
+			dumpOverflowState()
+			dumpBucketsPour()
+			os.Exit(0)
+		}
 		return nil
 	})
 }
 
+func dumpBucketsPour() {
+	fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "bucketpour-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
+	if err != nil {
+		log.Fatalf("open: %s", err)
+	}
+	out, err := yaml.Marshal(leaky.BucketPourCache)
+	if err != nil {
+		log.Fatalf("marshal: %s", err)
+	}
+	b, err := fd.Write(out)
+	if err != nil {
+		log.Fatalf("write: %s", err)
+	}
+	log.Tracef("wrote %d bytes", b)
+	if err := fd.Close(); err != nil {
+		log.Fatalf(" close: %s", err)
+	}
+}
+
+func dumpParserState() {
+
+	fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "parser-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
+	if err != nil {
+		log.Fatalf("open: %s", err)
+	}
+	out, err := yaml.Marshal(parser.StageParseCache)
+	if err != nil {
+		log.Fatalf("marshal: %s", err)
+	}
+	b, err := fd.Write(out)
+	if err != nil {
+		log.Fatalf("write: %s", err)
+	}
+	log.Tracef("wrote %d bytes", b)
+	if err := fd.Close(); err != nil {
+		log.Fatalf(" close: %s", err)
+	}
+}
+
+func dumpOverflowState() {
+
+	fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "bucket-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
+	if err != nil {
+		log.Fatalf("open: %s", err)
+	}
+	out, err := yaml.Marshal(bucketOverflows)
+	if err != nil {
+		log.Fatalf("marshal: %s", err)
+	}
+	b, err := fd.Write(out)
+	if err != nil {
+		log.Fatalf("write: %s", err)
+	}
+	log.Tracef("wrote %d bytes", b)
+	if err := fd.Close(); err != nil {
+		log.Fatalf(" close: %s", err)
+	}
+}
+
 func waitOnTomb() {
 	for {
 		select {

cmd/crowdsec/main.go

@@ -14,6 +14,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/csplugin"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"github.com/crowdsecurity/crowdsec/pkg/leakybucket"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
@@ -160,6 +161,9 @@ func LoadAcquisition(cConfig *csconfig.Config) error {
 	return nil
 }
 
+var dumpFolder string
+var dumpStates bool
+
 func (f *Flags) Parse() {
 
 	flag.StringVar(&f.ConfigFile, "c", "/etc/crowdsec/config.yaml", "configuration file")
@@ -172,6 +176,7 @@ func (f *Flags) Parse() {
 	flag.BoolVar(&f.TestMode, "t", false, "only test configs")
 	flag.BoolVar(&f.DisableAgent, "no-cs", false, "disable crowdsec agent")
 	flag.BoolVar(&f.DisableAPI, "no-api", false, "disable local API")
+	flag.StringVar(&dumpFolder, "dump-data", "", "dump parsers/buckets raw outputs")
 
 	flag.Parse()
 }
@@ -179,6 +184,13 @@ func (f *Flags) Parse() {
 // LoadConfig return configuration parsed from configuration file
 func LoadConfig(cConfig *csconfig.Config) error {
 
+	if dumpFolder != "" {
+		parser.ParseDump = true
+		parser.DumpFolder = dumpFolder
+		leakybucket.BucketPourTrack = true
+		dumpStates = true
+	}
+
 	if !flags.DisableAgent {
 		if err := cConfig.LoadCrowdsec(); err != nil {
 			return err

cmd/crowdsec/output.go

@@ -58,6 +58,8 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
 	return nil
 }
 
+var bucketOverflows []types.Event
+
 func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets,
 	postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, apiConfig csconfig.ApiCredentialsCfg) error {
 
@@ -129,7 +131,13 @@ LOOP:
 			}
 			break LOOP
 		case event := <-overflow:
-
+			//if the Alert is nil, it's to signal bucket is ready for GC, don't track this
+			if dumpStates && event.Overflow.Alert != nil {
+				if bucketOverflows == nil {
+					bucketOverflows = make([]types.Event, 0)
+				}
+				bucketOverflows = append(bucketOverflows, event)
+			}
 			/*if alert is empty and mapKey is present, the overflow is just to cleanup bucket*/
 			if event.Overflow.Alert == nil && event.Overflow.Mapkey != "" {
 				buckets.Bucket_map.Delete(event.Overflow.Mapkey)
@@ -149,10 +157,13 @@ LOOP:
 				log.Printf("[%s] is whitelisted, skip.", *event.Overflow.Alert.Message)
 				continue
 			}
+			if dumpStates {
+				continue
+			}
+
 			cacheMutex.Lock()
 			cache = append(cache, event.Overflow)
 			cacheMutex.Unlock()
-
 		}
 	}
 

pkg/acquisition/modules/file/file.go

@@ -399,6 +399,9 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom
 	}
 	scanner.Split(bufio.ScanLines)
 	for scanner.Scan() {
+		if scanner.Text() == "" {
+			continue
+		}
 		logger.Debugf("line %s", scanner.Text())
 		l := types.Line{}
 		l.Raw = scanner.Text()

pkg/csconfig/crowdsec_service.go

@@ -62,7 +62,7 @@ func (c *Config) LoadCrowdsec() error {
 		c.Crowdsec.AcquisitionFiles = append(c.Crowdsec.AcquisitionFiles, files...)
 	}
 	if c.Crowdsec.AcquisitionDirPath == "" && c.Crowdsec.AcquisitionFilePath == "" {
-		return fmt.Errorf("no acquisition_path nor acquisition_dir")
+		log.Warningf("no acquisition_path nor acquisition_dir")
 	}
 	if err := c.LoadSimulation(); err != nil {
 		return errors.Wrap(err, "load error (simulation)")

pkg/csconfig/crowdsec_service_test.go

@@ -139,9 +139,16 @@ func TestLoadCrowdsec(t *testing.T) {
 				Crowdsec: &CrowdsecServiceCfg{},
 			},
 			expectedResult: &CrowdsecServiceCfg{
-				BucketsRoutinesCount: 0,
-				ParserRoutinesCount:  0,
-				OutputRoutinesCount:  0,
+				BucketsRoutinesCount: 1,
+				ParserRoutinesCount:  1,
+				OutputRoutinesCount:  1,
+				ConfigDir:            configDirFullPath,
+				HubIndexFile:         hubIndexFileFullPath,
+				DataDir:              dataFullPath,
+				HubDir:               hubFullPath,
+				SimulationConfig: &SimulationConfig{
+					Simulation: &falseBoolPtr,
+				},
 			},
 		},
 		{

pkg/csconfig/profiles.go

@@ -19,8 +19,8 @@ type ProfileCfg struct {
 	Name           string                      `yaml:"name,omitempty"`
 	Debug          *bool                       `yaml:"debug,omitempty"`
 	Filters        []string                    `yaml:"filters,omitempty"` //A list of OR'ed expressions. the models.Alert object
-	RuntimeFilters []*vm.Program               `json:"-"`
-	DebugFilters   []*exprhelpers.ExprDebugger `json:"-"`
+	RuntimeFilters []*vm.Program               `json:"-" yaml:"-"`
+	DebugFilters   []*exprhelpers.ExprDebugger `json:"-" yaml:"-"`
 	Decisions      []models.Decision           `yaml:"decisions,omitempty"`
 	OnSuccess      string                      `yaml:"on_success,omitempty"` //continue or break
 	OnFailure      string                      `yaml:"on_failure,omitempty"` //continue or break

pkg/cstest/coverage.go

@@ -0,0 +1,177 @@
+package cstest
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	log "github.com/sirupsen/logrus"
+)
+
+type ParserCoverage struct {
+	Parser     string
+	TestsCount int
+	PresentIn  map[string]bool //poorman's set
+}
+
+type ScenarioCoverage struct {
+	Scenario   string
+	TestsCount int
+	PresentIn  map[string]bool
+}
+
+func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) {
+	var coverage []ParserCoverage
+	if _, ok := h.HubIndex.Data[cwhub.PARSERS]; !ok {
+		return coverage, fmt.Errorf("no parsers in hub index")
+	}
+	//populate from hub, iterate in alphabetical order
+	var pkeys []string
+	for pname := range h.HubIndex.Data[cwhub.PARSERS] {
+		pkeys = append(pkeys, pname)
+	}
+	sort.Strings(pkeys)
+	for _, pname := range pkeys {
+		coverage = append(coverage, ParserCoverage{
+			Parser:     pname,
+			TestsCount: 0,
+			PresentIn:  make(map[string]bool),
+		})
+	}
+
+	//parser the expressions a-la-oneagain
+	passerts, err := filepath.Glob(".tests/*/parser.assert")
+	if err != nil {
+		return coverage, fmt.Errorf("while find parser asserts : %s", err)
+	}
+	for _, assert := range passerts {
+		file, err := os.Open(assert)
+		if err != nil {
+			return coverage, fmt.Errorf("while reading %s : %s", assert, err)
+		}
+		scanner := bufio.NewScanner(file)
+		for scanner.Scan() {
+			assertLine := regexp.MustCompile(`^results\["[^"]+"\]\["(?P<parser>[^"]+)"\]\[[0-9]+\]\.Evt\..*`)
+			line := scanner.Text()
+			log.Debugf("assert line : %s", line)
+			match := assertLine.FindStringSubmatch(line)
+			if len(match) == 0 {
+				log.Debugf("%s doesn't match", line)
+				continue
+			}
+			sidx := assertLine.SubexpIndex("parser")
+			capturedParser := match[sidx]
+			for idx, pcover := range coverage {
+				if pcover.Parser == capturedParser {
+					coverage[idx].TestsCount++
+					coverage[idx].PresentIn[assert] = true
+					continue
+				}
+				parserNameSplit := strings.Split(pcover.Parser, "/")
+				parserNameOnly := parserNameSplit[len(parserNameSplit)-1]
+				if parserNameOnly == capturedParser {
+					coverage[idx].TestsCount++
+					coverage[idx].PresentIn[assert] = true
+					continue
+				}
+				capturedParserSplit := strings.Split(capturedParser, "/")
+				capturedParserName := capturedParserSplit[len(capturedParserSplit)-1]
+				if capturedParserName == parserNameOnly {
+					coverage[idx].TestsCount++
+					coverage[idx].PresentIn[assert] = true
+					continue
+				}
+				if capturedParserName == parserNameOnly+"-logs" {
+					coverage[idx].TestsCount++
+					coverage[idx].PresentIn[assert] = true
+					continue
+				}
+			}
+		}
+		file.Close()
+	}
+	return coverage, nil
+}
+
+func (h *HubTest) GetScenariosCoverage() ([]ScenarioCoverage, error) {
+	var coverage []ScenarioCoverage
+	if _, ok := h.HubIndex.Data[cwhub.SCENARIOS]; !ok {
+		return coverage, fmt.Errorf("no scenarios in hub index")
+	}
+	//populate from hub, iterate in alphabetical order
+	var pkeys []string
+	for scenarioName := range h.HubIndex.Data[cwhub.SCENARIOS] {
+		pkeys = append(pkeys, scenarioName)
+	}
+	sort.Strings(pkeys)
+	for _, scenarioName := range pkeys {
+		coverage = append(coverage, ScenarioCoverage{
+			Scenario:   scenarioName,
+			TestsCount: 0,
+			PresentIn:  make(map[string]bool),
+		})
+	}
+
+	//parser the expressions a-la-oneagain
+	passerts, err := filepath.Glob(".tests/*/scenario.assert")
+	if err != nil {
+		return coverage, fmt.Errorf("while find scenario asserts : %s", err)
+	}
+	for _, assert := range passerts {
+		file, err := os.Open(assert)
+		if err != nil {
+			return coverage, fmt.Errorf("while reading %s : %s", assert, err)
+		}
+		scanner := bufio.NewScanner(file)
+		for scanner.Scan() {
+			assertLine := regexp.MustCompile(`^results\[[0-9]+\].Overflow.Alert.GetScenario\(\) == "(?P<scenario>[^"]+)"`)
+			line := scanner.Text()
+			log.Debugf("assert line : %s", line)
+			match := assertLine.FindStringSubmatch(line)
+			if len(match) == 0 {
+				log.Debugf("%s doesn't match", line)
+				continue
+			}
+			sidx := assertLine.SubexpIndex("scenario")
+			scanner_name := match[sidx]
+			for idx, pcover := range coverage {
+				if pcover.Scenario == scanner_name {
+					coverage[idx].TestsCount++
+					coverage[idx].PresentIn[assert] = true
+					continue
+				}
+				scenarioNameSplit := strings.Split(pcover.Scenario, "/")
+				scenarioNameOnly := scenarioNameSplit[len(scenarioNameSplit)-1]
+				if scenarioNameOnly == scanner_name {
+					coverage[idx].TestsCount++
+					coverage[idx].PresentIn[assert] = true
+					continue
+				}
+				fixedProbingWord := strings.Replace(pcover.Scenario, "probbing", "probing", -1)
+				fixedProbingAssert := strings.Replace(scanner_name, "probbing", "probing", -1)
+				if fixedProbingWord == fixedProbingAssert {
+					coverage[idx].TestsCount++
+					coverage[idx].PresentIn[assert] = true
+					continue
+				}
+				if fmt.Sprintf("%s-detection", pcover.Scenario) == scanner_name {
+					coverage[idx].TestsCount++
+					coverage[idx].PresentIn[assert] = true
+					continue
+				}
+				if fmt.Sprintf("%s-detection", fixedProbingWord) == fixedProbingAssert {
+					coverage[idx].TestsCount++
+					coverage[idx].PresentIn[assert] = true
+					continue
+				}
+			}
+		}
+		file.Close()
+	}
+	return coverage, nil
+}

pkg/cstest/hubtest.go

@@ -0,0 +1,114 @@
+package cstest
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/pkg/errors"
+)
+
+type HubTest struct {
+	CrowdSecPath           string
+	CscliPath              string
+	HubPath                string
+	HubTestPath            string
+	HubIndexFile           string
+	TemplateConfigPath     string
+	TemplateProfilePath    string
+	TemplateSimulationPath string
+	HubIndex               *HubIndex
+	Tests                  []*HubTestItem
+}
+
+const (
+	templateConfigFile     = "template_config.yaml"
+	templateSimulationFile = "template_simulation.yaml"
+	templateProfileFile    = "template_profiles.yaml"
+)
+
+func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, error) {
+	var err error
+
+	hubPath, err = filepath.Abs(hubPath)
+	if err != nil {
+		return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err)
+	}
+	// we can't use hubtest without the hub
+	if _, err := os.Stat(hubPath); os.IsNotExist(err) {
+		return HubTest{}, fmt.Errorf("path to hub '%s' doesn't exist, can't run", hubPath)
+	}
+	HubTestPath := filepath.Join(hubPath, "./.tests/")
+
+	// we can't use hubtest without crowdsec binary
+	if _, err := exec.LookPath(crowdsecPath); err != nil {
+		if _, err := os.Stat(crowdsecPath); os.IsNotExist(err) {
+			return HubTest{}, fmt.Errorf("path to crowdsec binary '%s' doesn't exist or is not in $PATH, can't run", crowdsecPath)
+		}
+	}
+
+	// we can't use hubtest without cscli binary
+	if _, err := exec.LookPath(cscliPath); err != nil {
+		if _, err := os.Stat(cscliPath); os.IsNotExist(err) {
+			return HubTest{}, fmt.Errorf("path to cscli binary '%s' doesn't exist or is not in $PATH, can't run", cscliPath)
+		}
+	}
+
+	hubIndexFile := filepath.Join(hubPath, ".index.json")
+	bidx, err := ioutil.ReadFile(hubIndexFile)
+	if err != nil {
+		return HubTest{}, fmt.Errorf("unable to read index file: %s", err)
+	}
+
+	// load hub index
+	hubIndex, err := cwhub.LoadPkgIndex(bidx)
+	if err != nil {
+		return HubTest{}, fmt.Errorf("unable to load hub index file: %s", err)
+	}
+
+	templateConfigFilePath := filepath.Join(HubTestPath, templateConfigFile)
+	templateProfilePath := filepath.Join(HubTestPath, templateProfileFile)
+	templateSimulationPath := filepath.Join(HubTestPath, templateSimulationFile)
+
+	return HubTest{
+		CrowdSecPath:           crowdsecPath,
+		CscliPath:              cscliPath,
+		HubPath:                hubPath,
+		HubTestPath:            HubTestPath,
+		HubIndexFile:           hubIndexFile,
+		TemplateConfigPath:     templateConfigFilePath,
+		TemplateProfilePath:    templateProfilePath,
+		TemplateSimulationPath: templateSimulationPath,
+		HubIndex:               &HubIndex{Data: hubIndex},
+	}, nil
+}
+
+func (h *HubTest) LoadTestItem(name string) (*HubTestItem, error) {
+	HubTestItem := &HubTestItem{}
+	testItem, err := NewTest(name, h)
+	if err != nil {
+		return HubTestItem, err
+	}
+	h.Tests = append(h.Tests, testItem)
+
+	return testItem, nil
+}
+
+func (h *HubTest) LoadAllTests() error {
+	testsFolder, err := ioutil.ReadDir(h.HubTestPath)
+	if err != nil {
+		return err
+	}
+
+	for _, f := range testsFolder {
+		if f.IsDir() {
+			if _, err := h.LoadTestItem(f.Name()); err != nil {
+				return errors.Wrapf(err, "while loading %s", f.Name())
+			}
+		}
+	}
+	return nil
+}

pkg/cstest/hubtest_item.go

@@ -0,0 +1,604 @@
+package cstest
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+)
+
+type HubTestItemConfig struct {
+	Parsers       []string `yaml:"parsers"`
+	Scenarios     []string `yaml:"scenarios"`
+	PostOVerflows []string `yaml:"postoverflows"`
+	LogFile       string   `yaml:"log_file"`
+	LogType       string   `yaml:"log_type"`
+	IgnoreParsers bool     `yaml:"ignore_parsers"` // if we test a scenario, we don't want to assert on Parser
+}
+
+type HubIndex struct {
+	Data map[string]map[string]cwhub.Item
+}
+
+type HubTestItem struct {
+	Name string
+	Path string
+
+	CrowdSecPath string
+	CscliPath    string
+
+	RuntimePath               string
+	RuntimeHubPath            string
+	RuntimeDataPath           string
+	RuntimePatternsPath       string
+	RuntimeConfigFilePath     string
+	RuntimeProfileFilePath    string
+	RuntimeSimulationFilePath string
+	RuntimeHubConfig          *csconfig.Hub
+
+	ResultsPath          string
+	ParserResultFile     string
+	ScenarioResultFile   string
+	BucketPourResultFile string
+
+	HubPath                string
+	HubTestPath            string
+	HubIndexFile           string
+	TemplateConfigPath     string
+	TemplateProfilePath    string
+	TemplateSimulationPath string
+	HubIndex               *HubIndex
+
+	Config *HubTestItemConfig
+
+	Success    bool
+	ErrorsList []string
+
+	AutoGen        bool
+	ParserAssert   *ParserAssert
+	ScenarioAssert *ScenarioAssert
+
+	CustomItemsLocation []string
+}
+
+const (
+	ParserAssertFileName = "parser.assert"
+	ParserResultFileName = "parser-dump.yaml"
+
+	ScenarioAssertFileName = "scenario.assert"
+	ScenarioResultFileName = "bucket-dump.yaml"
+
+	BucketPourResultFileName = "bucketpour-dump.yaml"
+
+	crowdsecPatternsFolder = "/etc/crowdsec/patterns/"
+)
+
+func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
+	testPath := filepath.Join(hubTest.HubTestPath, name)
+	runtimeFolder := filepath.Join(testPath, "runtime")
+	runtimeHubFolder := filepath.Join(runtimeFolder, "hub")
+	configFilePath := filepath.Join(testPath, "config.yaml")
+	resultPath := filepath.Join(testPath, "results")
+
+	// read test configuration file
+	configFileData := &HubTestItemConfig{}
+	yamlFile, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		log.Printf("no config file found in '%s': %v", testPath, err)
+	}
+	err = yaml.Unmarshal(yamlFile, configFileData)
+	if err != nil {
+		return nil, fmt.Errorf("Unmarshal: %v", err)
+	}
+
+	parserAssertFilePath := filepath.Join(testPath, ParserAssertFileName)
+	ParserAssert := NewParserAssert(parserAssertFilePath)
+
+	scenarioAssertFilePath := filepath.Join(testPath, ScenarioAssertFileName)
+	ScenarioAssert := NewScenarioAssert(scenarioAssertFilePath)
+	return &HubTestItem{
+		Name:                      name,
+		Path:                      testPath,
+		CrowdSecPath:              hubTest.CrowdSecPath,
+		CscliPath:                 hubTest.CscliPath,
+		RuntimePath:               filepath.Join(testPath, "runtime"),
+		RuntimeHubPath:            runtimeHubFolder,
+		RuntimeDataPath:           filepath.Join(runtimeFolder, "data"),
+		RuntimePatternsPath:       filepath.Join(runtimeFolder, "patterns"),
+		RuntimeConfigFilePath:     filepath.Join(runtimeFolder, "config.yaml"),
+		RuntimeProfileFilePath:    filepath.Join(runtimeFolder, "profiles.yaml"),
+		RuntimeSimulationFilePath: filepath.Join(runtimeFolder, "simulation.yaml"),
+		ResultsPath:               resultPath,
+		ParserResultFile:          filepath.Join(resultPath, ParserResultFileName),
+		ScenarioResultFile:        filepath.Join(resultPath, ScenarioResultFileName),
+		BucketPourResultFile:      filepath.Join(resultPath, BucketPourResultFileName),
+		RuntimeHubConfig: &csconfig.Hub{
+			HubDir:       runtimeHubFolder,
+			ConfigDir:    runtimeFolder,
+			HubIndexFile: hubTest.HubIndexFile,
+			DataDir:      filepath.Join(runtimeFolder, "data"),
+		},
+		Config:                 configFileData,
+		HubPath:                hubTest.HubPath,
+		HubTestPath:            hubTest.HubTestPath,
+		HubIndexFile:           hubTest.HubIndexFile,
+		TemplateConfigPath:     hubTest.TemplateConfigPath,
+		TemplateProfilePath:    hubTest.TemplateProfilePath,
+		TemplateSimulationPath: hubTest.TemplateSimulationPath,
+		HubIndex:               hubTest.HubIndex,
+		ScenarioAssert:         ScenarioAssert,
+		ParserAssert:           ParserAssert,
+		CustomItemsLocation:    []string{hubTest.HubPath, testPath},
+	}, nil
+}
+
+func (t *HubTestItem) InstallHub() error {
+	// install parsers in runtime environment
+	for _, parser := range t.Config.Parsers {
+		if parser == "" {
+			continue
+		}
+		var parserDirDest string
+		if hubParser, ok := t.HubIndex.Data[cwhub.PARSERS][parser]; ok {
+			parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath))
+			if err != nil {
+				return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err)
+			}
+			parserFileName := filepath.Base(parserSource)
+
+			// runtime/hub/parsers/s00-raw/crowdsecurity/
+			hubDirParserDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubParser.RemotePath))
+
+			// runtime/parsers/s00-raw/
+			parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage)
+
+			if err := os.MkdirAll(hubDirParserDest, os.ModePerm); err != nil {
+				return fmt.Errorf("unable to create folder '%s': %s", hubDirParserDest, err)
+			}
+			if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil {
+				return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err)
+			}
+
+			// runtime/hub/parsers/s00-raw/crowdsecurity/syslog-logs.yaml
+			hubDirParserPath := filepath.Join(hubDirParserDest, parserFileName)
+			if err := Copy(parserSource, hubDirParserPath); err != nil {
+				return fmt.Errorf("unable to copy '%s' to '%s': %s", parserSource, hubDirParserPath, err)
+			}
+
+			// runtime/parsers/s00-raw/syslog-logs.yaml
+			parserDirParserPath := filepath.Join(parserDirDest, parserFileName)
+			if err := os.Symlink(hubDirParserPath, parserDirParserPath); err != nil {
+				if !os.IsExist(err) {
+					return fmt.Errorf("unable to symlink parser '%s' to '%s': %s", hubDirParserPath, parserDirParserPath, err)
+				}
+			}
+		} else {
+			customParserExist := false
+			for _, customPath := range t.CustomItemsLocation {
+				// we check if its a custom parser
+				customParserPath := filepath.Join(customPath, parser)
+				if _, err := os.Stat(customParserPath); os.IsNotExist(err) {
+					continue
+					//return fmt.Errorf("parser '%s' doesn't exist in the hub and doesn't appear to be a custom one.", parser)
+				}
+
+				customParserPathSplit := strings.Split(customParserPath, "/")
+				customParserName := customParserPathSplit[len(customParserPathSplit)-1]
+				// because path is parsers/<stage>/<author>/parser.yaml and we wan't the stage
+				customParserStage := customParserPathSplit[len(customParserPathSplit)-3]
+
+				// check if stage exist
+				hubStagePath := filepath.Join(t.HubPath, fmt.Sprintf("parsers/%s", customParserStage))
+
+				if _, err := os.Stat(hubStagePath); os.IsNotExist(err) {
+					continue
+					//return fmt.Errorf("stage '%s' extracted from '%s' doesn't exist in the hub", customParserStage, hubStagePath)
+				}
+
+				parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage)
+				if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil {
+					continue
+					//return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err)
+				}
+
+				customParserDest := filepath.Join(parserDirDest, customParserName)
+				// if path to parser exist, copy it
+				if err := Copy(customParserPath, customParserDest); err != nil {
+					continue
+					//return fmt.Errorf("unable to copy custom parser '%s' to '%s': %s", customParserPath, customParserDest, err)
+				}
+
+				customParserExist = true
+				break
+			}
+			if !customParserExist {
+				return fmt.Errorf("couldn't find custom parser '%s' in the following location: %+v", parser, t.CustomItemsLocation)
+			}
+		}
+	}
+
+	// install scenarios in runtime environment
+	for _, scenario := range t.Config.Scenarios {
+		if scenario == "" {
+			continue
+		}
+		var scenarioDirDest string
+		if hubScenario, ok := t.HubIndex.Data[cwhub.SCENARIOS][scenario]; ok {
+			scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath))
+			if err != nil {
+				return fmt.Errorf("can't get absolute path to: %s", scenarioSource)
+			}
+			scenarioFileName := filepath.Base(scenarioSource)
+
+			// runtime/hub/scenarios/crowdsecurity/
+			hubDirScenarioDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubScenario.RemotePath))
+
+			// runtime/parsers/scenarios/
+			scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath)
+
+			if err := os.MkdirAll(hubDirScenarioDest, os.ModePerm); err != nil {
+				return fmt.Errorf("unable to create folder '%s': %s", hubDirScenarioDest, err)
+			}
+			if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil {
+				return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err)
+			}
+
+			// runtime/hub/scenarios/crowdsecurity/ssh-bf.yaml
+			hubDirScenarioPath := filepath.Join(hubDirScenarioDest, scenarioFileName)
+			if err := Copy(scenarioSource, hubDirScenarioPath); err != nil {
+				return fmt.Errorf("unable to copy '%s' to '%s': %s", scenarioSource, hubDirScenarioPath, err)
+			}
+
+			// runtime/scenarios/ssh-bf.yaml
+			scenarioDirParserPath := filepath.Join(scenarioDirDest, scenarioFileName)
+			if err := os.Symlink(hubDirScenarioPath, scenarioDirParserPath); err != nil {
+				if !os.IsExist(err) {
+					return fmt.Errorf("unable to symlink scenario '%s' to '%s': %s", hubDirScenarioPath, scenarioDirParserPath, err)
+				}
+			}
+		} else {
+			customScenarioExist := false
+			for _, customPath := range t.CustomItemsLocation {
+				// we check if its a custom scenario
+				customScenarioPath := filepath.Join(customPath, scenario)
+				if _, err := os.Stat(customScenarioPath); os.IsNotExist(err) {
+					continue
+					//return fmt.Errorf("scenarios '%s' doesn't exist in the hub and doesn't appear to be a custom one.", scenario)
+				}
+
+				scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath)
+				if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil {
+					return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err)
+				}
+
+				scenarioFileName := filepath.Base(customScenarioPath)
+				scenarioFileDest := filepath.Join(scenarioDirDest, scenarioFileName)
+				if err := Copy(customScenarioPath, scenarioFileDest); err != nil {
+					continue
+					//return fmt.Errorf("unable to copy scenario from '%s' to '%s': %s", customScenarioPath, scenarioFileDest, err)
+				}
+				customScenarioExist = true
+				break
+			}
+			if !customScenarioExist {
+				return fmt.Errorf("couldn't find custom scenario '%s' in the following location: %+v", scenario, t.CustomItemsLocation)
+			}
+		}
+	}
+
+	// install postoverflows in runtime environment
+	for _, postoverflow := range t.Config.PostOVerflows {
+		if postoverflow == "" {
+			continue
+		}
+		var postoverflowDirDest string
+		if hubPostOverflow, ok := t.HubIndex.Data[cwhub.PARSERS_OVFLW][postoverflow]; ok {
+			postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath))
+			if err != nil {
+				return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err)
+			}
+			postoverflowFileName := filepath.Base(postoverflowSource)
+
+			// runtime/hub/postoverflows/s00-enrich/crowdsecurity/
+			hubDirPostoverflowDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubPostOverflow.RemotePath))
+
+			// runtime/postoverflows/s00-enrich
+			postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage)
+
+			if err := os.MkdirAll(hubDirPostoverflowDest, os.ModePerm); err != nil {
+				return fmt.Errorf("unable to create folder '%s': %s", hubDirPostoverflowDest, err)
+			}
+			if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil {
+				return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err)
+			}
+
+			// runtime/hub/postoverflows/s00-enrich/crowdsecurity/rdns.yaml
+			hubDirPostoverflowPath := filepath.Join(hubDirPostoverflowDest, postoverflowFileName)
+			if err := Copy(postoverflowSource, hubDirPostoverflowPath); err != nil {
+				return fmt.Errorf("unable to copy '%s' to '%s': %s", postoverflowSource, hubDirPostoverflowPath, err)
+			}
+
+			// runtime/postoverflows/s00-enrich/rdns.yaml
+			postoverflowDirParserPath := filepath.Join(postoverflowDirDest, postoverflowFileName)
+			if err := os.Symlink(hubDirPostoverflowPath, postoverflowDirParserPath); err != nil {
+				if !os.IsExist(err) {
+					return fmt.Errorf("unable to symlink postoverflow '%s' to '%s': %s", hubDirPostoverflowPath, postoverflowDirParserPath, err)
+				}
+			}
+		} else {
+			customPostoverflowExist := false
+			for _, customPath := range t.CustomItemsLocation {
+				// we check if its a custom postoverflow
+				customPostOverflowPath := filepath.Join(customPath, postoverflow)
+				if _, err := os.Stat(customPostOverflowPath); os.IsNotExist(err) {
+					continue
+					//return fmt.Errorf("postoverflow '%s' doesn't exist in the hub and doesn't appear to be a custom one.", postoverflow)
+				}
+
+				customPostOverflowPathSplit := strings.Split(customPostOverflowPath, "/")
+				customPostoverflowName := customPostOverflowPathSplit[len(customPostOverflowPathSplit)-1]
+				// because path is postoverflows/<stage>/<author>/parser.yaml and we wan't the stage
+				customPostoverflowStage := customPostOverflowPathSplit[len(customPostOverflowPathSplit)-3]
+
+				// check if stage exist
+				hubStagePath := filepath.Join(t.HubPath, fmt.Sprintf("postoverflows/%s", customPostoverflowStage))
+
+				if _, err := os.Stat(hubStagePath); os.IsNotExist(err) {
+					continue
+					//return fmt.Errorf("stage '%s' from extracted '%s' doesn't exist in the hub", customPostoverflowStage, hubStagePath)
+				}
+
+				postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage)
+				if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil {
+					continue
+					//return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err)
+				}
+
+				customPostoverflowDest := filepath.Join(postoverflowDirDest, customPostoverflowName)
+				// if path to postoverflow exist, copy it
+				if err := Copy(customPostOverflowPath, customPostoverflowDest); err != nil {
+					continue
+					//return fmt.Errorf("unable to copy custom parser '%s' to '%s': %s", customPostOverflowPath, customPostoverflowDest, err)
+				}
+				customPostoverflowExist = true
+				break
+			}
+			if !customPostoverflowExist {
+				return fmt.Errorf("couldn't find custom postoverflow '%s' in the following location: %+v", postoverflow, t.CustomItemsLocation)
+			}
+		}
+	}
+
+	// load installed hub
+	err := cwhub.GetHubIdx(t.RuntimeHubConfig)
+	if err != nil {
+		log.Fatalf("can't local sync the hub: %+v", err)
+	}
+
+	// install data for parsers if needed
+	ret := cwhub.GetItemMap(cwhub.PARSERS)
+	for parserName, item := range ret {
+		if item.Installed {
+			if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
+				return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err)
+			}
+			log.Debugf("parser '%s' installed succesfully in runtime environment", parserName)
+		}
+	}
+
+	// install data for scenarios if needed
+	ret = cwhub.GetItemMap(cwhub.SCENARIOS)
+	for scenarioName, item := range ret {
+		if item.Installed {
+			if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
+				return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err)
+			}
+			log.Debugf("scenario '%s' installed succesfully in runtime environment", scenarioName)
+		}
+	}
+
+	// install data for postoverflows if needed
+	ret = cwhub.GetItemMap(cwhub.PARSERS_OVFLW)
+	for postoverflowName, item := range ret {
+		if item.Installed {
+			if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
+				return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err)
+			}
+			log.Debugf("postoverflow '%s' installed succesfully in runtime environment", postoverflowName)
+		}
+	}
+
+	return nil
+}
+
+func (t *HubTestItem) Clean() error {
+	return os.RemoveAll(t.RuntimePath)
+}
+
+func (t *HubTestItem) Run() error {
+	t.Success = false
+	t.ErrorsList = make([]string, 0)
+
+	testPath := filepath.Join(t.HubTestPath, t.Name)
+	if _, err := os.Stat(testPath); os.IsNotExist(err) {
+		return fmt.Errorf("test '%s' doesn't exist in '%s', exiting", t.Name, t.HubTestPath)
+	}
+
+	currentDir, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("can't get current directory: %+v", err)
+	}
+
+	// create runtime folder
+	if err := os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil {
+		return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err)
+	}
+
+	// create runtime data folder
+	if err := os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil {
+		return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeDataPath, err)
+	}
+
+	// create runtime hub folder
+	if err := os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil {
+		return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeHubPath, err)
+	}
+
+	if err := Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil {
+		return fmt.Errorf("unable to copy .index.json file in '%s': %s", filepath.Join(t.RuntimeHubPath, ".index.json"), err)
+	}
+
+	// create results folder
+	if err := os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil {
+		return fmt.Errorf("unable to create folder '%s': %+v", t.ResultsPath, err)
+	}
+
+	// copy template config file to runtime folder
+	if err := Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil {
+		return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err)
+	}
+
+	// copy template profile file to runtime folder
+	if err := Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil {
+		return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err)
+	}
+
+	// copy template simulation file to runtime folder
+	if err := Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil {
+		return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err)
+	}
+
+	// copy template patterns folder to runtime folder
+	if err := CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil {
+		return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err)
+	}
+
+	// install the hub in the runtime folder
+	if err := t.InstallHub(); err != nil {
+		return fmt.Errorf("unable to install hub in '%s': %s", t.RuntimeHubPath, err)
+	}
+
+	logFile := t.Config.LogFile
+	logType := t.Config.LogType
+	dsn := fmt.Sprintf("file://%s", logFile)
+
+	if err := os.Chdir(testPath); err != nil {
+		return fmt.Errorf("can't 'cd' to '%s': %s", testPath, err)
+	}
+
+	logFileStat, err := os.Stat(logFile)
+	if err != nil {
+		return fmt.Errorf("unable to stat log file '%s': %s", logFile, err.Error())
+	}
+	if logFileStat.Size() == 0 {
+		return fmt.Errorf("Log file '%s' is empty, please fill it with log", logFile)
+	}
+
+	cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"}
+	cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...)
+	log.Debugf("%s", cscliRegisterCmd.String())
+	output, err := cscliRegisterCmd.CombinedOutput()
+	if err != nil {
+		if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") {
+			fmt.Println(string(output))
+			return fmt.Errorf("fail to run '%s' for test '%s': %v", cscliRegisterCmd.String(), t.Name, err)
+		}
+	}
+
+	cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", t.ResultsPath}
+	crowdsecCmd := exec.Command(t.CrowdSecPath, cmdArgs...)
+	log.Debugf("%s", crowdsecCmd.String())
+	output, err = crowdsecCmd.CombinedOutput()
+	if log.GetLevel() >= log.DebugLevel || err != nil {
+		fmt.Println(string(output))
+	}
+	if err != nil {
+		return fmt.Errorf("fail to run '%s' for test '%s': %v", crowdsecCmd.String(), t.Name, err)
+	}
+
+	if err := os.Chdir(currentDir); err != nil {
+		return fmt.Errorf("can't 'cd' to '%s': %s", currentDir, err)
+	}
+
+	// assert parsers
+	if !t.Config.IgnoreParsers {
+		assertFileStat, err := os.Stat(t.ParserAssert.File)
+		if os.IsNotExist(err) {
+			parserAssertFile, err := os.Create(t.ParserAssert.File)
+			if err != nil {
+				log.Fatal(err)
+			}
+			parserAssertFile.Close()
+		}
+		assertFileStat, err = os.Stat(t.ParserAssert.File)
+		if err != nil {
+			return fmt.Errorf("error while stats '%s': %s", t.ParserAssert.File, err)
+		}
+
+		if assertFileStat.Size() == 0 {
+			assertData, err := t.ParserAssert.AutoGenFromFile(t.ParserResultFile)
+			if err != nil {
+				return fmt.Errorf("couldn't generate assertion: %s", err.Error())
+			}
+			t.ParserAssert.AutoGenAssertData = assertData
+			t.ParserAssert.AutoGenAssert = true
+		} else {
+			if err := t.ParserAssert.AssertFile(t.ParserResultFile); err != nil {
+				return fmt.Errorf("unable to run assertion on file '%s': %s", t.ParserResultFile, err)
+			}
+		}
+	}
+
+	// assert scenarios
+	nbScenario := 0
+	for _, scenario := range t.Config.Scenarios {
+		if scenario == "" {
+			continue
+		}
+		nbScenario += 1
+	}
+	if nbScenario > 0 {
+		assertFileStat, err := os.Stat(t.ScenarioAssert.File)
+		if os.IsNotExist(err) {
+			scenarioAssertFile, err := os.Create(t.ScenarioAssert.File)
+			if err != nil {
+				log.Fatal(err)
+			}
+			scenarioAssertFile.Close()
+		}
+		assertFileStat, err = os.Stat(t.ScenarioAssert.File)
+		if err != nil {
+			return fmt.Errorf("error while stats '%s': %s", t.ScenarioAssert.File, err)
+		}
+
+		if assertFileStat.Size() == 0 {
+			assertData, err := t.ScenarioAssert.AutoGenFromFile(t.ScenarioResultFile)
+			if err != nil {
+				return fmt.Errorf("couldn't generate assertion: %s", err.Error())
+			}
+			t.ScenarioAssert.AutoGenAssertData = assertData
+			t.ScenarioAssert.AutoGenAssert = true
+		} else {
+			if err := t.ScenarioAssert.AssertFile(t.ScenarioResultFile); err != nil {
+				return fmt.Errorf("unable to run assertion on file '%s': %s", t.ScenarioResultFile, err)
+			}
+		}
+	}
+
+	if t.ParserAssert.AutoGenAssert || t.ScenarioAssert.AutoGenAssert {
+		t.AutoGen = true
+	}
+
+	if (t.ParserAssert.Success || t.Config.IgnoreParsers) && (nbScenario == 0 || t.ScenarioAssert.Success) {
+		t.Success = true
+	}
+
+	return nil
+}

pkg/cstest/parser_assert.go

@@ -0,0 +1,392 @@
+package cstest
+
+import (
+	"bufio"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/antonmedv/expr"
+	"github.com/antonmedv/expr/vm"
+	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/enescakir/emoji"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+)
+
+type AssertFail struct {
+	File       string
+	Line       int
+	Expression string
+	Debug      map[string]string
+}
+
+type ParserAssert struct {
+	File              string
+	AutoGenAssert     bool
+	AutoGenAssertData string
+	NbAssert          int
+	Fails             []AssertFail
+	Success           bool
+	TestData          *ParserResults
+}
+
+type ParserResult struct {
+	Evt     types.Event
+	Success bool
+}
+type ParserResults map[string]map[string][]ParserResult
+
+func NewParserAssert(file string) *ParserAssert {
+
+	ParserAssert := &ParserAssert{
+		File:          file,
+		NbAssert:      0,
+		Success:       false,
+		Fails:         make([]AssertFail, 0),
+		AutoGenAssert: false,
+		TestData:      &ParserResults{},
+	}
+	return ParserAssert
+}
+
+func (p *ParserAssert) AutoGenFromFile(filename string) (string, error) {
+	err := p.LoadTest(filename)
+	if err != nil {
+		return "", err
+	}
+	ret := p.AutoGenParserAssert()
+	return ret, nil
+}
+
+func (p *ParserAssert) LoadTest(filename string) error {
+	var err error
+	parserDump, err := LoadParserDump(filename)
+	if err != nil {
+		return fmt.Errorf("loading parser dump file: %+v", err)
+	}
+	p.TestData = parserDump
+	return nil
+}
+
+func (p *ParserAssert) AssertFile(testFile string) error {
+	file, err := os.Open(p.File)
+
+	if err != nil {
+		return fmt.Errorf("failed to open")
+	}
+
+	if err := p.LoadTest(testFile); err != nil {
+		return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err)
+	}
+	scanner := bufio.NewScanner(file)
+	scanner.Split(bufio.ScanLines)
+	nbLine := 0
+	for scanner.Scan() {
+		nbLine += 1
+		if scanner.Text() == "" {
+			continue
+		}
+		ok, err := p.Run(scanner.Text())
+		if err != nil {
+			return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err)
+		}
+		p.NbAssert += 1
+		if !ok {
+			log.Debugf("%s is FALSE", scanner.Text())
+			//fmt.SPrintf(" %s '%s'\n", emoji.RedSquare, scanner.Text())
+			failedAssert := &AssertFail{
+				File:       p.File,
+				Line:       nbLine,
+				Expression: scanner.Text(),
+				Debug:      make(map[string]string),
+			}
+			variableRE := regexp.MustCompile(`(?P<variable>[^  =]+) == .*`)
+			match := variableRE.FindStringSubmatch(scanner.Text())
+			if len(match) == 0 {
+				log.Infof("Couldn't get variable of line '%s'", scanner.Text())
+			}
+			variable := match[1]
+			result, err := p.EvalExpression(variable)
+			if err != nil {
+				log.Errorf("unable to evaluate variable '%s': %s", variable, err)
+				continue
+			}
+			failedAssert.Debug[variable] = result
+			p.Fails = append(p.Fails, *failedAssert)
+			continue
+		}
+		//fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text())
+
+	}
+	file.Close()
+	if p.NbAssert == 0 {
+		assertData, err := p.AutoGenFromFile(testFile)
+		if err != nil {
+			return fmt.Errorf("couldn't generate assertion: %s", err.Error())
+		}
+		p.AutoGenAssertData = assertData
+		p.AutoGenAssert = true
+	}
+	if len(p.Fails) == 0 {
+		p.Success = true
+	}
+
+	return nil
+}
+
+func (p *ParserAssert) RunExpression(expression string) (interface{}, error) {
+	var err error
+	//debug doesn't make much sense with the ability to evaluate "on the fly"
+	//var debugFilter *exprhelpers.ExprDebugger
+	var runtimeFilter *vm.Program
+	var output interface{}
+
+	env := map[string]interface{}{"results": *p.TestData}
+
+	if runtimeFilter, err = expr.Compile(expression, expr.Env(exprhelpers.GetExprEnv(env))); err != nil {
+		return output, err
+	}
+	// if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(exprhelpers.GetExprEnv(env))); err != nil {
+	// 	log.Warningf("Failed building debugher for %s : %s", assert, err)
+	// }
+
+	//dump opcode in trace level
+	log.Tracef("%s", runtimeFilter.Disassemble())
+
+	output, err = expr.Run(runtimeFilter, exprhelpers.GetExprEnv(map[string]interface{}{"results": *p.TestData}))
+	if err != nil {
+		log.Warningf("running : %s", expression)
+		log.Warningf("runtime error : %s", err)
+		return output, errors.Wrapf(err, "while running expression %s", expression)
+	}
+	return output, nil
+}
+
+func (p *ParserAssert) EvalExpression(expression string) (string, error) {
+	output, err := p.RunExpression(expression)
+	if err != nil {
+		return "", err
+	}
+	ret, err := yaml.Marshal(output)
+	if err != nil {
+		return "", err
+	}
+	return string(ret), nil
+}
+
+func (p *ParserAssert) Run(assert string) (bool, error) {
+	output, err := p.RunExpression(assert)
+	if err != nil {
+		return false, err
+	}
+	switch out := output.(type) {
+	case bool:
+		return out, nil
+	default:
+		return false, fmt.Errorf("assertion '%s' is not a condition", assert)
+	}
+}
+
+func Escape(val string) string {
+	val = strings.ReplaceAll(val, `\`, `\\`)
+	val = strings.ReplaceAll(val, `"`, `\"`)
+	return val
+}
+
+func (p *ParserAssert) AutoGenParserAssert() string {
+	//attempt to autogen parser asserts
+	var ret string
+
+	//sort map keys for consistent ordre
+	var stages []string
+	for stage := range *p.TestData {
+		stages = append(stages, stage)
+	}
+	sort.Strings(stages)
+	ret += fmt.Sprintf("len(results) == %d\n", len(*p.TestData))
+	for _, stage := range stages {
+		parsers := (*p.TestData)[stage]
+		//sort map keys for consistent ordre
+		var pnames []string
+		for pname := range parsers {
+			pnames = append(pnames, pname)
+		}
+		sort.Strings(pnames)
+		for _, parser := range pnames {
+			presults := parsers[parser]
+			ret += fmt.Sprintf(`len(results["%s"]["%s"]) == %d`+"\n", stage, parser, len(presults))
+			for pidx, result := range presults {
+				ret += fmt.Sprintf(`results["%s"]["%s"][%d].Success == %t`+"\n", stage, parser, pidx, result.Success)
+
+				if !result.Success {
+					continue
+				}
+				for pkey, pval := range result.Evt.Parsed {
+					if pval == "" {
+						continue
+					}
+					ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Parsed["%s"] == "%s"`+"\n", stage, parser, pidx, pkey, Escape(pval))
+				}
+				for mkey, mval := range result.Evt.Meta {
+					if mval == "" {
+						continue
+					}
+					ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval))
+				}
+				for ekey, eval := range result.Evt.Enriched {
+					if eval == "" {
+						continue
+					}
+					ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Enriched["%s"] == "%s"`+"\n", stage, parser, pidx, ekey, Escape(eval))
+				}
+			}
+		}
+	}
+	return ret
+}
+
+func LoadParserDump(filepath string) (*ParserResults, error) {
+	var pdump ParserResults
+
+	dumpData, err := os.Open(filepath)
+	if err != nil {
+		return nil, err
+	}
+	defer dumpData.Close()
+
+	results, err := ioutil.ReadAll(dumpData)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := yaml.Unmarshal(results, &pdump); err != nil {
+		return nil, err
+	}
+	return &pdump, nil
+}
+
+func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo) error {
+	//note : we can use line -> time as the unique identifier (of acquisition)
+
+	state := make(map[time.Time]map[string]map[string]bool, 0)
+	assoc := make(map[time.Time]string, 0)
+
+	for stage, parsers := range parser_results {
+		for parser, results := range parsers {
+			for _, parser_res := range results {
+				evt := parser_res.Evt
+				if _, ok := state[evt.Line.Time]; !ok {
+					state[evt.Line.Time] = make(map[string]map[string]bool)
+					assoc[evt.Line.Time] = evt.Line.Raw
+				}
+				if _, ok := state[evt.Line.Time][stage]; !ok {
+					state[evt.Line.Time][stage] = make(map[string]bool)
+				}
+				state[evt.Line.Time][stage][parser] = parser_res.Success
+			}
+		}
+	}
+
+	for bname, evtlist := range bucket_pour {
+		for _, evt := range evtlist {
+			if evt.Line.Raw == "" {
+				continue
+			}
+			//it might be bucket oveflow being reprocessed, skip this
+			if _, ok := state[evt.Line.Time]; !ok {
+				state[evt.Line.Time] = make(map[string]map[string]bool)
+				assoc[evt.Line.Time] = evt.Line.Raw
+			}
+			//there is a trick : to know if an event succesfully exit the parsers, we check if it reached the pour() phase
+			//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
+			if _, ok := state[evt.Line.Time]["buckets"]; !ok {
+				state[evt.Line.Time]["buckets"] = make(map[string]bool)
+			}
+			state[evt.Line.Time]["buckets"][bname] = true
+		}
+	}
+
+	//get each line
+	for tstamp, rawstr := range assoc {
+		fmt.Printf("line: %s\n", rawstr)
+		skeys := make([]string, 0, len(state[tstamp]))
+		for k := range state[tstamp] {
+			//there is a trick : to know if an event succesfully exit the parsers, we check if it reached the pour() phase
+			//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
+			if k == "buckets" {
+				continue
+			}
+			skeys = append(skeys, k)
+		}
+		sort.Strings(skeys)
+		//iterate stage
+		for _, stage := range skeys {
+			parsers := state[tstamp][stage]
+
+			sep := "├"
+			presep := "|"
+
+			fmt.Printf("\t%s %s\n", sep, stage)
+
+			pkeys := make([]string, 0, len(parsers))
+			for k := range parsers {
+				pkeys = append(pkeys, k)
+			}
+			sort.Strings(pkeys)
+
+			for idx, parser := range pkeys {
+				res := parsers[parser]
+				sep := "├"
+				if idx == len(pkeys)-1 {
+					sep = "└"
+				}
+				if res {
+					fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.GreenCircle, parser)
+				} else {
+					fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser)
+
+				}
+			}
+		}
+		sep := "└"
+		if len(state[tstamp]["buckets"]) > 0 {
+			sep = "├"
+		}
+		//did the event enter the bucket pour phase ?
+		if _, ok := state[tstamp]["buckets"]["OK"]; ok {
+			fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle)
+		} else {
+			fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle)
+		}
+		//now print bucket info
+		if len(state[tstamp]["buckets"]) > 0 {
+			fmt.Printf("\t├ Scenarios\n")
+		}
+		bnames := make([]string, 0, len(state[tstamp]["buckets"]))
+		for k, _ := range state[tstamp]["buckets"] {
+			//there is a trick : to know if an event succesfully exit the parsers, we check if it reached the pour() phase
+			//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
+			if k == "OK" {
+				continue
+			}
+			bnames = append(bnames, k)
+		}
+		sort.Strings(bnames)
+		for idx, bname := range bnames {
+			sep := "├"
+			if idx == len(bnames)-1 {
+				sep = "└"
+			}
+			fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname)
+		}
+		fmt.Println()
+	}
+	return nil
+}

pkg/cstest/scenario_assert.go

@@ -0,0 +1,272 @@
+package cstest
+
+import (
+	"bufio"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"sort"
+
+	"github.com/antonmedv/expr"
+	"github.com/antonmedv/expr/vm"
+	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+)
+
+type ScenarioAssert struct {
+	File              string
+	AutoGenAssert     bool
+	AutoGenAssertData string
+	NbAssert          int
+	Fails             []AssertFail
+	Success           bool
+	TestData          *BucketResults
+	PourData          *BucketPourInfo
+}
+
+type BucketResults []types.Event
+type BucketPourInfo map[string][]types.Event
+
+func NewScenarioAssert(file string) *ScenarioAssert {
+	ScenarioAssert := &ScenarioAssert{
+		File:          file,
+		NbAssert:      0,
+		Success:       false,
+		Fails:         make([]AssertFail, 0),
+		AutoGenAssert: false,
+		TestData:      &BucketResults{},
+		PourData:      &BucketPourInfo{},
+	}
+	return ScenarioAssert
+}
+
+func (s *ScenarioAssert) AutoGenFromFile(filename string) (string, error) {
+	err := s.LoadTest(filename, "")
+	if err != nil {
+		return "", err
+	}
+	ret := s.AutoGenScenarioAssert()
+	return ret, nil
+}
+
+func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error {
+	var err error
+	bucketDump, err := LoadScenarioDump(filename)
+	if err != nil {
+		return fmt.Errorf("loading scenario dump file '%s': %+v", filename, err)
+	}
+	s.TestData = bucketDump
+
+	if bucketpour != "" {
+		pourDump, err := LoadBucketPourDump(bucketpour)
+		if err != nil {
+			return fmt.Errorf("loading bucket pour dump file '%s': %+v", filename, err)
+		}
+		s.PourData = pourDump
+	}
+	return nil
+}
+
+func (s *ScenarioAssert) AssertFile(testFile string) error {
+	file, err := os.Open(s.File)
+
+	if err != nil {
+		return fmt.Errorf("failed to open")
+	}
+
+	if err := s.LoadTest(testFile, ""); err != nil {
+		return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err)
+	}
+	scanner := bufio.NewScanner(file)
+	scanner.Split(bufio.ScanLines)
+	nbLine := 0
+	for scanner.Scan() {
+		nbLine += 1
+		if scanner.Text() == "" {
+			continue
+		}
+		ok, err := s.Run(scanner.Text())
+		if err != nil {
+			return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err)
+		}
+		s.NbAssert += 1
+		if !ok {
+			log.Debugf("%s is FALSE", scanner.Text())
+			failedAssert := &AssertFail{
+				File:       s.File,
+				Line:       nbLine,
+				Expression: scanner.Text(),
+				Debug:      make(map[string]string),
+			}
+			variableRE := regexp.MustCompile(`(?P<variable>[^ ]+) == .*`)
+			match := variableRE.FindStringSubmatch(scanner.Text())
+			if len(match) == 0 {
+				log.Infof("Couldn't get variable of line '%s'", scanner.Text())
+				continue
+			}
+			variable := match[1]
+			result, err := s.EvalExpression(variable)
+			if err != nil {
+				log.Errorf("unable to evaluate variable '%s': %s", variable, err)
+				continue
+			}
+			failedAssert.Debug[variable] = result
+			s.Fails = append(s.Fails, *failedAssert)
+			continue
+		}
+		//fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text())
+
+	}
+	file.Close()
+	if s.NbAssert == 0 {
+		assertData, err := s.AutoGenFromFile(testFile)
+		if err != nil {
+			return fmt.Errorf("couldn't generate assertion: %s", err.Error())
+		}
+		s.AutoGenAssertData = assertData
+		s.AutoGenAssert = true
+	}
+
+	if len(s.Fails) == 0 {
+		s.Success = true
+	}
+
+	return nil
+}
+
+func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) {
+	var err error
+	//debug doesn't make much sense with the ability to evaluate "on the fly"
+	//var debugFilter *exprhelpers.ExprDebugger
+	var runtimeFilter *vm.Program
+	var output interface{}
+
+	env := map[string]interface{}{"results": *s.TestData}
+
+	if runtimeFilter, err = expr.Compile(expression, expr.Env(exprhelpers.GetExprEnv(env))); err != nil {
+		return output, err
+	}
+	// if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(exprhelpers.GetExprEnv(env))); err != nil {
+	// 	log.Warningf("Failed building debugher for %s : %s", assert, err)
+	// }
+
+	//dump opcode in trace level
+	log.Tracef("%s", runtimeFilter.Disassemble())
+
+	output, err = expr.Run(runtimeFilter, exprhelpers.GetExprEnv(map[string]interface{}{"results": *s.TestData}))
+	if err != nil {
+		log.Warningf("running : %s", expression)
+		log.Warningf("runtime error : %s", err)
+		return output, errors.Wrapf(err, "while running expression %s", expression)
+	}
+	return output, nil
+}
+
+func (s *ScenarioAssert) EvalExpression(expression string) (string, error) {
+	output, err := s.RunExpression(expression)
+	if err != nil {
+		return "", err
+	}
+	ret, err := yaml.Marshal(output)
+	if err != nil {
+		return "", err
+	}
+	return string(ret), nil
+}
+
+func (s *ScenarioAssert) Run(assert string) (bool, error) {
+	output, err := s.RunExpression(assert)
+	if err != nil {
+		return false, err
+	}
+	switch out := output.(type) {
+	case bool:
+		return out, nil
+	default:
+		return false, fmt.Errorf("assertion '%s' is not a condition", assert)
+	}
+}
+
+func (s *ScenarioAssert) AutoGenScenarioAssert() string {
+	//attempt to autogen parser asserts
+	var ret string
+	ret += fmt.Sprintf(`len(results) == %d`+"\n", len(*s.TestData))
+	for eventIndex, event := range *s.TestData {
+		for ipSrc, source := range event.Overflow.Sources {
+			ret += fmt.Sprintf(`"%s" in results[%d].Overflow.GetSources()`+"\n", ipSrc, eventIndex)
+			ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].IP == "%s"`+"\n", eventIndex, ipSrc, source.IP)
+			ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].Range == "%s"`+"\n", eventIndex, ipSrc, source.Range)
+			ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetScope() == "%s"`+"\n", eventIndex, ipSrc, *source.Scope)
+			ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetValue() == "%s"`+"\n", eventIndex, ipSrc, *source.Value)
+		}
+		for evtIndex, evt := range event.Overflow.Alert.Events {
+			for _, meta := range evt.Meta {
+				ret += fmt.Sprintf(`results[%d].Overflow.Alert.Events[%d].GetMeta("%s") == "%s"`+"\n", eventIndex, evtIndex, meta.Key, meta.Value)
+			}
+		}
+		ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetScenario() == "%s"`+"\n", eventIndex, *event.Overflow.Alert.Scenario)
+		ret += fmt.Sprintf(`results[%d].Overflow.Alert.Remediation == %t`+"\n", eventIndex, *&event.Overflow.Alert.Remediation)
+		ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetEventsCount() == %d`+"\n", eventIndex, *event.Overflow.Alert.EventsCount)
+	}
+	return ret
+}
+
+func (b BucketResults) Len() int {
+	return len(b)
+}
+
+func (b BucketResults) Less(i, j int) bool {
+	return b[i].Overflow.Alert.GetScenario() > b[j].Overflow.Alert.GetScenario()
+}
+
+func (b BucketResults) Swap(i, j int) {
+	b[i], b[j] = b[j], b[i]
+}
+
+func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
+	var bucketDump BucketPourInfo
+
+	dumpData, err := os.Open(filepath)
+	if err != nil {
+		return nil, err
+	}
+	defer dumpData.Close()
+
+	results, err := ioutil.ReadAll(dumpData)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := yaml.Unmarshal(results, &bucketDump); err != nil {
+		return nil, err
+	}
+
+	return &bucketDump, nil
+}
+
+func LoadScenarioDump(filepath string) (*BucketResults, error) {
+	var bucketDump BucketResults
+
+	dumpData, err := os.Open(filepath)
+	if err != nil {
+		return nil, err
+	}
+	defer dumpData.Close()
+
+	results, err := ioutil.ReadAll(dumpData)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := yaml.Unmarshal(results, &bucketDump); err != nil {
+		return nil, err
+	}
+
+	sort.Sort(BucketResults(bucketDump))
+
+	return &bucketDump, nil
+}

pkg/cstest/utils.go

@@ -0,0 +1,81 @@
+package cstest
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+)
+
+func Copy(sourceFile string, destinationFile string) error {
+	input, err := ioutil.ReadFile(sourceFile)
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(destinationFile, input, 0644)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func CopyDir(src string, dest string) error {
+
+	if dest[:len(src)] == src {
+		return fmt.Errorf("Cannot copy a folder into the folder itself!")
+	}
+
+	f, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+
+	file, err := f.Stat()
+	if err != nil {
+		return err
+	}
+	if !file.IsDir() {
+		return fmt.Errorf("Source " + file.Name() + " is not a directory!")
+	}
+
+	err = os.MkdirAll(dest, 0755)
+	if err != nil {
+		return err
+	}
+
+	files, err := ioutil.ReadDir(src)
+	if err != nil {
+		return err
+	}
+
+	for _, f := range files {
+
+		if f.IsDir() {
+
+			err = CopyDir(src+"/"+f.Name(), dest+"/"+f.Name())
+			if err != nil {
+				return err
+			}
+
+		}
+
+		if !f.IsDir() {
+
+			content, err := ioutil.ReadFile(src + "/" + f.Name())
+			if err != nil {
+				return err
+
+			}
+
+			err = ioutil.WriteFile(dest+"/"+f.Name(), content, 0755)
+			if err != nil {
+				return err
+
+			}
+
+		}
+
+	}
+
+	return nil
+}

pkg/cwhub/download.go

@@ -219,8 +219,7 @@ func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error {
 		itemFile   *os.File
 		err        error
 	)
-	itemFilePath := fmt.Sprintf("%s/%s", hub.HubDir, target.RemotePath)
-
+	itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.ConfigDir, target.Type, target.Stage, target.FileName)
 	if itemFile, err = os.Open(itemFilePath); err != nil {
 		return errors.Wrapf(err, "while opening %s", itemFilePath)
 	}

pkg/cwhub/loader.go

@@ -81,6 +81,7 @@ func parser_visit(path string, f os.FileInfo, err error) error {
 		return fmt.Errorf("File '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubdir, installdir)
 	}
 
+	log.Tracef("stage:%s ftype:%s", stage, ftype)
 	//log.Printf("%s -> name:%s stage:%s", path, fname, stage)
 	if stage == SCENARIOS {
 		ftype = SCENARIOS

pkg/exprhelpers/exprlib.go

@@ -141,3 +141,8 @@ func IpInRange(ip string, ipRange string) bool {
 func TimeNow() string {
 	return time.Now().Format(time.RFC3339)
 }
+
+func KeyExists(key string, dict map[string]interface{}) bool {
+	_, ok := dict[key]
+	return ok
+}

pkg/leakybucket/manager_run.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/mohae/deepcopy"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/antonmedv/expr"
@@ -18,6 +19,8 @@ import (
 )
 
 var serialized map[string]Leaky
+var BucketPourCache map[string][]types.Event
+var BucketPourTrack bool
 
 /*The leaky routines lifecycle are based on "real" time.
 But when we are running in time-machine mode, the reference time is in logs and not "real" time.
@@ -158,6 +161,18 @@ func PourItemToHolders(parsed types.Event, holders []BucketFactory, buckets *Buc
 	)
 	//synchronize with DumpBucketsStateAt
 
+	//to track bucket pour : track items that enter the pour routine
+	if BucketPourTrack {
+		if BucketPourCache == nil {
+			BucketPourCache = make(map[string][]types.Event)
+		}
+		if _, ok := BucketPourCache["OK"]; !ok {
+			BucketPourCache["OK"] = make([]types.Event, 0)
+		}
+		evt := deepcopy.Copy(parsed)
+		BucketPourCache["OK"] = append(BucketPourCache["OK"], evt.(types.Event))
+	}
+
 	for idx, holder := range holders {
 
 		if holder.RunTimeFilter != nil {
@@ -290,6 +305,15 @@ func PourItemToHolders(parsed types.Event, holders []BucketFactory, buckets *Buc
 			select {
 			case bucket.In <- parsed:
 				holder.logger.Tracef("Successfully sent !")
+				//and track item poured to each bucket
+				if BucketPourTrack {
+					if _, ok := BucketPourCache[bucket.Name]; !ok {
+						BucketPourCache[bucket.Name] = make([]types.Event, 0)
+					}
+					evt := deepcopy.Copy(parsed)
+					BucketPourCache[bucket.Name] = append(BucketPourCache[bucket.Name], evt.(types.Event))
+				}
+
 				//sent was successful !
 				sent = true
 				continue

pkg/leakybucket/overflows.go

@@ -3,6 +3,7 @@ package leakybucket
 import (
 	"fmt"
 	"net"
+	"sort"
 	"strconv"
 
 	"github.com/crowdsecurity/crowdsec/pkg/models"
@@ -144,7 +145,14 @@ func EventsFromQueue(queue *Queue) []*models.Event {
 			continue
 		}
 		meta := models.Meta{}
-		for k, v := range evt.Meta {
+		//we want consistence
+		skeys := make([]string, 0, len(evt.Meta))
+		for k := range evt.Meta {
+			skeys = append(skeys, k)
+		}
+		sort.Strings(skeys)
+		for _, k := range skeys {
+			v := evt.Meta[k]
 			subMeta := models.MetaItems0{Key: k, Value: v}
 			meta = append(meta, &subMeta)
 		}

pkg/models/helpers.go

@@ -17,3 +17,33 @@ func (a *Alert) GetScenario() string {
 	}
 	return *a.Scenario
 }
+
+func (a *Alert) GetEventsCount() int32 {
+	if a.EventsCount == nil {
+		return 0
+	}
+	return *a.EventsCount
+}
+
+func (e *Event) GetMeta(key string) string {
+	for _, meta := range e.Meta {
+		if meta.Key == key {
+			return meta.Value
+		}
+	}
+	return ""
+}
+
+func (s Source) GetValue() string {
+	if s.Value == nil {
+		return ""
+	}
+	return *s.Value
+}
+
+func (s Source) GetScope() string {
+	if s.Scope == nil {
+		return ""
+	}
+	return *s.Scope
+}

pkg/parser/node.go

@@ -210,7 +210,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
 		}
 	}
 	if isWhitelisted {
-		p.WhiteListReason = n.Whitelist.Reason
+		p.WhitelistReason = n.Whitelist.Reason
 		/*huglily wipe the ban order if the event is whitelisted and it's an overflow */
 		if p.Type == types.OVFLW { /*don't do this at home kids */
 			ips := []string{}

pkg/parser/runtime.go

@@ -220,8 +220,14 @@ func stageidx(stage string, stages []string) int {
 	return -1
 }
 
+type ParserResult struct {
+	Evt     types.Event
+	Success bool
+}
+
 var ParseDump bool
-var StageParseCache map[string]map[string]types.Event
+var DumpFolder string
+var StageParseCache map[string]map[string][]ParserResult
 
 func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error) {
 	var event types.Event = xp
@@ -250,12 +256,18 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error)
 	}
 
 	if ParseDump {
-		StageParseCache = make(map[string]map[string]types.Event)
+		if StageParseCache == nil {
+			StageParseCache = make(map[string]map[string][]ParserResult)
+			StageParseCache["success"] = make(map[string][]ParserResult)
+			StageParseCache["success"][""] = make([]ParserResult, 0)
+		}
 	}
 
 	for _, stage := range ctx.Stages {
 		if ParseDump {
-			StageParseCache[stage] = make(map[string]types.Event)
+			if _, ok := StageParseCache[stage]; !ok {
+				StageParseCache[stage] = make(map[string][]ParserResult)
+			}
 		}
 		/* if the node is forward in stages, seek to its stage */
 		/* this is for example used by testing system to inject logs in post-syslog-parsing phase*/
@@ -290,12 +302,16 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error)
 				clog.Fatalf("Error while processing node : %v", err)
 			}
 			clog.Tracef("node (%s) ret : %v", node.rn, ret)
+			if ParseDump {
+				if len(StageParseCache[stage][node.Name]) == 0 {
+					StageParseCache[stage][node.Name] = make([]ParserResult, 0)
+				}
+				evtcopy := deepcopy.Copy(event)
+				parserInfo := ParserResult{Evt: evtcopy.(types.Event), Success: ret}
+				StageParseCache[stage][node.Name] = append(StageParseCache[stage][node.Name], parserInfo)
+			}
 			if ret {
 				isStageOK = true
-				if ParseDump {
-					evtcopy := deepcopy.Copy(event)
-					StageParseCache[stage][node.Name] = evtcopy.(types.Event)
-				}
 			}
 			if ret && node.OnSuccess == "next_stage" {
 				clog.Debugf("node successful, stop end stage %s", stage)

pkg/types/event.go

@@ -20,7 +20,7 @@ type Event struct {
 	Type            int    `yaml:"Type,omitempty" json:"Type,omitempty"`             //Can be types.LOG (0) or types.OVFLOW (1)
 	ExpectMode      int    `yaml:"ExpectMode,omitempty" json:"ExpectMode,omitempty"` //how to buckets should handle event : leaky.TIMEMACHINE or leaky.LIVE
 	Whitelisted     bool   `yaml:"Whitelisted,omitempty" json:"Whitelisted,omitempty"`
-	WhiteListReason string `yaml:"whitelist_reason,omitempty" json:"whitelist_reason,omitempty"`
+	WhitelistReason string `yaml:"WhitelistReason,omitempty" json:"whitelist_reason,omitempty"`
 	//should add whitelist reason ?
 	/* the current stage of the line being parsed */
 	Stage string `yaml:"Stage,omitempty" json:"Stage,omitempty"`
@@ -31,7 +31,7 @@ type Event struct {
 	/* output of enrichment */
 	Enriched map[string]string `yaml:"Enriched,omitempty" json:"Enriched,omitempty"`
 	/* Overflow */
-	Overflow      RuntimeAlert `yaml:"Alert,omitempty" json:"Alert,omitempty"`
+	Overflow      RuntimeAlert `yaml:"Overflow,omitempty" json:"Alert,omitempty"`
 	Time          time.Time    `yaml:"Time,omitempty" json:"Time,omitempty"` //parsed time `json:"-"` ``
 	StrTime       string       `yaml:"StrTime,omitempty" json:"StrTime,omitempty"`
 	MarshaledTime string       `yaml:"MarshaledTime,omitempty" json:"MarshaledTime,omitempty"`
@@ -78,3 +78,11 @@ type RuntimeAlert struct {
 	//APIAlerts will be populated at the end when there is more than one source
 	APIAlerts []models.Alert `yaml:"APIAlerts,omitempty" json:"APIAlerts,omitempty"`
 }
+
+func (r RuntimeAlert) GetSources() []string {
+	ret := make([]string, 0)
+	for key, _ := range r.Sources {
+		ret = append(ret, key)
+	}
+	return ret
+}