CI: add a CI to test parsers (#67)

.github/workflows/hub-ci.yml

@@ -0,0 +1,48 @@
+name: Hub-CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    name: Hub Parser/Scenario tests
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go 1.13
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+      id: go
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+    - name: Build release
+      run: make release
+    - name: clone and build hub CI tool
+      run: |
+        git clone https://github.com/crowdsecurity/hub-tests.git
+        cd hub-tests
+        make
+    - name: Create crowdsec test env with all parsers from the release
+      run: |
+        cd crowdsec-pull
+        ./test_env.sh
+        cd tests
+        for i in `./cscli -c dev.yaml list parsers -a -o json | jq -r ".[].name" ` ; do 
+          ./cscli -c dev.yaml install parser $i ; 
+        done
+    - name: Setup hub ci in crowdsec
+      working-directory: ./crowdsec-pull/tests/
+      run: |
+        cp -R ../../hub-tests/tests .
+        cp ../../hub-tests/main .
+    - name: Run the HUB CI
+      working-directory: ./crowdsec-pull/tests/
+      run: |
+        for i in `find ./tests -mindepth 1 -maxdepth 1 -type d` ; do 
+          echo "::group::Test-${i}" ; 
+          ./main $i || (echo "::error file=${i}::Failed test for ${i}" ; diff ${i}"/results.yaml" ${i}"/results.yaml.fail") ;
+          echo "::endgroup::" ;
+        done ;

README.md

@@ -12,6 +12,7 @@
 <img src="https://github.com/crowdsecurity/crowdsec/workflows/build-binary-package/badge.svg">
 <img src="https://goreportcard.com/badge/github.com/crowdsecurity/crowdsec">
 <img src="https://img.shields.io/github/license/crowdsecurity/crowdsec">
+<img src="https://github.com/crowdsecurity/crowdsec/workflows/Hub-CI/badge.svg">
 </p>
 
 <p align="center">

cmd/crowdsec/acquisition.go

@@ -1,34 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
-)
-
-func loadAcquisition() (*acquisition.FileAcquisCtx, error) {
-	var acquisitionCTX *acquisition.FileAcquisCtx
-	var err error
-	/*Init the acqusition : from cli or from acquis.yaml file*/
-	if cConfig.SingleFile != "" {
-		var input acquisition.FileCtx
-		input.Filename = cConfig.SingleFile
-		input.Mode = acquisition.CATMODE
-		input.Labels = make(map[string]string)
-		input.Labels["type"] = cConfig.SingleFileLabel
-		acquisitionCTX, err = acquisition.InitReaderFromFileCtx([]acquisition.FileCtx{input})
-	} else { /* Init file reader if we tail */
-		acquisitionCTX, err = acquisition.InitReader(cConfig.AcquisitionFile)
-	}
-	if err != nil {
-		return nil, fmt.Errorf("unable to start file acquisition, bailout %v", err)
-	}
-	if acquisitionCTX == nil {
-		return nil, fmt.Errorf("no inputs to process")
-	}
-	if cConfig.Profiling {
-		acquisitionCTX.Profiling = true
-	}
-
-	return acquisitionCTX, nil
-}

cmd/crowdsec/main.go

@@ -294,7 +294,7 @@ func main() {
 	log.Warningf("Starting processing data")
 
 	//Init the acqusition : from cli or from acquis.yaml file
-	acquisitionCTX, err = loadAcquisition()
+	acquisitionCTX, err = acquisition.LoadAcquisitionConfig(cConfig)
 	if err != nil {
 		log.Fatalf("Failed to start acquisition : %s", err)
 	}

pkg/acquisition/file_reader.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 
@@ -66,6 +67,33 @@ var ReaderHits = prometheus.NewCounterVec(
 	[]string{"source"},
 )
 
+func LoadAcquisitionConfig(cConfig *csconfig.CrowdSec) (*FileAcquisCtx, error) {
+	var acquisitionCTX *FileAcquisCtx
+	var err error
+	/*Init the acqusition : from cli or from acquis.yaml file*/
+	if cConfig.SingleFile != "" {
+		var input FileCtx
+		input.Filename = cConfig.SingleFile
+		input.Mode = CATMODE
+		input.Labels = make(map[string]string)
+		input.Labels["type"] = cConfig.SingleFileLabel
+		acquisitionCTX, err = InitReaderFromFileCtx([]FileCtx{input})
+	} else { /* Init file reader if we tail */
+		acquisitionCTX, err = InitReader(cConfig.AcquisitionFile)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("unable to start file acquisition, bailout %v", err)
+	}
+	if acquisitionCTX == nil {
+		return nil, fmt.Errorf("no inputs to process")
+	}
+	if cConfig.Profiling {
+		acquisitionCTX.Profiling = true
+	}
+
+	return acquisitionCTX, nil
+}
+
 func InitReader(cfg string) (*FileAcquisCtx, error) {
 	var files []FileCtx
 

pkg/parser/runtime.go

@@ -227,6 +227,9 @@ func stageidx(stage string, stages []string) int {
 	return -1
 }
 
+var ParseDump bool
+var StageParseCache map[string]map[string]types.Event
+
 func /*(u types.UnixParser)*/ Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error) {
 	var event types.Event = xp
 
@@ -250,7 +253,14 @@ func /*(u types.UnixParser)*/ Parse(ctx UnixParserCtx, xp types.Event, nodes []N
 		log.Tracef("INPUT '%s'", event.Line.Raw)
 	}
 
+	if ParseDump {
+		StageParseCache = make(map[string]map[string]types.Event)
+	}
+
 	for _, stage := range ctx.Stages {
+		if ParseDump {
+			StageParseCache[stage] = make(map[string]types.Event)
+		}
 		/* if the node is forward in stages, seek to its stage */
 		/* this is for example used by testing system to inject logs in post-syslog-parsing phase*/
 		if stageidx(event.Stage, ctx.Stages) > stageidx(stage, ctx.Stages) {
@@ -267,14 +277,14 @@ func /*(u types.UnixParser)*/ Parse(ctx UnixParserCtx, xp types.Event, nodes []N
 
 		isStageOK := false
 		for idx, node := range nodes {
-			clog := log.WithFields(log.Fields{
-				"node-name": node.rn,
-				"stage":     event.Stage,
-			})
 			//Only process current stage's nodes
 			if event.Stage != node.Stage {
 				continue
 			}
+			clog := log.WithFields(log.Fields{
+				"node-name": node.rn,
+				"stage":     event.Stage,
+			})
 			clog.Tracef("Processing node %d/%d -> %s", idx, len(nodes), node.rn)
 			if ctx.Profiling {
 				node.Profiling = true
@@ -286,6 +296,13 @@ func /*(u types.UnixParser)*/ Parse(ctx UnixParserCtx, xp types.Event, nodes []N
 			clog.Tracef("node (%s) ret : %v", node.rn, ret)
 			if ret {
 				isStageOK = true
+				if ParseDump {
+					evtcopy := types.Event{}
+					if err := types.Clone(&event, &evtcopy); err != nil {
+						log.Fatalf("while cloning Event in parser : %s", err)
+					}
+					StageParseCache[stage][node.Name] = evtcopy
+				}
 			}
 			if ret && node.OnSuccess == "next_stage" {
 				clog.Debugf("node successful, stop end stage %s", stage)

pkg/types/utils.go

@@ -1,7 +1,9 @@
 package types
 
 import (
+	"bytes"
 	"encoding/binary"
+	"encoding/gob"
 	"fmt"
 	"io"
 	"net"
@@ -93,3 +95,17 @@ func ConfigureLogger(clog *log.Logger) error {
 	clog.SetLevel(logLevel)
 	return nil
 }
+
+func Clone(a, b interface{}) error {
+
+	buff := new(bytes.Buffer)
+	enc := gob.NewEncoder(buff)
+	dec := gob.NewDecoder(buff)
+	if err := enc.Encode(a); err != nil {
+		return fmt.Errorf("failed cloning %T", a)
+	}
+	if err := dec.Decode(b); err != nil {
+		return fmt.Errorf("failed cloning %T", b)
+	}
+	return nil
+}