Pattern syntax consistence (#675)

* fix #667

* improved error message

* mark the compability, ordered pattern_syntax will be tagged as 'version 2'

* fix tests + add tests to check grok subpattern dependencies

pkg/cwversion/version.go

@@ -27,7 +27,7 @@ var (
 	BuildDate           string // = "I don't remember exactly"
 	Tag                 string // = "dev"
 	GoVersion           string // = "1.13"
-	Constraint_parser   = ">= 1.0, < 2.0"
+	Constraint_parser   = ">= 1.0, <= 2.0"
 	Constraint_scenario = ">= 1.0, < 3.0"
 	Constraint_api      = "v1"
 	Constraint_acquis   = ">= 1.0, < 2.0"

pkg/parser/node.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/antonmedv/expr"
+	yaml "gopkg.in/yaml.v2"
 
 	"github.com/antonmedv/expr/vm"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
@@ -48,7 +49,8 @@ type Node struct {
 
 	/* If the node is actually a leaf, it can have : grok, enrich, statics */
 	//pattern_syntax are named grok patterns that are re-utilised over several grok patterns
-	SubGroks map[string]string `yaml:"pattern_syntax,omitempty"`
+	SubGroks yaml.MapSlice `yaml:"pattern_syntax,omitempty"`
+
 	//Holds a grok pattern
 	Grok types.GrokPattern `yaml:"grok,omitempty"`
 	//Statics can be present in any type of node and is executed last
@@ -402,13 +404,14 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx []EnricherCtx) error {
 	}
 
 	/* handle pattern_syntax and groks */
-	for node, pattern := range n.SubGroks {
-		n.Logger.Tracef("Adding subpattern '%s' : '%s'", node, pattern)
-		if err := pctx.Grok.Add(node, pattern); err != nil {
-			n.Logger.Errorf("Unable to compile subpattern %s : %v", node, err)
+	for _, pattern := range n.SubGroks {
+		n.Logger.Tracef("Adding subpattern '%s' : '%s'", pattern.Key, pattern.Value)
+		if err := pctx.Grok.Add(pattern.Key.(string), pattern.Value.(string)); err != nil {
+			n.Logger.Errorf("Unable to compile subpattern %s : %v", pattern.Key, err)
 			return err
 		}
 	}
+
 	/* load grok by name or compile in-place */
 	if n.Grok.RegexpName != "" {
 		n.Logger.Tracef("+ Regexp Compilation '%s'", n.Grok.RegexpName)

pkg/parser/node_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/crowdsecurity/crowdsec/pkg/types"
+	yaml "gopkg.in/yaml.v2"
 )
 
 func TestParserConfigs(t *testing.T) {
@@ -25,20 +26,27 @@ func TestParserConfigs(t *testing.T) {
 		//empty node
 		{&Node{Debug: true, Stage: "s00", Filter: "true"}, false, false},
 		//bad subgrok
-		{&Node{Debug: true, Stage: "s00", SubGroks: map[string]string{"FOOBAR": "[a-$"}}, false, true},
+		{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-$")}}}, false, true},
 		//valid node with grok pattern
-		{&Node{Debug: true, Stage: "s00", SubGroks: map[string]string{"FOOBAR": "[a-z]"}, Grok: types.GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true},
+		{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-z]")}}, Grok: types.GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true},
 		//bad node success
 		{&Node{Debug: true, Stage: "s00", OnSuccess: "ratat", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, false, false},
 		//ok node success
 		{&Node{Debug: true, Stage: "s00", OnSuccess: "continue", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true},
 		//valid node with grok sub-pattern used by name
-		{&Node{Debug: true, Stage: "s00", SubGroks: map[string]string{"FOOBARx": "[a-z] %{DATA:lol}$"}, Grok: types.GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true},
+		{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBARx"), Value: string("[a-z] %{DATA:lol}$")}}, Grok: types.GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true},
 		//node with unexisting grok pattern
 		{&Node{Debug: true, Stage: "s00", Grok: types.GrokPattern{RegexpName: "RATATA", TargetField: "t"}}, false, true},
-
-		//bad grok pattern
-		//{&Node{Debug: true, Grok: []GrokPattern{ GrokPattern{}, }}, false},
+		//node with grok pattern dependencies
+		{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{
+			{Key: string("SUBGROK"), Value: string("[a-z]")},
+			{Key: string("MYGROK"), Value: string("[a-z]%{SUBGROK}")},
+		}, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROK:extr}$", TargetField: "t"}}, true, true},
+		//node with broken grok pattern dependencies
+		{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{
+			{Key: string("SUBGROKBIS"), Value: string("[a-z]%{MYGROKBIS}")},
+			{Key: string("MYGROKBIS"), Value: string("[a-z]")},
+		}, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true},
 	}
 	for idx := range CfgTests {
 		err := CfgTests[idx].NodeCfg.compile(pctx, []EnricherCtx{})

pkg/parser/stage.go

@@ -90,7 +90,7 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx []EnricherCtx)
 				log.Fatalf("Failed to check version : %s", err)
 			}
 			if !ok {
-				log.Errorf("%s doesn't satisfy parser format %s, skip", node.FormatVersion, cwversion.Constraint_parser)
+				log.Errorf("%s : %s doesn't satisfy parser format %s, skip", node.Name, node.FormatVersion, cwversion.Constraint_parser)
 				continue
 			}