add for parsers

cmd/crowdsec/main.go

@@ -105,12 +105,12 @@ func main() {
 
 	log.Infof("Loading grok library")
 	/* load base regexps for two grok parsers */
-	parserCTX, err = p.Init(map[string]interface{}{"patterns": cConfig.ConfigFolder + string("/patterns/")})
+	parserCTX, err = p.Init(map[string]interface{}{"patterns": cConfig.ConfigFolder + string("/patterns/"), "data": cConfig.DataFolder})
 	if err != nil {
 		log.Errorf("failed to initialize parser : %v", err)
 		return
 	}
-	postOverflowCTX, err = p.Init(map[string]interface{}{"patterns": cConfig.ConfigFolder + string("/patterns/")})
+	postOverflowCTX, err = p.Init(map[string]interface{}{"patterns": cConfig.ConfigFolder + string("/patterns/"), "data": cConfig.DataFolder})
 	if err != nil {
 		log.Errorf("failed to initialize postoverflow : %v", err)
 		return

pkg/parser/node.go

@@ -53,7 +53,8 @@ type Node struct {
 	//Statics can be present in any type of node and is executed last
 	Statics []types.ExtraField `yaml:"statics,omitempty"`
 	//Whitelists
-	Whitelist types.Whitelist `yaml:"whitelist,omitempty"`
+	Whitelist types.Whitelist     `yaml:"whitelist,omitempty"`
+	Data      []*types.DataSource `yaml:"data,omitempty"`
 }
 
 func (n *Node) validate(pctx *UnixParserCtx) error {

pkg/parser/stage.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 
 	log "github.com/sirupsen/logrus"
 
@@ -42,6 +43,7 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx) ([]Node, error) {
 	tmpstages := make(map[string]bool)
 	pctx.Stages = []string{}
 
+	exprhelpers.Init()
 	for _, stageFile := range stageFiles {
 		if !strings.HasSuffix(stageFile.Filename, ".yaml") {
 			log.Warningf("skip non yaml : %s", stageFile.Filename)
@@ -109,6 +111,12 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx) ([]Node, error) {
 			if node.Stage == "" {
 				continue
 			}
+
+			if len(node.Data) > 0 {
+				for _, data := range node.Data {
+					err = exprhelpers.FileInit(pctx.DataFolder, data.DestPath)
+				}
+			}
 			nodes = append(nodes, node)
 			nodesCount++
 		}

pkg/parser/unix_parser.go

@@ -12,9 +12,10 @@ type UnixParser struct {
 }
 
 type UnixParserCtx struct {
-	Grok      grokky.Host
-	Stages    []string
-	Profiling bool
+	Grok       grokky.Host
+	Stages     []string
+	Profiling  bool
+	DataFolder string
 }
 
 func (u UnixParser) IsParsable(ctx interface{}, l types.Line) (bool, error) {
@@ -28,6 +29,7 @@ func (u UnixParser) Init(c map[string]interface{}) (*UnixParserCtx, error) {
 	if err != nil {
 		return nil, err
 	}
+	r.DataFolder = c["data"].(string)
 	for _, f := range files {
 		log.Debugf("Loading %s", f.Name())
 		if err := r.Grok.AddFromFile(c["patterns"].(string) + f.Name()); err != nil {