download data ok

cmd/crowdsec/main.go

@@ -204,11 +204,11 @@ func main() {
 		for _, scenarios := range CustomScenarios {
 			bucketFiles = append(bucketFiles, scenarios.Filename)
 		}
-		holders, outputEventChan, err = leaky.LoadBuckets(bucketFiles)
+		holders, outputEventChan, err = leaky.LoadBuckets(bucketFiles, cConfig.DataFolder)
 
 	} else {
 		log.Infof("Loading scenarios")
-		holders, outputEventChan, err = leaky.Init(map[string]string{"patterns": cConfig.ConfigFolder + "/scenarios/"})
+		holders, outputEventChan, err = leaky.Init(map[string]string{"patterns": cConfig.ConfigFolder + "/scenarios/", "data": cConfig.DataFolder})
 	}
 	if err != nil {
 		log.Fatalf("Scenario loading failed : %v", err)

pkg/cwhub/hubMgmt.go

@@ -749,7 +749,6 @@ func DownloadItem(target Item, tdir string, overwrite bool, dataFolder string) (
 	target.Tainted = false
 	target.UpToDate = true
 
-	log.Infof("Installing : %+v \n", string(body))
 	dec := yaml.NewDecoder(bytes.NewReader(body))
 	for {
 		data := &types.DataSet{}

pkg/exprhelpers/exprlib.go

@@ -1,12 +1,19 @@
 package exprhelpers
 
 import (
+	"bufio"
+	"os"
+	"path"
+	"regexp"
 	"strconv"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
 )
 
+var dataFile map[string][]string
+var dataFileRegex map[string][]*regexp.Regexp
+
 func Atof(x string) float64 {
 	log.Debugf("debug atof %s", x)
 	ret, err := strconv.ParseFloat(x, 64)
@@ -26,9 +33,68 @@ func EndsWith(s string, suff string) bool {
 
 func GetExprEnv(ctx map[string]interface{}) map[string]interface{} {
 
-	var ExprLib = map[string]interface{}{"Atof": Atof, "JsonExtract": JsonExtract, "JsonExtractLib": JsonExtractLib}
+	var ExprLib = map[string]interface{}{"Atof": Atof, "JsonExtract": JsonExtract, "JsonExtractLib": JsonExtractLib, "File": File, "RegexpInFile": RegexpInFile}
 	for k, v := range ctx {
 		ExprLib[k] = v
 	}
 	return ExprLib
 }
+
+func Init() error {
+	log.Infof("Expr helper initiated")
+	dataFile = make(map[string][]string)
+	dataFileRegex = make(map[string][]*regexp.Regexp)
+	return nil
+}
+
+func FileInit(fileFolder string, filename string) error {
+	filepath := path.Join(fileFolder, filename)
+	file, err := os.Open(filepath)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer file.Close()
+
+	if _, ok := dataFile[filename]; !ok {
+		dataFile[filename] = []string{}
+	}
+	fileType := "string"
+	scanner := bufio.NewScanner(file)
+	scanner.Scan()
+	if scanner.Text() == "#type: regex" { // if file contains, it should have this header
+		fileType = "regex"
+	}
+	for scanner.Scan() {
+		if fileType == "regex" {
+			dataFileRegex[filename] = append(dataFileRegex[filename], regexp.MustCompile(scanner.Text()))
+		} else {
+			dataFile[filename] = append(dataFile[filename], scanner.Text())
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+	return nil
+}
+
+func File(filename string) []string {
+	if _, ok := dataFile[filename]; ok {
+		return dataFile[filename]
+	}
+	log.Errorf("file '%s' not found for expr library", filename)
+	return []string{}
+}
+
+func RegexpInFile(data string, filename string) bool {
+	if _, ok := dataFileRegex[filename]; ok {
+		for _, re := range dataFileRegex[filename] {
+			if re.Match([]byte(data)) {
+				return true
+			}
+		}
+	} else {
+		log.Errorf("file '%s' not found for expr library", filename)
+	}
+	return false
+}

pkg/leakybucket/manager.go

@@ -55,6 +55,7 @@ type BucketFactory struct {
 	Filename       string              `yaml:"-"`
 	RunTimeFilter  *vm.Program         `json:"-"`
 	RunTimeGroupBy *vm.Program         `json:"-"`
+	Data           []*types.DataSource `yaml:"data,omitempty"`
 	leakspeed      time.Duration       //internal representation of `Leakspeed`
 	duration       time.Duration       //internal representation of `Duration`
 	ret            chan types.Event    //the bucket-specific output chan for overflows
@@ -101,16 +102,18 @@ func ValidateFactory(b *BucketFactory) error {
 
 /* Init recursively process yaml files from a directory and loads them as BucketFactory */
 func Init(cfg map[string]string) ([]BucketFactory, chan types.Event, error) {
-	return LoadBucketDir(cfg["patterns"])
+	return LoadBucketDir(cfg["patterns"], cfg["data"])
 }
 
-func LoadBuckets(files []string) ([]BucketFactory, chan types.Event, error) {
+func LoadBuckets(files []string, dataFolder string) ([]BucketFactory, chan types.Event, error) {
 	var (
 		ret      []BucketFactory = []BucketFactory{}
 		response chan types.Event
 	)
 
 	var seed namegenerator.Generator = namegenerator.NewNameGenerator(time.Now().UTC().UnixNano())
+	exprhelpers.Init()
+
 	response = make(chan types.Event, 1)
 	for _, f := range files {
 		log.Debugf("Loading %s", f)
@@ -160,7 +163,7 @@ func LoadBuckets(files []string) ([]BucketFactory, chan types.Event, error) {
 			g.Filename = filepath.Clean(f)
 			g.BucketName = seed.Generate()
 			g.ret = response
-			err = LoadBucket(&g)
+			err = LoadBucket(&g, dataFolder)
 			if err != nil {
 				log.Errorf("Failed to load bucket : %v", err)
 				return nil, nil, fmt.Errorf("loadBucket failed : %v", err)
@@ -172,7 +175,7 @@ func LoadBuckets(files []string) ([]BucketFactory, chan types.Event, error) {
 	return ret, response, nil
 }
 
-func LoadBucketDir(dir string) ([]BucketFactory, chan types.Event, error) {
+func LoadBucketDir(dir string, dataFolder string) ([]BucketFactory, chan types.Event, error) {
 	var (
 		filenames []string
 	)
@@ -183,11 +186,11 @@ func LoadBucketDir(dir string) ([]BucketFactory, chan types.Event, error) {
 	for _, f := range files {
 		filenames = append(filenames, dir+f.Name())
 	}
-	return LoadBuckets(filenames)
+	return LoadBuckets(filenames, dataFolder)
 }
 
 /* Init recursively process yaml files from a directory and loads them as BucketFactory */
-func LoadBucket(g *BucketFactory) error {
+func LoadBucket(g *BucketFactory, dataFolder string) error {
 	var err error
 	if g.Debug {
 		var clog = logrus.New()
@@ -275,6 +278,12 @@ func LoadBucket(g *BucketFactory) error {
 		g.processors = append(g.processors, blackhole)
 	}
 
+	if len(g.Data) > 0 {
+		for _, data := range g.Data {
+			err = exprhelpers.FileInit(dataFolder, data.DestPath)
+		}
+	}
+
 	g.output = false
 	if err := ValidateFactory(g); err != nil {
 		return fmt.Errorf("invalid bucket from %s : %v", g.Filename, err)

pkg/types/dataset.go

@@ -10,13 +10,13 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-type dataSource struct {
+type DataSource struct {
 	SourceURL string `yaml:"source_url"`
 	DestPath  string `yaml:"dest_file"`
 }
 
 type DataSet struct {
-	Data []*dataSource `yaml:"data",omitempty`
+	Data []*DataSource `yaml:"data",omitempty`
 }
 
 func downloadFile(url string, destPath string) error {
@@ -59,7 +59,7 @@ func downloadFile(url string, destPath string) error {
 	return nil
 }
 
-func GetData(data []*dataSource, dataDir string) error {
+func GetData(data []*DataSource, dataDir string) error {
 	for _, dataS := range data {
 		destPath := path.Join(dataDir, dataS.DestPath)
 		log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)