Allow for acquisition files to be specified from a directory as well (#619)

* allow a acquisition_dir in crowdsec's config + change the behaviour of config loading so that it's working with a list instead. keep backward compat with acquisition_path

* remove the default behaviour of 'guessing' acquis path if param isn't present, and error

cmd/crowdsec-cli/config.go

@@ -55,13 +55,40 @@ func backupConfigToDirectory(dirPath string) error {
 		}
 		log.Infof("Saved simulation to %s", backupSimulation)
 	}
+
+	/*
+	   - backup AcquisitionFilePath
+	   - backup the other files of acquisition directory
+	*/
 	if csConfig.Crowdsec != nil && csConfig.Crowdsec.AcquisitionFilePath != "" {
 		backupAcquisition := fmt.Sprintf("%s/acquis.yaml", dirPath)
 		if err = types.CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil {
 			return fmt.Errorf("failed copy %s to %s : %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err)
 		}
-		log.Infof("Saved acquis to %s", backupAcquisition)
 	}
+
+	acquisBackupDir := dirPath + "/acquis/"
+	if err = os.Mkdir(acquisBackupDir, 0600); err != nil {
+		return fmt.Errorf("error while creating %s : %s", acquisBackupDir, err)
+	}
+
+	if csConfig.Crowdsec != nil && len(csConfig.Crowdsec.AcquisitionFiles) > 0 {
+		for _, acquisFile := range csConfig.Crowdsec.AcquisitionFiles {
+			/*if it was the default one, it was already backup'ed*/
+			if csConfig.Crowdsec.AcquisitionFilePath == acquisFile {
+				continue
+			}
+			targetFname, err := filepath.Abs(acquisBackupDir + filepath.Base(acquisFile))
+			if err != nil {
+				return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir)
+			}
+			if err = types.CopyFile(acquisFile, targetFname); err != nil {
+				return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
+			}
+			log.Infof("Saved acquis %s to %s", acquisFile, targetFname)
+		}
+	}
+
 	if ConfigFilePath != "" {
 		backupMain := fmt.Sprintf("%s/config.yaml", dirPath)
 		if err = types.CopyFile(ConfigFilePath, backupMain); err != nil {
@@ -186,13 +213,57 @@ func restoreConfigFromDirectory(dirPath string) error {
 		}
 	}
 
+	/*if there is a acquisition dir, restore its content*/
+	if csConfig.Crowdsec.AcquisitionDirPath != "" {
+		if err = os.Mkdir(csConfig.Crowdsec.AcquisitionDirPath, 0600); err != nil {
+			return fmt.Errorf("error while creating %s : %s", csConfig.Crowdsec.AcquisitionDirPath, err)
+		}
+
+	}
+
+	//if there was a single one
 	backupAcquisition := fmt.Sprintf("%s/acquis.yaml", dirPath)
 	if _, err = os.Stat(backupAcquisition); err == nil {
+		log.Debugf("restoring backup'ed %s", backupAcquisition)
 		if err = types.CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil {
 			return fmt.Errorf("failed copy %s to %s : %s", backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath, err)
 		}
 	}
 
+	//if there is files in the acquis backup dir, restore them
+	acquisBackupDir := dirPath + "/acquis/*.yaml"
+	if acquisFiles, err := filepath.Glob(acquisBackupDir); err == nil {
+		for _, acquisFile := range acquisFiles {
+			targetFname, err := filepath.Abs(csConfig.Crowdsec.AcquisitionDirPath + "/" + filepath.Base(acquisFile))
+			if err != nil {
+				return errors.Wrapf(err, "while saving %s to %s", acquisFile, targetFname)
+			}
+			log.Debugf("restoring %s to %s", acquisFile, targetFname)
+			if err = types.CopyFile(acquisFile, targetFname); err != nil {
+				return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
+			}
+		}
+	}
+
+	if csConfig.Crowdsec != nil && len(csConfig.Crowdsec.AcquisitionFiles) > 0 {
+		for _, acquisFile := range csConfig.Crowdsec.AcquisitionFiles {
+			log.Infof("backup filepath from dir -> %s", acquisFile)
+			/*if it was the default one, it was already backup'ed*/
+			if csConfig.Crowdsec.AcquisitionFilePath == acquisFile {
+				log.Infof("skip this one")
+				continue
+			}
+			targetFname, err := filepath.Abs(acquisBackupDir + filepath.Base(acquisFile))
+			if err != nil {
+				return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir)
+			}
+			if err = types.CopyFile(acquisFile, targetFname); err != nil {
+				return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
+			}
+			log.Infof("Saved acquis %s to %s", acquisFile, targetFname)
+		}
+	}
+
 	if err = RestoreHub(dirPath); err != nil {
 		return fmt.Errorf("failed to restore hub config : %s", err)
 	}

config/dev.yaml

@@ -11,7 +11,7 @@ config_paths:
   #hub_dir: /etc/crowdsec/hub/
   #index_path: ./config/hub/.index.json
 crowdsec_service:
-  #acquisition_path: ./config/acquis.yaml
+  acquisition_path: ./config/acquis.yaml
   parser_routines: 1
 cscli:
   output: human

docs/v1.X/docs/references/acquisition.md

@@ -1,6 +1,14 @@
 # Acquisition format
 
-The `/etc/crowdsec/acquis.yaml` defines which files are read by crowdsec at runtime.
+The `crowdsec_service` section of configuration supports `acquisition_path` and `acquisition_dir` (>1.0.7).
+
+The default setting is to have `acquisition_path` pointing to `/etc/crowdsec/acquis.yaml`.
+
+`acquisition_dir` can be set to point to a directory where every `.yaml` file is considered as a valid acquisition configuration file.
+
+
+
+The acquisition file(s) define which source of information (ie. files or journald streams) are read by crowdsec at runtime.
 The file is a list of object representing groups of files to read, with the following properties.
 
 A least one of :

docs/v1.X/docs/references/crowdsec-config.md

@@ -24,6 +24,7 @@ config_paths:
   index_path: /etc/crowdsec/hub/.index.json
 crowdsec_service:
   acquisition_path: /etc/crowdsec/acquis.yaml
+  #acquisition_dir: /etc/crowdsec/acquis/
   parser_routines: 1
   buckets_routines: 1
   output_routines: 1
@@ -108,6 +109,7 @@ config_paths:
   index_path: <path_to_hub_index_file>
 crowdsec_service:
   acquisition_path: <acqusition_file_path>
+  acquisition_dir: <acquisition_dir_path>
   parser_routines: <number_of_parser_routines>
   buckets_routines: <number_of_buckets_routines>
   output_routines: <number_of_output_routines>
@@ -242,6 +244,7 @@ This section is only used by crowdsec agent.
 ```yaml
 crowdsec_service:
   acquisition_path: <acqusition_file_path>
+  acquisition_dir: <acqusition_dir_path>
   parser_routines: <number_of_parser_routines>
   buckets_routines: <number_of_buckets_routines>
   output_routines: <number_of_output_routines>
@@ -268,6 +271,11 @@ Number of dedicated goroutines for pushing data to local api.
 
 Path to the yaml file containing logs that needs to be read.
 
+#### `acquisition_dir`
+> string
+
+(>1.0.7) Path to a directory where each yaml is considered as a acquisition configuration file containing logs that needs to be read.
+
 
 ### `cscli`
 

pkg/acquisition/acquisition.go

@@ -108,30 +108,35 @@ func DataSourceConfigure(config DataSourceCfg) (DataSource, error) {
 func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, error) {
 
 	var sources []DataSource
+	var acquisSources = config.AcquisitionFiles
 
-	yamlFile, err := os.Open(config.AcquisitionFilePath)
-	if err != nil {
-		return nil, errors.Wrapf(err, "can't open %s", config.AcquisitionFilePath)
-	}
-	dec := yaml.NewDecoder(yamlFile)
-	dec.SetStrict(true)
-	for {
-		sub := DataSourceCfg{}
-		err = dec.Decode(&sub)
+	for _, acquisFile := range acquisSources {
+		log.Infof("loading acquisition file : %s", acquisFile)
+		yamlFile, err := os.Open(acquisFile)
 		if err != nil {
-			if err == io.EOF {
-				log.Tracef("End of yaml file")
-				break
-			}
-			return nil, errors.Wrap(err, fmt.Sprintf("failed to yaml decode %s", config.AcquisitionFilePath))
+			return nil, errors.Wrapf(err, "can't open %s", acquisFile)
 		}
-		src, err := DataSourceConfigure(sub)
-		if err != nil {
-			log.Warningf("while configuring datasource : %s", err)
-			continue
+		dec := yaml.NewDecoder(yamlFile)
+		dec.SetStrict(true)
+		for {
+			sub := DataSourceCfg{}
+			err = dec.Decode(&sub)
+			if err != nil {
+				if err == io.EOF {
+					log.Tracef("End of yaml file")
+					break
+				}
+				return nil, errors.Wrap(err, fmt.Sprintf("failed to yaml decode %s", acquisFile))
+			}
+			src, err := DataSourceConfigure(sub)
+			if err != nil {
+				log.Warningf("while configuring datasource : %s", err)
+				continue
+			}
+			sources = append(sources, src)
 		}
-		sources = append(sources, src)
 	}
+
 	return sources, nil
 }
 

pkg/acquisition/acquisition_test.go

@@ -16,19 +16,19 @@ import (
 func TestConfigLoading(t *testing.T) {
 	//bad filename
 	cfg := csconfig.CrowdsecServiceCfg{
-		AcquisitionFilePath: "./tests/xxx.yaml",
+		AcquisitionFiles: []string{"./tests/xxx.yaml"},
 	}
 	_, err := LoadAcquisitionFromFile(&cfg)
 	assert.Contains(t, fmt.Sprintf("%s", err), "can't open ./tests/xxx.yaml: open ./tests/xxx.yaml: no such file or directory")
 	//bad config file
 	cfg = csconfig.CrowdsecServiceCfg{
-		AcquisitionFilePath: "./tests/test.log",
+		AcquisitionFiles: []string{"./tests/test.log"},
 	}
 	_, err = LoadAcquisitionFromFile(&cfg)
 	assert.Contains(t, fmt.Sprintf("%s", err), "failed to yaml decode ./tests/test.log: yaml: unmarshal errors")
 	//correct config file
 	cfg = csconfig.CrowdsecServiceCfg{
-		AcquisitionFilePath: "./tests/acquis_test.yaml",
+		AcquisitionFiles: []string{"./tests/acquis_test.yaml"},
 	}
 	srcs, err := LoadAcquisitionFromFile(&cfg)
 	if err != nil {

pkg/csconfig/config.go

@@ -83,9 +83,24 @@ func (c *GlobalConfig) LoadConfiguration() error {
 	}
 
 	if c.Crowdsec != nil {
-		if c.Crowdsec.AcquisitionFilePath == "" {
-			c.Crowdsec.AcquisitionFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/acquis.yaml")
+		if c.Crowdsec.AcquisitionFilePath != "" {
+			log.Infof("non-empty acquisition file path %s", c.Crowdsec.AcquisitionFilePath)
+			if _, err := os.Stat(c.Crowdsec.AcquisitionFilePath); err != nil {
+				return errors.Wrapf(err, "while checking acquisition path %s", c.Crowdsec.AcquisitionFilePath)
+			}
+			c.Crowdsec.AcquisitionFiles = append(c.Crowdsec.AcquisitionFiles, c.Crowdsec.AcquisitionFilePath)
+		}
+		if c.Crowdsec.AcquisitionDirPath != "" {
+			files, err := filepath.Glob(c.Crowdsec.AcquisitionDirPath + "/*.yaml")
+			c.Crowdsec.AcquisitionFiles = append(c.Crowdsec.AcquisitionFiles, files...)
+			if err != nil {
+				return errors.Wrap(err, "while globing acquis_dir")
+			}
+		}
+		if c.Crowdsec.AcquisitionDirPath == "" && c.Crowdsec.AcquisitionFilePath == "" {
+			return fmt.Errorf("no acquisition_path nor acquisition_dir")
 		}
+
 		c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir
 		c.Crowdsec.DataDir = c.ConfigPaths.DataDir
 		c.Crowdsec.HubDir = c.ConfigPaths.HubDir
@@ -276,6 +291,9 @@ func (c *GlobalConfig) CleanupPaths() error {
 			&c.Common.WorkingDir,
 		}
 		for _, k := range CommonCleanup {
+			if *k == "" {
+				continue
+			}
 			*k, err = filepath.Abs(*k)
 			if err != nil {
 				return errors.Wrap(err, "failed to clean path")
@@ -288,6 +306,9 @@ func (c *GlobalConfig) CleanupPaths() error {
 			&c.Crowdsec.AcquisitionFilePath,
 		}
 		for _, k := range crowdsecCleanup {
+			if *k == "" {
+				continue
+			}
 			*k, err = filepath.Abs(*k)
 			if err != nil {
 				return errors.Wrap(err, "failed to clean path")
@@ -304,6 +325,9 @@ func (c *GlobalConfig) CleanupPaths() error {
 			&c.ConfigPaths.SimulationFilePath,
 		}
 		for _, k := range configPathsCleanup {
+			if *k == "" {
+				continue
+			}
 			*k, err = filepath.Abs(*k)
 			if err != nil {
 				return errors.Wrap(err, "failed to clean path")

pkg/csconfig/crowdsec_service.go

@@ -2,7 +2,10 @@ package csconfig
 
 /*Configurations needed for crowdsec to load parser/scenarios/... + acquisition*/
 type CrowdsecServiceCfg struct {
-	AcquisitionFilePath  string            `yaml:"acquisition_path,omitempty"`
+	AcquisitionFilePath string `yaml:"acquisition_path,omitempty"`
+	AcquisitionDirPath  string `yaml:"acquisition_dir,omitempty"`
+
+	AcquisitionFiles     []string          `yaml:"-"`
 	ParserRoutinesCount  int               `yaml:"parser_routines"`
 	BucketsRoutinesCount int               `yaml:"buckets_routines"`
 	OutputRoutinesCount  int               `yaml:"output_routines"`

pkg/csconfig/tests/config.yaml

@@ -8,7 +8,7 @@ prometheus:
   enabled: true
   level: full
 crowdsec_service:
-#  acquisition_path: ./config/acquis.yaml
+  acquisition_path: ./tests/acquis.yaml
   parser_routines: 1
 cscli:
   output: human