diff --git a/cmd/crowdsec-cli/config.go b/cmd/crowdsec-cli/config.go index dce85e659..54042b6bd 100644 --- a/cmd/crowdsec-cli/config.go +++ b/cmd/crowdsec-cli/config.go @@ -55,13 +55,40 @@ func backupConfigToDirectory(dirPath string) error { } log.Infof("Saved simulation to %s", backupSimulation) } + + /* + - backup AcquisitionFilePath + - backup the other files of acquisition directory + */ if csConfig.Crowdsec != nil && csConfig.Crowdsec.AcquisitionFilePath != "" { backupAcquisition := fmt.Sprintf("%s/acquis.yaml", dirPath) if err = types.CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil { return fmt.Errorf("failed copy %s to %s : %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err) } - log.Infof("Saved acquis to %s", backupAcquisition) } + + acquisBackupDir := dirPath + "/acquis/" + if err = os.Mkdir(acquisBackupDir, 0600); err != nil { + return fmt.Errorf("error while creating %s : %s", acquisBackupDir, err) + } + + if csConfig.Crowdsec != nil && len(csConfig.Crowdsec.AcquisitionFiles) > 0 { + for _, acquisFile := range csConfig.Crowdsec.AcquisitionFiles { + /*if it was the default one, it was already backup'ed*/ + if csConfig.Crowdsec.AcquisitionFilePath == acquisFile { + continue + } + targetFname, err := filepath.Abs(acquisBackupDir + filepath.Base(acquisFile)) + if err != nil { + return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir) + } + if err = types.CopyFile(acquisFile, targetFname); err != nil { + return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) + } + log.Infof("Saved acquis %s to %s", acquisFile, targetFname) + } + } + if ConfigFilePath != "" { backupMain := fmt.Sprintf("%s/config.yaml", dirPath) if err = types.CopyFile(ConfigFilePath, backupMain); err != nil { @@ -186,13 +213,57 @@ func restoreConfigFromDirectory(dirPath string) error { } } + /*if there is a acquisition dir, restore its content*/ + if csConfig.Crowdsec.AcquisitionDirPath != "" { + if err = os.Mkdir(csConfig.Crowdsec.AcquisitionDirPath, 0600); err != nil { + return fmt.Errorf("error while creating %s : %s", csConfig.Crowdsec.AcquisitionDirPath, err) + } + + } + + //if there was a single one backupAcquisition := fmt.Sprintf("%s/acquis.yaml", dirPath) if _, err = os.Stat(backupAcquisition); err == nil { + log.Debugf("restoring backup'ed %s", backupAcquisition) if err = types.CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath, err) } } + //if there is files in the acquis backup dir, restore them + acquisBackupDir := dirPath + "/acquis/*.yaml" + if acquisFiles, err := filepath.Glob(acquisBackupDir); err == nil { + for _, acquisFile := range acquisFiles { + targetFname, err := filepath.Abs(csConfig.Crowdsec.AcquisitionDirPath + "/" + filepath.Base(acquisFile)) + if err != nil { + return errors.Wrapf(err, "while saving %s to %s", acquisFile, targetFname) + } + log.Debugf("restoring %s to %s", acquisFile, targetFname) + if err = types.CopyFile(acquisFile, targetFname); err != nil { + return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) + } + } + } + + if csConfig.Crowdsec != nil && len(csConfig.Crowdsec.AcquisitionFiles) > 0 { + for _, acquisFile := range csConfig.Crowdsec.AcquisitionFiles { + log.Infof("backup filepath from dir -> %s", acquisFile) + /*if it was the default one, it was already backup'ed*/ + if csConfig.Crowdsec.AcquisitionFilePath == acquisFile { + log.Infof("skip this one") + continue + } + targetFname, err := filepath.Abs(acquisBackupDir + filepath.Base(acquisFile)) + if err != nil { + return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir) + } + if err = types.CopyFile(acquisFile, targetFname); err != nil { + return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) + } + log.Infof("Saved acquis %s to %s", acquisFile, targetFname) + } + } + if err = RestoreHub(dirPath); err != nil { return fmt.Errorf("failed to restore hub config : %s", err) } diff --git a/config/dev.yaml b/config/dev.yaml index 5011102a3..f8573fb6c 100644 --- a/config/dev.yaml +++ b/config/dev.yaml @@ -11,7 +11,7 @@ config_paths: #hub_dir: /etc/crowdsec/hub/ #index_path: ./config/hub/.index.json crowdsec_service: - #acquisition_path: ./config/acquis.yaml + acquisition_path: ./config/acquis.yaml parser_routines: 1 cscli: output: human diff --git a/docs/v1.X/docs/references/acquisition.md b/docs/v1.X/docs/references/acquisition.md index 19f5453e4..d37927078 100644 --- a/docs/v1.X/docs/references/acquisition.md +++ b/docs/v1.X/docs/references/acquisition.md @@ -1,6 +1,14 @@ # Acquisition format -The `/etc/crowdsec/acquis.yaml` defines which files are read by crowdsec at runtime. +The `crowdsec_service` section of configuration supports `acquisition_path` and `acquisition_dir` (>1.0.7). + +The default setting is to have `acquisition_path` pointing to `/etc/crowdsec/acquis.yaml`. + +`acquisition_dir` can be set to point to a directory where every `.yaml` file is considered as a valid acquisition configuration file. + + + +The acquisition file(s) define which source of information (ie. files or journald streams) are read by crowdsec at runtime. The file is a list of object representing groups of files to read, with the following properties. A least one of : diff --git a/docs/v1.X/docs/references/crowdsec-config.md b/docs/v1.X/docs/references/crowdsec-config.md index 36f40c843..bcf5fb8de 100644 --- a/docs/v1.X/docs/references/crowdsec-config.md +++ b/docs/v1.X/docs/references/crowdsec-config.md @@ -24,6 +24,7 @@ config_paths: index_path: /etc/crowdsec/hub/.index.json crowdsec_service: acquisition_path: /etc/crowdsec/acquis.yaml + #acquisition_dir: /etc/crowdsec/acquis/ parser_routines: 1 buckets_routines: 1 output_routines: 1 @@ -108,6 +109,7 @@ config_paths: index_path: crowdsec_service: acquisition_path: + acquisition_dir: parser_routines: buckets_routines: output_routines: @@ -242,6 +244,7 @@ This section is only used by crowdsec agent. ```yaml crowdsec_service: acquisition_path: + acquisition_dir: parser_routines: buckets_routines: output_routines: @@ -268,6 +271,11 @@ Number of dedicated goroutines for pushing data to local api. Path to the yaml file containing logs that needs to be read. +#### `acquisition_dir` +> string + +(>1.0.7) Path to a directory where each yaml is considered as a acquisition configuration file containing logs that needs to be read. + ### `cscli` diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 18476e871..988cdfbf6 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -108,30 +108,35 @@ func DataSourceConfigure(config DataSourceCfg) (DataSource, error) { func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, error) { var sources []DataSource + var acquisSources = config.AcquisitionFiles - yamlFile, err := os.Open(config.AcquisitionFilePath) - if err != nil { - return nil, errors.Wrapf(err, "can't open %s", config.AcquisitionFilePath) - } - dec := yaml.NewDecoder(yamlFile) - dec.SetStrict(true) - for { - sub := DataSourceCfg{} - err = dec.Decode(&sub) + for _, acquisFile := range acquisSources { + log.Infof("loading acquisition file : %s", acquisFile) + yamlFile, err := os.Open(acquisFile) if err != nil { - if err == io.EOF { - log.Tracef("End of yaml file") - break + return nil, errors.Wrapf(err, "can't open %s", acquisFile) + } + dec := yaml.NewDecoder(yamlFile) + dec.SetStrict(true) + for { + sub := DataSourceCfg{} + err = dec.Decode(&sub) + if err != nil { + if err == io.EOF { + log.Tracef("End of yaml file") + break + } + return nil, errors.Wrap(err, fmt.Sprintf("failed to yaml decode %s", acquisFile)) } - return nil, errors.Wrap(err, fmt.Sprintf("failed to yaml decode %s", config.AcquisitionFilePath)) + src, err := DataSourceConfigure(sub) + if err != nil { + log.Warningf("while configuring datasource : %s", err) + continue + } + sources = append(sources, src) } - src, err := DataSourceConfigure(sub) - if err != nil { - log.Warningf("while configuring datasource : %s", err) - continue - } - sources = append(sources, src) } + return sources, nil } diff --git a/pkg/acquisition/acquisition_test.go b/pkg/acquisition/acquisition_test.go index 9af19de3b..ea1dca8c8 100644 --- a/pkg/acquisition/acquisition_test.go +++ b/pkg/acquisition/acquisition_test.go @@ -16,19 +16,19 @@ import ( func TestConfigLoading(t *testing.T) { //bad filename cfg := csconfig.CrowdsecServiceCfg{ - AcquisitionFilePath: "./tests/xxx.yaml", + AcquisitionFiles: []string{"./tests/xxx.yaml"}, } _, err := LoadAcquisitionFromFile(&cfg) assert.Contains(t, fmt.Sprintf("%s", err), "can't open ./tests/xxx.yaml: open ./tests/xxx.yaml: no such file or directory") //bad config file cfg = csconfig.CrowdsecServiceCfg{ - AcquisitionFilePath: "./tests/test.log", + AcquisitionFiles: []string{"./tests/test.log"}, } _, err = LoadAcquisitionFromFile(&cfg) assert.Contains(t, fmt.Sprintf("%s", err), "failed to yaml decode ./tests/test.log: yaml: unmarshal errors") //correct config file cfg = csconfig.CrowdsecServiceCfg{ - AcquisitionFilePath: "./tests/acquis_test.yaml", + AcquisitionFiles: []string{"./tests/acquis_test.yaml"}, } srcs, err := LoadAcquisitionFromFile(&cfg) if err != nil { diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index 2c8303e89..769687fa5 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -83,9 +83,24 @@ func (c *GlobalConfig) LoadConfiguration() error { } if c.Crowdsec != nil { - if c.Crowdsec.AcquisitionFilePath == "" { - c.Crowdsec.AcquisitionFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/acquis.yaml") + if c.Crowdsec.AcquisitionFilePath != "" { + log.Infof("non-empty acquisition file path %s", c.Crowdsec.AcquisitionFilePath) + if _, err := os.Stat(c.Crowdsec.AcquisitionFilePath); err != nil { + return errors.Wrapf(err, "while checking acquisition path %s", c.Crowdsec.AcquisitionFilePath) + } + c.Crowdsec.AcquisitionFiles = append(c.Crowdsec.AcquisitionFiles, c.Crowdsec.AcquisitionFilePath) } + if c.Crowdsec.AcquisitionDirPath != "" { + files, err := filepath.Glob(c.Crowdsec.AcquisitionDirPath + "/*.yaml") + c.Crowdsec.AcquisitionFiles = append(c.Crowdsec.AcquisitionFiles, files...) + if err != nil { + return errors.Wrap(err, "while globing acquis_dir") + } + } + if c.Crowdsec.AcquisitionDirPath == "" && c.Crowdsec.AcquisitionFilePath == "" { + return fmt.Errorf("no acquisition_path nor acquisition_dir") + } + c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir c.Crowdsec.DataDir = c.ConfigPaths.DataDir c.Crowdsec.HubDir = c.ConfigPaths.HubDir @@ -276,6 +291,9 @@ func (c *GlobalConfig) CleanupPaths() error { &c.Common.WorkingDir, } for _, k := range CommonCleanup { + if *k == "" { + continue + } *k, err = filepath.Abs(*k) if err != nil { return errors.Wrap(err, "failed to clean path") @@ -288,6 +306,9 @@ func (c *GlobalConfig) CleanupPaths() error { &c.Crowdsec.AcquisitionFilePath, } for _, k := range crowdsecCleanup { + if *k == "" { + continue + } *k, err = filepath.Abs(*k) if err != nil { return errors.Wrap(err, "failed to clean path") @@ -304,6 +325,9 @@ func (c *GlobalConfig) CleanupPaths() error { &c.ConfigPaths.SimulationFilePath, } for _, k := range configPathsCleanup { + if *k == "" { + continue + } *k, err = filepath.Abs(*k) if err != nil { return errors.Wrap(err, "failed to clean path") diff --git a/pkg/csconfig/crowdsec_service.go b/pkg/csconfig/crowdsec_service.go index 9455a77a7..e9ff3f08b 100644 --- a/pkg/csconfig/crowdsec_service.go +++ b/pkg/csconfig/crowdsec_service.go @@ -2,7 +2,10 @@ package csconfig /*Configurations needed for crowdsec to load parser/scenarios/... + acquisition*/ type CrowdsecServiceCfg struct { - AcquisitionFilePath string `yaml:"acquisition_path,omitempty"` + AcquisitionFilePath string `yaml:"acquisition_path,omitempty"` + AcquisitionDirPath string `yaml:"acquisition_dir,omitempty"` + + AcquisitionFiles []string `yaml:"-"` ParserRoutinesCount int `yaml:"parser_routines"` BucketsRoutinesCount int `yaml:"buckets_routines"` OutputRoutinesCount int `yaml:"output_routines"` diff --git a/pkg/csconfig/tests/acquis.yaml b/pkg/csconfig/tests/acquis.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/csconfig/tests/config.yaml b/pkg/csconfig/tests/config.yaml index a8833f541..ffa993b77 100644 --- a/pkg/csconfig/tests/config.yaml +++ b/pkg/csconfig/tests/config.yaml @@ -8,7 +8,7 @@ prometheus: enabled: true level: full crowdsec_service: -# acquisition_path: ./config/acquis.yaml + acquisition_path: ./tests/acquis.yaml parser_routines: 1 cscli: output: human