non-fatal error if some datasource can't be run (i.e. journalctl but systemd is missing) (#2310)

This on the other hand, gives a new fatal error when there are no valid datasources.
In the previous version, crowdsec kept running with just a warning if no
acquisition yaml or dir were specified.

cmd/crowdsec/crowdsec.go

@@ -23,22 +23,22 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
 	var err error
 
 	// Populate cwhub package tools
-	if err := cwhub.GetHubIdx(cConfig.Hub); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err)
+	if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
+		return &parser.Parsers{}, fmt.Errorf("while loading hub index : %s", err)
 	}
 
 	// Start loading configs
 	csParsers := newParsers()
 	if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err)
+		return nil, fmt.Errorf("while loading parsers: %s", err)
 	}
 
 	if err := LoadBuckets(cConfig); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err)
+		return nil, fmt.Errorf("while loading scenarios: %s", err)
 	}
 
 	if err := LoadAcquisition(cConfig); err != nil {
-		return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err)
+		return nil, fmt.Errorf("while loading acquisition config: %s", err)
 	}
 	return csParsers, nil
 }

cmd/crowdsec/main.go

@@ -157,6 +157,10 @@ func LoadAcquisition(cConfig *csconfig.Config) error {
 		}
 	}
 
+	if len(dataSources) == 0 {
+		return fmt.Errorf("no datasource enabled")
+	}
+
 	return nil
 }
 

pkg/acquisition/acquisition.go

@@ -25,6 +25,20 @@ import (
 	tomb "gopkg.in/tomb.v2"
 )
 
+type DataSourceUnavailableError struct {
+	Name string
+	Err error
+}
+
+func (e *DataSourceUnavailableError) Error() string {
+	return fmt.Sprintf("datasource '%s' is not available: %v", e.Name, e.Err)
+}
+
+func (e *DataSourceUnavailableError) Unwrap() error {
+	return e.Err
+}
+
+
 // The interface each datasource must implement
 type DataSource interface {
 	GetMetrics() []prometheus.Collector                         // Returns pointers to metrics that are managed by the module
@@ -86,8 +100,11 @@ func GetDataSourceIface(dataSourceType string) DataSource {
 	return nil
 }
 
+// DataSourceConfigure creates and returns a DataSource object from a configuration,
+// if the configuration is not valid it returns an error.
+// If the datasource can't be run (eg. journalctl not available), it still returns an error which
+// can be checked for the appropriate action.
 func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) {
-
 	//we dump it back to []byte, because we want to decode the yaml blob twice :
 	//once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
 	yamlConfig, err := yaml.Marshal(commonConfig)
@@ -112,7 +129,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS
 		subLogger := clog.WithFields(customLog)
 		/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
 		if err := dataSrc.CanRun(); err != nil {
-			return nil, errors.Wrapf(err, "datasource %s cannot be run", commonConfig.Source)
+			return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err}
 		}
 		/* configure the actual datasource */
 		if err := dataSrc.Configure(yamlConfig, subLogger); err != nil {
@@ -179,10 +196,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
 		}
 		dec := yaml.NewDecoder(yamlFile)
 		dec.SetStrict(true)
+		idx := -1
 		for {
 			var sub configuration.DataSourceCommonCfg
-			var idx int
 			err = dec.Decode(&sub)
+			idx += 1
 			if err != nil {
 				if ! errors.Is(err, io.EOF) {
 					return nil, errors.Wrapf(err, "failed to yaml decode %s", acquisFile)
@@ -199,7 +217,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
 			if len(sub.Labels) == 0 {
 				if sub.Source == "" {
 					log.Debugf("skipping empty item in %s", acquisFile)
-					idx += 1
 					continue
 				}
 				return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx)
@@ -212,10 +229,14 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
 			}
 			src, err := DataSourceConfigure(sub)
 			if err != nil {
+				var dserr *DataSourceUnavailableError
+				if errors.As(err, &dserr) {
+					log.Error(err)
+					continue
+				}
 				return nil, errors.Wrapf(err, "while configuring datasource of type %s from %s (position: %d)", sub.Source, acquisFile, idx)
 			}
 			sources = append(sources, *src)
-			idx += 1
 		}
 	}
 	return sources, nil

pkg/acquisition/acquisition_test.go

@@ -171,7 +171,7 @@ log_level: debug
 source: mock_cant_run
 wowo: ajsajasjas
 `,
-			ExpectedError: "datasource mock_cant_run cannot be run: can't run bro",
+			ExpectedError: "datasource 'mock_cant_run' is not available: can't run bro",
 		},
 	}
 

tests/bats/01_crowdsec.bats

@@ -148,9 +148,10 @@ teardown() {
     rm -f "$ACQUIS_DIR"
 
     config_set '.common.log_media="stdout"'
-    run -124 --separate-stderr timeout 2s "${CROWDSEC}"
+    run -1 --separate-stderr timeout 2s "${CROWDSEC}"
     # check warning
-    assert_stderr_line --partial "no acquisition file found"
+    assert_stderr --partial "no acquisition file found"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 
 @test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" {
@@ -163,19 +164,55 @@ teardown() {
     config_set '.crowdsec_service.acquisition_dir=""'
 
     config_set '.common.log_media="stdout"'
-    run -124 --separate-stderr timeout 2s "${CROWDSEC}"
+    run -1 --separate-stderr timeout 2s "${CROWDSEC}"
     # check warning
-    assert_stderr_line --partial "no acquisition_path or acquisition_dir specified"
+    assert_stderr --partial "no acquisition_path or acquisition_dir specified"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 
 @test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
     ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
-    rm -f "$ACQUIS_YAML"
     config_set '.crowdsec_service.acquisition_path=""'
 
     ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
     mkdir -p "$ACQUIS_DIR"
-    touch "$ACQUIS_DIR"/foo.yaml
+    mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml
 
     run -124 --separate-stderr timeout 2s "${CROWDSEC}"
+
+    # now, if foo.yaml is empty instead, there won't be valid datasources.
+
+    cat /dev/null >"$ACQUIS_DIR"/foo.yaml
+
+    run --separate-stderr -1 timeout 2s "${CROWDSEC}"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
+
+@test "crowdsec (disabled datasources)" {
+    config_set '.common.log_media="stdout"'
+
+    # a datasource cannot run - missing journalctl command
+
+    ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
+    mkdir -p "$ACQUIS_DIR"
+    cat >"$ACQUIS_DIR"/foo.yaml <<-EOT
+	source: journalctl
+	journalctl_filter:
+	- "_SYSTEMD_UNIT=ssh.service"
+	labels:
+	  type: syslog
+	EOT
+
+    run --separate-stderr -124 timeout 2s env PATH='' "${CROWDSEC}"
+    #shellcheck disable=SC2016
+    assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH'
+
+    # if all datasources are disabled, crowdsec should exit
+
+    ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
+    rm -f "$ACQUIS_YAML"
+    config_set '.crowdsec_service.acquisition_path=""'
+
+    run --separate-stderr -1 timeout 2s env PATH='' "${CROWDSEC}"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
+ }

tests/bin/assert-crowdsec-not-running

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 is_crowdsec_running() {
-    PIDS=$(pgrep -x 'crowdsec|crowdsec.test|crowdsec.cover')
+    PIDS=$(pgrep -x 'crowdsec|crowdsec.test|crowdsec.cover' 2>/dev/null)
 }
 
 # The process can be slow, especially on CI and during test coverage.