diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 2a09142a0..1edd59a56 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -6,7 +6,6 @@ import ( "os" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" - file_acquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/pkg/errors" @@ -42,6 +41,8 @@ var ReaderHits = prometheus.NewCounterVec( filenames: - "/var/log/nginx/*.log" ``` + + !!! how to handle expect mode that is not directly linked to tail/cat mode */ /* Approach @@ -68,30 +69,30 @@ type DataSource interface { OneShotAcquisition(chan types.Event, *tomb.Tomb) error // Start one shot acquisition(eg, cat a file) LiveAcquisition(chan types.Event, *tomb.Tomb) error // Start live acquisition (eg, tail a file) CanRun() error // Whether the datasource can run or not (eg, journalctl on BSD is a non-sense) + Dump() interface{} + New() DataSource } var AcquisitionSources = []struct { name string - iface DataSource + iface func() DataSource }{ - { - name: "file", - iface: &file_acquisition.FileSource{}, - }, + // { + // name: "file", + // iface: &file_acquisition.FileSource{}, + // }, } func GetDataSourceIface(dataSourceType string) DataSource { for _, source := range AcquisitionSources { if source.name == dataSourceType { - newsrc := source.iface - return newsrc + return source.iface.New() } } return nil } func DataSourceConfigure(yamlConfig []byte, commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) { - if dataSrc := GetDataSourceIface(commonConfig.Type); dataSrc != nil { /* this logger will then be used by the datasource at runtime */ clog := log.New() @@ -104,7 +105,6 @@ func DataSourceConfigure(yamlConfig []byte, commonConfig configuration.DataSourc subLogger := clog.WithFields(log.Fields{ "type": commonConfig.Type, }) - /* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */ if err := dataSrc.CanRun(); err != nil { return nil, errors.Wrapf(err, "datasource %s cannot be run", commonConfig.Type) @@ -135,16 +135,15 @@ func DataSourceConfigure(yamlConfig []byte, commonConfig configuration.DataSourc func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, error) { var sources []DataSource - var acquisSources = config.AcquisitionFiles - for _, acquisFile := range acquisSources { + for _, acquisFile := range config.AcquisitionFiles { log.Infof("loading acquisition file : %s", acquisFile) yamlFile, err := os.Open(acquisFile) if err != nil { return nil, errors.Wrapf(err, "can't open %s", acquisFile) } dec := yaml.NewDecoder(yamlFile) - dec.SetStrict(false) + dec.SetStrict(true) for { var sub configuration.DataSourceCommonCfg var holder interface{} @@ -156,6 +155,20 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, } return nil, errors.Wrapf(err, "failed to yaml decode %s", sub.ConfigFile) } + log.Printf("%s -> %T", acquisFile, holder) + + switch holder.(type) { + case map[interface{}]interface{}: + //leftover empty item + if len(holder.(map[interface{}]interface{})) == 0 { + log.Printf("leftover empty item") + break + } + } + //the user let a trailing `---` at the end of the file, and the last item is empty + // if len(holder.(map[interface{}]interface{})) == 0 { + // continue + // } //we dump it back to []byte, because we want to decode the yaml blob twice : //once to DataSourceCommonCfg, and then later to the dedicated type of the datasource inBytes, err := yaml.Marshal(holder) @@ -176,7 +189,7 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, sub.Mode = configuration.TAIL_MODE } if GetDataSourceIface(sub.Type) == nil { - log.Errorf("unknown data source %s in %s", sub.Type, sub.ConfigFile) + return nil, fmt.Errorf("unknown data source %s in %s", sub.Type, sub.ConfigFile) } src, err := DataSourceConfigure(inBytes, sub) diff --git a/pkg/acquisition/acquisition_test.go b/pkg/acquisition/acquisition_test.go index e05cf32a5..111d1d873 100644 --- a/pkg/acquisition/acquisition_test.go +++ b/pkg/acquisition/acquisition_test.go @@ -2,33 +2,29 @@ package acquisition import ( "fmt" + "strings" "testing" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/davecgh/go-spew/spew" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" tomb "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" + "gotest.tools/v3/assert" ) type MockSource struct { - configuration.DataSourceCommonCfg - logger *log.Entry `yaml:"-"` - Toto string `yaml:"toto"` + configuration.DataSourceCommonCfg `yaml:",inline"` + Toto string `yaml:"toto"` + logger *log.Entry } -func (f *MockSource) GetMetrics() []prometheus.Collector { - return nil -} func (f *MockSource) Configure(cfg []byte, logger *log.Entry) error { f.logger = logger - f.logger.Infof("config called, bytes: %s", string(cfg)) - f.logger.Tracef("this is trace!") - - if err := yaml.Unmarshal(cfg, f); err != nil { + if err := yaml.UnmarshalStrict(cfg, &f); err != nil { return errors.Wrap(err, "while unmarshaling to reader specific config") } if f.Toto == "" { @@ -36,24 +32,22 @@ func (f *MockSource) Configure(cfg []byte, logger *log.Entry) error { } return nil } -func (f *MockSource) GetMode() string { - f.logger.Debugf("called") - return f.Mode -} -func (f *MockSource) SupportedModes() []string { - return []string{"tail", "cat"} -} -func (f *MockSource) OneShotAcquisition(chan types.Event, *tomb.Tomb) error { - return nil +func (f *MockSource) GetMode() string { return f.Mode } +func (f *MockSource) SupportedModes() []string { return []string{"tail", "cat"} } +func (f *MockSource) OneShotAcquisition(chan types.Event, *tomb.Tomb) error { return nil } +func (f *MockSource) LiveAcquisition(chan types.Event, *tomb.Tomb) error { return nil } +func (f *MockSource) CanRun() error { return nil } +func (f *MockSource) GetMetrics() []prometheus.Collector { return nil } +func (f *MockSource) Dump() interface{} { return f } +func (f *MockSource) New() DataSource { return &MockSource{} } + +//copy the mocksource, but this one can't run +type MockSourceCantRun struct { + MockSource } -func (f *MockSource) LiveAcquisition(chan types.Event, *tomb.Tomb) error { - return nil -} - -func (f *MockSource) CanRun() error { - return nil -} +func (f *MockSourceCantRun) CanRun() error { return fmt.Errorf("can't run bro") } +func (f *MockSourceCantRun) New() DataSource { return &MockSourceCantRun{} } //appendMockSource is only used to add mock source for tests func appendMockSource() { @@ -64,45 +58,197 @@ func appendMockSource() { }{name: "mock", iface: &MockSource{}} AcquisitionSources = append(AcquisitionSources, mock) } + if GetDataSourceIface("mock_cant_run") == nil { + mock := struct { + name string + iface DataSource + }{name: "mock_cant_run", iface: &MockSourceCantRun{}} + AcquisitionSources = append(AcquisitionSources, mock) + } } -func TestLoadAcquisition(t *testing.T) { - +func TestDataSourceConfigure(t *testing.T) { appendMockSource() - - config := []byte( - ` + tests := []struct { + TestName string + RawBytes []byte + ExpectedError string + }{ + { + TestName: "basic_valid_config", + RawBytes: []byte(` +mode: cat +labels: + test: foobar +log_level: info +type: mock +toto: test_value1 +`), + }, + { + TestName: "basic_debug_config", + RawBytes: []byte(` +mode: cat +labels: + test: foobar +log_level: debug +type: mock +toto: test_value1 +`), + }, + { + TestName: "basic_tailmode_config", + RawBytes: []byte(` mode: tail labels: - toto: tutu + test: foobar +log_level: debug type: mock -toto: foobar -log_level: trace -`) - var generic interface{} - var common configuration.DataSourceCommonCfg - - if err := yaml.Unmarshal(config, &generic); err != nil { - t.Fatalf("failed to unmarshal %s : %s", config, err) +toto: test_value1 +`), + }, + { + TestName: "bad_mode_config", + RawBytes: []byte(` +mode: ratata +labels: + test: foobar +log_level: debug +type: mock +toto: test_value1 +`), + ExpectedError: "ratata mode is not supported by mock", + }, + { + TestName: "bad_type_config", + RawBytes: []byte(` +mode: ratata +labels: + test: foobar +log_level: debug +type: tutu +`), + ExpectedError: "cannot find source tutu", + }, + { + TestName: "mismatch_config", + RawBytes: []byte(` +mode: cat +labels: + test: foobar +log_level: debug +type: mock +wowo: ajsajasjas +`), + ExpectedError: "field wowo not found in type acquisition.MockSource", + }, + { + TestName: "cant_run_error", + RawBytes: []byte(` +mode: cat +labels: + test: foobar +log_level: debug +type: mock_cant_run +wowo: ajsajasjas +`), + ExpectedError: "datasource mock_cant_run cannot be run: can't run bro", + }, } - outBytes, err := yaml.Marshal(generic) - if err != nil { - t.Fatal(err) + for _, test := range tests { + common := configuration.DataSourceCommonCfg{} + yaml.Unmarshal(test.RawBytes, &common) + ds, err := DataSourceConfigure(test.RawBytes, common) + if test.ExpectedError != "" { + if err == nil { + t.Fatalf("expected error %s, got none", test.ExpectedError) + } + if !strings.Contains(err.Error(), test.ExpectedError) { + t.Fatalf("%s : expected error '%s' in '%s'", test.TestName, test.ExpectedError, err.Error()) + } else { + continue + } + } else { + if err != nil { + t.Fatalf("%s : unexpected error '%s'", test.TestName, err) + } + } + + switch test.TestName { + case "basic_valid_config": + mock := (*ds).Dump().(*MockSource) + assert.Equal(t, mock.Toto, "test_value1") + assert.Equal(t, mock.Mode, "cat") + assert.Equal(t, mock.logger.Logger.Level, log.InfoLevel) + assert.DeepEqual(t, mock.Labels, map[string]string{"test": "foobar"}) + case "basic_debug_config": + mock := (*ds).Dump().(*MockSource) + assert.Equal(t, mock.Toto, "test_value1") + assert.Equal(t, mock.Mode, "cat") + assert.Equal(t, mock.logger.Logger.Level, log.DebugLevel) + assert.DeepEqual(t, mock.Labels, map[string]string{"test": "foobar"}) + case "basic_tailmode_config": + mock := (*ds).Dump().(*MockSource) + assert.Equal(t, mock.Toto, "test_value1") + assert.Equal(t, mock.Mode, "tail") + assert.Equal(t, mock.logger.Logger.Level, log.DebugLevel) + assert.DeepEqual(t, mock.Labels, map[string]string{"test": "foobar"}) + } + } +} + +func TestLoadAcquisitionFromFile(t *testing.T) { + appendMockSource() + tests := []struct { + TestName string + Config csconfig.CrowdsecServiceCfg + ExpectedError string + ExpectedLen int + }{ + { + TestName: "non_existent_file", + Config: csconfig.CrowdsecServiceCfg{ + AcquisitionFiles: []string{"does_not_exist"}, + }, + ExpectedError: "can't open does_not_exist", + ExpectedLen: 0, + }, + { + TestName: "invalid_yaml_file", + Config: csconfig.CrowdsecServiceCfg{ + AcquisitionFiles: []string{"test_files/badyaml.yaml"}, + }, + ExpectedError: "configuration isn't valid config in test_files/badyaml.yaml", + ExpectedLen: 0, + }, + { + TestName: "invalid_empty_yaml", + Config: csconfig.CrowdsecServiceCfg{ + AcquisitionFiles: []string{"test_files/emptyitem.yaml"}, + }, + ExpectedLen: 0, + }, + } + for _, test := range tests { + dss, err := LoadAcquisitionFromFile(&test.Config) + if test.ExpectedError != "" { + if err == nil { + t.Fatalf("expected error %s, got none", test.ExpectedError) + } + if !strings.Contains(err.Error(), test.ExpectedError) { + t.Fatalf("%s : expected error '%s' in '%s'", test.TestName, test.ExpectedError, err.Error()) + } else { + continue + } + } else { + if err != nil { + t.Fatalf("%s : unexpected error '%s'", test.TestName, err) + } + } + if len(dss) != test.ExpectedLen { + t.Fatalf("%s : expected %d datasources got %d", test.TestName, test.ExpectedLen, len(dss)) + } + } - log.Printf("-> %s", outBytes) - - if err := yaml.Unmarshal(outBytes, &common); err != nil { - t.Fatalf("cannot unmarshal to generic : %s", err) - } - - // if generic == nil { - // t.Fatalf("result of unmarshal is empty :(") - // } - // log.Printf("raw iface : %s", spew.Sdump(generic)) - // common = generic.(configuration.DataSourceCommonCfg) - - ds, err := DataSourceConfigure(outBytes, common) - log.Printf("-> ds : %s", spew.Sdump(ds)) - log.Printf("-> err : %s", err) } diff --git a/pkg/acquisition/configuration/configuration.go b/pkg/acquisition/configuration/configuration.go index b21532c21..25a6c2533 100644 --- a/pkg/acquisition/configuration/configuration.go +++ b/pkg/acquisition/configuration/configuration.go @@ -10,6 +10,7 @@ type DataSourceCommonCfg struct { LogLevel *log.Level `yaml:"log_level,omitempty"` Type string `yaml:"type,omitempty"` ConfigFile string `yaml:"-"` //filled at run time : the filepath from which the config was unmarshaled + //logger *log.Entry `yaml:"-"` } var TAIL_MODE = "tail" diff --git a/pkg/acquisition/datasourcemap.go b/pkg/acquisition/datasourcemap.go deleted file mode 100644 index 0895452d2..000000000 --- a/pkg/acquisition/datasourcemap.go +++ /dev/null @@ -1,17 +0,0 @@ -package acquisition - -// import file_acquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file" - -// type DataSourceMap struct { -// Datasources map[string]DataSource -// } - -// func (d *DataSourceMap) GetDataSource(dstype string) *DataSource { -// datasource := d.Datasources[dstype] -// return &datasource -// } - -// func (d *DataSourceMap) New() { -// m := make(map[string]DataSource) -// m["file"] = &file_acquisition.FileSource{} -// }