From 7f42f94fe3de05ba5e7618d0af9e630c4441c5fa Mon Sep 17 00:00:00 2001 From: alteredCoder Date: Fri, 15 Apr 2022 15:53:34 +0200 Subject: [PATCH] finalize --- cmd/crowdsec-cli/console.go | 73 +++++++++++++++------------- config/config.yaml | 2 +- config/{labels.yaml => context.yaml} | 0 debian/rules | 2 +- pkg/csconfig/crowdsec_service.go | 32 ++++++------ pkg/leakybucket/manager_load.go | 2 +- pkg/leakybucket/overflows.go | 50 ++++++++++++++----- rpm/SPECS/crowdsec.spec | 4 +- tests/bats/81_alerts-context.bats | 2 +- tests/lib/config/config-local | 4 +- wizard.sh | 2 +- 11 files changed, 101 insertions(+), 72 deletions(-) rename config/{labels.yaml => context.yaml} (100%) diff --git a/cmd/crowdsec-cli/console.go b/cmd/crowdsec-cli/console.go index fd299dc04..b3ffecdd4 100644 --- a/cmd/crowdsec-cli/console.go +++ b/cmd/crowdsec-cli/console.go @@ -263,9 +263,9 @@ Disable given information push to the central API.`, cmdConsole.AddCommand(cmdConsoleStatus) - cmdLabel := &cobra.Command{ - Use: "label [feature-flag]", - Short: "Manage label to send with alerts", + cmdContext := &cobra.Command{ + Use: "context [feature-flag]", + Short: "Manage context to send with alerts", DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { printHelp(cmd) @@ -274,40 +274,40 @@ Disable given information push to the central API.`, var keyToAdd string var valuesToAdd []string - cmdLabelAdd := &cobra.Command{ + cmdContextAdd := &cobra.Command{ Use: "add", - Short: "Add label to send with alerts", + Short: "Add context to send with alerts", DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { - if _, ok := csConfig.Crowdsec.LabelsToSend[keyToAdd]; !ok { - csConfig.Crowdsec.LabelsToSend[keyToAdd] = make([]string, 0) + if _, ok := csConfig.Crowdsec.ContextToSend[keyToAdd]; !ok { + csConfig.Crowdsec.ContextToSend[keyToAdd] = make([]string, 0) log.Infof("key '%s' added", keyToAdd) } - data := csConfig.Crowdsec.LabelsToSend[keyToAdd] + data := csConfig.Crowdsec.ContextToSend[keyToAdd] for _, val := range valuesToAdd { if !inSlice(val, data) { log.Infof("value '%s' added to key '%s'", val, keyToAdd) data = append(data, val) } - csConfig.Crowdsec.LabelsToSend[keyToAdd] = data + csConfig.Crowdsec.ContextToSend[keyToAdd] = data } - if err := csConfig.Crowdsec.DumpLabelConfigFile(); err != nil { + if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil { log.Fatalf(err.Error()) } }, } - cmdLabelAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send") - cmdLabelAdd.Flags().StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key") - cmdLabelAdd.MarkFlagRequired("key") - cmdLabelAdd.MarkFlagRequired("value") - cmdLabel.AddCommand(cmdLabelAdd) + cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send") + cmdContextAdd.Flags().StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key") + cmdContextAdd.MarkFlagRequired("key") + cmdContextAdd.MarkFlagRequired("value") + cmdContext.AddCommand(cmdContextAdd) - cmdLabelStatus := &cobra.Command{ + cmdContextStatus := &cobra.Command{ Use: "status", - Short: "List label to send with alerts", + Short: "List context to send with alerts", DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { - dump, err := yaml.Marshal(csConfig.Crowdsec.LabelsToSend) + dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend) if err != nil { log.Fatalf("unable to show labels status: %s", err) } @@ -315,16 +315,21 @@ Disable given information push to the central API.`, }, } - cmdLabel.AddCommand(cmdLabelStatus) + cmdContext.AddCommand(cmdContextStatus) var detectAll bool - cmdLabelDetect := &cobra.Command{ + cmdContextDetect := &cobra.Command{ Use: "detect", Short: "Detect available fields from the installed parsers", DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { var err error + if !detectAll && len(args) == 0 { + log.Infof("Please provide parsers to detect or --all flag.") + printHelp(cmd) + } + // to avoid all the log.Info from the loaders functions log.SetLevel(log.ErrorLevel) @@ -392,12 +397,12 @@ Disable given information push to the central API.`, } }, } - cmdLabelDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser") - cmdLabel.AddCommand(cmdLabelDetect) + cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser") + cmdContext.AddCommand(cmdContextDetect) var keysToDelete []string var valuesToDelete []string - cmdLabelDelete := &cobra.Command{ + cmdContextDelete := &cobra.Command{ Use: "delete", Short: "List label to send with alerts", DisableAutoGenTag: true, @@ -407,8 +412,8 @@ Disable given information push to the central API.`, } for _, key := range keysToDelete { - if _, ok := csConfig.Crowdsec.LabelsToSend[key]; ok { - delete(csConfig.Crowdsec.LabelsToSend, key) + if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok { + delete(csConfig.Crowdsec.ContextToSend, key) log.Infof("key '%s' has been removed", key) } else { log.Warningf("key '%s' doesn't exist", key) @@ -417,14 +422,14 @@ Disable given information push to the central API.`, for _, value := range valuesToDelete { valueFound := false - for key, labels := range csConfig.Crowdsec.LabelsToSend { + for key, labels := range csConfig.Crowdsec.ContextToSend { if inSlice(value, labels) { valueFound = true - csConfig.Crowdsec.LabelsToSend[key] = removeFromSlice(value, labels) + csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, labels) log.Infof("value '%s' has been removed from key '%s'", value, key) } - if len(csConfig.Crowdsec.LabelsToSend[key]) == 0 { - delete(csConfig.Crowdsec.LabelsToSend, key) + if len(csConfig.Crowdsec.ContextToSend[key]) == 0 { + delete(csConfig.Crowdsec.ContextToSend, key) } } if !valueFound { @@ -432,17 +437,17 @@ Disable given information push to the central API.`, } } - if err := csConfig.Crowdsec.DumpLabelConfigFile(); err != nil { + if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil { log.Fatalf(err.Error()) } }, } - cmdLabelDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete") - cmdLabelDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete") - cmdLabel.AddCommand(cmdLabelDelete) + cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete") + cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete") + cmdContext.AddCommand(cmdContextDelete) - cmdConsole.AddCommand(cmdLabel) + cmdConsole.AddCommand(cmdContext) return cmdConsole } diff --git a/config/config.yaml b/config/config.yaml index 9fcaad1a9..188f7fed5 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -14,7 +14,7 @@ config_paths: notification_dir: /etc/crowdsec/notifications/ plugin_dir: /usr/local/lib/crowdsec/plugins/ crowdsec_service: - console_labels_path: /etc/crowdsec/console/labels.yaml + console_context_path: /etc/crowdsec/console/context.yaml acquisition_path: /etc/crowdsec/acquis.yaml parser_routines: 1 cscli: diff --git a/config/labels.yaml b/config/context.yaml similarity index 100% rename from config/labels.yaml rename to config/context.yaml diff --git a/debian/rules b/debian/rules index ca6823f24..435179bd7 100755 --- a/debian/rules +++ b/debian/rules @@ -46,6 +46,6 @@ override_dh_auto_install: cp config/simulation.yaml debian/crowdsec/etc/crowdsec/simulation.yaml cp config/profiles.yaml debian/crowdsec/etc/crowdsec/profiles.yaml cp config/console.yaml debian/crowdsec/etc/crowdsec/console.yaml - cp config/labels.yaml debian/crowdsec/etc/crowdsec/console/labels.yaml + cp config/context.yaml debian/crowdsec/etc/crowdsec/console/context.yaml cp -a config/patterns debian/crowdsec/etc/crowdsec diff --git a/pkg/csconfig/crowdsec_service.go b/pkg/csconfig/crowdsec_service.go index bca616739..c4c3733eb 100644 --- a/pkg/csconfig/crowdsec_service.go +++ b/pkg/csconfig/crowdsec_service.go @@ -15,7 +15,7 @@ import ( type CrowdsecServiceCfg struct { AcquisitionFilePath string `yaml:"acquisition_path,omitempty"` AcquisitionDirPath string `yaml:"acquisition_dir,omitempty"` - ConsoleLabelsPath string `yaml:"console_labels_path"` + ConsoleContextPath string `yaml:"console_context_path"` AcquisitionFiles []string `yaml:"-"` ParserRoutinesCount int `yaml:"parser_routines"` BucketsRoutinesCount int `yaml:"buckets_routines"` @@ -31,10 +31,10 @@ type CrowdsecServiceCfg struct { ConfigDir string `yaml:"-"` HubIndexFile string `yaml:"-"` SimulationFilePath string `yaml:"-"` - LabelsToSend map[string][]string `yaml:"-"` + ContextToSend map[string][]string `yaml:"-"` } -var DefaultLabelsConfigFilePath = DefaultConfigPath("console", "labels.yaml") +var DefaultContextConfigFilePath = DefaultConfigPath("console", "context.yaml") func (c *Config) LoadCrowdsec() error { var err error @@ -94,17 +94,17 @@ func (c *Config) LoadCrowdsec() error { if c.Crowdsec.OutputRoutinesCount <= 0 { c.Crowdsec.OutputRoutinesCount = 1 } - if c.Crowdsec.ConsoleLabelsPath == "" { - c.Crowdsec.ConsoleLabelsPath = DefaultLabelsConfigFilePath + if c.Crowdsec.ConsoleContextPath == "" { + c.Crowdsec.ConsoleContextPath = DefaultContextConfigFilePath } - yamlFile, err := ioutil.ReadFile(c.Crowdsec.ConsoleLabelsPath) + yamlFile, err := ioutil.ReadFile(c.Crowdsec.ConsoleContextPath) if err != nil { - return fmt.Errorf("reading console label file '%s': %s", c.Crowdsec.ConsoleLabelsPath, err) + return fmt.Errorf("reading console label file '%s': %s", c.Crowdsec.ConsoleContextPath, err) } - c.Crowdsec.LabelsToSend = make(map[string][]string, 0) - err = yaml.Unmarshal(yamlFile, c.Crowdsec.LabelsToSend) + c.Crowdsec.ContextToSend = make(map[string][]string, 0) + err = yaml.Unmarshal(yamlFile, c.Crowdsec.ContextToSend) if err != nil { - return fmt.Errorf("unmarshaling labels console config file '%s': %s", DefaultLabelsConfigFilePath, err) + return fmt.Errorf("unmarshaling labels console config file '%s': %s", DefaultContextConfigFilePath, err) } var crowdsecCleanup = []*string{ @@ -136,19 +136,19 @@ func (c *Config) LoadCrowdsec() error { return nil } -func (c *CrowdsecServiceCfg) DumpLabelConfigFile() error { +func (c *CrowdsecServiceCfg) DumpContextConfigFile() error { var out []byte var err error - if out, err = yaml.Marshal(c.LabelsToSend); err != nil { - return errors.Wrapf(err, "while marshaling ConsoleConfig (for %s)", DefaultLabelsConfigFilePath) + if out, err = yaml.Marshal(c.ContextToSend); err != nil { + return errors.Wrapf(err, "while marshaling ConsoleConfig (for %s)", DefaultContextConfigFilePath) } - if err := os.WriteFile(c.ConsoleLabelsPath, out, 0600); err != nil { - return errors.Wrapf(err, "while dumping console config to %s", DefaultLabelsConfigFilePath) + if err := os.WriteFile(c.ConsoleContextPath, out, 0600); err != nil { + return errors.Wrapf(err, "while dumping console config to %s", DefaultContextConfigFilePath) } - log.Infof("%s file saved", c.ConsoleLabelsPath) + log.Infof("%s file saved", c.ConsoleContextPath) return nil } diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index d9a51105b..9c46838fe 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -218,7 +218,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb. bucketFactory.wgDumpState = buckets.wgDumpState bucketFactory.wgPour = buckets.wgPour - bucketFactory.LabelsToSend = cscfg.LabelsToSend + bucketFactory.LabelsToSend = cscfg.ContextToSend err = LoadBucket(&bucketFactory, tomb) if err != nil { log.Errorf("Failed to load bucket %s : %v", bucketFactory.Name, err) diff --git a/pkg/leakybucket/overflows.go b/pkg/leakybucket/overflows.go index 2e3a5bfe6..25dba7bad 100644 --- a/pkg/leakybucket/overflows.go +++ b/pkg/leakybucket/overflows.go @@ -19,6 +19,10 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" ) +const ( + maxContextValueLen = 4000 +) + //SourceFromEvent extracts and formats a valid models.Source object from an Event func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, error) { srcs := make(map[string]models.Source) @@ -234,13 +238,34 @@ func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, st return sources, source_type, nil } -func EventToLabel(labels map[string][]*vm.Program, queue *Queue) models.Meta { +func truncate(values []string) (string, error) { + var ret string + valueByte, err := json.Marshal(values) + if err != nil { + return "", fmt.Errorf("unable to dump metas: %s", err) + } + ret = string(valueByte) + for { + if len(ret) <= maxContextValueLen { + break + } + values = values[:len(values)-1] + valueByte, err = json.Marshal(values) + if err != nil { + return "", fmt.Errorf("unable to dump metas: %s", err) + } + ret = string(valueByte) + } + return ret, nil +} + +func EventToContext(labels map[string][]*vm.Program, queue *Queue) models.Meta { metas := make([]*models.MetaItems0, 0) - tmpLabels := make(map[string][]string) + tmpContext := make(map[string][]string) for _, evt := range queue.Queue { for key, values := range labels { - if _, ok := tmpLabels[key]; !ok { - tmpLabels[key] = make([]string, 0) + if _, ok := tmpContext[key]; !ok { + tmpContext[key] = make([]string, 0) } for _, value := range values { var val string @@ -255,27 +280,26 @@ func EventToLabel(labels map[string][]*vm.Program, queue *Queue) models.Meta { case int: val = strconv.Itoa(out) default: - log.Warningf("unexpected return type for label to send : %T", output) + log.Warningf("unexpected return type for context to send : %T", output) continue } - if val != "" && !types.InSlice(val, tmpLabels[key]) { - tmpLabels[key] = append(tmpLabels[key], val) + if val != "" && !types.InSlice(val, tmpContext[key]) { + tmpContext[key] = append(tmpContext[key], val) } } } } - for key, values := range tmpLabels { + for key, values := range tmpContext { if len(values) == 0 { continue } - valueByte, err := json.Marshal(values) + valueStr, err := truncate(values) if err != nil { - log.Warningf("unable to dump metas: %s", err) - continue + log.Warningf(err.Error()) } meta := models.MetaItems0{ Key: key, - Value: string(valueByte), + Value: valueStr, } metas = append(metas, &meta) } @@ -345,7 +369,7 @@ func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) { *apiAlert.Message = fmt.Sprintf("%s %s performed '%s' (%d events over %s) at %s", source_scope, sourceStr, leaky.Name, leaky.Total_count, leaky.Ovflw_ts.Sub(leaky.First_ts), leaky.Last_ts) //Get the events from Leaky/Queue apiAlert.Events = EventsFromQueue(queue) - apiAlert.Meta = EventToLabel(leaky.LabelsToSend, leaky.Queue) + apiAlert.Meta = EventToContext(leaky.LabelsToSend, leaky.Queue) //Loop over the Sources and generate appropriate number of ApiAlerts for _, srcValue := range sources { diff --git a/rpm/SPECS/crowdsec.spec b/rpm/SPECS/crowdsec.spec index 0e0fad343..5827c29ef 100644 --- a/rpm/SPECS/crowdsec.spec +++ b/rpm/SPECS/crowdsec.spec @@ -63,7 +63,7 @@ install -m 644 -D config/config.yaml %{buildroot}%{_sysconfdir}/crowdsec install -m 644 -D config/simulation.yaml %{buildroot}%{_sysconfdir}/crowdsec install -m 644 -D config/profiles.yaml %{buildroot}%{_sysconfdir}/crowdsec install -m 644 -D config/console.yaml %{buildroot}%{_sysconfdir}/crowdsec -install -m 644 -D config/console.yaml %{buildroot}%{_sysconfdir}/crowdsec/console/ +install -m 644 -D config/context.yaml %{buildroot}%{_sysconfdir}/crowdsec/console/ install -m 644 -D %{SOURCE1} %{buildroot}%{_presetdir} install -m 551 plugins/notifications/slack/notification-slack %{buildroot}%{_libdir}/%{name}/plugins/ @@ -116,7 +116,7 @@ rm -rf %{buildroot} %config(noreplace) %{_sysconfdir}/%{name}/simulation.yaml %config(noreplace) %{_sysconfdir}/%{name}/profiles.yaml %config(noreplace) %{_sysconfdir}/%{name}/console.yaml -%config(noreplace) %{_sysconfdir}/%{name}/console/labels.yaml +%config(noreplace) %{_sysconfdir}/%{name}/console/context.yaml %config(noreplace) %{_presetdir}/80-%{name}.preset %config(noreplace) %{_sysconfdir}/%{name}/notifications/http.yaml %config(noreplace) %{_sysconfdir}/%{name}/notifications/slack.yaml diff --git a/tests/bats/81_alerts-context.bats b/tests/bats/81_alerts-context.bats index 882a67c51..a79f41a0b 100644 --- a/tests/bats/81_alerts-context.bats +++ b/tests/bats/81_alerts-context.bats @@ -35,7 +35,7 @@ teardown() { ACQUIS_YAML=$(config_yq '.crowdsec_service.acquisition_path') echo -e "---\nfilename: $tmpfile\nlabels:\n type: syslog\n" >>"${ACQUIS_YAML}" - CONTEXT_YAML=$(config_yq '.crowdsec_service.console_labels_path') + CONTEXT_YAML=$(config_yq '.crowdsec_service.console_context_path') echo -e "---\ntarget_user:\n- evt.Parsed.sshd_invalid_user\nsource_ip:\n- evt.Parsed.sshd_client_ip\nsource_host:\n- evt.Meta.machine\n" >>"${CONTEXT_YAML}" ./instance-crowdsec start diff --git a/tests/lib/config/config-local b/tests/lib/config/config-local index cbaf67703..6294f001c 100755 --- a/tests/lib/config/config-local +++ b/tests/lib/config/config-local @@ -53,7 +53,7 @@ config_generate() { ../config/online_api_credentials.yaml \ "${CONFIG_DIR}/" - cp ../config/labels.yaml "${CONFIG_DIR}/console/" + cp ../config/context.yaml "${CONFIG_DIR}/console/" cp ../plugins/notifications/*/{http,email,slack,splunk,dummy}.yaml \ "${CONFIG_DIR}/notifications/" @@ -75,7 +75,7 @@ config_generate() { .api.client.credentials_path=strenv(CONFIG_DIR)+"/local_api_credentials.yaml" | .api.server.profiles_path=strenv(CONFIG_DIR)+"/profiles.yaml" | .api.server.console_path=strenv(CONFIG_DIR)+"/console.yaml" | - .crowdsec_service.console_labels_path=strenv(CONFIG_DIR) + "/console/labels.yaml" | + .crowdsec_service.console_context_path=strenv(CONFIG_DIR) + "/console/context.yaml" | .api.server.online_client.credentials_path=strenv(CONFIG_DIR)+"/online_api_credentials.yaml" ' <../config/config.yaml >"${CONFIG_DIR}/config.yaml" } diff --git a/wizard.sh b/wizard.sh index 141325349..2819d79b3 100755 --- a/wizard.sh +++ b/wizard.sh @@ -412,7 +412,7 @@ install_crowdsec() { install -v -m 644 -D ./config/profiles.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit install -v -m 644 -D ./config/simulation.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit install -v -m 644 -D ./config/"${CONSOLE_FILE}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit - install -v -m 644 -D ./config/labels.yaml "${CROWDSEC_CONSOLE_DIR}" 1> /dev/null || exit + install -v -m 644 -D ./config/context.yaml "${CROWDSEC_CONSOLE_DIR}" 1> /dev/null || exit DATA=${CROWDSEC_DATA_DIR} CFG=${CROWDSEC_CONFIG_PATH} envsubst '$CFG $DATA' < ./config/user.yaml > ${CROWDSEC_CONFIG_PATH}"/user.yaml" || log_fatal "unable to generate user configuration file" if [[ ${DOCKER_MODE} == "false" ]]; then